1 char_separator()

1 char_separator(// 不保留的分隔符
2                const Char* dropped_delims,
3                // 保留的分隔符
4                const Char* kept_delims = 0, 
5                // 默认不保留空格分隔符, 反之加上改参数keep_empty_tokens
6                empty_token_policy empty_tokens = drop_empty_tokens) 

1 explicit escaped_list_separator(Char e = ‘\\‘, Char c = ‘,‘,Char q = ‘\"‘); 

1 escaped_list_separator(string_type e, string_type c, string_type q):

1 template<typename Iter>
2   offset_separator(Iter begin,Iter end,bool bwrapoffsets = true, bool breturnpartiallast = true);

 1 void test_string_tokenizer()
 2 {
 3     using namespace boost;
 4 
 5     // 1. 使用缺省模板参数创建分词对象, 默认把所有的空格和标点作为分隔符. 
 6     {
 7         std::string str("Link raise the master-sword.");
 8 
 9         tokenizer<> tok(str);
10         for (BOOST_AUTO(pos, tok.begin()); pos != tok.end(); ++pos)
11             std::cout << "[" << *pos << "]";
12         std::cout << std::endl;
13         // [Link][raise][the][master][sword]
14     }
15 
16     // 2. char_separator()
17     {
18         std::string str("Link raise the master-sword.");
19 
20         // 一个char_separator对象, 默认构造函数(保留标点但将它看作分隔符)
21         char_separator<char> sep;
22         tokenizer<char_separator<char> > tok(str, sep);
23         for (BOOST_AUTO(pos, tok.begin()); pos != tok.end(); ++pos)
24             std::cout << "[" << *pos << "]";
25         std::cout << std::endl;
26         // [Link][raise][the][master][-][sword][.]
27     }
28 
29     // 3. char_separator(const Char* dropped_delims,
30     //                   const Char* kept_delims = 0, 
31     //                   empty_token_policy empty_tokens = drop_empty_tokens)
32     {
33         std::string str = ";!!;Hello|world||-foo--bar;yow;baz|";
34 
35         char_separator<char> sep1("-;|");
36         tokenizer<char_separator<char> > tok1(str, sep1);
37         for (BOOST_AUTO(pos, tok1.begin()); pos != tok1.end(); ++pos)
38             std::cout << "[" << *pos << "]";
39         std::cout << std::endl;
40         // [!!][Hello][world][foo][bar][yow][baz]
41 
42         char_separator<char> sep2("-;", "|", keep_empty_tokens);
43         tokenizer<char_separator<char> > tok2(str, sep2);
44         for (BOOST_AUTO(pos, tok2.begin()); pos != tok2.end(); ++pos)
45             std::cout << "[" << *pos << "]";
46         std::cout << std::endl;
47         // [][!!][Hello][|][world][|][][|][][foo][][bar][yow][baz][|][]
48     }
49 
50     // 4. escaped_list_separator
51     {
52         std::string str = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
53 
54         tokenizer<escaped_list_separator<char> > tok(str);
55         for (BOOST_AUTO(pos, tok.begin()); pos != tok.end(); ++pos)
56             std::cout << "[" << *pos << "]";
57         std::cout << std::endl;
58         // [Field 1][putting quotes around fields, allows commas][Field 3]
59         // 引号内的逗号不可做为分隔符.
60     }
61     
62     // 5. offset_separator
63     {
64         std::string str = "12252001400";
65 
66         int offsets[] = {2, 2, 4};
67         offset_separator f(offsets, offsets + 3);
68         tokenizer<offset_separator> tok(str, f);
69 
70         for (BOOST_AUTO(pos, tok.begin()); pos != tok.end(); ++pos)
71             std::cout << "[" << *pos << "]";
72         std::cout << std::endl;
73     }
74 }