123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150 |
- // Boost tokenizer examples -------------------------------------------------//
- // (c) Copyright John R. Bandela 2001.
- // Distributed under the Boost Software License, Version 1.0. (See
- // accompanying file LICENSE_1_0.txt or copy at
- // http://www.boost.org/LICENSE_1_0.txt)
- // See http://www.boost.org for updates, documentation, and revision history.
- #include <iostream>
- #include <iterator>
- #include <string>
- #include <algorithm>
- #include <boost/tokenizer.hpp>
- #include <boost/array.hpp>
- #include <boost/test/minimal.hpp>
- int test_main( int /*argc*/, char* /*argv*/[] )
- {
- using namespace boost;
- // Use tokenizer
- {
- const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
- std::string answer[] = { "Hello", "world", "foo", "bar", "yow", "baz" };
- typedef tokenizer<char_separator<char> > Tok;
- char_separator<char> sep("-;|");
- Tok t(test_string, sep);
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- {
- const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
- std::string answer[] = { "", "", "Hello", "|", "world", "|", "", "|", "",
- "foo", "", "bar", "yow", "baz", "|", "" };
- typedef tokenizer<char_separator<char> > Tok;
- char_separator<char> sep("-;", "|", boost::keep_empty_tokens);
- Tok t(test_string, sep);
- BOOST_REQUIRE(std::equal(t.begin(), t.end(), answer));
- }
- {
- const std::string test_string = "This,,is, a.test..";
- std::string answer[] = {"This","is","a","test"};
- typedef tokenizer<> Tok;
- Tok t(test_string);
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- {
- const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
- std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
- typedef tokenizer<escaped_list_separator<char> > Tok;
- Tok t(test_string);
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- {
- const std::string test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";
- std::string answer[] = {"","1","","2\"","3;","4","5\\","6,7",""};
- typedef tokenizer<escaped_list_separator<char> > Tok;
- escaped_list_separator<char> sep("\\^",",;","\"\'");
- Tok t(test_string,sep);
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- {
- const std::string test_string = "12252001";
- std::string answer[] = {"12","25","2001"};
- typedef tokenizer<offset_separator > Tok;
- boost::array<int,3> offsets = {{2,2,4}};
- offset_separator func(offsets.begin(),offsets.end());
- Tok t(test_string,func);
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- // Use token_iterator_generator
- {
- const std::string test_string = "This,,is, a.test..";
- std::string answer[] = {"This","is","a","test"};
- typedef token_iterator_generator<char_delimiters_separator<char> >::type Iter;
- Iter begin = make_token_iterator<std::string>(test_string.begin(),
- test_string.end(),char_delimiters_separator<char>());
- Iter end;
- BOOST_REQUIRE(std::equal(begin,end,answer));
- }
- {
- const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
- std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
- typedef token_iterator_generator<escaped_list_separator<char> >::type Iter;
- Iter begin = make_token_iterator<std::string>(test_string.begin(),
- test_string.end(),escaped_list_separator<char>());
- Iter begin_c(begin);
- Iter end;
- BOOST_REQUIRE(std::equal(begin,end,answer));
- while(begin_c != end)
- {
- BOOST_REQUIRE(begin_c.at_end() == 0);
- ++begin_c;
- }
- BOOST_REQUIRE(begin_c.at_end());
- }
- {
- const std::string test_string = "12252001";
- std::string answer[] = {"12","25","2001"};
- typedef token_iterator_generator<offset_separator>::type Iter;
- boost::array<int,3> offsets = {{2,2,4}};
- offset_separator func(offsets.begin(),offsets.end());
- Iter begin = make_token_iterator<std::string>(test_string.begin(),
- test_string.end(),func);
- Iter end= make_token_iterator<std::string>(test_string.end(),
- test_string.end(),func);
- BOOST_REQUIRE(std::equal(begin,end,answer));
- }
- // Test copying
- {
- const std::string test_string = "abcdef";
- token_iterator_generator<offset_separator>::type beg, end, other;
- boost::array<int,3> ar = {{1,2,3}};
- offset_separator f(ar.begin(),ar.end());
- beg = make_token_iterator<std::string>(test_string.begin(),test_string.end(),f);
- ++beg;
- other = beg;
- ++other;
- BOOST_REQUIRE(*beg=="bc");
- BOOST_REQUIRE(*other=="def");
- other = make_token_iterator<std::string>(test_string.begin(),
- test_string.end(),f);
- BOOST_REQUIRE(*other=="a");
- }
- // Test non-default constructed char_delimiters_separator
- {
- const std::string test_string = "how,are you, doing";
- std::string answer[] = {"how",",","are you",","," doing"};
- tokenizer<> t(test_string,char_delimiters_separator<char>(true,",",""));
- BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
- }
- return 0;
- }
|