tokenizer.hpp 2.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798
  1. // Boost tokenizer.hpp -----------------------------------------------------//
  2. // (c) Copyright Jeremy Siek and John R. Bandela 2001.
  3. // Distributed under the Boost Software License, Version 1.0. (See
  4. // accompanying file LICENSE_1_0.txt or copy at
  5. // http://www.boost.org/LICENSE_1_0.txt)
  6. // See http://www.boost.org/libs/tokenizer for documenation
  7. // Revision History:
  8. // 03 Jul 2003 John Bandela
  9. // Converted to new iterator adapter
  10. // 02 Feb 2002 Jeremy Siek
  11. // Removed tabs and a little cleanup.
  12. #ifndef BOOST_TOKENIZER_JRB070303_HPP_
  13. #define BOOST_TOKENIZER_JRB070303_HPP_
  14. #include <boost/token_iterator.hpp>
  15. namespace boost {
  16. //===========================================================================
  17. // A container-view of a tokenized "sequence"
  18. template <
  19. typename TokenizerFunc = char_delimiters_separator<char>,
  20. typename Iterator = std::string::const_iterator,
  21. typename Type = std::string
  22. >
  23. class tokenizer {
  24. private:
  25. typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
  26. // It seems that MSVC does not like the unqualified use of iterator,
  27. // Thus we use iter internally when it is used unqualified and
  28. // the users of this class will always qualify iterator.
  29. typedef typename TGen::type iter;
  30. public:
  31. typedef iter iterator;
  32. typedef iter const_iterator;
  33. typedef Type value_type;
  34. typedef value_type& reference;
  35. typedef const value_type& const_reference;
  36. typedef value_type* pointer;
  37. typedef const pointer const_pointer;
  38. typedef void size_type;
  39. typedef void difference_type;
  40. tokenizer(Iterator first, Iterator last,
  41. const TokenizerFunc& f = TokenizerFunc())
  42. : first_(first), last_(last), f_(f) { }
  43. template <typename Container>
  44. tokenizer(const Container& c)
  45. : first_(c.begin()), last_(c.end()), f_() { }
  46. template <typename Container>
  47. tokenizer(const Container& c,const TokenizerFunc& f)
  48. : first_(c.begin()), last_(c.end()), f_(f) { }
  49. void assign(Iterator first, Iterator last){
  50. first_ = first;
  51. last_ = last;
  52. }
  53. void assign(Iterator first, Iterator last, const TokenizerFunc& f){
  54. assign(first,last);
  55. f_ = f;
  56. }
  57. template <typename Container>
  58. void assign(const Container& c){
  59. assign(c.begin(),c.end());
  60. }
  61. template <typename Container>
  62. void assign(const Container& c, const TokenizerFunc& f){
  63. assign(c.begin(),c.end(),f);
  64. }
  65. iter begin() const { return iter(f_,first_,last_); }
  66. iter end() const { return iter(f_,last_,last_); }
  67. private:
  68. Iterator first_;
  69. Iterator last_;
  70. TokenizerFunc f_;
  71. };
  72. } // namespace boost
  73. #endif