General Utility Library for C++14  2.8
tokenize.h
Go to the documentation of this file.
1 
23 #ifndef GUL14_TOKENIZE_H_
24 #define GUL14_TOKENIZE_H_
25 
26 #include <string>
27 #include <vector>
28 #include "gul14/internal.h"
29 #include "gul14/string_util.h"
30 #include "gul14/string_view.h"
31 
32 namespace gul14 {
33 
107 template <typename StringContainer = std::vector<std::string>,
108  typename ContainerInsertFct = void (*)(StringContainer&, string_view)>
109 inline StringContainer
111  ContainerInsertFct insert_fct = detail::emplace_back<StringContainer>)
112 {
113  StringContainer tokens{ };
114 
115  string_view::size_type token_start = 0;
116  string_view::size_type token_end = 0;
117 
118  while (true)
119  {
120  token_start = str.find_first_not_of(delimiters, token_end);
121  if (token_start == string_view::npos)
122  break;
123 
124  token_end = str.find_first_of(delimiters, token_start);
125  if (token_end == string_view::npos)
126  {
127  insert_fct(tokens,
128  string_view{ str.data() + token_start, str.length() - token_start });
129  break;
130  }
131  else
132  {
133  insert_fct(tokens,
134  string_view{ str.data() + token_start, token_end - token_start });
135  }
136  }
137 
138  return tokens;
139 }
140 
161 template <typename StringContainer = std::vector<string_view>,
162  typename ContainerInsertFct = void (*)(StringContainer&, string_view)>
163 inline StringContainer
165  ContainerInsertFct insert_fct = detail::emplace_back<StringContainer>)
166 {
167  return tokenize<StringContainer>(str, delimiters, insert_fct);
168 }
169 
170 } // namespace gul14
171 
172 #endif
Definition of macros used internally by GUL.
Namespace gul14 contains all functions and classes of the General Utility Library.
Definition: doxygen.h:26
GUL_EXPORT const string_view default_whitespace_characters
The default characters that are treated as whitespace by GUL.
Definition: string_util.cc:28
StringContainer tokenize_sv(string_view str, string_view delimiters=default_whitespace_characters, ContainerInsertFct insert_fct=detail::emplace_back< StringContainer >)
Split the given string into a vector of substrings (tokens) delimited by any of the characters in the...
Definition: tokenize.h:164
StringContainer tokenize(string_view str, string_view delimiters=default_whitespace_characters, ContainerInsertFct insert_fct=detail::emplace_back< StringContainer >)
Split the given string into a vector of substrings (tokens) delimited by any of the characters in the...
Definition: tokenize.h:110
Declarations of string utility functions for the General Utility Library.
Provides a gul14::string_view that is fully compatible with C++17's std::string_view.