17void trimTokenFromStart(std::string &s) { s.erase(s.begin(), std::find_if_not(s.begin(), s.end(), ::isspace)); }
20void trimTokenFromEnd(std::string &s) { s.erase(std::find_if_not(s.rbegin(), s.rend(), ::isspace).base(), s.end()); }
23void trimToken(std::string &s) {
24 trimTokenFromStart(s);
30void addEmptyFinalToken(
const std::string &str,
const std::string &delims, std::vector<std::string> &tokens) {
32 const auto pos = std::find(delims.cbegin(), delims.cend(), str.back());
34 if (pos != delims.cend()) {
35 tokens.emplace_back();
42template <
class InputIt,
class ForwardIt,
class BinOp>
43void for_each_token(InputIt first, InputIt last, ForwardIt s_first, ForwardIt s_last, BinOp binary_op) {
44 while (first != last) {
45 const auto pos = std::find_first_of(first, last, s_first, s_last);
46 binary_op(first, pos);
49 first = std::next(pos);
53void splitKeepingWhitespaceEmptyTokens(
const std::string &str,
const std::string &delims,
54 std::vector<std::string> &output) {
56 for_each_token(str.cbegin(), str.cend(), delims.cbegin(), delims.cend(),
57 [&output](std::string::const_iterator first, std::string::const_iterator second) {
58 output.emplace_back(first, second);
62void splitKeepingWhitespaceIgnoringEmptyTokens(
const std::string &str,
const std::string &delims,
63 std::vector<std::string> &output) {
65 for_each_token(str.cbegin(), str.cend(), delims.cbegin(), delims.cend(),
66 [&output](std::string::const_iterator first, std::string::const_iterator second) {
68 output.emplace_back(first, second);
72void splitIgnoringWhitespaceKeepingEmptyTokens(
const std::string &str,
const std::string &delims,
73 std::vector<std::string> &output) {
75 for_each_token(str.cbegin(), str.cend(), delims.cbegin(), delims.cend(),
76 [&output](std::string::const_iterator first, std::string::const_iterator second) {
77 output.emplace_back(first, second);
78 trimToken(output.back());
82void splitIgnoringWhitespaceEmptyTokens(
const std::string &str,
const std::string &delims,
83 std::vector<std::string> &output) {
85 for_each_token(str.cbegin(), str.cend(), delims.cbegin(), delims.cend(),
86 [&output](std::string::const_iterator first, std::string::const_iterator second) {
87 if (first != second) {
88 output.emplace_back(first, second);
89 trimToken(output.back());
90 if (output.back().empty())
120 splitKeepingWhitespaceEmptyTokens(str, separators,
m_tokens);
121 addEmptyFinalToken(str, separators,
m_tokens);
124 splitKeepingWhitespaceIgnoringEmptyTokens(str, separators,
m_tokens);
127 splitIgnoringWhitespaceKeepingEmptyTokens(str, separators,
m_tokens);
128 addEmptyFinalToken(str, separators,
m_tokens);
131 splitIgnoringWhitespaceEmptyTokens(str, separators,
m_tokens);
134 splitKeepingWhitespaceEmptyTokens(str, separators,
m_tokens);
137 splitKeepingWhitespaceIgnoringEmptyTokens(str, separators,
m_tokens);
140 splitIgnoringWhitespaceKeepingEmptyTokens(str, separators,
m_tokens);
143 splitIgnoringWhitespaceEmptyTokens(str, separators,
m_tokens);
148 throw std::runtime_error(
"Invalid option passed to Mantid::Kernel::StringTokenizer:" +
std::to_string(options));
@ TOK_IGNORE_FINAL_EMPTY_TOKEN
ignore an empty token at the end of the string.
@ TOK_IGNORE_EMPTY
ignore empty tokens
@ TOK_TRIM
remove leading and trailing whitespace from tokens
std::vector< std::string > m_tokens
StringTokenizer()=default
Constructs an object from an empty string.
std::string to_string(const wide_integer< Bits, Signed > &n)