kgrams  0.1.0
WordStream.h
1 #ifndef WORD_STREAM_H
2 #define WORD_STREAM_H
3 #include <string>
4 #include "special_tokens.h"
5 
6 class WordStream {
7  const std::string & str_;
8  size_t len_;
9  size_t start_; // start position of current word
10  bool eos_;
11  size_t end_; // end position of current word
12  size_t first (std::string start, size_t pos = 0)
13  { return str_.find_first_of(start, pos); }
14  size_t first_not (std::string start, size_t pos = 0)
15  { return str_.find_first_not_of(start, pos); }
16 public:
17  WordStream (const std::string & str)
18  : str_(str),
19  len_(str.length()),
20  start_(first_not(" ")),
21  eos_(start_ >= len_),
22  end_(eos_ ? len_ : 0)
23  {}
24  // Disallow initialization by rvalue reference!
25  WordStream(const std::string &&) = delete;
26  bool eos () { return eos_; }
27 
28  std::string pop_word() {
29  if ((end_ >= len_) or ((start_ = first_not(" ", end_)) >= len_))
30  { eos_ = true; return EOS_TOK; }
31 
32  if ((end_ = first(" ", start_)) >= len_)
33  return str_.substr(start_);
34  return str_.substr(start_, end_ - start_);
35  }
36 }; // class WordStream
37 
38 #endif // WORD_STREAM_H
WordStream
Definition: WordStream.h:6