kgrams  0.1.0
special_tokens.h
1 #ifndef SPECIAL_TOKENS_H
2 #define SPECIAL_TOKENS_H
3 
4 #include <string>
5 
6 const std::string EOS_TOK = "___EOS___";
7 const std::string EOS_IND = "0";
8 const std::string BOS_TOK = "___BOS___";
9 const std::string BOS_IND = "-1";
10 const std::string UNK_TOK = "___UNK___";
11 const std::string UNK_IND = "-2";
12 
13 #endif // SPECIAL_TOKENS_H