8 #include "special_tokens.h"
14 template<
class Smoother>
25 std::string sample_word(std::string context,
double T = 1.0) {
30 for (
size_t i = 1; i <= prob_.V_; ++i) {
31 word = prob_.f_.dictionary()->word(std::to_string(i));
32 tmp = std::pow(prob_(word, context), 1 / T);
40 tmp = std::pow(prob_(EOS_TOK, context), 1 / T) / R::rexp(1.);
76 std::string res =
"", context =
"";
77 for (
size_t i = 1; i < prob_.f_.N(); ++i) {
78 context += BOS_TOK +
" ";
81 std::string new_word;
size_t start = 0;
82 while (n_words < max_length) {
84 new_word = sample_word(context, T);
85 if (new_word == EOS_TOK)
87 res += new_word +
" ";
88 context +=
" " + new_word;
89 start = context.find_first_not_of(
" ");
90 start = context.find_first_of(
" ", start);
91 context = context.substr(start + 1);
93 return res +
"[...] (truncated output)";