diff options
Diffstat (limited to 'src/tokenizer.cpp')
| -rw-r--r-- | src/tokenizer.cpp | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 22d63568bf..b4e9d910d9 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1494,9 +1494,17 @@ void tokenize(Buf *buf, Tokenization *out) { tokenize_error(&t, "unexpected EOF"); break; case TokenizeStateLineComment: + break; case TokenizeStateSawSlash2: + cancel_token(&t); + break; case TokenizeStateSawSlash3: + set_token_id(&t, t.cur_tok, TokenIdDocComment); + end_token(&t); + break; case TokenizeStateSawSlashBang: + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + end_token(&t); break; } if (t.state != TokenizeStateError) { |