Merge branch 'master' into translate-c-userland

1 files changed, 21 insertions, 0 deletions

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 7bae45f477..a0acde52e9 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -224,6 +224,7 @@ enum TokenizeState {
     TokenizeStateLBracket,
     TokenizeStateLBracketStar,
     TokenizeStateLBracketStarC,
+    TokenizeStateLBracketUnderscore,
 };
 
 
@@ -774,6 +775,9 @@ void tokenize(Buf *buf, Tokenization *out) {
                     case '*':
                         t.state = TokenizeStateLBracketStar;
                         break;
+                    case '_':
+                        t.state = TokenizeStateLBracketUnderscore;
+                        break;
                     default:
                         // reinterpret as just an lbracket
                         t.pos -= 1;
@@ -782,6 +786,21 @@ void tokenize(Buf *buf, Tokenization *out) {
                         continue;
                 }
                 break;
+            case TokenizeStateLBracketUnderscore:
+                switch (c) {
+                    case ']':
+                        set_token_id(&t, t.cur_tok, TokenIdBracketUnderscoreBracket);
+                        end_token(&t);
+                        t.state = TokenizeStateStart;
+                        break;
+                    default:
+                        // reinterpret as just an lbracket
+                        t.pos -= 2;
+                        end_token(&t);
+                        t.state = TokenizeStateStart;
+                        continue;
+                }
+                break;
             case TokenizeStateLBracketStar:
                 switch (c) {
                     case 'c':
@@ -1443,6 +1462,7 @@ void tokenize(Buf *buf, Tokenization *out) {
         case TokenizeStateLineStringContinueC:
         case TokenizeStateLBracketStar:
         case TokenizeStateLBracketStarC:
+        case TokenizeStateLBracketUnderscore:
             tokenize_error(&t, "unexpected EOF");
             break;
         case TokenizeStateLineComment:
@@ -1581,6 +1601,7 @@ const char * token_name(TokenId id) {
         case TokenIdTimesPercent: return "*%";
         case TokenIdTimesPercentEq: return "*%=";
         case TokenIdBarBarEq: return "||=";
+        case TokenIdBracketUnderscoreBracket: return "[_]";
         case TokenIdCount:
             zig_unreachable();
     }