Merge pull request #5336 from Vexu/parser

Make self-hosted parser more error tolerant

5 files changed, 943 insertions, 568 deletions

diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig
index 8dec2de15c..b1441d5b25 100644
--- a/lib/std/zig/ast.zig
+++ b/lib/std/zig/ast.zig
@@ -129,6 +129,7 @@ pub const Error = union(enum) {
     ExpectedStatement: ExpectedStatement,
     ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,
     ExpectedVarDecl: ExpectedVarDecl,
+    ExpectedFn: ExpectedFn,
     ExpectedReturnType: ExpectedReturnType,
     ExpectedAggregateKw: ExpectedAggregateKw,
     UnattachedDocComment: UnattachedDocComment,
@@ -165,6 +166,7 @@ pub const Error = union(enum) {
     ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,
     ExpectedSuffixOp: ExpectedSuffixOp,
     DeclBetweenFields: DeclBetweenFields,
+    InvalidAnd: InvalidAnd,
 
     pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
         switch (self.*) {
@@ -177,6 +179,7 @@ pub const Error = union(enum) {
             .ExpectedStatement => |*x| return x.render(tokens, stream),
             .ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream),
             .ExpectedVarDecl => |*x| return x.render(tokens, stream),
+            .ExpectedFn => |*x| return x.render(tokens, stream),
             .ExpectedReturnType => |*x| return x.render(tokens, stream),
             .ExpectedAggregateKw => |*x| return x.render(tokens, stream),
             .UnattachedDocComment => |*x| return x.render(tokens, stream),
@@ -213,6 +216,7 @@ pub const Error = union(enum) {
             .ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream),
             .ExpectedSuffixOp => |*x| return x.render(tokens, stream),
             .DeclBetweenFields => |*x| return x.render(tokens, stream),
+            .InvalidAnd => |*x| return x.render(tokens, stream),
         }
     }
 
@@ -227,6 +231,7 @@ pub const Error = union(enum) {
             .ExpectedStatement => |x| return x.token,
             .ExpectedVarDeclOrFn => |x| return x.token,
             .ExpectedVarDecl => |x| return x.token,
+            .ExpectedFn => |x| return x.token,
             .ExpectedReturnType => |x| return x.token,
             .ExpectedAggregateKw => |x| return x.token,
             .UnattachedDocComment => |x| return x.token,
@@ -263,6 +268,7 @@ pub const Error = union(enum) {
             .ExpectedDerefOrUnwrap => |x| return x.token,
             .ExpectedSuffixOp => |x| return x.token,
             .DeclBetweenFields => |x| return x.token,
+            .InvalidAnd => |x| return x.token,
         }
     }
 
@@ -274,6 +280,7 @@ pub const Error = union(enum) {
     pub const ExpectedStatement = SingleTokenError("Expected statement, found '{}'");
     pub const ExpectedVarDeclOrFn = SingleTokenError("Expected variable declaration or function, found '{}'");
     pub const ExpectedVarDecl = SingleTokenError("Expected variable declaration, found '{}'");
+    pub const ExpectedFn = SingleTokenError("Expected function, found '{}'");
     pub const ExpectedReturnType = SingleTokenError("Expected 'var' or return type expression, found '{}'");
     pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Id.Keyword_struct.symbol() ++ "', '" ++ Token.Id.Keyword_union.symbol() ++ "', or '" ++ Token.Id.Keyword_enum.symbol() ++ "', found '{}'");
     pub const ExpectedEqOrSemi = SingleTokenError("Expected '=' or ';', found '{}'");
@@ -308,6 +315,7 @@ pub const Error = union(enum) {
     pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");
     pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
     pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
+    pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
 
     pub const ExpectedCall = struct {
         node: *Node,
@@ -335,9 +343,6 @@ pub const Error = union(enum) {
         pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
             const found_token = tokens.at(self.token);
             switch (found_token.id) {
-                .Invalid_ampersands => {
-                    return stream.print("`&&` is invalid. Note that `and` is boolean AND.", .{});
-                },
                 .Invalid => {
                     return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
                 },
@@ -888,6 +893,7 @@ pub const Node = struct {
         pub const ReturnType = union(enum) {
             Explicit: *Node,
             InferErrorSet: *Node,
+            Invalid: TokenIndex,
         };
 
         pub fn iterate(self: *FnProto, index: usize) ?*Node {
@@ -916,6 +922,7 @@ pub const Node = struct {
                     if (i < 1) return node;
                     i -= 1;
                 },
+                .Invalid => {},
             }
 
             if (self.body_node) |body_node| {
@@ -937,6 +944,7 @@ pub const Node = struct {
             if (self.body_node) |body_node| return body_node.lastToken();
             switch (self.return_type) {
                 .Explicit, .InferErrorSet => |node| return node.lastToken(),
+                .Invalid => |tok| return tok,
             }
         }
     };
diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig
index 031fd9c160..a269dc616c 100644
--- a/lib/std/zig/parse.zig
+++ b/lib/std/zig/parse.zig
@@ -48,31 +48,24 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
 
     while (it.peek().?.id == .LineComment) _ = it.next();
 
-    tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
-        switch (err) {
-            error.ParseError => {
-                assert(tree.errors.len != 0);
-                break :blk undefined;
-            },
-            error.OutOfMemory => {
-                return error.OutOfMemory;
-            },
-        }
-    };
+    tree.root_node = try parseRoot(arena, &it, tree);
 
     return tree;
 }
 
 /// Root <- skip ContainerMembers eof
-fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
+fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
     const node = try arena.create(Node.Root);
     node.* = .{
         .decls = try parseContainerMembers(arena, it, tree),
-        .eof_token = eatToken(it, .Eof) orelse {
+        .eof_token = eatToken(it, .Eof) orelse blk: {
+            // parseContainerMembers will try to skip as much
+            // invalid tokens as it can so this can only be a '}'
+            const tok = eatToken(it, .RBrace).?;
             try tree.errors.push(.{
-                .ExpectedContainerMembers = .{ .token = it.index },
+                .ExpectedContainerMembers = .{ .token = tok },
             });
-            return error.ParseError;
+            break :blk tok;
         },
     };
     return node;
@@ -108,7 +101,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
 
         const doc_comments = try parseDocComment(arena, it, tree);
 
-        if (try parseTestDecl(arena, it, tree)) |node| {
+        if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.ParseError => {
+                findNextContainerMember(it);
+                continue;
+            },
+        }) |node| {
             if (field_state == .seen) {
                 field_state = .{ .end = node.firstToken() };
             }
@@ -117,7 +116,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
             continue;
         }
 
-        if (try parseTopLevelComptime(arena, it, tree)) |node| {
+        if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.ParseError => {
+                findNextContainerMember(it);
+                continue;
+            },
+        }) |node| {
             if (field_state == .seen) {
                 field_state = .{ .end = node.firstToken() };
             }
@@ -128,7 +133,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
 
         const visib_token = eatToken(it, .Keyword_pub);
 
-        if (try parseTopLevelDecl(arena, it, tree)) |node| {
+        if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.ParseError => {
+                findNextContainerMember(it);
+                continue;
+            },
+        }) |node| {
             if (field_state == .seen) {
                 field_state = .{ .end = visib_token orelse node.firstToken() };
             }
@@ -163,10 +174,18 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
             try tree.errors.push(.{
                 .ExpectedPubItem = .{ .token = it.index },
             });
-            return error.ParseError;
+            // ignore this pub
+            continue;
         }
 
-        if (try parseContainerField(arena, it, tree)) |node| {
+        if (parseContainerField(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.ParseError => {
+                // attempt to recover
+                findNextContainerMember(it);
+                continue;
+            },
+        }) |node| {
             switch (field_state) {
                 .none => field_state = .seen,
                 .err, .seen => {},
@@ -182,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
             const field = node.cast(Node.ContainerField).?;
             field.doc_comments = doc_comments;
             try list.push(node);
-            const comma = eatToken(it, .Comma) orelse break;
+            const comma = eatToken(it, .Comma) orelse {
+                // try to continue parsing
+                const index = it.index;
+                findNextContainerMember(it);
+                switch (it.peek().?.id) {
+                    .Eof, .RBrace => break,
+                    else => {
+                        // add error and continue
+                        try tree.errors.push(.{
+                            .ExpectedToken = .{ .token = index, .expected_id = .Comma },
+                        });
+                        continue;
+                    },
+                }
+            };
             if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
                 field.doc_comments = appended_comment;
             continue;
@@ -194,12 +227,102 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
                 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
             });
         }
-        break;
+
+        switch (it.peek().?.id) {
+            .Eof, .RBrace => break,
+            else => {
+                // this was likely not supposed to end yet,
+                // try to find the next declaration
+                const index = it.index;
+                findNextContainerMember(it);
+                try tree.errors.push(.{
+                    .ExpectedContainerMembers = .{ .token = index },
+                });
+            },
+        }
     }
 
     return list;
 }
 
+/// Attempts to find next container member by searching for certain tokens
+fn findNextContainerMember(it: *TokenIterator) void {
+    var level: u32 = 0;
+    while (true) {
+        const tok = nextToken(it);
+        switch (tok.ptr.id) {
+            // any of these can start a new top level declaration
+            .Keyword_test,
+            .Keyword_comptime,
+            .Keyword_pub,
+            .Keyword_export,
+            .Keyword_extern,
+            .Keyword_inline,
+            .Keyword_noinline,
+            .Keyword_usingnamespace,
+            .Keyword_threadlocal,
+            .Keyword_const,
+            .Keyword_var,
+            .Keyword_fn,
+            .Identifier,
+            => {
+                if (level == 0) {
+                    putBackToken(it, tok.index);
+                    return;
+                }
+            },
+            .Comma, .Semicolon => {
+                // this decl was likely meant to end here
+                if (level == 0) {
+                    return;
+                }
+            },
+            .LParen, .LBracket, .LBrace => level += 1,
+            .RParen, .RBracket, .RBrace => {
+                if (level == 0) {
+                    // end of container, exit
+                    putBackToken(it, tok.index);
+                    return;
+                }
+                level -= 1;
+            },
+            .Eof => {
+                putBackToken(it, tok.index);
+                return;
+            },
+            else => {},
+        }
+    }
+}
+
+/// Attempts to find the next statement by searching for a semicolon
+fn findNextStmt(it: *TokenIterator) void {
+    var level: u32 = 0;
+    while (true) {
+        const tok = nextToken(it);
+        switch (tok.ptr.id) {
+            .LBrace => level += 1,
+            .RBrace => {
+                if (level == 0) {
+                    putBackToken(it, tok.index);
+                    return;
+                }
+                level -= 1;
+            },
+            .Semicolon => {
+                if (level == 0) {
+                    return;
+                }
+            },
+            .Eof => {
+                putBackToken(it, tok.index);
+                return;
+            },
+            else => {},
+        }
+    }
+}
+
 /// Eat a multiline container doc comment
 fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
     var lines = Node.DocComment.LineList.init(arena);
@@ -279,22 +402,30 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         fn_node.*.extern_export_inline_token = extern_export_inline_token;
         fn_node.*.lib_name = lib_name;
         if (eatToken(it, .Semicolon)) |_| return node;
-        if (try parseBlock(arena, it, tree)) |body_node| {
+        if (parseBlock(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            // since parseBlock only return error.ParseError on
+            // a missing '}' we can assume this function was
+            // supposed to end here.
+            error.ParseError => return node,
+        }) |body_node| {
             fn_node.body_node = body_node;
             return node;
         }
         try tree.errors.push(.{
             .ExpectedSemiOrLBrace = .{ .token = it.index },
         });
-        return null;
+        return error.ParseError;
     }
 
     if (extern_export_inline_token) |token| {
         if (tree.tokens.at(token).id == .Keyword_inline or
             tree.tokens.at(token).id == .Keyword_noinline)
         {
-            putBackToken(it, token);
-            return null;
+            try tree.errors.push(.{
+                .ExpectedFn = .{ .token = it.index },
+            });
+            return error.ParseError;
         }
     }
 
@@ -313,26 +444,19 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         try tree.errors.push(.{
             .ExpectedVarDecl = .{ .token = it.index },
         });
+        // ignore this and try again;
         return error.ParseError;
     }
 
     if (extern_export_inline_token) |token| {
-        if (lib_name) |string_literal_node|
-            putBackToken(it, string_literal_node.cast(Node.StringLiteral).?.token);
-        putBackToken(it, token);
-        return null;
+        try tree.errors.push(.{
+            .ExpectedVarDeclOrFn = .{ .token = it.index },
+        });
+        // ignore this and try again;
+        return error.ParseError;
     }
 
-    const use_node = (try parseUse(arena, it, tree)) orelse return null;
-    const expr_node = try expectNode(arena, it, tree, parseExpr, .{
-        .ExpectedExpr = .{ .token = it.index },
-    });
-    const semicolon_token = try expectToken(it, tree, .Semicolon);
-    const use_node_raw = use_node.cast(Node.Use).?;
-    use_node_raw.*.expr = expr_node;
-    use_node_raw.*.semicolon_token = semicolon_token;
-
-    return use_node;
+    return try parseUse(arena, it, tree);
 }
 
 /// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
@@ -366,18 +490,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
     const exclamation_token = eatToken(it, .Bang);
 
     const return_type_expr = (try parseVarType(arena, it, tree)) orelse
-        try expectNode(arena, it, tree, parseTypeExpr, .{
-        .ExpectedReturnType = .{ .token = it.index },
-    });
+        (try parseTypeExpr(arena, it, tree)) orelse blk: {
+        try tree.errors.push(.{
+            .ExpectedReturnType = .{ .token = it.index },
+        });
+        // most likely the user forgot to specify the return type.
+        // Mark return type as invalid and try to continue.
+        break :blk null;
+    };
 
-    const return_type: Node.FnProto.ReturnType = if (exclamation_token != null)
-        .{
-            .InferErrorSet = return_type_expr,
-        }
+    // TODO https://github.com/ziglang/zig/issues/3750
+    const R = Node.FnProto.ReturnType;
+    const return_type = if (return_type_expr == null)
+        R{ .Invalid = rparen }
+    else if (exclamation_token != null)
+        R{ .InferErrorSet = return_type_expr.? }
     else
-        .{
-            .Explicit = return_type_expr,
-        };
+        R{ .Explicit = return_type_expr.? };
 
     const var_args_token = if (params.len > 0)
         params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token
@@ -578,7 +707,12 @@ fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*No
     if (try parseLabeledStatement(arena, it, tree)) |node| return node;
     if (try parseSwitchExpr(arena, it, tree)) |node| return node;
     if (try parseAssignExpr(arena, it, tree)) |node| {
-        _ = try expectToken(it, tree, .Semicolon);
+        _ = eatToken(it, .Semicolon) orelse {
+            try tree.errors.push(.{
+                .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
+            });
+            // pretend we saw a semicolon and continue parsing
+        };
         return node;
     }
 
@@ -687,8 +821,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
         node.cast(Node.While).?.inline_token = inline_token;
         return node;
     }
+    if (inline_token == null) return null;
 
-    return null;
+    // If we've seen "inline", there should have been a "for" or "while"
+    try tree.errors.push(.{
+        .ExpectedInlinable = .{ .token = it.index },
+    });
+    return error.ParseError;
 }
 
 /// ForStatement
@@ -817,7 +956,12 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
 fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
     if (try parseBlockExpr(arena, it, tree)) |node| return node;
     if (try parseAssignExpr(arena, it, tree)) |node| {
-        _ = try expectToken(it, tree, .Semicolon);
+        _ = eatToken(it, .Semicolon) orelse {
+            try tree.errors.push(.{
+                .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
+            });
+            // pretend we saw a semicolon and continue parsing
+        };
         return node;
     }
     return null;
@@ -924,7 +1068,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         const node = try arena.create(Node.ControlFlowExpression);
         node.* = .{
             .ltoken = token,
-            .kind = Node.ControlFlowExpression.Kind{ .Break = label },
+            .kind = .{ .Break = label },
             .rhs = expr_node,
         };
         return &node.base;
@@ -960,7 +1104,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         const node = try arena.create(Node.ControlFlowExpression);
         node.* = .{
             .ltoken = token,
-            .kind = Node.ControlFlowExpression.Kind{ .Continue = label },
+            .kind = .{ .Continue = label },
             .rhs = null,
         };
         return &node.base;
@@ -984,7 +1128,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         const node = try arena.create(Node.ControlFlowExpression);
         node.* = .{
             .ltoken = token,
-            .kind = Node.ControlFlowExpression.Kind.Return,
+            .kind = .Return,
             .rhs = expr_node,
         };
         return &node.base;
@@ -1022,7 +1166,14 @@ fn parseBlock(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
 
     var statements = Node.Block.StatementList.init(arena);
     while (true) {
-        const statement = (try parseStatement(arena, it, tree)) orelse break;
+        const statement = (parseStatement(arena, it, tree) catch |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.ParseError => {
+                // try to skip to the next statement
+                findNextStmt(it);
+                continue;
+            },
+        }) orelse break;
         try statements.push(statement);
     }
 
@@ -1222,7 +1373,8 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
             try tree.errors.push(.{
                 .ExpectedParamList = .{ .token = it.index },
             });
-            return null;
+            // ignore this, continue parsing
+            return res;
         };
         const node = try arena.create(Node.SuffixOp);
         node.* = .{
@@ -1287,7 +1439,6 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
 ///      / IfTypeExpr
 ///      / INTEGER
 ///      / KEYWORD_comptime TypeExpr
-///      / KEYWORD_nosuspend TypeExpr
 ///      / KEYWORD_error DOT IDENTIFIER
 ///      / KEYWORD_false
 ///      / KEYWORD_null
@@ -1326,15 +1477,6 @@ fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*N
         };
         return &node.base;
     }
-    if (eatToken(it, .Keyword_nosuspend)) |token| {
-        const expr = (try parseTypeExpr(arena, it, tree)) orelse return null;
-        const node = try arena.create(Node.Nosuspend);
-        node.* = .{
-            .nosuspend_token = token,
-            .expr = expr,
-        };
-        return &node.base;
-    }
     if (eatToken(it, .Keyword_error)) |token| {
         const period = try expectToken(it, tree, .Period);
         const identifier = try expectNode(arena, it, tree, parseIdentifier, .{
@@ -2271,7 +2413,7 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         const node = try arena.create(Node.AnyFrameType);
         node.* = .{
             .anyframe_token = token,
-            .result = Node.AnyFrameType.Result{
+            .result = .{
                 .arrow_token = arrow,
                 .return_type = undefined, // set by caller
             },
@@ -2312,6 +2454,13 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                 } else null;
                 _ = try expectToken(it, tree, .RParen);
 
+                if (ptr_info.align_info != null) {
+                    try tree.errors.push(.{
+                        .ExtraAlignQualifier = .{ .token = it.index - 1 },
+                    });
+                    continue;
+                }
+
                 ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{
                     .node = expr_node,
                     .bit_range = bit_range,
@@ -2320,14 +2469,32 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                 continue;
             }
             if (eatToken(it, .Keyword_const)) |const_token| {
+                if (ptr_info.const_token != null) {
+                    try tree.errors.push(.{
+                        .ExtraConstQualifier = .{ .token = it.index - 1 },
+                    });
+                    continue;
+                }
                 ptr_info.const_token = const_token;
                 continue;
             }
             if (eatToken(it, .Keyword_volatile)) |volatile_token| {
+                if (ptr_info.volatile_token != null) {
+                    try tree.errors.push(.{
+                        .ExtraVolatileQualifier = .{ .token = it.index - 1 },
+                    });
+                    continue;
+                }
                 ptr_info.volatile_token = volatile_token;
                 continue;
             }
             if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
+                if (ptr_info.allowzero_token != null) {
+                    try tree.errors.push(.{
+                        .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
+                    });
+                    continue;
+                }
                 ptr_info.allowzero_token = allowzero_token;
                 continue;
             }
@@ -2346,9 +2513,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                     if (try parseByteAlign(arena, it, tree)) |align_expr| {
                         if (slice_type.align_info != null) {
                             try tree.errors.push(.{
-                                .ExtraAlignQualifier = .{ .token = it.index },
+                                .ExtraAlignQualifier = .{ .token = it.index - 1 },
                             });
-                            return error.ParseError;
+                            continue;
                         }
                         slice_type.align_info = Node.PrefixOp.PtrInfo.Align{
                             .node = align_expr,
@@ -2359,9 +2526,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                     if (eatToken(it, .Keyword_const)) |const_token| {
                         if (slice_type.const_token != null) {
                             try tree.errors.push(.{
-                                .ExtraConstQualifier = .{ .token = it.index },
+                                .ExtraConstQualifier = .{ .token = it.index - 1 },
                             });
-                            return error.ParseError;
+                            continue;
                         }
                         slice_type.const_token = const_token;
                         continue;
@@ -2369,9 +2536,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                     if (eatToken(it, .Keyword_volatile)) |volatile_token| {
                         if (slice_type.volatile_token != null) {
                             try tree.errors.push(.{
-                                .ExtraVolatileQualifier = .{ .token = it.index },
+                                .ExtraVolatileQualifier = .{ .token = it.index - 1 },
                             });
-                            return error.ParseError;
+                            continue;
                         }
                         slice_type.volatile_token = volatile_token;
                         continue;
@@ -2379,9 +2546,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
                     if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
                         if (slice_type.allowzero_token != null) {
                             try tree.errors.push(.{
-                                .ExtraAllowZeroQualifier = .{ .token = it.index },
+                                .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
                             });
-                            return error.ParseError;
+                            continue;
                         }
                         slice_type.allowzero_token = allowzero_token;
                         continue;
@@ -2730,7 +2897,19 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
             var list = L.init(arena);
             while (try nodeParseFn(arena, it, tree)) |node| {
                 try list.push(node);
-                if (eatToken(it, .Comma) == null) break;
+
+                switch (it.peek().?.id) {
+                    .Comma => _ = nextToken(it),
+                    // all possible delimiters
+                    .Colon, .RParen, .RBrace, .RBracket => break,
+                    else => {
+                        // this is likely just a missing comma,
+                        // continue parsing this list and give an error
+                        try tree.errors.push(.{
+                            .ExpectedToken = .{ .token = it.index, .expected_id = .Comma },
+                        });
+                    },
+                }
             }
             return list;
         }
@@ -2740,7 +2919,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
 fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
     return struct {
         pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {
-            const op_token = eatToken(it, token) orelse return null;
+            const op_token = if (token == .Keyword_and) switch (it.peek().?.id) {
+                .Keyword_and => nextToken(it).index,
+                .Invalid_ampersands => blk: {
+                    try tree.errors.push(.{
+                        .InvalidAnd = .{ .token = it.index },
+                    });
+                    break :blk nextToken(it).index;
+                },
+                else => return null,
+            } else eatToken(it, token) orelse return null;
+
             const node = try arena.create(Node.InfixOp);
             node.* = .{
                 .op_token = op_token,
@@ -2761,7 +2950,13 @@ fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
         try tree.errors.push(.{
             .ExpectedParamList = .{ .token = it.index },
         });
-        return error.ParseError;
+
+        // lets pretend this was an identifier so we can continue parsing
+        const node = try arena.create(Node.Identifier);
+        node.* = .{
+            .token = token,
+        };
+        return &node.base;
     };
     const node = try arena.create(Node.BuiltinCall);
     node.* = .{
@@ -2877,8 +3072,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
         .doc_comments = null,
         .visib_token = null,
         .use_token = token,
-        .expr = undefined, // set by caller
-        .semicolon_token = undefined, // set by caller
+        .expr = try expectNode(arena, it, tree, parseExpr, .{
+            .ExpectedExpr = .{ .token = it.index },
+        }),
+        .semicolon_token = try expectToken(it, tree, .Semicolon),
     };
     return &node.base;
 }
@@ -3058,6 +3255,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
         try tree.errors.push(.{
             .ExpectedToken = .{ .token = token.index, .expected_id = id },
         });
+        // go back so that we can recover properly
+        putBackToken(it, token.index);
         return error.ParseError;
     }
     return token.index;
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index b98e8c69c3..6adc44a5b7 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -1,3 +1,153 @@
+test "recovery: top level" {
+    try testError(
+        \\test "" {inline}
+        \\test "" {inline}
+    , &[_]Error{
+        .ExpectedInlinable,
+        .ExpectedInlinable,
+    });
+}
+
+test "recovery: block statements" {
+    try testError(
+        \\test "" {
+        \\    foo + +;
+        \\    inline;
+        \\}
+    , &[_]Error{
+        .InvalidToken,
+        .ExpectedInlinable,
+    });
+}
+
+test "recovery: missing comma" {
+    try testError(
+        \\test "" {
+        \\    switch (foo) {
+        \\        2 => {}
+        \\        3 => {}
+        \\        else => {
+        \\            foo && bar +;
+        \\        }
+        \\    }
+        \\}
+    , &[_]Error{
+        .ExpectedToken,
+        .ExpectedToken,
+        .InvalidAnd,
+        .InvalidToken,
+    });
+}
+
+test "recovery: extra qualifier" {
+    try testError(
+        \\const a: *const const u8;
+        \\test ""
+    , &[_]Error{
+        .ExtraConstQualifier,
+        .ExpectedLBrace,
+    });
+}
+
+test "recovery: missing return type" {
+    try testError(
+        \\fn foo() {
+        \\    a && b;
+        \\}
+        \\test ""
+    , &[_]Error{
+        .ExpectedReturnType,
+        .InvalidAnd,
+        .ExpectedLBrace,
+    });
+}
+
+test "recovery: continue after invalid decl" {
+    try testError(
+        \\fn foo {
+        \\    inline;
+        \\}
+        \\pub test "" {
+        \\    async a && b;
+        \\}
+    , &[_]Error{
+        .ExpectedToken,
+        .ExpectedPubItem,
+        .ExpectedParamList,
+        .InvalidAnd,
+    });
+    try testError(
+        \\threadlocal test "" {
+        \\    @a && b;
+        \\}
+    , &[_]Error{
+        .ExpectedVarDecl,
+        .ExpectedParamList,
+        .InvalidAnd,
+    });
+}
+
+test "recovery: invalid extern/inline" {
+    try testError(
+        \\inline test "" { a && b; }
+    , &[_]Error{
+        .ExpectedFn,
+        .InvalidAnd,
+    });
+    try testError(
+        \\extern "" test "" { a && b; }
+    , &[_]Error{
+        .ExpectedVarDeclOrFn,
+        .InvalidAnd,
+    });
+}
+
+test "recovery: missing semicolon" {
+    try testError(
+        \\test "" {
+        \\    comptime a && b
+        \\    c && d
+        \\    @foo
+        \\}
+    , &[_]Error{
+        .InvalidAnd,
+        .ExpectedToken,
+        .InvalidAnd,
+        .ExpectedToken,
+        .ExpectedParamList,
+        .ExpectedToken,
+    });
+}
+
+test "recovery: invalid container members" {
+    try testError(
+        \\usingnamespace;
+        \\foo+
+        \\bar@,
+        \\while (a == 2) { test "" {}}
+        \\test "" {
+        \\    a && b
+        \\}
+    , &[_]Error{
+        .ExpectedExpr,
+        .ExpectedToken,
+        .ExpectedToken,
+        .ExpectedContainerMembers,
+        .InvalidAnd,
+        .ExpectedToken,
+    });
+}
+
+test "recovery: invalid parameter" {
+    try testError(
+        \\fn main() void {
+        \\    a(comptime T: type)
+        \\}
+    , &[_]Error{
+        .ExpectedToken,
+    });
+}
+
 test "zig fmt: top-level fields" {
     try testCanonical(
         \\a: did_you_know,
@@ -19,7 +169,9 @@ test "zig fmt: decl between fields" {
         \\    const baz1 = 2;
         \\    b: usize,
         \\};
-    );
+    , &[_]Error{
+        .DeclBetweenFields,
+    });
 }
 
 test "zig fmt: errdefer with payload" {
@@ -2828,7 +2980,10 @@ test "zig fmt: extern without container keyword returns error" {
     try testError(
         \\const container = extern {};
         \\
-    );
+    , &[_]Error{
+        .ExpectedExpr,
+        .ExpectedVarDeclOrFn,
+    });
 }
 
 test "zig fmt: integer literals with underscore separators" {
@@ -3030,9 +3185,17 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
 fn testCanonical(source: []const u8) !void {
     return testTransform(source, source);
 }
-fn testError(source: []const u8) !void {
+
+const Error = @TagType(std.zig.ast.Error);
+
+fn testError(source: []const u8, expected_errors: []const Error) !void {
     const tree = try std.zig.parse(std.testing.allocator, source);
     defer tree.deinit();
 
-    std.testing.expect(tree.errors.len != 0);
+    std.testing.expect(tree.errors.len == expected_errors.len);
+    for (expected_errors) |expected, i| {
+        const err = tree.errors.at(i);
+
+        std.testing.expect(expected == err.*);
+    }
 }
diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig
index bcdc08d43a..9ca6d4450c 100644
--- a/lib/std/zig/render.zig
+++ b/lib/std/zig/render.zig
@@ -13,6 +13,9 @@ pub const Error = error{
 
 /// Returns whether anything changed
 pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool {
+    // cannot render an invalid tree
+    std.debug.assert(tree.errors.len == 0);
+
     // make a passthrough stream that checks whether something changed
     const MyStream = struct {
         const MyStream = @This();
@@ -1444,6 +1447,7 @@ fn renderExpression(
             else switch (fn_proto.return_type) {
                 .Explicit => |node| node.firstToken(),
                 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
+                .Invalid => unreachable,
             });
             assert(tree.tokens.at(rparen).id == .RParen);
 
@@ -1518,13 +1522,14 @@ fn renderExpression(
             }
 
             switch (fn_proto.return_type) {
-                ast.Node.FnProto.ReturnType.Explicit => |node| {
+                .Explicit => |node| {
                     return renderExpression(allocator, stream, tree, indent, start_col, node, space);
                 },
-                ast.Node.FnProto.ReturnType.InferErrorSet => |node| {
+                .InferErrorSet => |node| {
                     try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !
                     return renderExpression(allocator, stream, tree, indent, start_col, node, space);
                 },
+                .Invalid => unreachable,
             }
         },
 
diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
index 089711871e..160530f459 100644
--- a/lib/std/zig/tokenizer.zig
+++ b/lib/std/zig/tokenizer.zig
@@ -353,64 +353,64 @@ pub const Tokenizer = struct {
     }
 
     const State = enum {
-        Start,
-        Identifier,
-        Builtin,
-        StringLiteral,
-        StringLiteralBackslash,
-        MultilineStringLiteralLine,
-        CharLiteral,
-        CharLiteralBackslash,
-        CharLiteralHexEscape,
-        CharLiteralUnicodeEscapeSawU,
-        CharLiteralUnicodeEscape,
-        CharLiteralUnicodeInvalid,
-        CharLiteralUnicode,
-        CharLiteralEnd,
-        Backslash,
-        Equal,
-        Bang,
-        Pipe,
-        Minus,
-        MinusPercent,
-        Asterisk,
-        AsteriskPercent,
-        Slash,
-        LineCommentStart,
-        LineComment,
-        DocCommentStart,
-        DocComment,
-        ContainerDocComment,
-        Zero,
-        IntegerLiteralDec,
-        IntegerLiteralDecNoUnderscore,
-        IntegerLiteralBin,
-        IntegerLiteralBinNoUnderscore,
-        IntegerLiteralOct,
-        IntegerLiteralOctNoUnderscore,
-        IntegerLiteralHex,
-        IntegerLiteralHexNoUnderscore,
-        NumberDotDec,
-        NumberDotHex,
-        FloatFractionDec,
-        FloatFractionDecNoUnderscore,
-        FloatFractionHex,
-        FloatFractionHexNoUnderscore,
-        FloatExponentUnsigned,
-        FloatExponentNumber,
-        FloatExponentNumberNoUnderscore,
-        Ampersand,
-        Caret,
-        Percent,
-        Plus,
-        PlusPercent,
-        AngleBracketLeft,
-        AngleBracketAngleBracketLeft,
-        AngleBracketRight,
-        AngleBracketAngleBracketRight,
-        Period,
-        Period2,
-        SawAtSign,
+        start,
+        identifier,
+        builtin,
+        string_literal,
+        string_literal_backslash,
+        multiline_string_literal_line,
+        char_literal,
+        char_literal_backslash,
+        char_literal_hex_escape,
+        char_literal_unicode_escape_saw_u,
+        char_literal_unicode_escape,
+        char_literal_unicode_invalid,
+        char_literal_unicode,
+        char_literal_end,
+        backslash,
+        equal,
+        bang,
+        pipe,
+        minus,
+        minus_percent,
+        asterisk,
+        asterisk_percent,
+        slash,
+        line_comment_start,
+        line_comment,
+        doc_comment_start,
+        doc_comment,
+        container_doc_comment,
+        zero,
+        int_literal_dec,
+        int_literal_dec_no_underscore,
+        int_literal_bin,
+        int_literal_bin_no_underscore,
+        int_literal_oct,
+        int_literal_oct_no_underscore,
+        int_literal_hex,
+        int_literal_hex_no_underscore,
+        num_dot_dec,
+        num_dot_hex,
+        float_fraction_dec,
+        float_fraction_dec_no_underscore,
+        float_fraction_hex,
+        float_fraction_hex_no_underscore,
+        float_exponent_unsigned,
+        float_exponent_num,
+        float_exponent_num_no_underscore,
+        ampersand,
+        caret,
+        percent,
+        plus,
+        plus_percent,
+        angle_bracket_left,
+        angle_bracket_angle_bracket_left,
+        angle_bracket_right,
+        angle_bracket_angle_bracket_right,
+        period,
+        period_2,
+        saw_at_sign,
     };
 
     fn isIdentifierChar(char: u8) bool {
@@ -423,9 +423,9 @@ pub const Tokenizer = struct {
             return token;
         }
         const start_index = self.index;
-        var state = State.Start;
+        var state: State = .start;
         var result = Token{
-            .id = Token.Id.Eof,
+            .id = .Eof,
             .start = self.index,
             .end = undefined,
         };
@@ -434,40 +434,40 @@ pub const Tokenizer = struct {
         while (self.index < self.buffer.len) : (self.index += 1) {
             const c = self.buffer[self.index];
             switch (state) {
-                State.Start => switch (c) {
+                .start => switch (c) {
                     ' ', '\n', '\t', '\r' => {
                         result.start = self.index + 1;
                     },
                     '"' => {
-                        state = State.StringLiteral;
-                        result.id = Token.Id.StringLiteral;
+                        state = .string_literal;
+                        result.id = .StringLiteral;
                     },
                     '\'' => {
-                        state = State.CharLiteral;
+                        state = .char_literal;
                     },
                     'a'...'z', 'A'...'Z', '_' => {
-                        state = State.Identifier;
-                        result.id = Token.Id.Identifier;
+                        state = .identifier;
+                        result.id = .Identifier;
                     },
                     '@' => {
-                        state = State.SawAtSign;
+                        state = .saw_at_sign;
                     },
                     '=' => {
-                        state = State.Equal;
+                        state = .equal;
                     },
                     '!' => {
-                        state = State.Bang;
+                        state = .bang;
                     },
                     '|' => {
-                        state = State.Pipe;
+                        state = .pipe;
                     },
                     '(' => {
-                        result.id = Token.Id.LParen;
+                        result.id = .LParen;
                         self.index += 1;
                         break;
                     },
                     ')' => {
-                        result.id = Token.Id.RParen;
+                        result.id = .RParen;
                         self.index += 1;
                         break;
                     },
@@ -477,213 +477,213 @@ pub const Tokenizer = struct {
                         break;
                     },
                     ']' => {
-                        result.id = Token.Id.RBracket;
+                        result.id = .RBracket;
                         self.index += 1;
                         break;
                     },
                     ';' => {
-                        result.id = Token.Id.Semicolon;
+                        result.id = .Semicolon;
                         self.index += 1;
                         break;
                     },
                     ',' => {
-                        result.id = Token.Id.Comma;
+                        result.id = .Comma;
                         self.index += 1;
                         break;
                     },
                     '?' => {
-                        result.id = Token.Id.QuestionMark;
+                        result.id = .QuestionMark;
                         self.index += 1;
                         break;
                     },
                     ':' => {
-                        result.id = Token.Id.Colon;
+                        result.id = .Colon;
                         self.index += 1;
                         break;
                     },
                     '%' => {
-                        state = State.Percent;
+                        state = .percent;
                     },
                     '*' => {
-                        state = State.Asterisk;
+                        state = .asterisk;
                     },
                     '+' => {
-                        state = State.Plus;
+                        state = .plus;
                     },
                     '<' => {
-                        state = State.AngleBracketLeft;
+                        state = .angle_bracket_left;
                     },
                     '>' => {
-                        state = State.AngleBracketRight;
+                        state = .angle_bracket_right;
                     },
                     '^' => {
-                        state = State.Caret;
+                        state = .caret;
                     },
                     '\\' => {
-                        state = State.Backslash;
-                        result.id = Token.Id.MultilineStringLiteralLine;
+                        state = .backslash;
+                        result.id = .MultilineStringLiteralLine;
                     },
                     '{' => {
-                        result.id = Token.Id.LBrace;
+                        result.id = .LBrace;
                         self.index += 1;
                         break;
                     },
                     '}' => {
-                        result.id = Token.Id.RBrace;
+                        result.id = .RBrace;
                         self.index += 1;
                         break;
                     },
                     '~' => {
-                        result.id = Token.Id.Tilde;
+                        result.id = .Tilde;
                         self.index += 1;
                         break;
                     },
                     '.' => {
-                        state = State.Period;
+                        state = .period;
                     },
                     '-' => {
-                        state = State.Minus;
+                        state = .minus;
                     },
                     '/' => {
-                        state = State.Slash;
+                        state = .slash;
                     },
                     '&' => {
-                        state = State.Ampersand;
+                        state = .ampersand;
                     },
                     '0' => {
-                        state = State.Zero;
-                        result.id = Token.Id.IntegerLiteral;
+                        state = .zero;
+                        result.id = .IntegerLiteral;
                     },
                     '1'...'9' => {
-                        state = State.IntegerLiteralDec;
-                        result.id = Token.Id.IntegerLiteral;
+                        state = .int_literal_dec;
+                        result.id = .IntegerLiteral;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         self.index += 1;
                         break;
                     },
                 },
 
-                State.SawAtSign => switch (c) {
+                .saw_at_sign => switch (c) {
                     '"' => {
-                        result.id = Token.Id.Identifier;
-                        state = State.StringLiteral;
+                        result.id = .Identifier;
+                        state = .string_literal;
                     },
                     else => {
                         // reinterpret as a builtin
                         self.index -= 1;
-                        state = State.Builtin;
-                        result.id = Token.Id.Builtin;
+                        state = .builtin;
+                        result.id = .Builtin;
                     },
                 },
 
-                State.Ampersand => switch (c) {
+                .ampersand => switch (c) {
                     '&' => {
-                        result.id = Token.Id.Invalid_ampersands;
+                        result.id = .Invalid_ampersands;
                         self.index += 1;
                         break;
                     },
                     '=' => {
-                        result.id = Token.Id.AmpersandEqual;
+                        result.id = .AmpersandEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Ampersand;
+                        result.id = .Ampersand;
                         break;
                     },
                 },
 
-                State.Asterisk => switch (c) {
+                .asterisk => switch (c) {
                     '=' => {
-                        result.id = Token.Id.AsteriskEqual;
+                        result.id = .AsteriskEqual;
                         self.index += 1;
                         break;
                     },
                     '*' => {
-                        result.id = Token.Id.AsteriskAsterisk;
+                        result.id = .AsteriskAsterisk;
                         self.index += 1;
                         break;
                     },
                     '%' => {
-                        state = State.AsteriskPercent;
+                        state = .asterisk_percent;
                     },
                     else => {
-                        result.id = Token.Id.Asterisk;
+                        result.id = .Asterisk;
                         break;
                     },
                 },
 
-                State.AsteriskPercent => switch (c) {
+                .asterisk_percent => switch (c) {
                     '=' => {
-                        result.id = Token.Id.AsteriskPercentEqual;
+                        result.id = .AsteriskPercentEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.AsteriskPercent;
+                        result.id = .AsteriskPercent;
                         break;
                     },
                 },
 
-                State.Percent => switch (c) {
+                .percent => switch (c) {
                     '=' => {
-                        result.id = Token.Id.PercentEqual;
+                        result.id = .PercentEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Percent;
+                        result.id = .Percent;
                         break;
                     },
                 },
 
-                State.Plus => switch (c) {
+                .plus => switch (c) {
                     '=' => {
-                        result.id = Token.Id.PlusEqual;
+                        result.id = .PlusEqual;
                         self.index += 1;
                         break;
                     },
                     '+' => {
-                        result.id = Token.Id.PlusPlus;
+                        result.id = .PlusPlus;
                         self.index += 1;
                         break;
                     },
                     '%' => {
-                        state = State.PlusPercent;
+                        state = .plus_percent;
                     },
                     else => {
-                        result.id = Token.Id.Plus;
+                        result.id = .Plus;
                         break;
                     },
                 },
 
-                State.PlusPercent => switch (c) {
+                .plus_percent => switch (c) {
                     '=' => {
-                        result.id = Token.Id.PlusPercentEqual;
+                        result.id = .PlusPercentEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.PlusPercent;
+                        result.id = .PlusPercent;
                         break;
                     },
                 },
 
-                State.Caret => switch (c) {
+                .caret => switch (c) {
                     '=' => {
-                        result.id = Token.Id.CaretEqual;
+                        result.id = .CaretEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Caret;
+                        result.id = .Caret;
                         break;
                     },
                 },
 
-                State.Identifier => switch (c) {
+                .identifier => switch (c) {
                     'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
                     else => {
                         if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
@@ -692,19 +692,19 @@ pub const Tokenizer = struct {
                         break;
                     },
                 },
-                State.Builtin => switch (c) {
+                .builtin => switch (c) {
                     'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
                     else => break,
                 },
-                State.Backslash => switch (c) {
+                .backslash => switch (c) {
                     '\\' => {
-                        state = State.MultilineStringLiteralLine;
+                        state = .multiline_string_literal_line;
                     },
                     else => break,
                 },
-                State.StringLiteral => switch (c) {
+                .string_literal => switch (c) {
                     '\\' => {
-                        state = State.StringLiteralBackslash;
+                        state = .string_literal_backslash;
                     },
                     '"' => {
                         self.index += 1;
@@ -714,98 +714,98 @@ pub const Tokenizer = struct {
                     else => self.checkLiteralCharacter(),
                 },
 
-                State.StringLiteralBackslash => switch (c) {
+                .string_literal_backslash => switch (c) {
                     '\n', '\r' => break, // Look for this error later.
                     else => {
-                        state = State.StringLiteral;
+                        state = .string_literal;
                     },
                 },
 
-                State.CharLiteral => switch (c) {
+                .char_literal => switch (c) {
                     '\\' => {
-                        state = State.CharLiteralBackslash;
+                        state = .char_literal_backslash;
                     },
                     '\'', 0x80...0xbf, 0xf8...0xff => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                     0xc0...0xdf => { // 110xxxxx
                         remaining_code_units = 1;
-                        state = State.CharLiteralUnicode;
+                        state = .char_literal_unicode;
                     },
                     0xe0...0xef => { // 1110xxxx
                         remaining_code_units = 2;
-                        state = State.CharLiteralUnicode;
+                        state = .char_literal_unicode;
                     },
                     0xf0...0xf7 => { // 11110xxx
                         remaining_code_units = 3;
-                        state = State.CharLiteralUnicode;
+                        state = .char_literal_unicode;
                     },
                     else => {
-                        state = State.CharLiteralEnd;
+                        state = .char_literal_end;
                     },
                 },
 
-                State.CharLiteralBackslash => switch (c) {
+                .char_literal_backslash => switch (c) {
                     '\n' => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                     'x' => {
-                        state = State.CharLiteralHexEscape;
+                        state = .char_literal_hex_escape;
                         seen_escape_digits = 0;
                     },
                     'u' => {
-                        state = State.CharLiteralUnicodeEscapeSawU;
+                        state = .char_literal_unicode_escape_saw_u;
                     },
                     else => {
-                        state = State.CharLiteralEnd;
+                        state = .char_literal_end;
                     },
                 },
 
-                State.CharLiteralHexEscape => switch (c) {
+                .char_literal_hex_escape => switch (c) {
                     '0'...'9', 'a'...'f', 'A'...'F' => {
                         seen_escape_digits += 1;
                         if (seen_escape_digits == 2) {
-                            state = State.CharLiteralEnd;
+                            state = .char_literal_end;
                         }
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
 
-                State.CharLiteralUnicodeEscapeSawU => switch (c) {
+                .char_literal_unicode_escape_saw_u => switch (c) {
                     '{' => {
-                        state = State.CharLiteralUnicodeEscape;
+                        state = .char_literal_unicode_escape;
                         seen_escape_digits = 0;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
-                        state = State.CharLiteralUnicodeInvalid;
+                        result.id = .Invalid;
+                        state = .char_literal_unicode_invalid;
                     },
                 },
 
-                State.CharLiteralUnicodeEscape => switch (c) {
+                .char_literal_unicode_escape => switch (c) {
                     '0'...'9', 'a'...'f', 'A'...'F' => {
                         seen_escape_digits += 1;
                     },
                     '}' => {
                         if (seen_escape_digits == 0) {
-                            result.id = Token.Id.Invalid;
-                            state = State.CharLiteralUnicodeInvalid;
+                            result.id = .Invalid;
+                            state = .char_literal_unicode_invalid;
                         } else {
-                            state = State.CharLiteralEnd;
+                            state = .char_literal_end;
                         }
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
-                        state = State.CharLiteralUnicodeInvalid;
+                        result.id = .Invalid;
+                        state = .char_literal_unicode_invalid;
                     },
                 },
 
-                State.CharLiteralUnicodeInvalid => switch (c) {
+                .char_literal_unicode_invalid => switch (c) {
                     // Keep consuming characters until an obvious stopping point.
                     // This consolidates e.g. `u{0ab1Q}` into a single invalid token
                     // instead of creating the tokens `u{0ab1`, `Q`, `}`
@@ -813,32 +813,32 @@ pub const Tokenizer = struct {
                     else => break,
                 },
 
-                State.CharLiteralEnd => switch (c) {
+                .char_literal_end => switch (c) {
                     '\'' => {
-                        result.id = Token.Id.CharLiteral;
+                        result.id = .CharLiteral;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
 
-                State.CharLiteralUnicode => switch (c) {
+                .char_literal_unicode => switch (c) {
                     0x80...0xbf => {
                         remaining_code_units -= 1;
                         if (remaining_code_units == 0) {
-                            state = State.CharLiteralEnd;
+                            state = .char_literal_end;
                         }
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
 
-                State.MultilineStringLiteralLine => switch (c) {
+                .multiline_string_literal_line => switch (c) {
                     '\n' => {
                         self.index += 1;
                         break;
@@ -847,449 +847,449 @@ pub const Tokenizer = struct {
                     else => self.checkLiteralCharacter(),
                 },
 
-                State.Bang => switch (c) {
+                .bang => switch (c) {
                     '=' => {
-                        result.id = Token.Id.BangEqual;
+                        result.id = .BangEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Bang;
+                        result.id = .Bang;
                         break;
                     },
                 },
 
-                State.Pipe => switch (c) {
+                .pipe => switch (c) {
                     '=' => {
-                        result.id = Token.Id.PipeEqual;
+                        result.id = .PipeEqual;
                         self.index += 1;
                         break;
                     },
                     '|' => {
-                        result.id = Token.Id.PipePipe;
+                        result.id = .PipePipe;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Pipe;
+                        result.id = .Pipe;
                         break;
                     },
                 },
 
-                State.Equal => switch (c) {
+                .equal => switch (c) {
                     '=' => {
-                        result.id = Token.Id.EqualEqual;
+                        result.id = .EqualEqual;
                         self.index += 1;
                         break;
                     },
                     '>' => {
-                        result.id = Token.Id.EqualAngleBracketRight;
+                        result.id = .EqualAngleBracketRight;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Equal;
+                        result.id = .Equal;
                         break;
                     },
                 },
 
-                State.Minus => switch (c) {
+                .minus => switch (c) {
                     '>' => {
-                        result.id = Token.Id.Arrow;
+                        result.id = .Arrow;
                         self.index += 1;
                         break;
                     },
                     '=' => {
-                        result.id = Token.Id.MinusEqual;
+                        result.id = .MinusEqual;
                         self.index += 1;
                         break;
                     },
                     '%' => {
-                        state = State.MinusPercent;
+                        state = .minus_percent;
                     },
                     else => {
-                        result.id = Token.Id.Minus;
+                        result.id = .Minus;
                         break;
                     },
                 },
 
-                State.MinusPercent => switch (c) {
+                .minus_percent => switch (c) {
                     '=' => {
-                        result.id = Token.Id.MinusPercentEqual;
+                        result.id = .MinusPercentEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.MinusPercent;
+                        result.id = .MinusPercent;
                         break;
                     },
                 },
 
-                State.AngleBracketLeft => switch (c) {
+                .angle_bracket_left => switch (c) {
                     '<' => {
-                        state = State.AngleBracketAngleBracketLeft;
+                        state = .angle_bracket_angle_bracket_left;
                     },
                     '=' => {
-                        result.id = Token.Id.AngleBracketLeftEqual;
+                        result.id = .AngleBracketLeftEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.AngleBracketLeft;
+                        result.id = .AngleBracketLeft;
                         break;
                     },
                 },
 
-                State.AngleBracketAngleBracketLeft => switch (c) {
+                .angle_bracket_angle_bracket_left => switch (c) {
                     '=' => {
-                        result.id = Token.Id.AngleBracketAngleBracketLeftEqual;
+                        result.id = .AngleBracketAngleBracketLeftEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.AngleBracketAngleBracketLeft;
+                        result.id = .AngleBracketAngleBracketLeft;
                         break;
                     },
                 },
 
-                State.AngleBracketRight => switch (c) {
+                .angle_bracket_right => switch (c) {
                     '>' => {
-                        state = State.AngleBracketAngleBracketRight;
+                        state = .angle_bracket_angle_bracket_right;
                     },
                     '=' => {
-                        result.id = Token.Id.AngleBracketRightEqual;
+                        result.id = .AngleBracketRightEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.AngleBracketRight;
+                        result.id = .AngleBracketRight;
                         break;
                     },
                 },
 
-                State.AngleBracketAngleBracketRight => switch (c) {
+                .angle_bracket_angle_bracket_right => switch (c) {
                     '=' => {
-                        result.id = Token.Id.AngleBracketAngleBracketRightEqual;
+                        result.id = .AngleBracketAngleBracketRightEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.AngleBracketAngleBracketRight;
+                        result.id = .AngleBracketAngleBracketRight;
                         break;
                     },
                 },
 
-                State.Period => switch (c) {
+                .period => switch (c) {
                     '.' => {
-                        state = State.Period2;
+                        state = .period_2;
                     },
                     '*' => {
-                        result.id = Token.Id.PeriodAsterisk;
+                        result.id = .PeriodAsterisk;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Period;
+                        result.id = .Period;
                         break;
                     },
                 },
 
-                State.Period2 => switch (c) {
+                .period_2 => switch (c) {
                     '.' => {
-                        result.id = Token.Id.Ellipsis3;
+                        result.id = .Ellipsis3;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Ellipsis2;
+                        result.id = .Ellipsis2;
                         break;
                     },
                 },
 
-                State.Slash => switch (c) {
+                .slash => switch (c) {
                     '/' => {
-                        state = State.LineCommentStart;
-                        result.id = Token.Id.LineComment;
+                        state = .line_comment_start;
+                        result.id = .LineComment;
                     },
                     '=' => {
-                        result.id = Token.Id.SlashEqual;
+                        result.id = .SlashEqual;
                         self.index += 1;
                         break;
                     },
                     else => {
-                        result.id = Token.Id.Slash;
+                        result.id = .Slash;
                         break;
                     },
                 },
-                State.LineCommentStart => switch (c) {
+                .line_comment_start => switch (c) {
                     '/' => {
-                        state = State.DocCommentStart;
+                        state = .doc_comment_start;
                     },
                     '!' => {
-                        result.id = Token.Id.ContainerDocComment;
-                        state = State.ContainerDocComment;
+                        result.id = .ContainerDocComment;
+                        state = .container_doc_comment;
                     },
                     '\n' => break,
                     else => {
-                        state = State.LineComment;
+                        state = .line_comment;
                         self.checkLiteralCharacter();
                     },
                 },
-                State.DocCommentStart => switch (c) {
+                .doc_comment_start => switch (c) {
                     '/' => {
-                        state = State.LineComment;
+                        state = .line_comment;
                     },
                     '\n' => {
-                        result.id = Token.Id.DocComment;
+                        result.id = .DocComment;
                         break;
                     },
                     else => {
-                        state = State.DocComment;
-                        result.id = Token.Id.DocComment;
+                        state = .doc_comment;
+                        result.id = .DocComment;
                         self.checkLiteralCharacter();
                     },
                 },
-                State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
+                .line_comment, .doc_comment, .container_doc_comment => switch (c) {
                     '\n' => break,
                     else => self.checkLiteralCharacter(),
                 },
-                State.Zero => switch (c) {
+                .zero => switch (c) {
                     'b' => {
-                        state = State.IntegerLiteralBinNoUnderscore;
+                        state = .int_literal_bin_no_underscore;
                     },
                     'o' => {
-                        state = State.IntegerLiteralOctNoUnderscore;
+                        state = .int_literal_oct_no_underscore;
                     },
                     'x' => {
-                        state = State.IntegerLiteralHexNoUnderscore;
+                        state = .int_literal_hex_no_underscore;
                     },
                     '0'...'9', '_', '.', 'e', 'E' => {
                         // reinterpret as a decimal number
                         self.index -= 1;
-                        state = State.IntegerLiteralDec;
+                        state = .int_literal_dec;
                     },
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.IntegerLiteralBinNoUnderscore => switch (c) {
+                .int_literal_bin_no_underscore => switch (c) {
                     '0'...'1' => {
-                        state = State.IntegerLiteralBin;
+                        state = .int_literal_bin;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.IntegerLiteralBin => switch (c) {
+                .int_literal_bin => switch (c) {
                     '_' => {
-                        state = State.IntegerLiteralBinNoUnderscore;
+                        state = .int_literal_bin_no_underscore;
                     },
                     '0'...'1' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.IntegerLiteralOctNoUnderscore => switch (c) {
+                .int_literal_oct_no_underscore => switch (c) {
                     '0'...'7' => {
-                        state = State.IntegerLiteralOct;
+                        state = .int_literal_oct;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.IntegerLiteralOct => switch (c) {
+                .int_literal_oct => switch (c) {
                     '_' => {
-                        state = State.IntegerLiteralOctNoUnderscore;
+                        state = .int_literal_oct_no_underscore;
                     },
                     '0'...'7' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.IntegerLiteralDecNoUnderscore => switch (c) {
+                .int_literal_dec_no_underscore => switch (c) {
                     '0'...'9' => {
-                        state = State.IntegerLiteralDec;
+                        state = .int_literal_dec;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.IntegerLiteralDec => switch (c) {
+                .int_literal_dec => switch (c) {
                     '_' => {
-                        state = State.IntegerLiteralDecNoUnderscore;
+                        state = .int_literal_dec_no_underscore;
                     },
                     '.' => {
-                        state = State.NumberDotDec;
-                        result.id = Token.Id.FloatLiteral;
+                        state = .num_dot_dec;
+                        result.id = .FloatLiteral;
                     },
                     'e', 'E' => {
-                        state = State.FloatExponentUnsigned;
-                        result.id = Token.Id.FloatLiteral;
+                        state = .float_exponent_unsigned;
+                        result.id = .FloatLiteral;
                     },
                     '0'...'9' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.IntegerLiteralHexNoUnderscore => switch (c) {
+                .int_literal_hex_no_underscore => switch (c) {
                     '0'...'9', 'a'...'f', 'A'...'F' => {
-                        state = State.IntegerLiteralHex;
+                        state = .int_literal_hex;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.IntegerLiteralHex => switch (c) {
+                .int_literal_hex => switch (c) {
                     '_' => {
-                        state = State.IntegerLiteralHexNoUnderscore;
+                        state = .int_literal_hex_no_underscore;
                     },
                     '.' => {
-                        state = State.NumberDotHex;
-                        result.id = Token.Id.FloatLiteral;
+                        state = .num_dot_hex;
+                        result.id = .FloatLiteral;
                     },
                     'p', 'P' => {
-                        state = State.FloatExponentUnsigned;
-                        result.id = Token.Id.FloatLiteral;
+                        state = .float_exponent_unsigned;
+                        result.id = .FloatLiteral;
                     },
                     '0'...'9', 'a'...'f', 'A'...'F' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.NumberDotDec => switch (c) {
+                .num_dot_dec => switch (c) {
                     '.' => {
                         self.index -= 1;
-                        state = State.Start;
+                        state = .start;
                         break;
                     },
                     'e', 'E' => {
-                        state = State.FloatExponentUnsigned;
+                        state = .float_exponent_unsigned;
                     },
                     '0'...'9' => {
-                        result.id = Token.Id.FloatLiteral;
-                        state = State.FloatFractionDec;
+                        result.id = .FloatLiteral;
+                        state = .float_fraction_dec;
                     },
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.NumberDotHex => switch (c) {
+                .num_dot_hex => switch (c) {
                     '.' => {
                         self.index -= 1;
-                        state = State.Start;
+                        state = .start;
                         break;
                     },
                     'p', 'P' => {
-                        state = State.FloatExponentUnsigned;
+                        state = .float_exponent_unsigned;
                     },
                     '0'...'9', 'a'...'f', 'A'...'F' => {
-                        result.id = Token.Id.FloatLiteral;
-                        state = State.FloatFractionHex;
+                        result.id = .FloatLiteral;
+                        state = .float_fraction_hex;
                     },
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.FloatFractionDecNoUnderscore => switch (c) {
+                .float_fraction_dec_no_underscore => switch (c) {
                     '0'...'9' => {
-                        state = State.FloatFractionDec;
+                        state = .float_fraction_dec;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.FloatFractionDec => switch (c) {
+                .float_fraction_dec => switch (c) {
                     '_' => {
-                        state = State.FloatFractionDecNoUnderscore;
+                        state = .float_fraction_dec_no_underscore;
                     },
                     'e', 'E' => {
-                        state = State.FloatExponentUnsigned;
+                        state = .float_exponent_unsigned;
                     },
                     '0'...'9' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.FloatFractionHexNoUnderscore => switch (c) {
+                .float_fraction_hex_no_underscore => switch (c) {
                     '0'...'9', 'a'...'f', 'A'...'F' => {
-                        state = State.FloatFractionHex;
+                        state = .float_fraction_hex;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.FloatFractionHex => switch (c) {
+                .float_fraction_hex => switch (c) {
                     '_' => {
-                        state = State.FloatFractionHexNoUnderscore;
+                        state = .float_fraction_hex_no_underscore;
                     },
                     'p', 'P' => {
-                        state = State.FloatExponentUnsigned;
+                        state = .float_exponent_unsigned;
                     },
                     '0'...'9', 'a'...'f', 'A'...'F' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
                 },
-                State.FloatExponentUnsigned => switch (c) {
+                .float_exponent_unsigned => switch (c) {
                     '+', '-' => {
-                        state = State.FloatExponentNumberNoUnderscore;
+                        state = .float_exponent_num_no_underscore;
                     },
                     else => {
                         // reinterpret as a normal exponent number
                         self.index -= 1;
-                        state = State.FloatExponentNumberNoUnderscore;
+                        state = .float_exponent_num_no_underscore;
                     },
                 },
-                State.FloatExponentNumberNoUnderscore => switch (c) {
+                .float_exponent_num_no_underscore => switch (c) {
                     '0'...'9' => {
-                        state = State.FloatExponentNumber;
+                        state = .float_exponent_num;
                     },
                     else => {
-                        result.id = Token.Id.Invalid;
+                        result.id = .Invalid;
                         break;
                     },
                 },
-                State.FloatExponentNumber => switch (c) {
+                .float_exponent_num => switch (c) {
                     '_' => {
-                        state = State.FloatExponentNumberNoUnderscore;
+                        state = .float_exponent_num_no_underscore;
                     },
                     '0'...'9' => {},
                     else => {
                         if (isIdentifierChar(c)) {
-                            result.id = Token.Id.Invalid;
+                            result.id = .Invalid;
                         }
                         break;
                     },
@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {
             }
         } else if (self.index == self.buffer.len) {
             switch (state) {
-                State.Start,
-                State.IntegerLiteralDec,
-                State.IntegerLiteralBin,
-                State.IntegerLiteralOct,
-                State.IntegerLiteralHex,
-                State.NumberDotDec,
-                State.NumberDotHex,
-                State.FloatFractionDec,
-                State.FloatFractionHex,
-                State.FloatExponentNumber,
-                State.StringLiteral, // find this error later
-                State.MultilineStringLiteralLine,
-                State.Builtin,
+                .start,
+                .int_literal_dec,
+                .int_literal_bin,
+                .int_literal_oct,
+                .int_literal_hex,
+                .num_dot_dec,
+                .num_dot_hex,
+                .float_fraction_dec,
+                .float_fraction_hex,
+                .float_exponent_num,
+                .string_literal, // find this error later
+                .multiline_string_literal_line,
+                .builtin,
                 => {},
 
-                State.Identifier => {
+                .identifier => {
                     if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
                         result.id = id;
                     }
                 },
-                State.LineCommentStart, State.LineComment => {
-                    result.id = Token.Id.LineComment;
-                },
-                State.DocComment, State.DocCommentStart => {
-                    result.id = Token.Id.DocComment;
-                },
-                State.ContainerDocComment => {
-                    result.id = Token.Id.ContainerDocComment;
-                },
-
-                State.IntegerLiteralDecNoUnderscore,
-                State.IntegerLiteralBinNoUnderscore,
-                State.IntegerLiteralOctNoUnderscore,
-                State.IntegerLiteralHexNoUnderscore,
-                State.FloatFractionDecNoUnderscore,
-                State.FloatFractionHexNoUnderscore,
-                State.FloatExponentNumberNoUnderscore,
-                State.FloatExponentUnsigned,
-                State.SawAtSign,
-                State.Backslash,
-                State.CharLiteral,
-                State.CharLiteralBackslash,
-                State.CharLiteralHexEscape,
-                State.CharLiteralUnicodeEscapeSawU,
-                State.CharLiteralUnicodeEscape,
-                State.CharLiteralUnicodeInvalid,
-                State.CharLiteralEnd,
-                State.CharLiteralUnicode,
-                State.StringLiteralBackslash,
+                .line_comment, .line_comment_start => {
+                    result.id = .LineComment;
+                },
+                .doc_comment, .doc_comment_start => {
+                    result.id = .DocComment;
+                },
+                .container_doc_comment => {
+                    result.id = .ContainerDocComment;
+                },
+
+                .int_literal_dec_no_underscore,
+                .int_literal_bin_no_underscore,
+                .int_literal_oct_no_underscore,
+                .int_literal_hex_no_underscore,
+                .float_fraction_dec_no_underscore,
+                .float_fraction_hex_no_underscore,
+                .float_exponent_num_no_underscore,
+                .float_exponent_unsigned,
+                .saw_at_sign,
+                .backslash,
+                .char_literal,
+                .char_literal_backslash,
+                .char_literal_hex_escape,
+                .char_literal_unicode_escape_saw_u,
+                .char_literal_unicode_escape,
+                .char_literal_unicode_invalid,
+                .char_literal_end,
+                .char_literal_unicode,
+                .string_literal_backslash,
                 => {
-                    result.id = Token.Id.Invalid;
+                    result.id = .Invalid;
                 },
 
-                State.Equal => {
-                    result.id = Token.Id.Equal;
+                .equal => {
+                    result.id = .Equal;
                 },
-                State.Bang => {
-                    result.id = Token.Id.Bang;
+                .bang => {
+                    result.id = .Bang;
                 },
-                State.Minus => {
-                    result.id = Token.Id.Minus;
+                .minus => {
+                    result.id = .Minus;
                 },
-                State.Slash => {
-                    result.id = Token.Id.Slash;
+                .slash => {
+                    result.id = .Slash;
                 },
-                State.Zero => {
-                    result.id = Token.Id.IntegerLiteral;
+                .zero => {
+                    result.id = .IntegerLiteral;
                 },
-                State.Ampersand => {
-                    result.id = Token.Id.Ampersand;
+                .ampersand => {
+                    result.id = .Ampersand;
                 },
-                State.Period => {
-                    result.id = Token.Id.Period;
+                .period => {
+                    result.id = .Period;
                 },
-                State.Period2 => {
-                    result.id = Token.Id.Ellipsis2;
+                .period_2 => {
+                    result.id = .Ellipsis2;
                 },
-                State.Pipe => {
-                    result.id = Token.Id.Pipe;
+                .pipe => {
+                    result.id = .Pipe;
                 },
-                State.AngleBracketAngleBracketRight => {
-                    result.id = Token.Id.AngleBracketAngleBracketRight;
+                .angle_bracket_angle_bracket_right => {
+                    result.id = .AngleBracketAngleBracketRight;
                 },
-                State.AngleBracketRight => {
-                    result.id = Token.Id.AngleBracketRight;
+                .angle_bracket_right => {
+                    result.id = .AngleBracketRight;
                 },
-                State.AngleBracketAngleBracketLeft => {
-                    result.id = Token.Id.AngleBracketAngleBracketLeft;
+                .angle_bracket_angle_bracket_left => {
+                    result.id = .AngleBracketAngleBracketLeft;
                 },
-                State.AngleBracketLeft => {
-                    result.id = Token.Id.AngleBracketLeft;
+                .angle_bracket_left => {
+                    result.id = .AngleBracketLeft;
                 },
-                State.PlusPercent => {
-                    result.id = Token.Id.PlusPercent;
+                .plus_percent => {
+                    result.id = .PlusPercent;
                 },
-                State.Plus => {
-                    result.id = Token.Id.Plus;
+                .plus => {
+                    result.id = .Plus;
                 },
-                State.Percent => {
-                    result.id = Token.Id.Percent;
+                .percent => {
+                    result.id = .Percent;
                 },
-                State.Caret => {
-                    result.id = Token.Id.Caret;
+                .caret => {
+                    result.id = .Caret;
                 },
-                State.AsteriskPercent => {
-                    result.id = Token.Id.AsteriskPercent;
+                .asterisk_percent => {
+                    result.id = .AsteriskPercent;
                 },
-                State.Asterisk => {
-                    result.id = Token.Id.Asterisk;
+                .asterisk => {
+                    result.id = .Asterisk;
                 },
-                State.MinusPercent => {
-                    result.id = Token.Id.MinusPercent;
+                .minus_percent => {
+                    result.id = .MinusPercent;
                 },
             }
         }
 
-        if (result.id == Token.Id.Eof) {
+        if (result.id == .Eof) {
             if (self.pending_invalid_token) |token| {
                 self.pending_invalid_token = null;
                 return token;
@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {
         if (self.pending_invalid_token != null) return;
         const invalid_length = self.getInvalidCharacterLength();
         if (invalid_length == 0) return;
-        self.pending_invalid_token = Token{
-            .id = Token.Id.Invalid,
+        self.pending_invalid_token = .{
+            .id = .Invalid,
             .start = self.index,
             .end = self.index + invalid_length,
         };
@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {
 };
 
 test "tokenizer" {
-    testTokenize("test", &[_]Token.Id{Token.Id.Keyword_test});
+    testTokenize("test", &[_]Token.Id{.Keyword_test});
 }
 
 test "tokenizer - unknown length pointer and then c pointer" {
@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {
         \\[*]u8
         \\[*c]u8
     , &[_]Token.Id{
-        Token.Id.LBracket,
-        Token.Id.Asterisk,
-        Token.Id.RBracket,
-        Token.Id.Identifier,
-        Token.Id.LBracket,
-        Token.Id.Asterisk,
-        Token.Id.Identifier,
-        Token.Id.RBracket,
-        Token.Id.Identifier,
+        .LBracket,
+        .Asterisk,
+        .RBracket,
+        .Identifier,
+        .LBracket,
+        .Asterisk,
+        .Identifier,
+        .RBracket,
+        .Identifier,
     });
 }
 
@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {
 
 test "tokenizer - float literal e exponent" {
     testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
-        Token.Id.Identifier,
-        Token.Id.Equal,
-        Token.Id.FloatLiteral,
-        Token.Id.Semicolon,
+        .Identifier,
+        .Equal,
+        .FloatLiteral,
+        .Semicolon,
     });
 }
 
 test "tokenizer - float literal p exponent" {
     testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
-        Token.Id.Identifier,
-        Token.Id.Equal,
-        Token.Id.FloatLiteral,
-        Token.Id.Semicolon,
+        .Identifier,
+        .Equal,
+        .FloatLiteral,
+        .Semicolon,
     });
 }
 
 test "tokenizer - chars" {
-    testTokenize("'c'", &[_]Token.Id{Token.Id.CharLiteral});
+    testTokenize("'c'", &[_]Token.Id{.CharLiteral});
 }
 
 test "tokenizer - invalid token characters" {
-    testTokenize("#", &[_]Token.Id{Token.Id.Invalid});
-    testTokenize("`", &[_]Token.Id{Token.Id.Invalid});
-    testTokenize("'c", &[_]Token.Id{Token.Id.Invalid});
-    testTokenize("'", &[_]Token.Id{Token.Id.Invalid});
-    testTokenize("''", &[_]Token.Id{ Token.Id.Invalid, Token.Id.Invalid });
+    testTokenize("#", &[_]Token.Id{.Invalid});
+    testTokenize("`", &[_]Token.Id{.Invalid});
+    testTokenize("'c", &[_]Token.Id{.Invalid});
+    testTokenize("'", &[_]Token.Id{.Invalid});
+    testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
 }
 
 test "tokenizer - invalid literal/comment characters" {
     testTokenize("\"\x00\"", &[_]Token.Id{
-        Token.Id.StringLiteral,
-        Token.Id.Invalid,
+        .StringLiteral,
+        .Invalid,
     });
     testTokenize("//\x00", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\x1f", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\x7f", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
 }
 
 test "tokenizer - utf8" {
-    testTokenize("//\xc2\x80", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{Token.Id.LineComment});
+    testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
+    testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
 }
 
 test "tokenizer - invalid utf8" {
     testTokenize("//\x80", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xbf", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xf8", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xff", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xc2\xc0", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xe0", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xf0", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
 }
 
 test "tokenizer - illegal unicode codepoints" {
     // unicode newline characters.U+0085, U+2028, U+2029
-    testTokenize("//\xc2\x84", &[_]Token.Id{Token.Id.LineComment});
+    testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
     testTokenize("//\xc2\x85", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
-    testTokenize("//\xc2\x86", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("//\xe2\x80\xa7", &[_]Token.Id{Token.Id.LineComment});
+    testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
+    testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
     testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
     testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
-        Token.Id.LineComment,
-        Token.Id.Invalid,
+        .LineComment,
+        .Invalid,
     });
-    testTokenize("//\xe2\x80\xaa", &[_]Token.Id{Token.Id.LineComment});
+    testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
 }
 
 test "tokenizer - string identifier and builtin fns" {
     testTokenize(
         \\const @"if" = @import("std");
     , &[_]Token.Id{
-        Token.Id.Keyword_const,
-        Token.Id.Identifier,
-        Token.Id.Equal,
-        Token.Id.Builtin,
-        Token.Id.LParen,
-        Token.Id.StringLiteral,
-        Token.Id.RParen,
-        Token.Id.Semicolon,
+        .Keyword_const,
+        .Identifier,
+        .Equal,
+        .Builtin,
+        .LParen,
+        .StringLiteral,
+        .RParen,
+        .Semicolon,
     });
 }
 
@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {
     testTokenize(
         \\\\foo	bar
     , &[_]Token.Id{
-        Token.Id.MultilineStringLiteralLine,
+        .MultilineStringLiteralLine,
     });
 }
 
 test "tokenizer - pipe and then invalid" {
     testTokenize("||=", &[_]Token.Id{
-        Token.Id.PipePipe,
-        Token.Id.Equal,
+        .PipePipe,
+        .Equal,
     });
 }
 
 test "tokenizer - line comment and doc comment" {
-    testTokenize("//", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("// a / b", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("// /", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("/// a", &[_]Token.Id{Token.Id.DocComment});
-    testTokenize("///", &[_]Token.Id{Token.Id.DocComment});
-    testTokenize("////", &[_]Token.Id{Token.Id.LineComment});
-    testTokenize("//!", &[_]Token.Id{Token.Id.ContainerDocComment});
-    testTokenize("//!!", &[_]Token.Id{Token.Id.ContainerDocComment});
+    testTokenize("//", &[_]Token.Id{.LineComment});
+    testTokenize("// a / b", &[_]Token.Id{.LineComment});
+    testTokenize("// /", &[_]Token.Id{.LineComment});
+    testTokenize("/// a", &[_]Token.Id{.DocComment});
+    testTokenize("///", &[_]Token.Id{.DocComment});
+    testTokenize("////", &[_]Token.Id{.LineComment});
+    testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
+    testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
 }
 
 test "tokenizer - line comment followed by identifier" {
@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {
         \\    // another
         \\    Another,
     , &[_]Token.Id{
-        Token.Id.Identifier,
-        Token.Id.Comma,
-        Token.Id.LineComment,
-        Token.Id.Identifier,
-        Token.Id.Comma,
+        .Identifier,
+        .Comma,
+        .LineComment,
+        .Identifier,
+        .Comma,
     });
 }
 
 test "tokenizer - UTF-8 BOM is recognized and skipped" {
     testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
-        Token.Id.Identifier,
-        Token.Id.Semicolon,
+        .Identifier,
+        .Semicolon,
     });
 }
 
 test "correctly parse pointer assignment" {
     testTokenize("b.*=3;\n", &[_]Token.Id{
-        Token.Id.Identifier,
-        Token.Id.PeriodAsterisk,
-        Token.Id.Equal,
-        Token.Id.IntegerLiteral,
-        Token.Id.Semicolon,
+        .Identifier,
+        .PeriodAsterisk,
+        .Equal,
+        .IntegerLiteral,
+        .Semicolon,
     });
 }
 
@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
         }
     }
     const last_token = tokenizer.next();
-    std.testing.expect(last_token.id == Token.Id.Eof);
+    std.testing.expect(last_token.id == .Eof);
 }