From 23b7d28896609e3f01765730599119baf53a56c9 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Sun, 22 Jan 2023 16:40:00 +0100 Subject: std: restrict mem.span() and mem.len() to sentinel terminated pointers These functions are currently footgunny when working with pointers to arrays and slices. They just return the stated length of the array/slice without iterating and looking for the first sentinel, even if the array/slice is a sentinel terminated type. From looking at the quite small list of places in the standard library/compiler that this change breaks existing code, the new code looks to be more readable in all cases. The usage of std.mem.span/len was totally unneeded in most of the cases affected by this breaking change. We could remove these functions entirely in favor of other existing functions in std.mem such as std.mem.sliceTo(), but that would be a somewhat nasty breaking change as std.mem.span() is very widely used for converting sentinel terminated pointers to slices. It is however not at all widely used for anything else. Therefore I think it is better to break these few non-standard and potentially incorrect usages of these functions now and at some later time, if deemed worthwhile, finally remove these functions. If we wait for at least a full release cycle so that everyone adapts to this change first, updating for the removal could be a simple find and replace without needing to worry about the semantics. --- src/main.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index fdc761ac92..72e7e094e6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -893,7 +893,7 @@ fn buildOutputType( i: usize = 0, fn next(it: *@This()) ?[]const u8 { if (it.i >= it.args.len) { - if (it.resp_file) |*resp| return if (resp.next()) |sentinel| std.mem.span(sentinel) else null; + if (it.resp_file) |*resp| return resp.next(); return null; } defer it.i += 1; @@ -901,7 +901,7 @@ fn buildOutputType( } fn nextOrFatal(it: *@This()) []const u8 { if (it.i >= it.args.len) { - if (it.resp_file) |*resp| if (resp.next()) |sentinel| return std.mem.span(sentinel); + if (it.resp_file) |*resp| if (resp.next()) |ret| return ret; fatal("expected parameter after {s}", .{it.args[it.i - 1]}); } defer it.i += 1; @@ -4973,7 +4973,7 @@ pub const ClangArgIterator = struct { // rather than an argument to a parameter. // We adjust the len below when necessary. self.other_args = (self.argv.ptr + self.next_index)[0..1]; - var arg = mem.span(self.argv[self.next_index]); + var arg = self.argv[self.next_index]; self.incrementArgIndex(); if (mem.startsWith(u8, arg, "@")) { @@ -5017,7 +5017,7 @@ pub const ClangArgIterator = struct { self.has_next = true; self.other_args = (self.argv.ptr + self.next_index)[0..1]; // We adjust len below when necessary. - arg = mem.span(self.argv[self.next_index]); + arg = self.argv[self.next_index]; self.incrementArgIndex(); } -- cgit v1.2.3 From 873bb29c984b976021fb9ca95ad3298e03a8b3ff Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Feb 2023 17:32:06 -0700 Subject: introduce ZON: Zig Object Notation * std.zig.parse is moved to std.zig.Ast.parse * the new function has an additional parameter that requires passing Mode.zig or Mode.zon * moved parser.zig code to Parse.zig * added parseZon function next to parseRoot function --- CMakeLists.txt | 2 +- lib/std/Build/OptionsStep.zig | 2 +- lib/std/zig.zig | 1 - lib/std/zig/Ast.zig | 78 +- lib/std/zig/Parse.zig | 3816 ++++++++++++++++++++++++++++++++++++++++ lib/std/zig/parse.zig | 3852 ----------------------------------------- lib/std/zig/parser_test.zig | 4 +- lib/std/zig/perf_test.zig | 3 +- src/Module.zig | 6 +- src/main.zig | 10 +- 10 files changed, 3898 insertions(+), 3876 deletions(-) create mode 100644 lib/std/zig/Parse.zig delete mode 100644 lib/std/zig/parse.zig (limited to 'src/main.zig') diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a9de7bdc1..f8029fdcde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,7 +513,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/zig/Ast.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/CrossTarget.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/c_builtins.zig" - "${CMAKE_SOURCE_DIR}/lib/std/zig/parse.zig" + "${CMAKE_SOURCE_DIR}/lib/std/zig/Parse.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/render.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/string_literal.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/system.zig" diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index a353737512..c1d2c8454a 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -367,5 +367,5 @@ test "OptionsStep" { \\ , options.contents.items); - _ = try std.zig.parse(arena.allocator(), try options.contents.toOwnedSliceSentinel(0)); + _ = try std.zig.Ast.parse(arena.allocator(), try options.contents.toOwnedSliceSentinel(0), .zig); } diff --git a/lib/std/zig.zig b/lib/std/zig.zig index bce8f6ce3c..f85cf75e60 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -8,7 +8,6 @@ pub const Tokenizer = tokenizer.Tokenizer; pub const fmtId = fmt.fmtId; pub const fmtEscapes = fmt.fmtEscapes; pub const isValidId = fmt.isValidId; -pub const parse = @import("zig/parse.zig").parse; pub const string_literal = @import("zig/string_literal.zig"); pub const number_literal = @import("zig/number_literal.zig"); pub const primitives = @import("zig/primitives.zig"); diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index f312093aa3..a9a02606eb 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -11,13 +11,6 @@ extra_data: []Node.Index, errors: []const Error, -const std = @import("../std.zig"); -const assert = std.debug.assert; -const testing = std.testing; -const mem = std.mem; -const Token = std.zig.Token; -const Ast = @This(); - pub const TokenIndex = u32; pub const ByteOffset = u32; @@ -34,7 +27,7 @@ pub const Location = struct { line_end: usize, }; -pub fn deinit(tree: *Ast, gpa: mem.Allocator) void { +pub fn deinit(tree: *Ast, gpa: Allocator) void { tree.tokens.deinit(gpa); tree.nodes.deinit(gpa); gpa.free(tree.extra_data); @@ -48,11 +41,69 @@ pub const RenderError = error{ OutOfMemory, }; +pub const Mode = enum { zig, zon }; + +/// Result should be freed with tree.deinit() when there are +/// no more references to any of the tokens or nodes. +pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!Ast { + var tokens = Ast.TokenList{}; + defer tokens.deinit(gpa); + + // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count. + const estimated_token_count = source.len / 8; + try tokens.ensureTotalCapacity(gpa, estimated_token_count); + + var tokenizer = std.zig.Tokenizer.init(source); + while (true) { + const token = tokenizer.next(); + try tokens.append(gpa, .{ + .tag = token.tag, + .start = @intCast(u32, token.loc.start), + }); + if (token.tag == .eof) break; + } + + var parser: Parse = .{ + .source = source, + .gpa = gpa, + .token_tags = tokens.items(.tag), + .token_starts = tokens.items(.start), + .errors = .{}, + .nodes = .{}, + .extra_data = .{}, + .scratch = .{}, + .tok_i = 0, + }; + defer parser.errors.deinit(gpa); + defer parser.nodes.deinit(gpa); + defer parser.extra_data.deinit(gpa); + defer parser.scratch.deinit(gpa); + + // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes. + // Make sure at least 1 so we can use appendAssumeCapacity on the root node below. + const estimated_node_count = (tokens.len + 2) / 2; + try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count); + + switch (mode) { + .zig => try parser.parseRoot(), + .zon => try parser.parseZon(), + } + + // TODO experiment with compacting the MultiArrayList slices here + return Ast{ + .source = source, + .tokens = tokens.toOwnedSlice(), + .nodes = parser.nodes.toOwnedSlice(), + .extra_data = try parser.extra_data.toOwnedSlice(gpa), + .errors = try parser.errors.toOwnedSlice(gpa), + }; +} + /// `gpa` is used for allocating the resulting formatted source code, as well as /// for allocating extra stack memory if needed, because this function utilizes recursion. /// Note: that's not actually true yet, see https://github.com/ziglang/zig/issues/1006. /// Caller owns the returned slice of bytes, allocated with `gpa`. -pub fn render(tree: Ast, gpa: mem.Allocator) RenderError![]u8 { +pub fn render(tree: Ast, gpa: Allocator) RenderError![]u8 { var buffer = std.ArrayList(u8).init(gpa); defer buffer.deinit(); @@ -3347,3 +3398,12 @@ pub const Node = struct { rparen: TokenIndex, }; }; + +const std = @import("../std.zig"); +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const Token = std.zig.Token; +const Ast = @This(); +const Allocator = std.mem.Allocator; +const Parse = @import("Parse.zig"); diff --git a/lib/std/zig/Parse.zig b/lib/std/zig/Parse.zig new file mode 100644 index 0000000000..f599a08f55 --- /dev/null +++ b/lib/std/zig/Parse.zig @@ -0,0 +1,3816 @@ +//! Represents in-progress parsing, will be converted to an Ast after completion. + +pub const Error = error{ParseError} || Allocator.Error; + +gpa: Allocator, +source: []const u8, +token_tags: []const Token.Tag, +token_starts: []const Ast.ByteOffset, +tok_i: TokenIndex, +errors: std.ArrayListUnmanaged(AstError), +nodes: Ast.NodeList, +extra_data: std.ArrayListUnmanaged(Node.Index), +scratch: std.ArrayListUnmanaged(Node.Index), + +const SmallSpan = union(enum) { + zero_or_one: Node.Index, + multi: Node.SubRange, +}; + +const Members = struct { + len: usize, + lhs: Node.Index, + rhs: Node.Index, + trailing: bool, + + fn toSpan(self: Members, p: *Parse) !Node.SubRange { + if (self.len <= 2) { + const nodes = [2]Node.Index{ self.lhs, self.rhs }; + return p.listToSpan(nodes[0..self.len]); + } else { + return Node.SubRange{ .start = self.lhs, .end = self.rhs }; + } + } +}; + +fn listToSpan(p: *Parse, list: []const Node.Index) !Node.SubRange { + try p.extra_data.appendSlice(p.gpa, list); + return Node.SubRange{ + .start = @intCast(Node.Index, p.extra_data.items.len - list.len), + .end = @intCast(Node.Index, p.extra_data.items.len), + }; +} + +fn addNode(p: *Parse, elem: Ast.NodeList.Elem) Allocator.Error!Node.Index { + const result = @intCast(Node.Index, p.nodes.len); + try p.nodes.append(p.gpa, elem); + return result; +} + +fn setNode(p: *Parse, i: usize, elem: Ast.NodeList.Elem) Node.Index { + p.nodes.set(i, elem); + return @intCast(Node.Index, i); +} + +fn reserveNode(p: *Parse, tag: Ast.Node.Tag) !usize { + try p.nodes.resize(p.gpa, p.nodes.len + 1); + p.nodes.items(.tag)[p.nodes.len - 1] = tag; + return p.nodes.len - 1; +} + +fn unreserveNode(p: *Parse, node_index: usize) void { + if (p.nodes.len == node_index) { + p.nodes.resize(p.gpa, p.nodes.len - 1) catch unreachable; + } else { + // There is zombie node left in the tree, let's make it as inoffensive as possible + // (sadly there's no no-op node) + p.nodes.items(.tag)[node_index] = .unreachable_literal; + p.nodes.items(.main_token)[node_index] = p.tok_i; + } +} + +fn addExtra(p: *Parse, extra: anytype) Allocator.Error!Node.Index { + const fields = std.meta.fields(@TypeOf(extra)); + try p.extra_data.ensureUnusedCapacity(p.gpa, fields.len); + const result = @intCast(u32, p.extra_data.items.len); + inline for (fields) |field| { + comptime assert(field.type == Node.Index); + p.extra_data.appendAssumeCapacity(@field(extra, field.name)); + } + return result; +} + +fn warnExpected(p: *Parse, expected_token: Token.Tag) error{OutOfMemory}!void { + @setCold(true); + try p.warnMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = expected_token }, + }); +} + +fn warn(p: *Parse, error_tag: AstError.Tag) error{OutOfMemory}!void { + @setCold(true); + try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i }); +} + +fn warnMsg(p: *Parse, msg: Ast.Error) error{OutOfMemory}!void { + @setCold(true); + switch (msg.tag) { + .expected_semi_after_decl, + .expected_semi_after_stmt, + .expected_comma_after_field, + .expected_comma_after_arg, + .expected_comma_after_param, + .expected_comma_after_initializer, + .expected_comma_after_switch_prong, + .expected_semi_or_else, + .expected_semi_or_lbrace, + .expected_token, + .expected_block, + .expected_block_or_assignment, + .expected_block_or_expr, + .expected_block_or_field, + .expected_expr, + .expected_expr_or_assignment, + .expected_fn, + .expected_inlinable, + .expected_labelable, + .expected_param_list, + .expected_prefix_expr, + .expected_primary_type_expr, + .expected_pub_item, + .expected_return_type, + .expected_suffix_op, + .expected_type_expr, + .expected_var_decl, + .expected_var_decl_or_fn, + .expected_loop_payload, + .expected_container, + => if (msg.token != 0 and !p.tokensOnSameLine(msg.token - 1, msg.token)) { + var copy = msg; + copy.token_is_prev = true; + copy.token -= 1; + return p.errors.append(p.gpa, copy); + }, + else => {}, + } + try p.errors.append(p.gpa, msg); +} + +fn fail(p: *Parse, tag: Ast.Error.Tag) error{ ParseError, OutOfMemory } { + @setCold(true); + return p.failMsg(.{ .tag = tag, .token = p.tok_i }); +} + +fn failExpected(p: *Parse, expected_token: Token.Tag) error{ ParseError, OutOfMemory } { + @setCold(true); + return p.failMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = expected_token }, + }); +} + +fn failMsg(p: *Parse, msg: Ast.Error) error{ ParseError, OutOfMemory } { + @setCold(true); + try p.warnMsg(msg); + return error.ParseError; +} + +/// Root <- skip container_doc_comment? ContainerMembers eof +pub fn parseRoot(p: *Parse) !void { + // Root node must be index 0. + p.nodes.appendAssumeCapacity(.{ + .tag = .root, + .main_token = 0, + .data = undefined, + }); + const root_members = try p.parseContainerMembers(); + const root_decls = try root_members.toSpan(p); + if (p.token_tags[p.tok_i] != .eof) { + try p.warnExpected(.eof); + } + p.nodes.items(.data)[0] = .{ + .lhs = root_decls.start, + .rhs = root_decls.end, + }; +} + +/// Parse in ZON mode. Subset of the language. +/// TODO: set a flag in Parse struct, and honor that flag +/// by emitting compilation errors when non-zon nodes are encountered. +pub fn parseZon(p: *Parse) !void { + const node_index = p.parseExpr() catch |err| switch (err) { + error.ParseError => { + assert(p.errors.items.len > 0); + return; + }, + else => |e| return e, + }; + assert(node_index == 0); + if (p.token_tags[p.tok_i] != .eof) { + try p.warnExpected(.eof); + } +} + +/// ContainerMembers <- ContainerDeclarations (ContainerField COMMA)* (ContainerField / ContainerDeclarations) +/// +/// ContainerDeclarations +/// <- TestDecl ContainerDeclarations +/// / ComptimeDecl ContainerDeclarations +/// / doc_comment? KEYWORD_pub? Decl ContainerDeclarations +/// / +/// +/// ComptimeDecl <- KEYWORD_comptime Block +fn parseContainerMembers(p: *Parse) !Members { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + var field_state: union(enum) { + /// No fields have been seen. + none, + /// Currently parsing fields. + seen, + /// Saw fields and then a declaration after them. + /// Payload is first token of previous declaration. + end: Node.Index, + /// There was a declaration between fields, don't report more errors. + err, + } = .none; + + var last_field: TokenIndex = undefined; + + // Skip container doc comments. + while (p.eatToken(.container_doc_comment)) |_| {} + + var trailing = false; + while (true) { + const doc_comment = try p.eatDocComments(); + + switch (p.token_tags[p.tok_i]) { + .keyword_test => { + if (doc_comment) |some| { + try p.warnMsg(.{ .tag = .test_doc_comment, .token = some }); + } + const test_decl_node = try p.expectTestDeclRecoverable(); + if (test_decl_node != 0) { + if (field_state == .seen) { + field_state = .{ .end = test_decl_node }; + } + try p.scratch.append(p.gpa, test_decl_node); + } + trailing = false; + }, + .keyword_comptime => switch (p.token_tags[p.tok_i + 1]) { + .l_brace => { + if (doc_comment) |some| { + try p.warnMsg(.{ .tag = .comptime_doc_comment, .token = some }); + } + const comptime_token = p.nextToken(); + const block = p.parseBlock() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => blk: { + p.findNextContainerMember(); + break :blk null_node; + }, + }; + if (block != 0) { + const comptime_node = try p.addNode(.{ + .tag = .@"comptime", + .main_token = comptime_token, + .data = .{ + .lhs = block, + .rhs = undefined, + }, + }); + if (field_state == .seen) { + field_state = .{ .end = comptime_node }; + } + try p.scratch.append(p.gpa, comptime_node); + } + trailing = false; + }, + else => { + const identifier = p.tok_i; + defer last_field = identifier; + const container_field = p.expectContainerField() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }; + switch (field_state) { + .none => field_state = .seen, + .err, .seen => {}, + .end => |node| { + try p.warnMsg(.{ + .tag = .decl_between_fields, + .token = p.nodes.items(.main_token)[node], + }); + try p.warnMsg(.{ + .tag = .previous_field, + .is_note = true, + .token = last_field, + }); + try p.warnMsg(.{ + .tag = .next_field, + .is_note = true, + .token = identifier, + }); + // Continue parsing; error will be reported later. + field_state = .err; + }, + } + try p.scratch.append(p.gpa, container_field); + switch (p.token_tags[p.tok_i]) { + .comma => { + p.tok_i += 1; + trailing = true; + continue; + }, + .r_brace, .eof => { + trailing = false; + break; + }, + else => {}, + } + // There is not allowed to be a decl after a field with no comma. + // Report error but recover parser. + try p.warn(.expected_comma_after_field); + p.findNextContainerMember(); + }, + }, + .keyword_pub => { + p.tok_i += 1; + const top_level_decl = try p.expectTopLevelDeclRecoverable(); + if (top_level_decl != 0) { + if (field_state == .seen) { + field_state = .{ .end = top_level_decl }; + } + try p.scratch.append(p.gpa, top_level_decl); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .keyword_usingnamespace => { + const node = try p.expectUsingNamespaceRecoverable(); + if (node != 0) { + if (field_state == .seen) { + field_state = .{ .end = node }; + } + try p.scratch.append(p.gpa, node); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .keyword_const, + .keyword_var, + .keyword_threadlocal, + .keyword_export, + .keyword_extern, + .keyword_inline, + .keyword_noinline, + .keyword_fn, + => { + const top_level_decl = try p.expectTopLevelDeclRecoverable(); + if (top_level_decl != 0) { + if (field_state == .seen) { + field_state = .{ .end = top_level_decl }; + } + try p.scratch.append(p.gpa, top_level_decl); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .eof, .r_brace => { + if (doc_comment) |tok| { + try p.warnMsg(.{ + .tag = .unattached_doc_comment, + .token = tok, + }); + } + break; + }, + else => { + const c_container = p.parseCStyleContainer() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => false, + }; + if (c_container) continue; + + const identifier = p.tok_i; + defer last_field = identifier; + const container_field = p.expectContainerField() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }; + switch (field_state) { + .none => field_state = .seen, + .err, .seen => {}, + .end => |node| { + try p.warnMsg(.{ + .tag = .decl_between_fields, + .token = p.nodes.items(.main_token)[node], + }); + try p.warnMsg(.{ + .tag = .previous_field, + .is_note = true, + .token = last_field, + }); + try p.warnMsg(.{ + .tag = .next_field, + .is_note = true, + .token = identifier, + }); + // Continue parsing; error will be reported later. + field_state = .err; + }, + } + try p.scratch.append(p.gpa, container_field); + switch (p.token_tags[p.tok_i]) { + .comma => { + p.tok_i += 1; + trailing = true; + continue; + }, + .r_brace, .eof => { + trailing = false; + break; + }, + else => {}, + } + // There is not allowed to be a decl after a field with no comma. + // Report error but recover parser. + try p.warn(.expected_comma_after_field); + if (p.token_tags[p.tok_i] == .semicolon and p.token_tags[identifier] == .identifier) { + try p.warnMsg(.{ + .tag = .var_const_decl, + .is_note = true, + .token = identifier, + }); + } + p.findNextContainerMember(); + continue; + }, + } + } + + const items = p.scratch.items[scratch_top..]; + switch (items.len) { + 0 => return Members{ + .len = 0, + .lhs = 0, + .rhs = 0, + .trailing = trailing, + }, + 1 => return Members{ + .len = 1, + .lhs = items[0], + .rhs = 0, + .trailing = trailing, + }, + 2 => return Members{ + .len = 2, + .lhs = items[0], + .rhs = items[1], + .trailing = trailing, + }, + else => { + const span = try p.listToSpan(items); + return Members{ + .len = items.len, + .lhs = span.start, + .rhs = span.end, + .trailing = trailing, + }; + }, + } +} + +/// Attempts to find next container member by searching for certain tokens +fn findNextContainerMember(p: *Parse) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (p.token_tags[tok]) { + // Any of these can start a new top level declaration. + .keyword_test, + .keyword_comptime, + .keyword_pub, + .keyword_export, + .keyword_extern, + .keyword_inline, + .keyword_noinline, + .keyword_usingnamespace, + .keyword_threadlocal, + .keyword_const, + .keyword_var, + .keyword_fn, + => { + if (level == 0) { + p.tok_i -= 1; + return; + } + }, + .identifier => { + if (p.token_tags[tok + 1] == .comma and level == 0) { + p.tok_i -= 1; + return; + } + }, + .comma, .semicolon => { + // this decl was likely meant to end here + if (level == 0) { + return; + } + }, + .l_paren, .l_bracket, .l_brace => level += 1, + .r_paren, .r_bracket => { + if (level != 0) level -= 1; + }, + .r_brace => { + if (level == 0) { + // end of container, exit + p.tok_i -= 1; + return; + } + level -= 1; + }, + .eof => { + p.tok_i -= 1; + return; + }, + else => {}, + } + } +} + +/// Attempts to find the next statement by searching for a semicolon +fn findNextStmt(p: *Parse) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (p.token_tags[tok]) { + .l_brace => level += 1, + .r_brace => { + if (level == 0) { + p.tok_i -= 1; + return; + } + level -= 1; + }, + .semicolon => { + if (level == 0) { + return; + } + }, + .eof => { + p.tok_i -= 1; + return; + }, + else => {}, + } + } +} + +/// TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block +fn expectTestDecl(p: *Parse) !Node.Index { + const test_token = p.assertToken(.keyword_test); + const name_token = switch (p.token_tags[p.nextToken()]) { + .string_literal, .identifier => p.tok_i - 1, + else => blk: { + p.tok_i -= 1; + break :blk null; + }, + }; + const block_node = try p.parseBlock(); + if (block_node == 0) return p.fail(.expected_block); + return p.addNode(.{ + .tag = .test_decl, + .main_token = test_token, + .data = .{ + .lhs = name_token orelse 0, + .rhs = block_node, + }, + }); +} + +fn expectTestDeclRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectTestDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +/// Decl +/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block) +/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl +/// / KEYWORD_usingnamespace Expr SEMICOLON +fn expectTopLevelDecl(p: *Parse) !Node.Index { + const extern_export_inline_token = p.nextToken(); + var is_extern: bool = false; + var expect_fn: bool = false; + var expect_var_or_fn: bool = false; + switch (p.token_tags[extern_export_inline_token]) { + .keyword_extern => { + _ = p.eatToken(.string_literal); + is_extern = true; + expect_var_or_fn = true; + }, + .keyword_export => expect_var_or_fn = true, + .keyword_inline, .keyword_noinline => expect_fn = true, + else => p.tok_i -= 1, + } + const fn_proto = try p.parseFnProto(); + if (fn_proto != 0) { + switch (p.token_tags[p.tok_i]) { + .semicolon => { + p.tok_i += 1; + return fn_proto; + }, + .l_brace => { + if (is_extern) { + try p.warnMsg(.{ .tag = .extern_fn_body, .token = extern_export_inline_token }); + return null_node; + } + const fn_decl_index = try p.reserveNode(.fn_decl); + errdefer p.unreserveNode(fn_decl_index); + + const body_block = try p.parseBlock(); + assert(body_block != 0); + return p.setNode(fn_decl_index, .{ + .tag = .fn_decl, + .main_token = p.nodes.items(.main_token)[fn_proto], + .data = .{ + .lhs = fn_proto, + .rhs = body_block, + }, + }); + }, + else => { + // Since parseBlock only return error.ParseError on + // a missing '}' we can assume this function was + // supposed to end here. + try p.warn(.expected_semi_or_lbrace); + return null_node; + }, + } + } + if (expect_fn) { + try p.warn(.expected_fn); + return error.ParseError; + } + + const thread_local_token = p.eatToken(.keyword_threadlocal); + const var_decl = try p.parseVarDecl(); + if (var_decl != 0) { + try p.expectSemicolon(.expected_semi_after_decl, false); + return var_decl; + } + if (thread_local_token != null) { + return p.fail(.expected_var_decl); + } + if (expect_var_or_fn) { + return p.fail(.expected_var_decl_or_fn); + } + if (p.token_tags[p.tok_i] != .keyword_usingnamespace) { + return p.fail(.expected_pub_item); + } + return p.expectUsingNamespace(); +} + +fn expectTopLevelDeclRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectTopLevelDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +fn expectUsingNamespace(p: *Parse) !Node.Index { + const usingnamespace_token = p.assertToken(.keyword_usingnamespace); + const expr = try p.expectExpr(); + try p.expectSemicolon(.expected_semi_after_decl, false); + return p.addNode(.{ + .tag = .@"usingnamespace", + .main_token = usingnamespace_token, + .data = .{ + .lhs = expr, + .rhs = undefined, + }, + }); +} + +fn expectUsingNamespaceRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectUsingNamespace() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr +fn parseFnProto(p: *Parse) !Node.Index { + const fn_token = p.eatToken(.keyword_fn) orelse return null_node; + + // We want the fn proto node to be before its children in the array. + const fn_proto_index = try p.reserveNode(.fn_proto); + errdefer p.unreserveNode(fn_proto_index); + + _ = p.eatToken(.identifier); + const params = try p.parseParamDeclList(); + const align_expr = try p.parseByteAlign(); + const addrspace_expr = try p.parseAddrSpace(); + const section_expr = try p.parseLinkSection(); + const callconv_expr = try p.parseCallconv(); + _ = p.eatToken(.bang); + + const return_type_expr = try p.parseTypeExpr(); + if (return_type_expr == 0) { + // most likely the user forgot to specify the return type. + // Mark return type as invalid and try to continue. + try p.warn(.expected_return_type); + } + + if (align_expr == 0 and section_expr == 0 and callconv_expr == 0 and addrspace_expr == 0) { + switch (params) { + .zero_or_one => |param| return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_simple, + .main_token = fn_token, + .data = .{ + .lhs = param, + .rhs = return_type_expr, + }, + }), + .multi => |span| { + return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_multi, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.SubRange{ + .start = span.start, + .end = span.end, + }), + .rhs = return_type_expr, + }, + }); + }, + } + } + switch (params) { + .zero_or_one => |param| return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_one, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.FnProtoOne{ + .param = param, + .align_expr = align_expr, + .addrspace_expr = addrspace_expr, + .section_expr = section_expr, + .callconv_expr = callconv_expr, + }), + .rhs = return_type_expr, + }, + }), + .multi => |span| { + return p.setNode(fn_proto_index, .{ + .tag = .fn_proto, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.FnProto{ + .params_start = span.start, + .params_end = span.end, + .align_expr = align_expr, + .addrspace_expr = addrspace_expr, + .section_expr = section_expr, + .callconv_expr = callconv_expr, + }), + .rhs = return_type_expr, + }, + }); + }, + } +} + +/// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection? (EQUAL Expr)? SEMICOLON +fn parseVarDecl(p: *Parse) !Node.Index { + const mut_token = p.eatToken(.keyword_const) orelse + p.eatToken(.keyword_var) orelse + return null_node; + + _ = try p.expectToken(.identifier); + const type_node: Node.Index = if (p.eatToken(.colon) == null) 0 else try p.expectTypeExpr(); + const align_node = try p.parseByteAlign(); + const addrspace_node = try p.parseAddrSpace(); + const section_node = try p.parseLinkSection(); + const init_node: Node.Index = switch (p.token_tags[p.tok_i]) { + .equal_equal => blk: { + try p.warn(.wrong_equal_var_decl); + p.tok_i += 1; + break :blk try p.expectExpr(); + }, + .equal => blk: { + p.tok_i += 1; + break :blk try p.expectExpr(); + }, + else => 0, + }; + if (section_node == 0 and addrspace_node == 0) { + if (align_node == 0) { + return p.addNode(.{ + .tag = .simple_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = type_node, + .rhs = init_node, + }, + }); + } else if (type_node == 0) { + return p.addNode(.{ + .tag = .aligned_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = align_node, + .rhs = init_node, + }, + }); + } else { + return p.addNode(.{ + .tag = .local_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = try p.addExtra(Node.LocalVarDecl{ + .type_node = type_node, + .align_node = align_node, + }), + .rhs = init_node, + }, + }); + } + } else { + return p.addNode(.{ + .tag = .global_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = try p.addExtra(Node.GlobalVarDecl{ + .type_node = type_node, + .align_node = align_node, + .addrspace_node = addrspace_node, + .section_node = section_node, + }), + .rhs = init_node, + }, + }); + } +} + +/// ContainerField +/// <- doc_comment? KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? ByteAlign? (EQUAL Expr)? +/// / doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? !KEYWORD_fn TypeExpr ByteAlign? (EQUAL Expr)? +fn expectContainerField(p: *Parse) !Node.Index { + var main_token = p.tok_i; + _ = p.eatToken(.keyword_comptime); + const tuple_like = p.token_tags[p.tok_i] != .identifier or p.token_tags[p.tok_i + 1] != .colon; + if (!tuple_like) { + main_token = p.assertToken(.identifier); + } + + var align_expr: Node.Index = 0; + var type_expr: Node.Index = 0; + if (p.eatToken(.colon) != null or tuple_like) { + type_expr = try p.expectTypeExpr(); + align_expr = try p.parseByteAlign(); + } + + const value_expr: Node.Index = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); + + if (align_expr == 0) { + return p.addNode(.{ + .tag = .container_field_init, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = value_expr, + }, + }); + } else if (value_expr == 0) { + return p.addNode(.{ + .tag = .container_field_align, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = align_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .container_field, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = try p.addExtra(Node.ContainerField{ + .value_expr = value_expr, + .align_expr = align_expr, + }), + }, + }); + } +} + +/// Statement +/// <- KEYWORD_comptime? VarDecl +/// / KEYWORD_comptime BlockExprStatement +/// / KEYWORD_nosuspend BlockExprStatement +/// / KEYWORD_suspend BlockExprStatement +/// / KEYWORD_defer BlockExprStatement +/// / KEYWORD_errdefer Payload? BlockExprStatement +/// / IfStatement +/// / LabeledStatement +/// / SwitchExpr +/// / AssignExpr SEMICOLON +fn parseStatement(p: *Parse, allow_defer_var: bool) Error!Node.Index { + const comptime_token = p.eatToken(.keyword_comptime); + + if (allow_defer_var) { + const var_decl = try p.parseVarDecl(); + if (var_decl != 0) { + try p.expectSemicolon(.expected_semi_after_decl, true); + return var_decl; + } + } + + if (comptime_token) |token| { + return p.addNode(.{ + .tag = .@"comptime", + .main_token = token, + .data = .{ + .lhs = try p.expectBlockExprStatement(), + .rhs = undefined, + }, + }); + } + + switch (p.token_tags[p.tok_i]) { + .keyword_nosuspend => { + return p.addNode(.{ + .tag = .@"nosuspend", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectBlockExprStatement(), + .rhs = undefined, + }, + }); + }, + .keyword_suspend => { + const token = p.nextToken(); + const block_expr = try p.expectBlockExprStatement(); + return p.addNode(.{ + .tag = .@"suspend", + .main_token = token, + .data = .{ + .lhs = block_expr, + .rhs = undefined, + }, + }); + }, + .keyword_defer => if (allow_defer_var) return p.addNode(.{ + .tag = .@"defer", + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = try p.expectBlockExprStatement(), + }, + }), + .keyword_errdefer => if (allow_defer_var) return p.addNode(.{ + .tag = .@"errdefer", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.parsePayload(), + .rhs = try p.expectBlockExprStatement(), + }, + }), + .keyword_switch => return p.expectSwitchExpr(), + .keyword_if => return p.expectIfStatement(), + .keyword_enum, .keyword_struct, .keyword_union => { + const identifier = p.tok_i + 1; + if (try p.parseCStyleContainer()) { + // Return something so that `expectStatement` is happy. + return p.addNode(.{ + .tag = .identifier, + .main_token = identifier, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + } + }, + else => {}, + } + + const labeled_statement = try p.parseLabeledStatement(); + if (labeled_statement != 0) return labeled_statement; + + const assign_expr = try p.parseAssignExpr(); + if (assign_expr != 0) { + try p.expectSemicolon(.expected_semi_after_stmt, true); + return assign_expr; + } + + return null_node; +} + +fn expectStatement(p: *Parse, allow_defer_var: bool) !Node.Index { + const statement = try p.parseStatement(allow_defer_var); + if (statement == 0) { + return p.fail(.expected_statement); + } + return statement; +} + +/// If a parse error occurs, reports an error, but then finds the next statement +/// and returns that one instead. If a parse error occurs but there is no following +/// statement, returns 0. +fn expectStatementRecoverable(p: *Parse) Error!Node.Index { + while (true) { + return p.expectStatement(true) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextStmt(); // Try to skip to the next statement. + switch (p.token_tags[p.tok_i]) { + .r_brace => return null_node, + .eof => return error.ParseError, + else => continue, + } + }, + }; + } +} + +/// IfStatement +/// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? +/// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) +fn expectIfStatement(p: *Parse) !Node.Index { + const if_token = p.assertToken(.keyword_if); + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + + // TODO propose to change the syntax so that semicolons are always required + // inside if statements, even if there is an `else`. + var else_required = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = assign_expr, + }, + }); + } + else_required = true; + break :blk assign_expr; + }; + _ = p.eatToken(.keyword_else) orelse { + if (else_required) { + try p.warn(.expected_semi_or_else); + } + return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + }; + _ = try p.parsePayload(); + const else_expr = try p.expectStatement(false); + return p.addNode(.{ + .tag = .@"if", + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// LabeledStatement <- BlockLabel? (Block / LoopStatement) +fn parseLabeledStatement(p: *Parse) !Node.Index { + const label_token = p.parseBlockLabel(); + const block = try p.parseBlock(); + if (block != 0) return block; + + const loop_stmt = try p.parseLoopStatement(); + if (loop_stmt != 0) return loop_stmt; + + if (label_token != 0) { + const after_colon = p.tok_i; + const node = try p.parseTypeExpr(); + if (node != 0) { + const a = try p.parseByteAlign(); + const b = try p.parseAddrSpace(); + const c = try p.parseLinkSection(); + const d = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); + if (a != 0 or b != 0 or c != 0 or d != 0) { + return p.failMsg(.{ .tag = .expected_var_const, .token = label_token }); + } + } + return p.failMsg(.{ .tag = .expected_labelable, .token = after_colon }); + } + + return null_node; +} + +/// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) +fn parseLoopStatement(p: *Parse) !Node.Index { + const inline_token = p.eatToken(.keyword_inline); + + const for_statement = try p.parseForStatement(); + if (for_statement != 0) return for_statement; + + const while_statement = try p.parseWhileStatement(); + if (while_statement != 0) return while_statement; + + if (inline_token == null) return null_node; + + // If we've seen "inline", there should have been a "for" or "while" + return p.fail(.expected_inlinable); +} + +/// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload +/// +/// ForStatement +/// <- ForPrefix BlockExpr ( KEYWORD_else Statement )? +/// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement ) +fn parseForStatement(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + _ = try p.expectToken(.l_paren); + const array_expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + const found_payload = try p.parsePtrIndexPayload(); + if (found_payload == 0) try p.warn(.expected_loop_payload); + + // TODO propose to change the syntax so that semicolons are always required + // inside while statements, even if there is an `else`. + var else_required = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = assign_expr, + }, + }); + } + else_required = true; + break :blk assign_expr; + }; + _ = p.eatToken(.keyword_else) orelse { + if (else_required) { + try p.warn(.expected_semi_or_else); + } + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = then_expr, + }, + }); + }; + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = try p.expectStatement(false), + }), + }, + }); +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileStatement +/// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? +/// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) +fn parseWhileStatement(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + // TODO propose to change the syntax so that semicolons are always required + // inside while statements, even if there is an `else`. + var else_required = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = assign_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = assign_expr, + }), + }, + }); + } + } + else_required = true; + break :blk assign_expr; + }; + _ = p.eatToken(.keyword_else) orelse { + if (else_required) { + try p.warn(.expected_semi_or_else); + } + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectStatement(false); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// BlockExprStatement +/// <- BlockExpr +/// / AssignExpr SEMICOLON +fn parseBlockExprStatement(p: *Parse) !Node.Index { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) { + return block_expr; + } + const assign_expr = try p.parseAssignExpr(); + if (assign_expr != 0) { + try p.expectSemicolon(.expected_semi_after_stmt, true); + return assign_expr; + } + return null_node; +} + +fn expectBlockExprStatement(p: *Parse) !Node.Index { + const node = try p.parseBlockExprStatement(); + if (node == 0) { + return p.fail(.expected_block_or_expr); + } + return node; +} + +/// BlockExpr <- BlockLabel? Block +fn parseBlockExpr(p: *Parse) Error!Node.Index { + switch (p.token_tags[p.tok_i]) { + .identifier => { + if (p.token_tags[p.tok_i + 1] == .colon and + p.token_tags[p.tok_i + 2] == .l_brace) + { + p.tok_i += 2; + return p.parseBlock(); + } else { + return null_node; + } + }, + .l_brace => return p.parseBlock(), + else => return null_node, + } +} + +/// AssignExpr <- Expr (AssignOp Expr)? +/// +/// AssignOp +/// <- ASTERISKEQUAL +/// / ASTERISKPIPEEQUAL +/// / SLASHEQUAL +/// / PERCENTEQUAL +/// / PLUSEQUAL +/// / PLUSPIPEEQUAL +/// / MINUSEQUAL +/// / MINUSPIPEEQUAL +/// / LARROW2EQUAL +/// / LARROW2PIPEEQUAL +/// / RARROW2EQUAL +/// / AMPERSANDEQUAL +/// / CARETEQUAL +/// / PIPEEQUAL +/// / ASTERISKPERCENTEQUAL +/// / PLUSPERCENTEQUAL +/// / MINUSPERCENTEQUAL +/// / EQUAL +fn parseAssignExpr(p: *Parse) !Node.Index { + const expr = try p.parseExpr(); + if (expr == 0) return null_node; + + const tag: Node.Tag = switch (p.token_tags[p.tok_i]) { + .asterisk_equal => .assign_mul, + .slash_equal => .assign_div, + .percent_equal => .assign_mod, + .plus_equal => .assign_add, + .minus_equal => .assign_sub, + .angle_bracket_angle_bracket_left_equal => .assign_shl, + .angle_bracket_angle_bracket_left_pipe_equal => .assign_shl_sat, + .angle_bracket_angle_bracket_right_equal => .assign_shr, + .ampersand_equal => .assign_bit_and, + .caret_equal => .assign_bit_xor, + .pipe_equal => .assign_bit_or, + .asterisk_percent_equal => .assign_mul_wrap, + .plus_percent_equal => .assign_add_wrap, + .minus_percent_equal => .assign_sub_wrap, + .asterisk_pipe_equal => .assign_mul_sat, + .plus_pipe_equal => .assign_add_sat, + .minus_pipe_equal => .assign_sub_sat, + .equal => .assign, + else => return expr, + }; + return p.addNode(.{ + .tag = tag, + .main_token = p.nextToken(), + .data = .{ + .lhs = expr, + .rhs = try p.expectExpr(), + }, + }); +} + +fn expectAssignExpr(p: *Parse) !Node.Index { + const expr = try p.parseAssignExpr(); + if (expr == 0) { + return p.fail(.expected_expr_or_assignment); + } + return expr; +} + +fn parseExpr(p: *Parse) Error!Node.Index { + return p.parseExprPrecedence(0); +} + +fn expectExpr(p: *Parse) Error!Node.Index { + const node = try p.parseExpr(); + if (node == 0) { + return p.fail(.expected_expr); + } else { + return node; + } +} + +const Assoc = enum { + left, + none, +}; + +const OperInfo = struct { + prec: i8, + tag: Node.Tag, + assoc: Assoc = Assoc.left, +}; + +// A table of binary operator information. Higher precedence numbers are +// stickier. All operators at the same precedence level should have the same +// associativity. +const operTable = std.enums.directEnumArrayDefault(Token.Tag, OperInfo, .{ .prec = -1, .tag = Node.Tag.root }, 0, .{ + .keyword_or = .{ .prec = 10, .tag = .bool_or }, + + .keyword_and = .{ .prec = 20, .tag = .bool_and }, + + .equal_equal = .{ .prec = 30, .tag = .equal_equal, .assoc = Assoc.none }, + .bang_equal = .{ .prec = 30, .tag = .bang_equal, .assoc = Assoc.none }, + .angle_bracket_left = .{ .prec = 30, .tag = .less_than, .assoc = Assoc.none }, + .angle_bracket_right = .{ .prec = 30, .tag = .greater_than, .assoc = Assoc.none }, + .angle_bracket_left_equal = .{ .prec = 30, .tag = .less_or_equal, .assoc = Assoc.none }, + .angle_bracket_right_equal = .{ .prec = 30, .tag = .greater_or_equal, .assoc = Assoc.none }, + + .ampersand = .{ .prec = 40, .tag = .bit_and }, + .caret = .{ .prec = 40, .tag = .bit_xor }, + .pipe = .{ .prec = 40, .tag = .bit_or }, + .keyword_orelse = .{ .prec = 40, .tag = .@"orelse" }, + .keyword_catch = .{ .prec = 40, .tag = .@"catch" }, + + .angle_bracket_angle_bracket_left = .{ .prec = 50, .tag = .shl }, + .angle_bracket_angle_bracket_left_pipe = .{ .prec = 50, .tag = .shl_sat }, + .angle_bracket_angle_bracket_right = .{ .prec = 50, .tag = .shr }, + + .plus = .{ .prec = 60, .tag = .add }, + .minus = .{ .prec = 60, .tag = .sub }, + .plus_plus = .{ .prec = 60, .tag = .array_cat }, + .plus_percent = .{ .prec = 60, .tag = .add_wrap }, + .minus_percent = .{ .prec = 60, .tag = .sub_wrap }, + .plus_pipe = .{ .prec = 60, .tag = .add_sat }, + .minus_pipe = .{ .prec = 60, .tag = .sub_sat }, + + .pipe_pipe = .{ .prec = 70, .tag = .merge_error_sets }, + .asterisk = .{ .prec = 70, .tag = .mul }, + .slash = .{ .prec = 70, .tag = .div }, + .percent = .{ .prec = 70, .tag = .mod }, + .asterisk_asterisk = .{ .prec = 70, .tag = .array_mult }, + .asterisk_percent = .{ .prec = 70, .tag = .mul_wrap }, + .asterisk_pipe = .{ .prec = 70, .tag = .mul_sat }, +}); + +fn parseExprPrecedence(p: *Parse, min_prec: i32) Error!Node.Index { + assert(min_prec >= 0); + var node = try p.parsePrefixExpr(); + if (node == 0) { + return null_node; + } + + var banned_prec: i8 = -1; + + while (true) { + const tok_tag = p.token_tags[p.tok_i]; + const info = operTable[@intCast(usize, @enumToInt(tok_tag))]; + if (info.prec < min_prec) { + break; + } + if (info.prec == banned_prec) { + return p.fail(.chained_comparison_operators); + } + + const oper_token = p.nextToken(); + // Special-case handling for "catch" + if (tok_tag == .keyword_catch) { + _ = try p.parsePayload(); + } + const rhs = try p.parseExprPrecedence(info.prec + 1); + if (rhs == 0) { + try p.warn(.expected_expr); + return node; + } + + { + const tok_len = tok_tag.lexeme().?.len; + const char_before = p.source[p.token_starts[oper_token] - 1]; + const char_after = p.source[p.token_starts[oper_token] + tok_len]; + if (tok_tag == .ampersand and char_after == '&') { + // without types we don't know if '&&' was intended as 'bitwise_and address_of', or a c-style logical_and + // The best the parser can do is recommend changing it to 'and' or ' & &' + try p.warnMsg(.{ .tag = .invalid_ampersand_ampersand, .token = oper_token }); + } else if (std.ascii.isWhitespace(char_before) != std.ascii.isWhitespace(char_after)) { + try p.warnMsg(.{ .tag = .mismatched_binary_op_whitespace, .token = oper_token }); + } + } + + node = try p.addNode(.{ + .tag = info.tag, + .main_token = oper_token, + .data = .{ + .lhs = node, + .rhs = rhs, + }, + }); + + if (info.assoc == Assoc.none) { + banned_prec = info.prec; + } + } + + return node; +} + +/// PrefixExpr <- PrefixOp* PrimaryExpr +/// +/// PrefixOp +/// <- EXCLAMATIONMARK +/// / MINUS +/// / TILDE +/// / MINUSPERCENT +/// / AMPERSAND +/// / KEYWORD_try +/// / KEYWORD_await +fn parsePrefixExpr(p: *Parse) Error!Node.Index { + const tag: Node.Tag = switch (p.token_tags[p.tok_i]) { + .bang => .bool_not, + .minus => .negation, + .tilde => .bit_not, + .minus_percent => .negation_wrap, + .ampersand => .address_of, + .keyword_try => .@"try", + .keyword_await => .@"await", + else => return p.parsePrimaryExpr(), + }; + return p.addNode(.{ + .tag = tag, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectPrefixExpr(), + .rhs = undefined, + }, + }); +} + +fn expectPrefixExpr(p: *Parse) Error!Node.Index { + const node = try p.parsePrefixExpr(); + if (node == 0) { + return p.fail(.expected_prefix_expr); + } + return node; +} + +/// TypeExpr <- PrefixTypeOp* ErrorUnionExpr +/// +/// PrefixTypeOp +/// <- QUESTIONMARK +/// / KEYWORD_anyframe MINUSRARROW +/// / SliceTypeStart (ByteAlign / AddrSpace / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* +/// / PtrTypeStart (AddrSpace / KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* +/// / ArrayTypeStart +/// +/// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET +/// +/// PtrTypeStart +/// <- ASTERISK +/// / ASTERISK2 +/// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET +/// +/// ArrayTypeStart <- LBRACKET Expr (COLON Expr)? RBRACKET +fn parseTypeExpr(p: *Parse) Error!Node.Index { + switch (p.token_tags[p.tok_i]) { + .question_mark => return p.addNode(.{ + .tag = .optional_type, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectTypeExpr(), + .rhs = undefined, + }, + }), + .keyword_anyframe => switch (p.token_tags[p.tok_i + 1]) { + .arrow => return p.addNode(.{ + .tag = .anyframe_type, + .main_token = p.nextToken(), + .data = .{ + .lhs = p.nextToken(), + .rhs = try p.expectTypeExpr(), + }, + }), + else => return p.parseErrorUnionExpr(), + }, + .asterisk => { + const asterisk = p.nextToken(); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start != 0) { + return p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } else if (mods.addrspace_node != 0) { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } + }, + .asterisk_asterisk => { + const asterisk = p.nextToken(); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + const inner: Node.Index = inner: { + if (mods.bit_range_start != 0) { + break :inner try p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } else if (mods.addrspace_node != 0) { + break :inner try p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } else { + break :inner try p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } + }; + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = 0, + .rhs = inner, + }, + }); + }, + .l_bracket => switch (p.token_tags[p.tok_i + 1]) { + .asterisk => { + _ = p.nextToken(); + const asterisk = p.nextToken(); + var sentinel: Node.Index = 0; + if (p.eatToken(.identifier)) |ident| { + const ident_slice = p.source[p.token_starts[ident]..p.token_starts[ident + 1]]; + if (!std.mem.eql(u8, std.mem.trimRight(u8, ident_slice, &std.ascii.whitespace), "c")) { + p.tok_i -= 1; + } + } else if (p.eatToken(.colon)) |_| { + sentinel = try p.expectExpr(); + } + _ = try p.expectToken(.r_bracket); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start == 0) { + if (sentinel == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } else if (mods.align_node == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_sentinel, + .main_token = asterisk, + .data = .{ + .lhs = sentinel, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } + } else { + return p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } + }, + else => { + const lbracket = p.nextToken(); + const len_expr = try p.parseExpr(); + const sentinel: Node.Index = if (p.eatToken(.colon)) |_| + try p.expectExpr() + else + 0; + _ = try p.expectToken(.r_bracket); + if (len_expr == 0) { + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start != 0) { + try p.warnMsg(.{ + .tag = .invalid_bit_range, + .token = p.nodes.items(.main_token)[mods.bit_range_start], + }); + } + if (sentinel == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = lbracket, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } else if (mods.align_node == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = sentinel, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = lbracket, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } + } else { + switch (p.token_tags[p.tok_i]) { + .keyword_align, + .keyword_const, + .keyword_volatile, + .keyword_allowzero, + .keyword_addrspace, + => return p.fail(.ptr_mod_on_array_child_type), + else => {}, + } + const elem_type = try p.expectTypeExpr(); + if (sentinel == 0) { + return p.addNode(.{ + .tag = .array_type, + .main_token = lbracket, + .data = .{ + .lhs = len_expr, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .array_type_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = len_expr, + .rhs = try p.addExtra(.{ + .elem_type = elem_type, + .sentinel = sentinel, + }), + }, + }); + } + } + }, + }, + else => return p.parseErrorUnionExpr(), + } +} + +fn expectTypeExpr(p: *Parse) Error!Node.Index { + const node = try p.parseTypeExpr(); + if (node == 0) { + return p.fail(.expected_type_expr); + } + return node; +} + +/// PrimaryExpr +/// <- AsmExpr +/// / IfExpr +/// / KEYWORD_break BreakLabel? Expr? +/// / KEYWORD_comptime Expr +/// / KEYWORD_nosuspend Expr +/// / KEYWORD_continue BreakLabel? +/// / KEYWORD_resume Expr +/// / KEYWORD_return Expr? +/// / BlockLabel? LoopExpr +/// / Block +/// / CurlySuffixExpr +fn parsePrimaryExpr(p: *Parse) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .keyword_asm => return p.expectAsmExpr(), + .keyword_if => return p.parseIfExpr(), + .keyword_break => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"break", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseBreakLabel(), + .rhs = try p.parseExpr(), + }, + }); + }, + .keyword_continue => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"continue", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseBreakLabel(), + .rhs = undefined, + }, + }); + }, + .keyword_comptime => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"comptime", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_nosuspend => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"nosuspend", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_resume => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"resume", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_return => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"return", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseExpr(), + .rhs = undefined, + }, + }); + }, + .identifier => { + if (p.token_tags[p.tok_i + 1] == .colon) { + switch (p.token_tags[p.tok_i + 2]) { + .keyword_inline => { + p.tok_i += 3; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => { + p.tok_i += 2; + return p.parseForExpr(); + }, + .keyword_while => { + p.tok_i += 2; + return p.parseWhileExpr(); + }, + .l_brace => { + p.tok_i += 2; + return p.parseBlock(); + }, + else => return p.parseCurlySuffixExpr(), + } + } else { + return p.parseCurlySuffixExpr(); + } + }, + .keyword_inline => { + p.tok_i += 1; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + .l_brace => return p.parseBlock(), + else => return p.parseCurlySuffixExpr(), + } +} + +/// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? +fn parseIfExpr(p: *Parse) !Node.Index { + return p.parseIf(expectExpr); +} + +/// Block <- LBRACE Statement* RBRACE +fn parseBlock(p: *Parse) !Node.Index { + const lbrace = p.eatToken(.l_brace) orelse return null_node; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.token_tags[p.tok_i] == .r_brace) break; + const statement = try p.expectStatementRecoverable(); + if (statement == 0) break; + try p.scratch.append(p.gpa, statement); + } + _ = try p.expectToken(.r_brace); + const semicolon = (p.token_tags[p.tok_i - 2] == .semicolon); + const statements = p.scratch.items[scratch_top..]; + switch (statements.len) { + 0 => return p.addNode(.{ + .tag = .block_two, + .main_token = lbrace, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (semicolon) .block_two_semicolon else .block_two, + .main_token = lbrace, + .data = .{ + .lhs = statements[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (semicolon) .block_two_semicolon else .block_two, + .main_token = lbrace, + .data = .{ + .lhs = statements[0], + .rhs = statements[1], + }, + }), + else => { + const span = try p.listToSpan(statements); + return p.addNode(.{ + .tag = if (semicolon) .block_semicolon else .block, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } +} + +/// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload +/// +/// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)? +fn parseForExpr(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + _ = try p.expectToken(.l_paren); + const array_expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + const found_payload = try p.parsePtrIndexPayload(); + if (found_payload == 0) try p.warn(.expected_loop_payload); + + const then_expr = try p.expectExpr(); + _ = p.eatToken(.keyword_else) orelse { + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = then_expr, + }, + }); + }; + const else_expr = try p.expectExpr(); + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? +fn parseWhileExpr(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + const then_expr = try p.expectExpr(); + _ = p.eatToken(.keyword_else) orelse { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectExpr(); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// CurlySuffixExpr <- TypeExpr InitList? +/// +/// InitList +/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE +/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE +/// / LBRACE RBRACE +fn parseCurlySuffixExpr(p: *Parse) !Node.Index { + const lhs = try p.parseTypeExpr(); + if (lhs == 0) return null_node; + const lbrace = p.eatToken(.l_brace) orelse return lhs; + + // If there are 0 or 1 items, we can use ArrayInitOne/StructInitOne; + // otherwise we use the full ArrayInit/StructInit. + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const field_init = try p.parseFieldInit(); + if (field_init != 0) { + try p.scratch.append(p.gpa, field_init); + while (true) { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + if (p.eatToken(.r_brace)) |_| break; + const next = try p.expectFieldInit(); + try p.scratch.append(p.gpa, next); + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => unreachable, + 1 => return p.addNode(.{ + .tag = if (comma) .struct_init_one_comma else .struct_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = inits[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .struct_init_comma else .struct_init, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(try p.listToSpan(inits)), + }, + }), + } + } + + while (true) { + if (p.eatToken(.r_brace)) |_| break; + const elem_init = try p.expectExpr(); + try p.scratch.append(p.gpa, elem_init); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => return p.addNode(.{ + .tag = .struct_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .array_init_one_comma else .array_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = inits[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .array_init_comma else .array_init, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(try p.listToSpan(inits)), + }, + }), + } +} + +/// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? +fn parseErrorUnionExpr(p: *Parse) !Node.Index { + const suffix_expr = try p.parseSuffixExpr(); + if (suffix_expr == 0) return null_node; + const bang = p.eatToken(.bang) orelse return suffix_expr; + return p.addNode(.{ + .tag = .error_union, + .main_token = bang, + .data = .{ + .lhs = suffix_expr, + .rhs = try p.expectTypeExpr(), + }, + }); +} + +/// SuffixExpr +/// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments +/// / PrimaryTypeExpr (SuffixOp / FnCallArguments)* +/// +/// FnCallArguments <- LPAREN ExprList RPAREN +/// +/// ExprList <- (Expr COMMA)* Expr? +fn parseSuffixExpr(p: *Parse) !Node.Index { + if (p.eatToken(.keyword_async)) |_| { + var res = try p.expectPrimaryTypeExpr(); + while (true) { + const node = try p.parseSuffixOp(res); + if (node == 0) break; + res = node; + } + const lparen = p.eatToken(.l_paren) orelse { + try p.warn(.expected_param_list); + return res; + }; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + switch (params.len) { + 0 => return p.addNode(.{ + .tag = if (comma) .async_call_one_comma else .async_call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .async_call_one_comma else .async_call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = params[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .async_call_comma else .async_call, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = try p.addExtra(try p.listToSpan(params)), + }, + }), + } + } + + var res = try p.parsePrimaryTypeExpr(); + if (res == 0) return res; + while (true) { + const suffix_op = try p.parseSuffixOp(res); + if (suffix_op != 0) { + res = suffix_op; + continue; + } + const lparen = p.eatToken(.l_paren) orelse return res; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + res = switch (params.len) { + 0 => try p.addNode(.{ + .tag = if (comma) .call_one_comma else .call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = 0, + }, + }), + 1 => try p.addNode(.{ + .tag = if (comma) .call_one_comma else .call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = params[0], + }, + }), + else => try p.addNode(.{ + .tag = if (comma) .call_comma else .call, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = try p.addExtra(try p.listToSpan(params)), + }, + }), + }; + } +} + +/// PrimaryTypeExpr +/// <- BUILTINIDENTIFIER FnCallArguments +/// / CHAR_LITERAL +/// / ContainerDecl +/// / DOT IDENTIFIER +/// / DOT InitList +/// / ErrorSetDecl +/// / FLOAT +/// / FnProto +/// / GroupedExpr +/// / LabeledTypeExpr +/// / IDENTIFIER +/// / IfTypeExpr +/// / INTEGER +/// / KEYWORD_comptime TypeExpr +/// / KEYWORD_error DOT IDENTIFIER +/// / KEYWORD_anyframe +/// / KEYWORD_unreachable +/// / STRINGLITERAL +/// / SwitchExpr +/// +/// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto +/// +/// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE +/// +/// InitList +/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE +/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE +/// / LBRACE RBRACE +/// +/// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE +/// +/// GroupedExpr <- LPAREN Expr RPAREN +/// +/// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? +/// +/// LabeledTypeExpr +/// <- BlockLabel Block +/// / BlockLabel? LoopTypeExpr +/// +/// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) +fn parsePrimaryTypeExpr(p: *Parse) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .char_literal => return p.addNode(.{ + .tag = .char_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .number_literal => return p.addNode(.{ + .tag = .number_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .keyword_unreachable => return p.addNode(.{ + .tag = .unreachable_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .keyword_anyframe => return p.addNode(.{ + .tag = .anyframe_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .string_literal => { + const main_token = p.nextToken(); + return p.addNode(.{ + .tag = .string_literal, + .main_token = main_token, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + }, + + .builtin => return p.parseBuiltinCall(), + .keyword_fn => return p.parseFnProto(), + .keyword_if => return p.parseIf(expectTypeExpr), + .keyword_switch => return p.expectSwitchExpr(), + + .keyword_extern, + .keyword_packed, + => { + p.tok_i += 1; + return p.parseContainerDeclAuto(); + }, + + .keyword_struct, + .keyword_opaque, + .keyword_enum, + .keyword_union, + => return p.parseContainerDeclAuto(), + + .keyword_comptime => return p.addNode(.{ + .tag = .@"comptime", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectTypeExpr(), + .rhs = undefined, + }, + }), + .multiline_string_literal_line => { + const first_line = p.nextToken(); + while (p.token_tags[p.tok_i] == .multiline_string_literal_line) { + p.tok_i += 1; + } + return p.addNode(.{ + .tag = .multiline_string_literal, + .main_token = first_line, + .data = .{ + .lhs = first_line, + .rhs = p.tok_i - 1, + }, + }); + }, + .identifier => switch (p.token_tags[p.tok_i + 1]) { + .colon => switch (p.token_tags[p.tok_i + 2]) { + .keyword_inline => { + p.tok_i += 3; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => { + p.tok_i += 2; + return p.parseForTypeExpr(); + }, + .keyword_while => { + p.tok_i += 2; + return p.parseWhileTypeExpr(); + }, + .l_brace => { + p.tok_i += 2; + return p.parseBlock(); + }, + else => return p.addNode(.{ + .tag = .identifier, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + }, + else => return p.addNode(.{ + .tag = .identifier, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + }, + .keyword_inline => { + p.tok_i += 1; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + .period => switch (p.token_tags[p.tok_i + 1]) { + .identifier => return p.addNode(.{ + .tag = .enum_literal, + .data = .{ + .lhs = p.nextToken(), // dot + .rhs = undefined, + }, + .main_token = p.nextToken(), // identifier + }), + .l_brace => { + const lbrace = p.tok_i + 1; + p.tok_i = lbrace + 1; + + // If there are 0, 1, or 2 items, we can use ArrayInitDotTwo/StructInitDotTwo; + // otherwise we use the full ArrayInitDot/StructInitDot. + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const field_init = try p.parseFieldInit(); + if (field_init != 0) { + try p.scratch.append(p.gpa, field_init); + while (true) { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + if (p.eatToken(.r_brace)) |_| break; + const next = try p.expectFieldInit(); + try p.scratch.append(p.gpa, next); + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => unreachable, + 1 => return p.addNode(.{ + .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = inits[1], + }, + }), + else => { + const span = try p.listToSpan(inits); + return p.addNode(.{ + .tag = if (comma) .struct_init_dot_comma else .struct_init_dot, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } + } + + while (true) { + if (p.eatToken(.r_brace)) |_| break; + const elem_init = try p.expectExpr(); + try p.scratch.append(p.gpa, elem_init); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => return p.addNode(.{ + .tag = .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = inits[1], + }, + }), + else => { + const span = try p.listToSpan(inits); + return p.addNode(.{ + .tag = if (comma) .array_init_dot_comma else .array_init_dot, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } + }, + else => return null_node, + }, + .keyword_error => switch (p.token_tags[p.tok_i + 1]) { + .l_brace => { + const error_token = p.tok_i; + p.tok_i += 2; + while (true) { + if (p.eatToken(.r_brace)) |_| break; + _ = try p.eatDocComments(); + _ = try p.expectToken(.identifier); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_field), + } + } + return p.addNode(.{ + .tag = .error_set_decl, + .main_token = error_token, + .data = .{ + .lhs = undefined, + .rhs = p.tok_i - 1, // rbrace + }, + }); + }, + else => { + const main_token = p.nextToken(); + const period = p.eatToken(.period); + if (period == null) try p.warnExpected(.period); + const identifier = p.eatToken(.identifier); + if (identifier == null) try p.warnExpected(.identifier); + return p.addNode(.{ + .tag = .error_value, + .main_token = main_token, + .data = .{ + .lhs = period orelse 0, + .rhs = identifier orelse 0, + }, + }); + }, + }, + .l_paren => return p.addNode(.{ + .tag = .grouped_expression, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectExpr(), + .rhs = try p.expectToken(.r_paren), + }, + }), + else => return null_node, + } +} + +fn expectPrimaryTypeExpr(p: *Parse) !Node.Index { + const node = try p.parsePrimaryTypeExpr(); + if (node == 0) { + return p.fail(.expected_primary_type_expr); + } + return node; +} + +/// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload +/// +/// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)? +fn parseForTypeExpr(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + _ = try p.expectToken(.l_paren); + const array_expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + const found_payload = try p.parsePtrIndexPayload(); + if (found_payload == 0) try p.warn(.expected_loop_payload); + + const then_expr = try p.expectTypeExpr(); + _ = p.eatToken(.keyword_else) orelse { + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = then_expr, + }, + }); + }; + const else_expr = try p.expectTypeExpr(); + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = array_expr, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? +fn parseWhileTypeExpr(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + const then_expr = try p.expectTypeExpr(); + _ = p.eatToken(.keyword_else) orelse { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectTypeExpr(); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE +fn expectSwitchExpr(p: *Parse) !Node.Index { + const switch_token = p.assertToken(.keyword_switch); + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.expectToken(.l_brace); + const cases = try p.parseSwitchProngList(); + const trailing_comma = p.token_tags[p.tok_i - 1] == .comma; + _ = try p.expectToken(.r_brace); + + return p.addNode(.{ + .tag = if (trailing_comma) .switch_comma else .@"switch", + .main_token = switch_token, + .data = .{ + .lhs = expr_node, + .rhs = try p.addExtra(Node.SubRange{ + .start = cases.start, + .end = cases.end, + }), + }, + }); +} + +/// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN +/// +/// AsmOutput <- COLON AsmOutputList AsmInput? +/// +/// AsmInput <- COLON AsmInputList AsmClobbers? +/// +/// AsmClobbers <- COLON StringList +/// +/// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? +/// +/// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? +/// +/// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? +fn expectAsmExpr(p: *Parse) !Node.Index { + const asm_token = p.assertToken(.keyword_asm); + _ = p.eatToken(.keyword_volatile); + _ = try p.expectToken(.l_paren); + const template = try p.expectExpr(); + + if (p.eatToken(.r_paren)) |rparen| { + return p.addNode(.{ + .tag = .asm_simple, + .main_token = asm_token, + .data = .{ + .lhs = template, + .rhs = rparen, + }, + }); + } + + _ = try p.expectToken(.colon); + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const output_item = try p.parseAsmOutputItem(); + if (output_item == 0) break; + try p.scratch.append(p.gpa, output_item); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + if (p.eatToken(.colon)) |_| { + while (true) { + const input_item = try p.parseAsmInputItem(); + if (input_item == 0) break; + try p.scratch.append(p.gpa, input_item); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + if (p.eatToken(.colon)) |_| { + while (p.eatToken(.string_literal)) |_| { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + } + } + const rparen = try p.expectToken(.r_paren); + const span = try p.listToSpan(p.scratch.items[scratch_top..]); + return p.addNode(.{ + .tag = .@"asm", + .main_token = asm_token, + .data = .{ + .lhs = template, + .rhs = try p.addExtra(Node.Asm{ + .items_start = span.start, + .items_end = span.end, + .rparen = rparen, + }), + }, + }); +} + +/// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN +fn parseAsmOutputItem(p: *Parse) !Node.Index { + _ = p.eatToken(.l_bracket) orelse return null_node; + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.r_bracket); + _ = try p.expectToken(.string_literal); + _ = try p.expectToken(.l_paren); + const type_expr: Node.Index = blk: { + if (p.eatToken(.arrow)) |_| { + break :blk try p.expectTypeExpr(); + } else { + _ = try p.expectToken(.identifier); + break :blk null_node; + } + }; + const rparen = try p.expectToken(.r_paren); + return p.addNode(.{ + .tag = .asm_output, + .main_token = identifier, + .data = .{ + .lhs = type_expr, + .rhs = rparen, + }, + }); +} + +/// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN +fn parseAsmInputItem(p: *Parse) !Node.Index { + _ = p.eatToken(.l_bracket) orelse return null_node; + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.r_bracket); + _ = try p.expectToken(.string_literal); + _ = try p.expectToken(.l_paren); + const expr = try p.expectExpr(); + const rparen = try p.expectToken(.r_paren); + return p.addNode(.{ + .tag = .asm_input, + .main_token = identifier, + .data = .{ + .lhs = expr, + .rhs = rparen, + }, + }); +} + +/// BreakLabel <- COLON IDENTIFIER +fn parseBreakLabel(p: *Parse) !TokenIndex { + _ = p.eatToken(.colon) orelse return @as(TokenIndex, 0); + return p.expectToken(.identifier); +} + +/// BlockLabel <- IDENTIFIER COLON +fn parseBlockLabel(p: *Parse) TokenIndex { + if (p.token_tags[p.tok_i] == .identifier and + p.token_tags[p.tok_i + 1] == .colon) + { + const identifier = p.tok_i; + p.tok_i += 2; + return identifier; + } + return null_node; +} + +/// FieldInit <- DOT IDENTIFIER EQUAL Expr +fn parseFieldInit(p: *Parse) !Node.Index { + if (p.token_tags[p.tok_i + 0] == .period and + p.token_tags[p.tok_i + 1] == .identifier and + p.token_tags[p.tok_i + 2] == .equal) + { + p.tok_i += 3; + return p.expectExpr(); + } else { + return null_node; + } +} + +fn expectFieldInit(p: *Parse) !Node.Index { + if (p.token_tags[p.tok_i] != .period or + p.token_tags[p.tok_i + 1] != .identifier or + p.token_tags[p.tok_i + 2] != .equal) + return p.fail(.expected_initializer); + + p.tok_i += 3; + return p.expectExpr(); +} + +/// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN +fn parseWhileContinueExpr(p: *Parse) !Node.Index { + _ = p.eatToken(.colon) orelse { + if (p.token_tags[p.tok_i] == .l_paren and + p.tokensOnSameLine(p.tok_i - 1, p.tok_i)) + return p.fail(.expected_continue_expr); + return null_node; + }; + _ = try p.expectToken(.l_paren); + const node = try p.parseAssignExpr(); + if (node == 0) return p.fail(.expected_expr_or_assignment); + _ = try p.expectToken(.r_paren); + return node; +} + +/// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN +fn parseLinkSection(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_linksection) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN +fn parseCallconv(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_callconv) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN +fn parseAddrSpace(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_addrspace) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// This function can return null nodes and then still return nodes afterwards, +/// such as in the case of anytype and `...`. Caller must look for rparen to find +/// out when there are no more param decls left. +/// +/// ParamDecl +/// <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType +/// / DOT3 +/// +/// ParamType +/// <- KEYWORD_anytype +/// / TypeExpr +fn expectParamDecl(p: *Parse) !Node.Index { + _ = try p.eatDocComments(); + switch (p.token_tags[p.tok_i]) { + .keyword_noalias, .keyword_comptime => p.tok_i += 1, + .ellipsis3 => { + p.tok_i += 1; + return null_node; + }, + else => {}, + } + if (p.token_tags[p.tok_i] == .identifier and + p.token_tags[p.tok_i + 1] == .colon) + { + p.tok_i += 2; + } + switch (p.token_tags[p.tok_i]) { + .keyword_anytype => { + p.tok_i += 1; + return null_node; + }, + else => return p.expectTypeExpr(), + } +} + +/// Payload <- PIPE IDENTIFIER PIPE +fn parsePayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.pipe); + return identifier; +} + +/// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE +fn parsePtrPayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + _ = p.eatToken(.asterisk); + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.pipe); + return identifier; +} + +/// Returns the first identifier token, if any. +/// +/// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE +fn parsePtrIndexPayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + _ = p.eatToken(.asterisk); + const identifier = try p.expectToken(.identifier); + if (p.eatToken(.comma) != null) { + _ = try p.expectToken(.identifier); + } + _ = try p.expectToken(.pipe); + return identifier; +} + +/// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? AssignExpr +/// +/// SwitchCase +/// <- SwitchItem (COMMA SwitchItem)* COMMA? +/// / KEYWORD_else +fn parseSwitchProng(p: *Parse) !Node.Index { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + const is_inline = p.eatToken(.keyword_inline) != null; + + if (p.eatToken(.keyword_else) == null) { + while (true) { + const item = try p.parseSwitchItem(); + if (item == 0) break; + try p.scratch.append(p.gpa, item); + if (p.eatToken(.comma) == null) break; + } + if (scratch_top == p.scratch.items.len) { + if (is_inline) p.tok_i -= 1; + return null_node; + } + } + const arrow_token = try p.expectToken(.equal_angle_bracket_right); + _ = try p.parsePtrIndexPayload(); + + const items = p.scratch.items[scratch_top..]; + switch (items.len) { + 0 => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, + .main_token = arrow_token, + .data = .{ + .lhs = 0, + .rhs = try p.expectAssignExpr(), + }, + }), + 1 => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, + .main_token = arrow_token, + .data = .{ + .lhs = items[0], + .rhs = try p.expectAssignExpr(), + }, + }), + else => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline else .switch_case, + .main_token = arrow_token, + .data = .{ + .lhs = try p.addExtra(try p.listToSpan(items)), + .rhs = try p.expectAssignExpr(), + }, + }), + } +} + +/// SwitchItem <- Expr (DOT3 Expr)? +fn parseSwitchItem(p: *Parse) !Node.Index { + const expr = try p.parseExpr(); + if (expr == 0) return null_node; + + if (p.eatToken(.ellipsis3)) |token| { + return p.addNode(.{ + .tag = .switch_range, + .main_token = token, + .data = .{ + .lhs = expr, + .rhs = try p.expectExpr(), + }, + }); + } + return expr; +} + +const PtrModifiers = struct { + align_node: Node.Index, + addrspace_node: Node.Index, + bit_range_start: Node.Index, + bit_range_end: Node.Index, +}; + +fn parsePtrModifiers(p: *Parse) !PtrModifiers { + var result: PtrModifiers = .{ + .align_node = 0, + .addrspace_node = 0, + .bit_range_start = 0, + .bit_range_end = 0, + }; + var saw_const = false; + var saw_volatile = false; + var saw_allowzero = false; + var saw_addrspace = false; + while (true) { + switch (p.token_tags[p.tok_i]) { + .keyword_align => { + if (result.align_node != 0) { + try p.warn(.extra_align_qualifier); + } + p.tok_i += 1; + _ = try p.expectToken(.l_paren); + result.align_node = try p.expectExpr(); + + if (p.eatToken(.colon)) |_| { + result.bit_range_start = try p.expectExpr(); + _ = try p.expectToken(.colon); + result.bit_range_end = try p.expectExpr(); + } + + _ = try p.expectToken(.r_paren); + }, + .keyword_const => { + if (saw_const) { + try p.warn(.extra_const_qualifier); + } + p.tok_i += 1; + saw_const = true; + }, + .keyword_volatile => { + if (saw_volatile) { + try p.warn(.extra_volatile_qualifier); + } + p.tok_i += 1; + saw_volatile = true; + }, + .keyword_allowzero => { + if (saw_allowzero) { + try p.warn(.extra_allowzero_qualifier); + } + p.tok_i += 1; + saw_allowzero = true; + }, + .keyword_addrspace => { + if (saw_addrspace) { + try p.warn(.extra_addrspace_qualifier); + } + result.addrspace_node = try p.parseAddrSpace(); + }, + else => return result, + } + } +} + +/// SuffixOp +/// <- LBRACKET Expr (DOT2 (Expr? (COLON Expr)?)?)? RBRACKET +/// / DOT IDENTIFIER +/// / DOTASTERISK +/// / DOTQUESTIONMARK +fn parseSuffixOp(p: *Parse, lhs: Node.Index) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .l_bracket => { + const lbracket = p.nextToken(); + const index_expr = try p.expectExpr(); + + if (p.eatToken(.ellipsis2)) |_| { + const end_expr = try p.parseExpr(); + if (p.eatToken(.colon)) |_| { + const sentinel = try p.expectExpr(); + _ = try p.expectToken(.r_bracket); + return p.addNode(.{ + .tag = .slice_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(Node.SliceSentinel{ + .start = index_expr, + .end = end_expr, + .sentinel = sentinel, + }), + }, + }); + } + _ = try p.expectToken(.r_bracket); + if (end_expr == 0) { + return p.addNode(.{ + .tag = .slice_open, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = index_expr, + }, + }); + } + return p.addNode(.{ + .tag = .slice, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(Node.Slice{ + .start = index_expr, + .end = end_expr, + }), + }, + }); + } + _ = try p.expectToken(.r_bracket); + return p.addNode(.{ + .tag = .array_access, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = index_expr, + }, + }); + }, + .period_asterisk => return p.addNode(.{ + .tag = .deref, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = undefined, + }, + }), + .invalid_periodasterisks => { + try p.warn(.asterisk_after_ptr_deref); + return p.addNode(.{ + .tag = .deref, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = undefined, + }, + }); + }, + .period => switch (p.token_tags[p.tok_i + 1]) { + .identifier => return p.addNode(.{ + .tag = .field_access, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = p.nextToken(), + }, + }), + .question_mark => return p.addNode(.{ + .tag = .unwrap_optional, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = p.nextToken(), + }, + }), + .l_brace => { + // this a misplaced `.{`, handle the error somewhere else + return null_node; + }, + else => { + p.tok_i += 1; + try p.warn(.expected_suffix_op); + return null_node; + }, + }, + else => return null_node, + } +} + +/// Caller must have already verified the first token. +/// +/// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE +/// +/// ContainerDeclType +/// <- KEYWORD_struct (LPAREN Expr RPAREN)? +/// / KEYWORD_opaque +/// / KEYWORD_enum (LPAREN Expr RPAREN)? +/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? +fn parseContainerDeclAuto(p: *Parse) !Node.Index { + const main_token = p.nextToken(); + const arg_expr = switch (p.token_tags[main_token]) { + .keyword_opaque => null_node, + .keyword_struct, .keyword_enum => blk: { + if (p.eatToken(.l_paren)) |_| { + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + break :blk expr; + } else { + break :blk null_node; + } + }, + .keyword_union => blk: { + if (p.eatToken(.l_paren)) |_| { + if (p.eatToken(.keyword_enum)) |_| { + if (p.eatToken(.l_paren)) |_| { + const enum_tag_expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.expectToken(.r_paren); + + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + const members_span = try members.toSpan(p); + _ = try p.expectToken(.r_brace); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_enum_tag_trailing, + false => .tagged_union_enum_tag, + }, + .main_token = main_token, + .data = .{ + .lhs = enum_tag_expr, + .rhs = try p.addExtra(members_span), + }, + }); + } else { + _ = try p.expectToken(.r_paren); + + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + if (members.len <= 2) { + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_two_trailing, + false => .tagged_union_two, + }, + .main_token = main_token, + .data = .{ + .lhs = members.lhs, + .rhs = members.rhs, + }, + }); + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_trailing, + false => .tagged_union, + }, + .main_token = main_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + } + } + } else { + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + break :blk expr; + } + } else { + break :blk null_node; + } + }, + else => { + p.tok_i -= 1; + return p.fail(.expected_container); + }, + }; + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + if (arg_expr == 0) { + if (members.len <= 2) { + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_two_trailing, + false => .container_decl_two, + }, + .main_token = main_token, + .data = .{ + .lhs = members.lhs, + .rhs = members.rhs, + }, + }); + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_trailing, + false => .container_decl, + }, + .main_token = main_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + } + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_arg_trailing, + false => .container_decl_arg, + }, + .main_token = main_token, + .data = .{ + .lhs = arg_expr, + .rhs = try p.addExtra(Node.SubRange{ + .start = span.start, + .end = span.end, + }), + }, + }); + } +} + +/// Give a helpful error message for those transitioning from +/// C's 'struct Foo {};' to Zig's 'const Foo = struct {};'. +fn parseCStyleContainer(p: *Parse) Error!bool { + const main_token = p.tok_i; + switch (p.token_tags[p.tok_i]) { + .keyword_enum, .keyword_union, .keyword_struct => {}, + else => return false, + } + const identifier = p.tok_i + 1; + if (p.token_tags[identifier] != .identifier) return false; + p.tok_i += 2; + + try p.warnMsg(.{ + .tag = .c_style_container, + .token = identifier, + .extra = .{ .expected_tag = p.token_tags[main_token] }, + }); + try p.warnMsg(.{ + .tag = .zig_style_container, + .is_note = true, + .token = identifier, + .extra = .{ .expected_tag = p.token_tags[main_token] }, + }); + + _ = try p.expectToken(.l_brace); + _ = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + try p.expectSemicolon(.expected_semi_after_decl, true); + return true; +} + +/// Holds temporary data until we are ready to construct the full ContainerDecl AST node. +/// +/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN +fn parseByteAlign(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_align) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr; +} + +/// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? +fn parseSwitchProngList(p: *Parse) !Node.SubRange { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const item = try parseSwitchProng(p); + if (item == 0) break; + + try p.scratch.append(p.gpa, item); + + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_switch_prong), + } + } + return p.listToSpan(p.scratch.items[scratch_top..]); +} + +/// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? +fn parseParamDeclList(p: *Parse) !SmallSpan { + _ = try p.expectToken(.l_paren); + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + var varargs: union(enum) { none, seen, nonfinal: TokenIndex } = .none; + while (true) { + if (p.eatToken(.r_paren)) |_| break; + if (varargs == .seen) varargs = .{ .nonfinal = p.tok_i }; + const param = try p.expectParamDecl(); + if (param != 0) { + try p.scratch.append(p.gpa, param); + } else if (p.token_tags[p.tok_i - 1] == .ellipsis3) { + if (varargs == .none) varargs = .seen; + } + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_param), + } + } + if (varargs == .nonfinal) { + try p.warnMsg(.{ .tag = .varargs_nonfinal, .token = varargs.nonfinal }); + } + const params = p.scratch.items[scratch_top..]; + return switch (params.len) { + 0 => SmallSpan{ .zero_or_one = 0 }, + 1 => SmallSpan{ .zero_or_one = params[0] }, + else => SmallSpan{ .multi = try p.listToSpan(params) }, + }; +} + +/// FnCallArguments <- LPAREN ExprList RPAREN +/// +/// ExprList <- (Expr COMMA)* Expr? +fn parseBuiltinCall(p: *Parse) !Node.Index { + const builtin_token = p.assertToken(.builtin); + if (p.token_tags[p.nextToken()] != .l_paren) { + p.tok_i -= 1; + try p.warn(.expected_param_list); + // Pretend this was an identifier so we can continue parsing. + return p.addNode(.{ + .tag = .identifier, + .main_token = builtin_token, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + } + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + switch (params.len) { + 0 => return p.addNode(.{ + .tag = .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = params[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = params[0], + .rhs = params[1], + }, + }), + else => { + const span = try p.listToSpan(params); + return p.addNode(.{ + .tag = if (comma) .builtin_call_comma else .builtin_call, + .main_token = builtin_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } +} + +/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? +fn parseIf(p: *Parse, comptime bodyParseFn: fn (p: *Parse) Error!Node.Index) !Node.Index { + const if_token = p.eatToken(.keyword_if) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + + const then_expr = try bodyParseFn(p); + assert(then_expr != 0); + + _ = p.eatToken(.keyword_else) orelse return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + _ = try p.parsePayload(); + const else_expr = try bodyParseFn(p); + assert(then_expr != 0); + + return p.addNode(.{ + .tag = .@"if", + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// Skips over doc comment tokens. Returns the first one, if any. +fn eatDocComments(p: *Parse) !?TokenIndex { + if (p.eatToken(.doc_comment)) |tok| { + var first_line = tok; + if (tok > 0 and tokensOnSameLine(p, tok - 1, tok)) { + try p.warnMsg(.{ + .tag = .same_line_doc_comment, + .token = tok, + }); + first_line = p.eatToken(.doc_comment) orelse return null; + } + while (p.eatToken(.doc_comment)) |_| {} + return first_line; + } + return null; +} + +fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool { + return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null; +} + +fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex { + return if (p.token_tags[p.tok_i] == tag) p.nextToken() else null; +} + +fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex { + const token = p.nextToken(); + assert(p.token_tags[token] == tag); + return token; +} + +fn expectToken(p: *Parse, tag: Token.Tag) Error!TokenIndex { + if (p.token_tags[p.tok_i] != tag) { + return p.failMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = tag }, + }); + } + return p.nextToken(); +} + +fn expectSemicolon(p: *Parse, error_tag: AstError.Tag, recoverable: bool) Error!void { + if (p.token_tags[p.tok_i] == .semicolon) { + _ = p.nextToken(); + return; + } + try p.warn(error_tag); + if (!recoverable) return error.ParseError; +} + +fn nextToken(p: *Parse) TokenIndex { + const result = p.tok_i; + p.tok_i += 1; + return result; +} + +const null_node: Node.Index = 0; + +const Parse = @This(); +const std = @import("../std.zig"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const Ast = std.zig.Ast; +const Node = Ast.Node; +const AstError = Ast.Error; +const TokenIndex = Ast.TokenIndex; +const Token = std.zig.Token; + +test { + _ = @import("parser_test.zig"); +} diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig deleted file mode 100644 index fdb122b19d..0000000000 --- a/lib/std/zig/parse.zig +++ /dev/null @@ -1,3852 +0,0 @@ -const std = @import("../std.zig"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const Ast = std.zig.Ast; -const Node = Ast.Node; -const AstError = Ast.Error; -const TokenIndex = Ast.TokenIndex; -const Token = std.zig.Token; - -pub const Error = error{ParseError} || Allocator.Error; - -/// Result should be freed with tree.deinit() when there are -/// no more references to any of the tokens or nodes. -pub fn parse(gpa: Allocator, source: [:0]const u8) Allocator.Error!Ast { - var tokens = Ast.TokenList{}; - defer tokens.deinit(gpa); - - // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count. - const estimated_token_count = source.len / 8; - try tokens.ensureTotalCapacity(gpa, estimated_token_count); - - var tokenizer = std.zig.Tokenizer.init(source); - while (true) { - const token = tokenizer.next(); - try tokens.append(gpa, .{ - .tag = token.tag, - .start = @intCast(u32, token.loc.start), - }); - if (token.tag == .eof) break; - } - - var parser: Parser = .{ - .source = source, - .gpa = gpa, - .token_tags = tokens.items(.tag), - .token_starts = tokens.items(.start), - .errors = .{}, - .nodes = .{}, - .extra_data = .{}, - .scratch = .{}, - .tok_i = 0, - }; - defer parser.errors.deinit(gpa); - defer parser.nodes.deinit(gpa); - defer parser.extra_data.deinit(gpa); - defer parser.scratch.deinit(gpa); - - // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes. - // Make sure at least 1 so we can use appendAssumeCapacity on the root node below. - const estimated_node_count = (tokens.len + 2) / 2; - try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count); - - try parser.parseRoot(); - - // TODO experiment with compacting the MultiArrayList slices here - return Ast{ - .source = source, - .tokens = tokens.toOwnedSlice(), - .nodes = parser.nodes.toOwnedSlice(), - .extra_data = try parser.extra_data.toOwnedSlice(gpa), - .errors = try parser.errors.toOwnedSlice(gpa), - }; -} - -const null_node: Node.Index = 0; - -/// Represents in-progress parsing, will be converted to an Ast after completion. -const Parser = struct { - gpa: Allocator, - source: []const u8, - token_tags: []const Token.Tag, - token_starts: []const Ast.ByteOffset, - tok_i: TokenIndex, - errors: std.ArrayListUnmanaged(AstError), - nodes: Ast.NodeList, - extra_data: std.ArrayListUnmanaged(Node.Index), - scratch: std.ArrayListUnmanaged(Node.Index), - - const SmallSpan = union(enum) { - zero_or_one: Node.Index, - multi: Node.SubRange, - }; - - const Members = struct { - len: usize, - lhs: Node.Index, - rhs: Node.Index, - trailing: bool, - - fn toSpan(self: Members, p: *Parser) !Node.SubRange { - if (self.len <= 2) { - const nodes = [2]Node.Index{ self.lhs, self.rhs }; - return p.listToSpan(nodes[0..self.len]); - } else { - return Node.SubRange{ .start = self.lhs, .end = self.rhs }; - } - } - }; - - fn listToSpan(p: *Parser, list: []const Node.Index) !Node.SubRange { - try p.extra_data.appendSlice(p.gpa, list); - return Node.SubRange{ - .start = @intCast(Node.Index, p.extra_data.items.len - list.len), - .end = @intCast(Node.Index, p.extra_data.items.len), - }; - } - - fn addNode(p: *Parser, elem: Ast.NodeList.Elem) Allocator.Error!Node.Index { - const result = @intCast(Node.Index, p.nodes.len); - try p.nodes.append(p.gpa, elem); - return result; - } - - fn setNode(p: *Parser, i: usize, elem: Ast.NodeList.Elem) Node.Index { - p.nodes.set(i, elem); - return @intCast(Node.Index, i); - } - - fn reserveNode(p: *Parser, tag: Ast.Node.Tag) !usize { - try p.nodes.resize(p.gpa, p.nodes.len + 1); - p.nodes.items(.tag)[p.nodes.len - 1] = tag; - return p.nodes.len - 1; - } - - fn unreserveNode(p: *Parser, node_index: usize) void { - if (p.nodes.len == node_index) { - p.nodes.resize(p.gpa, p.nodes.len - 1) catch unreachable; - } else { - // There is zombie node left in the tree, let's make it as inoffensive as possible - // (sadly there's no no-op node) - p.nodes.items(.tag)[node_index] = .unreachable_literal; - p.nodes.items(.main_token)[node_index] = p.tok_i; - } - } - - fn addExtra(p: *Parser, extra: anytype) Allocator.Error!Node.Index { - const fields = std.meta.fields(@TypeOf(extra)); - try p.extra_data.ensureUnusedCapacity(p.gpa, fields.len); - const result = @intCast(u32, p.extra_data.items.len); - inline for (fields) |field| { - comptime assert(field.type == Node.Index); - p.extra_data.appendAssumeCapacity(@field(extra, field.name)); - } - return result; - } - - fn warnExpected(p: *Parser, expected_token: Token.Tag) error{OutOfMemory}!void { - @setCold(true); - try p.warnMsg(.{ - .tag = .expected_token, - .token = p.tok_i, - .extra = .{ .expected_tag = expected_token }, - }); - } - - fn warn(p: *Parser, error_tag: AstError.Tag) error{OutOfMemory}!void { - @setCold(true); - try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i }); - } - - fn warnMsg(p: *Parser, msg: Ast.Error) error{OutOfMemory}!void { - @setCold(true); - switch (msg.tag) { - .expected_semi_after_decl, - .expected_semi_after_stmt, - .expected_comma_after_field, - .expected_comma_after_arg, - .expected_comma_after_param, - .expected_comma_after_initializer, - .expected_comma_after_switch_prong, - .expected_semi_or_else, - .expected_semi_or_lbrace, - .expected_token, - .expected_block, - .expected_block_or_assignment, - .expected_block_or_expr, - .expected_block_or_field, - .expected_expr, - .expected_expr_or_assignment, - .expected_fn, - .expected_inlinable, - .expected_labelable, - .expected_param_list, - .expected_prefix_expr, - .expected_primary_type_expr, - .expected_pub_item, - .expected_return_type, - .expected_suffix_op, - .expected_type_expr, - .expected_var_decl, - .expected_var_decl_or_fn, - .expected_loop_payload, - .expected_container, - => if (msg.token != 0 and !p.tokensOnSameLine(msg.token - 1, msg.token)) { - var copy = msg; - copy.token_is_prev = true; - copy.token -= 1; - return p.errors.append(p.gpa, copy); - }, - else => {}, - } - try p.errors.append(p.gpa, msg); - } - - fn fail(p: *Parser, tag: Ast.Error.Tag) error{ ParseError, OutOfMemory } { - @setCold(true); - return p.failMsg(.{ .tag = tag, .token = p.tok_i }); - } - - fn failExpected(p: *Parser, expected_token: Token.Tag) error{ ParseError, OutOfMemory } { - @setCold(true); - return p.failMsg(.{ - .tag = .expected_token, - .token = p.tok_i, - .extra = .{ .expected_tag = expected_token }, - }); - } - - fn failMsg(p: *Parser, msg: Ast.Error) error{ ParseError, OutOfMemory } { - @setCold(true); - try p.warnMsg(msg); - return error.ParseError; - } - - /// Root <- skip container_doc_comment? ContainerMembers eof - fn parseRoot(p: *Parser) !void { - // Root node must be index 0. - p.nodes.appendAssumeCapacity(.{ - .tag = .root, - .main_token = 0, - .data = undefined, - }); - const root_members = try p.parseContainerMembers(); - const root_decls = try root_members.toSpan(p); - if (p.token_tags[p.tok_i] != .eof) { - try p.warnExpected(.eof); - } - p.nodes.items(.data)[0] = .{ - .lhs = root_decls.start, - .rhs = root_decls.end, - }; - } - - /// ContainerMembers <- ContainerDeclarations (ContainerField COMMA)* (ContainerField / ContainerDeclarations) - /// - /// ContainerDeclarations - /// <- TestDecl ContainerDeclarations - /// / ComptimeDecl ContainerDeclarations - /// / doc_comment? KEYWORD_pub? Decl ContainerDeclarations - /// / - /// - /// ComptimeDecl <- KEYWORD_comptime Block - fn parseContainerMembers(p: *Parser) !Members { - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - var field_state: union(enum) { - /// No fields have been seen. - none, - /// Currently parsing fields. - seen, - /// Saw fields and then a declaration after them. - /// Payload is first token of previous declaration. - end: Node.Index, - /// There was a declaration between fields, don't report more errors. - err, - } = .none; - - var last_field: TokenIndex = undefined; - - // Skip container doc comments. - while (p.eatToken(.container_doc_comment)) |_| {} - - var trailing = false; - while (true) { - const doc_comment = try p.eatDocComments(); - - switch (p.token_tags[p.tok_i]) { - .keyword_test => { - if (doc_comment) |some| { - try p.warnMsg(.{ .tag = .test_doc_comment, .token = some }); - } - const test_decl_node = try p.expectTestDeclRecoverable(); - if (test_decl_node != 0) { - if (field_state == .seen) { - field_state = .{ .end = test_decl_node }; - } - try p.scratch.append(p.gpa, test_decl_node); - } - trailing = false; - }, - .keyword_comptime => switch (p.token_tags[p.tok_i + 1]) { - .l_brace => { - if (doc_comment) |some| { - try p.warnMsg(.{ .tag = .comptime_doc_comment, .token = some }); - } - const comptime_token = p.nextToken(); - const block = p.parseBlock() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => blk: { - p.findNextContainerMember(); - break :blk null_node; - }, - }; - if (block != 0) { - const comptime_node = try p.addNode(.{ - .tag = .@"comptime", - .main_token = comptime_token, - .data = .{ - .lhs = block, - .rhs = undefined, - }, - }); - if (field_state == .seen) { - field_state = .{ .end = comptime_node }; - } - try p.scratch.append(p.gpa, comptime_node); - } - trailing = false; - }, - else => { - const identifier = p.tok_i; - defer last_field = identifier; - const container_field = p.expectContainerField() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextContainerMember(); - continue; - }, - }; - switch (field_state) { - .none => field_state = .seen, - .err, .seen => {}, - .end => |node| { - try p.warnMsg(.{ - .tag = .decl_between_fields, - .token = p.nodes.items(.main_token)[node], - }); - try p.warnMsg(.{ - .tag = .previous_field, - .is_note = true, - .token = last_field, - }); - try p.warnMsg(.{ - .tag = .next_field, - .is_note = true, - .token = identifier, - }); - // Continue parsing; error will be reported later. - field_state = .err; - }, - } - try p.scratch.append(p.gpa, container_field); - switch (p.token_tags[p.tok_i]) { - .comma => { - p.tok_i += 1; - trailing = true; - continue; - }, - .r_brace, .eof => { - trailing = false; - break; - }, - else => {}, - } - // There is not allowed to be a decl after a field with no comma. - // Report error but recover parser. - try p.warn(.expected_comma_after_field); - p.findNextContainerMember(); - }, - }, - .keyword_pub => { - p.tok_i += 1; - const top_level_decl = try p.expectTopLevelDeclRecoverable(); - if (top_level_decl != 0) { - if (field_state == .seen) { - field_state = .{ .end = top_level_decl }; - } - try p.scratch.append(p.gpa, top_level_decl); - } - trailing = p.token_tags[p.tok_i - 1] == .semicolon; - }, - .keyword_usingnamespace => { - const node = try p.expectUsingNamespaceRecoverable(); - if (node != 0) { - if (field_state == .seen) { - field_state = .{ .end = node }; - } - try p.scratch.append(p.gpa, node); - } - trailing = p.token_tags[p.tok_i - 1] == .semicolon; - }, - .keyword_const, - .keyword_var, - .keyword_threadlocal, - .keyword_export, - .keyword_extern, - .keyword_inline, - .keyword_noinline, - .keyword_fn, - => { - const top_level_decl = try p.expectTopLevelDeclRecoverable(); - if (top_level_decl != 0) { - if (field_state == .seen) { - field_state = .{ .end = top_level_decl }; - } - try p.scratch.append(p.gpa, top_level_decl); - } - trailing = p.token_tags[p.tok_i - 1] == .semicolon; - }, - .eof, .r_brace => { - if (doc_comment) |tok| { - try p.warnMsg(.{ - .tag = .unattached_doc_comment, - .token = tok, - }); - } - break; - }, - else => { - const c_container = p.parseCStyleContainer() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => false, - }; - if (c_container) continue; - - const identifier = p.tok_i; - defer last_field = identifier; - const container_field = p.expectContainerField() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextContainerMember(); - continue; - }, - }; - switch (field_state) { - .none => field_state = .seen, - .err, .seen => {}, - .end => |node| { - try p.warnMsg(.{ - .tag = .decl_between_fields, - .token = p.nodes.items(.main_token)[node], - }); - try p.warnMsg(.{ - .tag = .previous_field, - .is_note = true, - .token = last_field, - }); - try p.warnMsg(.{ - .tag = .next_field, - .is_note = true, - .token = identifier, - }); - // Continue parsing; error will be reported later. - field_state = .err; - }, - } - try p.scratch.append(p.gpa, container_field); - switch (p.token_tags[p.tok_i]) { - .comma => { - p.tok_i += 1; - trailing = true; - continue; - }, - .r_brace, .eof => { - trailing = false; - break; - }, - else => {}, - } - // There is not allowed to be a decl after a field with no comma. - // Report error but recover parser. - try p.warn(.expected_comma_after_field); - if (p.token_tags[p.tok_i] == .semicolon and p.token_tags[identifier] == .identifier) { - try p.warnMsg(.{ - .tag = .var_const_decl, - .is_note = true, - .token = identifier, - }); - } - p.findNextContainerMember(); - continue; - }, - } - } - - const items = p.scratch.items[scratch_top..]; - switch (items.len) { - 0 => return Members{ - .len = 0, - .lhs = 0, - .rhs = 0, - .trailing = trailing, - }, - 1 => return Members{ - .len = 1, - .lhs = items[0], - .rhs = 0, - .trailing = trailing, - }, - 2 => return Members{ - .len = 2, - .lhs = items[0], - .rhs = items[1], - .trailing = trailing, - }, - else => { - const span = try p.listToSpan(items); - return Members{ - .len = items.len, - .lhs = span.start, - .rhs = span.end, - .trailing = trailing, - }; - }, - } - } - - /// Attempts to find next container member by searching for certain tokens - fn findNextContainerMember(p: *Parser) void { - var level: u32 = 0; - while (true) { - const tok = p.nextToken(); - switch (p.token_tags[tok]) { - // Any of these can start a new top level declaration. - .keyword_test, - .keyword_comptime, - .keyword_pub, - .keyword_export, - .keyword_extern, - .keyword_inline, - .keyword_noinline, - .keyword_usingnamespace, - .keyword_threadlocal, - .keyword_const, - .keyword_var, - .keyword_fn, - => { - if (level == 0) { - p.tok_i -= 1; - return; - } - }, - .identifier => { - if (p.token_tags[tok + 1] == .comma and level == 0) { - p.tok_i -= 1; - return; - } - }, - .comma, .semicolon => { - // this decl was likely meant to end here - if (level == 0) { - return; - } - }, - .l_paren, .l_bracket, .l_brace => level += 1, - .r_paren, .r_bracket => { - if (level != 0) level -= 1; - }, - .r_brace => { - if (level == 0) { - // end of container, exit - p.tok_i -= 1; - return; - } - level -= 1; - }, - .eof => { - p.tok_i -= 1; - return; - }, - else => {}, - } - } - } - - /// Attempts to find the next statement by searching for a semicolon - fn findNextStmt(p: *Parser) void { - var level: u32 = 0; - while (true) { - const tok = p.nextToken(); - switch (p.token_tags[tok]) { - .l_brace => level += 1, - .r_brace => { - if (level == 0) { - p.tok_i -= 1; - return; - } - level -= 1; - }, - .semicolon => { - if (level == 0) { - return; - } - }, - .eof => { - p.tok_i -= 1; - return; - }, - else => {}, - } - } - } - - /// TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block - fn expectTestDecl(p: *Parser) !Node.Index { - const test_token = p.assertToken(.keyword_test); - const name_token = switch (p.token_tags[p.nextToken()]) { - .string_literal, .identifier => p.tok_i - 1, - else => blk: { - p.tok_i -= 1; - break :blk null; - }, - }; - const block_node = try p.parseBlock(); - if (block_node == 0) return p.fail(.expected_block); - return p.addNode(.{ - .tag = .test_decl, - .main_token = test_token, - .data = .{ - .lhs = name_token orelse 0, - .rhs = block_node, - }, - }); - } - - fn expectTestDeclRecoverable(p: *Parser) error{OutOfMemory}!Node.Index { - return p.expectTestDecl() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextContainerMember(); - return null_node; - }, - }; - } - - /// Decl - /// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block) - /// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl - /// / KEYWORD_usingnamespace Expr SEMICOLON - fn expectTopLevelDecl(p: *Parser) !Node.Index { - const extern_export_inline_token = p.nextToken(); - var is_extern: bool = false; - var expect_fn: bool = false; - var expect_var_or_fn: bool = false; - switch (p.token_tags[extern_export_inline_token]) { - .keyword_extern => { - _ = p.eatToken(.string_literal); - is_extern = true; - expect_var_or_fn = true; - }, - .keyword_export => expect_var_or_fn = true, - .keyword_inline, .keyword_noinline => expect_fn = true, - else => p.tok_i -= 1, - } - const fn_proto = try p.parseFnProto(); - if (fn_proto != 0) { - switch (p.token_tags[p.tok_i]) { - .semicolon => { - p.tok_i += 1; - return fn_proto; - }, - .l_brace => { - if (is_extern) { - try p.warnMsg(.{ .tag = .extern_fn_body, .token = extern_export_inline_token }); - return null_node; - } - const fn_decl_index = try p.reserveNode(.fn_decl); - errdefer p.unreserveNode(fn_decl_index); - - const body_block = try p.parseBlock(); - assert(body_block != 0); - return p.setNode(fn_decl_index, .{ - .tag = .fn_decl, - .main_token = p.nodes.items(.main_token)[fn_proto], - .data = .{ - .lhs = fn_proto, - .rhs = body_block, - }, - }); - }, - else => { - // Since parseBlock only return error.ParseError on - // a missing '}' we can assume this function was - // supposed to end here. - try p.warn(.expected_semi_or_lbrace); - return null_node; - }, - } - } - if (expect_fn) { - try p.warn(.expected_fn); - return error.ParseError; - } - - const thread_local_token = p.eatToken(.keyword_threadlocal); - const var_decl = try p.parseVarDecl(); - if (var_decl != 0) { - try p.expectSemicolon(.expected_semi_after_decl, false); - return var_decl; - } - if (thread_local_token != null) { - return p.fail(.expected_var_decl); - } - if (expect_var_or_fn) { - return p.fail(.expected_var_decl_or_fn); - } - if (p.token_tags[p.tok_i] != .keyword_usingnamespace) { - return p.fail(.expected_pub_item); - } - return p.expectUsingNamespace(); - } - - fn expectTopLevelDeclRecoverable(p: *Parser) error{OutOfMemory}!Node.Index { - return p.expectTopLevelDecl() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextContainerMember(); - return null_node; - }, - }; - } - - fn expectUsingNamespace(p: *Parser) !Node.Index { - const usingnamespace_token = p.assertToken(.keyword_usingnamespace); - const expr = try p.expectExpr(); - try p.expectSemicolon(.expected_semi_after_decl, false); - return p.addNode(.{ - .tag = .@"usingnamespace", - .main_token = usingnamespace_token, - .data = .{ - .lhs = expr, - .rhs = undefined, - }, - }); - } - - fn expectUsingNamespaceRecoverable(p: *Parser) error{OutOfMemory}!Node.Index { - return p.expectUsingNamespace() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextContainerMember(); - return null_node; - }, - }; - } - - /// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr - fn parseFnProto(p: *Parser) !Node.Index { - const fn_token = p.eatToken(.keyword_fn) orelse return null_node; - - // We want the fn proto node to be before its children in the array. - const fn_proto_index = try p.reserveNode(.fn_proto); - errdefer p.unreserveNode(fn_proto_index); - - _ = p.eatToken(.identifier); - const params = try p.parseParamDeclList(); - const align_expr = try p.parseByteAlign(); - const addrspace_expr = try p.parseAddrSpace(); - const section_expr = try p.parseLinkSection(); - const callconv_expr = try p.parseCallconv(); - _ = p.eatToken(.bang); - - const return_type_expr = try p.parseTypeExpr(); - if (return_type_expr == 0) { - // most likely the user forgot to specify the return type. - // Mark return type as invalid and try to continue. - try p.warn(.expected_return_type); - } - - if (align_expr == 0 and section_expr == 0 and callconv_expr == 0 and addrspace_expr == 0) { - switch (params) { - .zero_or_one => |param| return p.setNode(fn_proto_index, .{ - .tag = .fn_proto_simple, - .main_token = fn_token, - .data = .{ - .lhs = param, - .rhs = return_type_expr, - }, - }), - .multi => |span| { - return p.setNode(fn_proto_index, .{ - .tag = .fn_proto_multi, - .main_token = fn_token, - .data = .{ - .lhs = try p.addExtra(Node.SubRange{ - .start = span.start, - .end = span.end, - }), - .rhs = return_type_expr, - }, - }); - }, - } - } - switch (params) { - .zero_or_one => |param| return p.setNode(fn_proto_index, .{ - .tag = .fn_proto_one, - .main_token = fn_token, - .data = .{ - .lhs = try p.addExtra(Node.FnProtoOne{ - .param = param, - .align_expr = align_expr, - .addrspace_expr = addrspace_expr, - .section_expr = section_expr, - .callconv_expr = callconv_expr, - }), - .rhs = return_type_expr, - }, - }), - .multi => |span| { - return p.setNode(fn_proto_index, .{ - .tag = .fn_proto, - .main_token = fn_token, - .data = .{ - .lhs = try p.addExtra(Node.FnProto{ - .params_start = span.start, - .params_end = span.end, - .align_expr = align_expr, - .addrspace_expr = addrspace_expr, - .section_expr = section_expr, - .callconv_expr = callconv_expr, - }), - .rhs = return_type_expr, - }, - }); - }, - } - } - - /// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection? (EQUAL Expr)? SEMICOLON - fn parseVarDecl(p: *Parser) !Node.Index { - const mut_token = p.eatToken(.keyword_const) orelse - p.eatToken(.keyword_var) orelse - return null_node; - - _ = try p.expectToken(.identifier); - const type_node: Node.Index = if (p.eatToken(.colon) == null) 0 else try p.expectTypeExpr(); - const align_node = try p.parseByteAlign(); - const addrspace_node = try p.parseAddrSpace(); - const section_node = try p.parseLinkSection(); - const init_node: Node.Index = switch (p.token_tags[p.tok_i]) { - .equal_equal => blk: { - try p.warn(.wrong_equal_var_decl); - p.tok_i += 1; - break :blk try p.expectExpr(); - }, - .equal => blk: { - p.tok_i += 1; - break :blk try p.expectExpr(); - }, - else => 0, - }; - if (section_node == 0 and addrspace_node == 0) { - if (align_node == 0) { - return p.addNode(.{ - .tag = .simple_var_decl, - .main_token = mut_token, - .data = .{ - .lhs = type_node, - .rhs = init_node, - }, - }); - } else if (type_node == 0) { - return p.addNode(.{ - .tag = .aligned_var_decl, - .main_token = mut_token, - .data = .{ - .lhs = align_node, - .rhs = init_node, - }, - }); - } else { - return p.addNode(.{ - .tag = .local_var_decl, - .main_token = mut_token, - .data = .{ - .lhs = try p.addExtra(Node.LocalVarDecl{ - .type_node = type_node, - .align_node = align_node, - }), - .rhs = init_node, - }, - }); - } - } else { - return p.addNode(.{ - .tag = .global_var_decl, - .main_token = mut_token, - .data = .{ - .lhs = try p.addExtra(Node.GlobalVarDecl{ - .type_node = type_node, - .align_node = align_node, - .addrspace_node = addrspace_node, - .section_node = section_node, - }), - .rhs = init_node, - }, - }); - } - } - - /// ContainerField - /// <- doc_comment? KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? ByteAlign? (EQUAL Expr)? - /// / doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? !KEYWORD_fn TypeExpr ByteAlign? (EQUAL Expr)? - fn expectContainerField(p: *Parser) !Node.Index { - var main_token = p.tok_i; - _ = p.eatToken(.keyword_comptime); - const tuple_like = p.token_tags[p.tok_i] != .identifier or p.token_tags[p.tok_i + 1] != .colon; - if (!tuple_like) { - main_token = p.assertToken(.identifier); - } - - var align_expr: Node.Index = 0; - var type_expr: Node.Index = 0; - if (p.eatToken(.colon) != null or tuple_like) { - type_expr = try p.expectTypeExpr(); - align_expr = try p.parseByteAlign(); - } - - const value_expr: Node.Index = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); - - if (align_expr == 0) { - return p.addNode(.{ - .tag = .container_field_init, - .main_token = main_token, - .data = .{ - .lhs = type_expr, - .rhs = value_expr, - }, - }); - } else if (value_expr == 0) { - return p.addNode(.{ - .tag = .container_field_align, - .main_token = main_token, - .data = .{ - .lhs = type_expr, - .rhs = align_expr, - }, - }); - } else { - return p.addNode(.{ - .tag = .container_field, - .main_token = main_token, - .data = .{ - .lhs = type_expr, - .rhs = try p.addExtra(Node.ContainerField{ - .value_expr = value_expr, - .align_expr = align_expr, - }), - }, - }); - } - } - - /// Statement - /// <- KEYWORD_comptime? VarDecl - /// / KEYWORD_comptime BlockExprStatement - /// / KEYWORD_nosuspend BlockExprStatement - /// / KEYWORD_suspend BlockExprStatement - /// / KEYWORD_defer BlockExprStatement - /// / KEYWORD_errdefer Payload? BlockExprStatement - /// / IfStatement - /// / LabeledStatement - /// / SwitchExpr - /// / AssignExpr SEMICOLON - fn parseStatement(p: *Parser, allow_defer_var: bool) Error!Node.Index { - const comptime_token = p.eatToken(.keyword_comptime); - - if (allow_defer_var) { - const var_decl = try p.parseVarDecl(); - if (var_decl != 0) { - try p.expectSemicolon(.expected_semi_after_decl, true); - return var_decl; - } - } - - if (comptime_token) |token| { - return p.addNode(.{ - .tag = .@"comptime", - .main_token = token, - .data = .{ - .lhs = try p.expectBlockExprStatement(), - .rhs = undefined, - }, - }); - } - - switch (p.token_tags[p.tok_i]) { - .keyword_nosuspend => { - return p.addNode(.{ - .tag = .@"nosuspend", - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.expectBlockExprStatement(), - .rhs = undefined, - }, - }); - }, - .keyword_suspend => { - const token = p.nextToken(); - const block_expr = try p.expectBlockExprStatement(); - return p.addNode(.{ - .tag = .@"suspend", - .main_token = token, - .data = .{ - .lhs = block_expr, - .rhs = undefined, - }, - }); - }, - .keyword_defer => if (allow_defer_var) return p.addNode(.{ - .tag = .@"defer", - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = try p.expectBlockExprStatement(), - }, - }), - .keyword_errdefer => if (allow_defer_var) return p.addNode(.{ - .tag = .@"errdefer", - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.parsePayload(), - .rhs = try p.expectBlockExprStatement(), - }, - }), - .keyword_switch => return p.expectSwitchExpr(), - .keyword_if => return p.expectIfStatement(), - .keyword_enum, .keyword_struct, .keyword_union => { - const identifier = p.tok_i + 1; - if (try p.parseCStyleContainer()) { - // Return something so that `expectStatement` is happy. - return p.addNode(.{ - .tag = .identifier, - .main_token = identifier, - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }); - } - }, - else => {}, - } - - const labeled_statement = try p.parseLabeledStatement(); - if (labeled_statement != 0) return labeled_statement; - - const assign_expr = try p.parseAssignExpr(); - if (assign_expr != 0) { - try p.expectSemicolon(.expected_semi_after_stmt, true); - return assign_expr; - } - - return null_node; - } - - fn expectStatement(p: *Parser, allow_defer_var: bool) !Node.Index { - const statement = try p.parseStatement(allow_defer_var); - if (statement == 0) { - return p.fail(.expected_statement); - } - return statement; - } - - /// If a parse error occurs, reports an error, but then finds the next statement - /// and returns that one instead. If a parse error occurs but there is no following - /// statement, returns 0. - fn expectStatementRecoverable(p: *Parser) Error!Node.Index { - while (true) { - return p.expectStatement(true) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - p.findNextStmt(); // Try to skip to the next statement. - switch (p.token_tags[p.tok_i]) { - .r_brace => return null_node, - .eof => return error.ParseError, - else => continue, - } - }, - }; - } - } - - /// IfStatement - /// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? - /// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) - fn expectIfStatement(p: *Parser) !Node.Index { - const if_token = p.assertToken(.keyword_if); - _ = try p.expectToken(.l_paren); - const condition = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.parsePtrPayload(); - - // TODO propose to change the syntax so that semicolons are always required - // inside if statements, even if there is an `else`. - var else_required = false; - const then_expr = blk: { - const block_expr = try p.parseBlockExpr(); - if (block_expr != 0) break :blk block_expr; - const assign_expr = try p.parseAssignExpr(); - if (assign_expr == 0) { - return p.fail(.expected_block_or_assignment); - } - if (p.eatToken(.semicolon)) |_| { - return p.addNode(.{ - .tag = .if_simple, - .main_token = if_token, - .data = .{ - .lhs = condition, - .rhs = assign_expr, - }, - }); - } - else_required = true; - break :blk assign_expr; - }; - _ = p.eatToken(.keyword_else) orelse { - if (else_required) { - try p.warn(.expected_semi_or_else); - } - return p.addNode(.{ - .tag = .if_simple, - .main_token = if_token, - .data = .{ - .lhs = condition, - .rhs = then_expr, - }, - }); - }; - _ = try p.parsePayload(); - const else_expr = try p.expectStatement(false); - return p.addNode(.{ - .tag = .@"if", - .main_token = if_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.If{ - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// LabeledStatement <- BlockLabel? (Block / LoopStatement) - fn parseLabeledStatement(p: *Parser) !Node.Index { - const label_token = p.parseBlockLabel(); - const block = try p.parseBlock(); - if (block != 0) return block; - - const loop_stmt = try p.parseLoopStatement(); - if (loop_stmt != 0) return loop_stmt; - - if (label_token != 0) { - const after_colon = p.tok_i; - const node = try p.parseTypeExpr(); - if (node != 0) { - const a = try p.parseByteAlign(); - const b = try p.parseAddrSpace(); - const c = try p.parseLinkSection(); - const d = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); - if (a != 0 or b != 0 or c != 0 or d != 0) { - return p.failMsg(.{ .tag = .expected_var_const, .token = label_token }); - } - } - return p.failMsg(.{ .tag = .expected_labelable, .token = after_colon }); - } - - return null_node; - } - - /// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) - fn parseLoopStatement(p: *Parser) !Node.Index { - const inline_token = p.eatToken(.keyword_inline); - - const for_statement = try p.parseForStatement(); - if (for_statement != 0) return for_statement; - - const while_statement = try p.parseWhileStatement(); - if (while_statement != 0) return while_statement; - - if (inline_token == null) return null_node; - - // If we've seen "inline", there should have been a "for" or "while" - return p.fail(.expected_inlinable); - } - - /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload - /// - /// ForStatement - /// <- ForPrefix BlockExpr ( KEYWORD_else Statement )? - /// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement ) - fn parseForStatement(p: *Parser) !Node.Index { - const for_token = p.eatToken(.keyword_for) orelse return null_node; - _ = try p.expectToken(.l_paren); - const array_expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - const found_payload = try p.parsePtrIndexPayload(); - if (found_payload == 0) try p.warn(.expected_loop_payload); - - // TODO propose to change the syntax so that semicolons are always required - // inside while statements, even if there is an `else`. - var else_required = false; - const then_expr = blk: { - const block_expr = try p.parseBlockExpr(); - if (block_expr != 0) break :blk block_expr; - const assign_expr = try p.parseAssignExpr(); - if (assign_expr == 0) { - return p.fail(.expected_block_or_assignment); - } - if (p.eatToken(.semicolon)) |_| { - return p.addNode(.{ - .tag = .for_simple, - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = assign_expr, - }, - }); - } - else_required = true; - break :blk assign_expr; - }; - _ = p.eatToken(.keyword_else) orelse { - if (else_required) { - try p.warn(.expected_semi_or_else); - } - return p.addNode(.{ - .tag = .for_simple, - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = then_expr, - }, - }); - }; - return p.addNode(.{ - .tag = .@"for", - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = try p.addExtra(Node.If{ - .then_expr = then_expr, - .else_expr = try p.expectStatement(false), - }), - }, - }); - } - - /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? - /// - /// WhileStatement - /// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? - /// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) - fn parseWhileStatement(p: *Parser) !Node.Index { - const while_token = p.eatToken(.keyword_while) orelse return null_node; - _ = try p.expectToken(.l_paren); - const condition = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.parsePtrPayload(); - const cont_expr = try p.parseWhileContinueExpr(); - - // TODO propose to change the syntax so that semicolons are always required - // inside while statements, even if there is an `else`. - var else_required = false; - const then_expr = blk: { - const block_expr = try p.parseBlockExpr(); - if (block_expr != 0) break :blk block_expr; - const assign_expr = try p.parseAssignExpr(); - if (assign_expr == 0) { - return p.fail(.expected_block_or_assignment); - } - if (p.eatToken(.semicolon)) |_| { - if (cont_expr == 0) { - return p.addNode(.{ - .tag = .while_simple, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = assign_expr, - }, - }); - } else { - return p.addNode(.{ - .tag = .while_cont, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.WhileCont{ - .cont_expr = cont_expr, - .then_expr = assign_expr, - }), - }, - }); - } - } - else_required = true; - break :blk assign_expr; - }; - _ = p.eatToken(.keyword_else) orelse { - if (else_required) { - try p.warn(.expected_semi_or_else); - } - if (cont_expr == 0) { - return p.addNode(.{ - .tag = .while_simple, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = then_expr, - }, - }); - } else { - return p.addNode(.{ - .tag = .while_cont, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.WhileCont{ - .cont_expr = cont_expr, - .then_expr = then_expr, - }), - }, - }); - } - }; - _ = try p.parsePayload(); - const else_expr = try p.expectStatement(false); - return p.addNode(.{ - .tag = .@"while", - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.While{ - .cont_expr = cont_expr, - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// BlockExprStatement - /// <- BlockExpr - /// / AssignExpr SEMICOLON - fn parseBlockExprStatement(p: *Parser) !Node.Index { - const block_expr = try p.parseBlockExpr(); - if (block_expr != 0) { - return block_expr; - } - const assign_expr = try p.parseAssignExpr(); - if (assign_expr != 0) { - try p.expectSemicolon(.expected_semi_after_stmt, true); - return assign_expr; - } - return null_node; - } - - fn expectBlockExprStatement(p: *Parser) !Node.Index { - const node = try p.parseBlockExprStatement(); - if (node == 0) { - return p.fail(.expected_block_or_expr); - } - return node; - } - - /// BlockExpr <- BlockLabel? Block - fn parseBlockExpr(p: *Parser) Error!Node.Index { - switch (p.token_tags[p.tok_i]) { - .identifier => { - if (p.token_tags[p.tok_i + 1] == .colon and - p.token_tags[p.tok_i + 2] == .l_brace) - { - p.tok_i += 2; - return p.parseBlock(); - } else { - return null_node; - } - }, - .l_brace => return p.parseBlock(), - else => return null_node, - } - } - - /// AssignExpr <- Expr (AssignOp Expr)? - /// - /// AssignOp - /// <- ASTERISKEQUAL - /// / ASTERISKPIPEEQUAL - /// / SLASHEQUAL - /// / PERCENTEQUAL - /// / PLUSEQUAL - /// / PLUSPIPEEQUAL - /// / MINUSEQUAL - /// / MINUSPIPEEQUAL - /// / LARROW2EQUAL - /// / LARROW2PIPEEQUAL - /// / RARROW2EQUAL - /// / AMPERSANDEQUAL - /// / CARETEQUAL - /// / PIPEEQUAL - /// / ASTERISKPERCENTEQUAL - /// / PLUSPERCENTEQUAL - /// / MINUSPERCENTEQUAL - /// / EQUAL - fn parseAssignExpr(p: *Parser) !Node.Index { - const expr = try p.parseExpr(); - if (expr == 0) return null_node; - - const tag: Node.Tag = switch (p.token_tags[p.tok_i]) { - .asterisk_equal => .assign_mul, - .slash_equal => .assign_div, - .percent_equal => .assign_mod, - .plus_equal => .assign_add, - .minus_equal => .assign_sub, - .angle_bracket_angle_bracket_left_equal => .assign_shl, - .angle_bracket_angle_bracket_left_pipe_equal => .assign_shl_sat, - .angle_bracket_angle_bracket_right_equal => .assign_shr, - .ampersand_equal => .assign_bit_and, - .caret_equal => .assign_bit_xor, - .pipe_equal => .assign_bit_or, - .asterisk_percent_equal => .assign_mul_wrap, - .plus_percent_equal => .assign_add_wrap, - .minus_percent_equal => .assign_sub_wrap, - .asterisk_pipe_equal => .assign_mul_sat, - .plus_pipe_equal => .assign_add_sat, - .minus_pipe_equal => .assign_sub_sat, - .equal => .assign, - else => return expr, - }; - return p.addNode(.{ - .tag = tag, - .main_token = p.nextToken(), - .data = .{ - .lhs = expr, - .rhs = try p.expectExpr(), - }, - }); - } - - fn expectAssignExpr(p: *Parser) !Node.Index { - const expr = try p.parseAssignExpr(); - if (expr == 0) { - return p.fail(.expected_expr_or_assignment); - } - return expr; - } - - fn parseExpr(p: *Parser) Error!Node.Index { - return p.parseExprPrecedence(0); - } - - fn expectExpr(p: *Parser) Error!Node.Index { - const node = try p.parseExpr(); - if (node == 0) { - return p.fail(.expected_expr); - } else { - return node; - } - } - - const Assoc = enum { - left, - none, - }; - - const OperInfo = struct { - prec: i8, - tag: Node.Tag, - assoc: Assoc = Assoc.left, - }; - - // A table of binary operator information. Higher precedence numbers are - // stickier. All operators at the same precedence level should have the same - // associativity. - const operTable = std.enums.directEnumArrayDefault(Token.Tag, OperInfo, .{ .prec = -1, .tag = Node.Tag.root }, 0, .{ - .keyword_or = .{ .prec = 10, .tag = .bool_or }, - - .keyword_and = .{ .prec = 20, .tag = .bool_and }, - - .equal_equal = .{ .prec = 30, .tag = .equal_equal, .assoc = Assoc.none }, - .bang_equal = .{ .prec = 30, .tag = .bang_equal, .assoc = Assoc.none }, - .angle_bracket_left = .{ .prec = 30, .tag = .less_than, .assoc = Assoc.none }, - .angle_bracket_right = .{ .prec = 30, .tag = .greater_than, .assoc = Assoc.none }, - .angle_bracket_left_equal = .{ .prec = 30, .tag = .less_or_equal, .assoc = Assoc.none }, - .angle_bracket_right_equal = .{ .prec = 30, .tag = .greater_or_equal, .assoc = Assoc.none }, - - .ampersand = .{ .prec = 40, .tag = .bit_and }, - .caret = .{ .prec = 40, .tag = .bit_xor }, - .pipe = .{ .prec = 40, .tag = .bit_or }, - .keyword_orelse = .{ .prec = 40, .tag = .@"orelse" }, - .keyword_catch = .{ .prec = 40, .tag = .@"catch" }, - - .angle_bracket_angle_bracket_left = .{ .prec = 50, .tag = .shl }, - .angle_bracket_angle_bracket_left_pipe = .{ .prec = 50, .tag = .shl_sat }, - .angle_bracket_angle_bracket_right = .{ .prec = 50, .tag = .shr }, - - .plus = .{ .prec = 60, .tag = .add }, - .minus = .{ .prec = 60, .tag = .sub }, - .plus_plus = .{ .prec = 60, .tag = .array_cat }, - .plus_percent = .{ .prec = 60, .tag = .add_wrap }, - .minus_percent = .{ .prec = 60, .tag = .sub_wrap }, - .plus_pipe = .{ .prec = 60, .tag = .add_sat }, - .minus_pipe = .{ .prec = 60, .tag = .sub_sat }, - - .pipe_pipe = .{ .prec = 70, .tag = .merge_error_sets }, - .asterisk = .{ .prec = 70, .tag = .mul }, - .slash = .{ .prec = 70, .tag = .div }, - .percent = .{ .prec = 70, .tag = .mod }, - .asterisk_asterisk = .{ .prec = 70, .tag = .array_mult }, - .asterisk_percent = .{ .prec = 70, .tag = .mul_wrap }, - .asterisk_pipe = .{ .prec = 70, .tag = .mul_sat }, - }); - - fn parseExprPrecedence(p: *Parser, min_prec: i32) Error!Node.Index { - assert(min_prec >= 0); - var node = try p.parsePrefixExpr(); - if (node == 0) { - return null_node; - } - - var banned_prec: i8 = -1; - - while (true) { - const tok_tag = p.token_tags[p.tok_i]; - const info = operTable[@intCast(usize, @enumToInt(tok_tag))]; - if (info.prec < min_prec) { - break; - } - if (info.prec == banned_prec) { - return p.fail(.chained_comparison_operators); - } - - const oper_token = p.nextToken(); - // Special-case handling for "catch" - if (tok_tag == .keyword_catch) { - _ = try p.parsePayload(); - } - const rhs = try p.parseExprPrecedence(info.prec + 1); - if (rhs == 0) { - try p.warn(.expected_expr); - return node; - } - - { - const tok_len = tok_tag.lexeme().?.len; - const char_before = p.source[p.token_starts[oper_token] - 1]; - const char_after = p.source[p.token_starts[oper_token] + tok_len]; - if (tok_tag == .ampersand and char_after == '&') { - // without types we don't know if '&&' was intended as 'bitwise_and address_of', or a c-style logical_and - // The best the parser can do is recommend changing it to 'and' or ' & &' - try p.warnMsg(.{ .tag = .invalid_ampersand_ampersand, .token = oper_token }); - } else if (std.ascii.isWhitespace(char_before) != std.ascii.isWhitespace(char_after)) { - try p.warnMsg(.{ .tag = .mismatched_binary_op_whitespace, .token = oper_token }); - } - } - - node = try p.addNode(.{ - .tag = info.tag, - .main_token = oper_token, - .data = .{ - .lhs = node, - .rhs = rhs, - }, - }); - - if (info.assoc == Assoc.none) { - banned_prec = info.prec; - } - } - - return node; - } - - /// PrefixExpr <- PrefixOp* PrimaryExpr - /// - /// PrefixOp - /// <- EXCLAMATIONMARK - /// / MINUS - /// / TILDE - /// / MINUSPERCENT - /// / AMPERSAND - /// / KEYWORD_try - /// / KEYWORD_await - fn parsePrefixExpr(p: *Parser) Error!Node.Index { - const tag: Node.Tag = switch (p.token_tags[p.tok_i]) { - .bang => .bool_not, - .minus => .negation, - .tilde => .bit_not, - .minus_percent => .negation_wrap, - .ampersand => .address_of, - .keyword_try => .@"try", - .keyword_await => .@"await", - else => return p.parsePrimaryExpr(), - }; - return p.addNode(.{ - .tag = tag, - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.expectPrefixExpr(), - .rhs = undefined, - }, - }); - } - - fn expectPrefixExpr(p: *Parser) Error!Node.Index { - const node = try p.parsePrefixExpr(); - if (node == 0) { - return p.fail(.expected_prefix_expr); - } - return node; - } - - /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr - /// - /// PrefixTypeOp - /// <- QUESTIONMARK - /// / KEYWORD_anyframe MINUSRARROW - /// / SliceTypeStart (ByteAlign / AddrSpace / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* - /// / PtrTypeStart (AddrSpace / KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* - /// / ArrayTypeStart - /// - /// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET - /// - /// PtrTypeStart - /// <- ASTERISK - /// / ASTERISK2 - /// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET - /// - /// ArrayTypeStart <- LBRACKET Expr (COLON Expr)? RBRACKET - fn parseTypeExpr(p: *Parser) Error!Node.Index { - switch (p.token_tags[p.tok_i]) { - .question_mark => return p.addNode(.{ - .tag = .optional_type, - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.expectTypeExpr(), - .rhs = undefined, - }, - }), - .keyword_anyframe => switch (p.token_tags[p.tok_i + 1]) { - .arrow => return p.addNode(.{ - .tag = .anyframe_type, - .main_token = p.nextToken(), - .data = .{ - .lhs = p.nextToken(), - .rhs = try p.expectTypeExpr(), - }, - }), - else => return p.parseErrorUnionExpr(), - }, - .asterisk => { - const asterisk = p.nextToken(); - const mods = try p.parsePtrModifiers(); - const elem_type = try p.expectTypeExpr(); - if (mods.bit_range_start != 0) { - return p.addNode(.{ - .tag = .ptr_type_bit_range, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrTypeBitRange{ - .sentinel = 0, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - .bit_range_start = mods.bit_range_start, - .bit_range_end = mods.bit_range_end, - }), - .rhs = elem_type, - }, - }); - } else if (mods.addrspace_node != 0) { - return p.addNode(.{ - .tag = .ptr_type, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrType{ - .sentinel = 0, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - }), - .rhs = elem_type, - }, - }); - } else { - return p.addNode(.{ - .tag = .ptr_type_aligned, - .main_token = asterisk, - .data = .{ - .lhs = mods.align_node, - .rhs = elem_type, - }, - }); - } - }, - .asterisk_asterisk => { - const asterisk = p.nextToken(); - const mods = try p.parsePtrModifiers(); - const elem_type = try p.expectTypeExpr(); - const inner: Node.Index = inner: { - if (mods.bit_range_start != 0) { - break :inner try p.addNode(.{ - .tag = .ptr_type_bit_range, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrTypeBitRange{ - .sentinel = 0, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - .bit_range_start = mods.bit_range_start, - .bit_range_end = mods.bit_range_end, - }), - .rhs = elem_type, - }, - }); - } else if (mods.addrspace_node != 0) { - break :inner try p.addNode(.{ - .tag = .ptr_type, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrType{ - .sentinel = 0, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - }), - .rhs = elem_type, - }, - }); - } else { - break :inner try p.addNode(.{ - .tag = .ptr_type_aligned, - .main_token = asterisk, - .data = .{ - .lhs = mods.align_node, - .rhs = elem_type, - }, - }); - } - }; - return p.addNode(.{ - .tag = .ptr_type_aligned, - .main_token = asterisk, - .data = .{ - .lhs = 0, - .rhs = inner, - }, - }); - }, - .l_bracket => switch (p.token_tags[p.tok_i + 1]) { - .asterisk => { - _ = p.nextToken(); - const asterisk = p.nextToken(); - var sentinel: Node.Index = 0; - if (p.eatToken(.identifier)) |ident| { - const ident_slice = p.source[p.token_starts[ident]..p.token_starts[ident + 1]]; - if (!std.mem.eql(u8, std.mem.trimRight(u8, ident_slice, &std.ascii.whitespace), "c")) { - p.tok_i -= 1; - } - } else if (p.eatToken(.colon)) |_| { - sentinel = try p.expectExpr(); - } - _ = try p.expectToken(.r_bracket); - const mods = try p.parsePtrModifiers(); - const elem_type = try p.expectTypeExpr(); - if (mods.bit_range_start == 0) { - if (sentinel == 0 and mods.addrspace_node == 0) { - return p.addNode(.{ - .tag = .ptr_type_aligned, - .main_token = asterisk, - .data = .{ - .lhs = mods.align_node, - .rhs = elem_type, - }, - }); - } else if (mods.align_node == 0 and mods.addrspace_node == 0) { - return p.addNode(.{ - .tag = .ptr_type_sentinel, - .main_token = asterisk, - .data = .{ - .lhs = sentinel, - .rhs = elem_type, - }, - }); - } else { - return p.addNode(.{ - .tag = .ptr_type, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrType{ - .sentinel = sentinel, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - }), - .rhs = elem_type, - }, - }); - } - } else { - return p.addNode(.{ - .tag = .ptr_type_bit_range, - .main_token = asterisk, - .data = .{ - .lhs = try p.addExtra(Node.PtrTypeBitRange{ - .sentinel = sentinel, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - .bit_range_start = mods.bit_range_start, - .bit_range_end = mods.bit_range_end, - }), - .rhs = elem_type, - }, - }); - } - }, - else => { - const lbracket = p.nextToken(); - const len_expr = try p.parseExpr(); - const sentinel: Node.Index = if (p.eatToken(.colon)) |_| - try p.expectExpr() - else - 0; - _ = try p.expectToken(.r_bracket); - if (len_expr == 0) { - const mods = try p.parsePtrModifiers(); - const elem_type = try p.expectTypeExpr(); - if (mods.bit_range_start != 0) { - try p.warnMsg(.{ - .tag = .invalid_bit_range, - .token = p.nodes.items(.main_token)[mods.bit_range_start], - }); - } - if (sentinel == 0 and mods.addrspace_node == 0) { - return p.addNode(.{ - .tag = .ptr_type_aligned, - .main_token = lbracket, - .data = .{ - .lhs = mods.align_node, - .rhs = elem_type, - }, - }); - } else if (mods.align_node == 0 and mods.addrspace_node == 0) { - return p.addNode(.{ - .tag = .ptr_type_sentinel, - .main_token = lbracket, - .data = .{ - .lhs = sentinel, - .rhs = elem_type, - }, - }); - } else { - return p.addNode(.{ - .tag = .ptr_type, - .main_token = lbracket, - .data = .{ - .lhs = try p.addExtra(Node.PtrType{ - .sentinel = sentinel, - .align_node = mods.align_node, - .addrspace_node = mods.addrspace_node, - }), - .rhs = elem_type, - }, - }); - } - } else { - switch (p.token_tags[p.tok_i]) { - .keyword_align, - .keyword_const, - .keyword_volatile, - .keyword_allowzero, - .keyword_addrspace, - => return p.fail(.ptr_mod_on_array_child_type), - else => {}, - } - const elem_type = try p.expectTypeExpr(); - if (sentinel == 0) { - return p.addNode(.{ - .tag = .array_type, - .main_token = lbracket, - .data = .{ - .lhs = len_expr, - .rhs = elem_type, - }, - }); - } else { - return p.addNode(.{ - .tag = .array_type_sentinel, - .main_token = lbracket, - .data = .{ - .lhs = len_expr, - .rhs = try p.addExtra(.{ - .elem_type = elem_type, - .sentinel = sentinel, - }), - }, - }); - } - } - }, - }, - else => return p.parseErrorUnionExpr(), - } - } - - fn expectTypeExpr(p: *Parser) Error!Node.Index { - const node = try p.parseTypeExpr(); - if (node == 0) { - return p.fail(.expected_type_expr); - } - return node; - } - - /// PrimaryExpr - /// <- AsmExpr - /// / IfExpr - /// / KEYWORD_break BreakLabel? Expr? - /// / KEYWORD_comptime Expr - /// / KEYWORD_nosuspend Expr - /// / KEYWORD_continue BreakLabel? - /// / KEYWORD_resume Expr - /// / KEYWORD_return Expr? - /// / BlockLabel? LoopExpr - /// / Block - /// / CurlySuffixExpr - fn parsePrimaryExpr(p: *Parser) !Node.Index { - switch (p.token_tags[p.tok_i]) { - .keyword_asm => return p.expectAsmExpr(), - .keyword_if => return p.parseIfExpr(), - .keyword_break => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"break", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.parseBreakLabel(), - .rhs = try p.parseExpr(), - }, - }); - }, - .keyword_continue => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"continue", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.parseBreakLabel(), - .rhs = undefined, - }, - }); - }, - .keyword_comptime => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"comptime", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.expectExpr(), - .rhs = undefined, - }, - }); - }, - .keyword_nosuspend => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"nosuspend", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.expectExpr(), - .rhs = undefined, - }, - }); - }, - .keyword_resume => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"resume", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.expectExpr(), - .rhs = undefined, - }, - }); - }, - .keyword_return => { - p.tok_i += 1; - return p.addNode(.{ - .tag = .@"return", - .main_token = p.tok_i - 1, - .data = .{ - .lhs = try p.parseExpr(), - .rhs = undefined, - }, - }); - }, - .identifier => { - if (p.token_tags[p.tok_i + 1] == .colon) { - switch (p.token_tags[p.tok_i + 2]) { - .keyword_inline => { - p.tok_i += 3; - switch (p.token_tags[p.tok_i]) { - .keyword_for => return p.parseForExpr(), - .keyword_while => return p.parseWhileExpr(), - else => return p.fail(.expected_inlinable), - } - }, - .keyword_for => { - p.tok_i += 2; - return p.parseForExpr(); - }, - .keyword_while => { - p.tok_i += 2; - return p.parseWhileExpr(); - }, - .l_brace => { - p.tok_i += 2; - return p.parseBlock(); - }, - else => return p.parseCurlySuffixExpr(), - } - } else { - return p.parseCurlySuffixExpr(); - } - }, - .keyword_inline => { - p.tok_i += 1; - switch (p.token_tags[p.tok_i]) { - .keyword_for => return p.parseForExpr(), - .keyword_while => return p.parseWhileExpr(), - else => return p.fail(.expected_inlinable), - } - }, - .keyword_for => return p.parseForExpr(), - .keyword_while => return p.parseWhileExpr(), - .l_brace => return p.parseBlock(), - else => return p.parseCurlySuffixExpr(), - } - } - - /// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? - fn parseIfExpr(p: *Parser) !Node.Index { - return p.parseIf(expectExpr); - } - - /// Block <- LBRACE Statement* RBRACE - fn parseBlock(p: *Parser) !Node.Index { - const lbrace = p.eatToken(.l_brace) orelse return null_node; - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - while (true) { - if (p.token_tags[p.tok_i] == .r_brace) break; - const statement = try p.expectStatementRecoverable(); - if (statement == 0) break; - try p.scratch.append(p.gpa, statement); - } - _ = try p.expectToken(.r_brace); - const semicolon = (p.token_tags[p.tok_i - 2] == .semicolon); - const statements = p.scratch.items[scratch_top..]; - switch (statements.len) { - 0 => return p.addNode(.{ - .tag = .block_two, - .main_token = lbrace, - .data = .{ - .lhs = 0, - .rhs = 0, - }, - }), - 1 => return p.addNode(.{ - .tag = if (semicolon) .block_two_semicolon else .block_two, - .main_token = lbrace, - .data = .{ - .lhs = statements[0], - .rhs = 0, - }, - }), - 2 => return p.addNode(.{ - .tag = if (semicolon) .block_two_semicolon else .block_two, - .main_token = lbrace, - .data = .{ - .lhs = statements[0], - .rhs = statements[1], - }, - }), - else => { - const span = try p.listToSpan(statements); - return p.addNode(.{ - .tag = if (semicolon) .block_semicolon else .block, - .main_token = lbrace, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - }, - } - } - - /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload - /// - /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)? - fn parseForExpr(p: *Parser) !Node.Index { - const for_token = p.eatToken(.keyword_for) orelse return null_node; - _ = try p.expectToken(.l_paren); - const array_expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - const found_payload = try p.parsePtrIndexPayload(); - if (found_payload == 0) try p.warn(.expected_loop_payload); - - const then_expr = try p.expectExpr(); - _ = p.eatToken(.keyword_else) orelse { - return p.addNode(.{ - .tag = .for_simple, - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = then_expr, - }, - }); - }; - const else_expr = try p.expectExpr(); - return p.addNode(.{ - .tag = .@"for", - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = try p.addExtra(Node.If{ - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? - /// - /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? - fn parseWhileExpr(p: *Parser) !Node.Index { - const while_token = p.eatToken(.keyword_while) orelse return null_node; - _ = try p.expectToken(.l_paren); - const condition = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.parsePtrPayload(); - const cont_expr = try p.parseWhileContinueExpr(); - - const then_expr = try p.expectExpr(); - _ = p.eatToken(.keyword_else) orelse { - if (cont_expr == 0) { - return p.addNode(.{ - .tag = .while_simple, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = then_expr, - }, - }); - } else { - return p.addNode(.{ - .tag = .while_cont, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.WhileCont{ - .cont_expr = cont_expr, - .then_expr = then_expr, - }), - }, - }); - } - }; - _ = try p.parsePayload(); - const else_expr = try p.expectExpr(); - return p.addNode(.{ - .tag = .@"while", - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.While{ - .cont_expr = cont_expr, - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// CurlySuffixExpr <- TypeExpr InitList? - /// - /// InitList - /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE - /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE - /// / LBRACE RBRACE - fn parseCurlySuffixExpr(p: *Parser) !Node.Index { - const lhs = try p.parseTypeExpr(); - if (lhs == 0) return null_node; - const lbrace = p.eatToken(.l_brace) orelse return lhs; - - // If there are 0 or 1 items, we can use ArrayInitOne/StructInitOne; - // otherwise we use the full ArrayInit/StructInit. - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - const field_init = try p.parseFieldInit(); - if (field_init != 0) { - try p.scratch.append(p.gpa, field_init); - while (true) { - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_brace => { - p.tok_i += 1; - break; - }, - .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_initializer), - } - if (p.eatToken(.r_brace)) |_| break; - const next = try p.expectFieldInit(); - try p.scratch.append(p.gpa, next); - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const inits = p.scratch.items[scratch_top..]; - switch (inits.len) { - 0 => unreachable, - 1 => return p.addNode(.{ - .tag = if (comma) .struct_init_one_comma else .struct_init_one, - .main_token = lbrace, - .data = .{ - .lhs = lhs, - .rhs = inits[0], - }, - }), - else => return p.addNode(.{ - .tag = if (comma) .struct_init_comma else .struct_init, - .main_token = lbrace, - .data = .{ - .lhs = lhs, - .rhs = try p.addExtra(try p.listToSpan(inits)), - }, - }), - } - } - - while (true) { - if (p.eatToken(.r_brace)) |_| break; - const elem_init = try p.expectExpr(); - try p.scratch.append(p.gpa, elem_init); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_brace => { - p.tok_i += 1; - break; - }, - .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_initializer), - } - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const inits = p.scratch.items[scratch_top..]; - switch (inits.len) { - 0 => return p.addNode(.{ - .tag = .struct_init_one, - .main_token = lbrace, - .data = .{ - .lhs = lhs, - .rhs = 0, - }, - }), - 1 => return p.addNode(.{ - .tag = if (comma) .array_init_one_comma else .array_init_one, - .main_token = lbrace, - .data = .{ - .lhs = lhs, - .rhs = inits[0], - }, - }), - else => return p.addNode(.{ - .tag = if (comma) .array_init_comma else .array_init, - .main_token = lbrace, - .data = .{ - .lhs = lhs, - .rhs = try p.addExtra(try p.listToSpan(inits)), - }, - }), - } - } - - /// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? - fn parseErrorUnionExpr(p: *Parser) !Node.Index { - const suffix_expr = try p.parseSuffixExpr(); - if (suffix_expr == 0) return null_node; - const bang = p.eatToken(.bang) orelse return suffix_expr; - return p.addNode(.{ - .tag = .error_union, - .main_token = bang, - .data = .{ - .lhs = suffix_expr, - .rhs = try p.expectTypeExpr(), - }, - }); - } - - /// SuffixExpr - /// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments - /// / PrimaryTypeExpr (SuffixOp / FnCallArguments)* - /// - /// FnCallArguments <- LPAREN ExprList RPAREN - /// - /// ExprList <- (Expr COMMA)* Expr? - fn parseSuffixExpr(p: *Parser) !Node.Index { - if (p.eatToken(.keyword_async)) |_| { - var res = try p.expectPrimaryTypeExpr(); - while (true) { - const node = try p.parseSuffixOp(res); - if (node == 0) break; - res = node; - } - const lparen = p.eatToken(.l_paren) orelse { - try p.warn(.expected_param_list); - return res; - }; - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - while (true) { - if (p.eatToken(.r_paren)) |_| break; - const param = try p.expectExpr(); - try p.scratch.append(p.gpa, param); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_paren => { - p.tok_i += 1; - break; - }, - .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_arg), - } - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const params = p.scratch.items[scratch_top..]; - switch (params.len) { - 0 => return p.addNode(.{ - .tag = if (comma) .async_call_one_comma else .async_call_one, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = 0, - }, - }), - 1 => return p.addNode(.{ - .tag = if (comma) .async_call_one_comma else .async_call_one, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = params[0], - }, - }), - else => return p.addNode(.{ - .tag = if (comma) .async_call_comma else .async_call, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = try p.addExtra(try p.listToSpan(params)), - }, - }), - } - } - - var res = try p.parsePrimaryTypeExpr(); - if (res == 0) return res; - while (true) { - const suffix_op = try p.parseSuffixOp(res); - if (suffix_op != 0) { - res = suffix_op; - continue; - } - const lparen = p.eatToken(.l_paren) orelse return res; - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - while (true) { - if (p.eatToken(.r_paren)) |_| break; - const param = try p.expectExpr(); - try p.scratch.append(p.gpa, param); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_paren => { - p.tok_i += 1; - break; - }, - .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_arg), - } - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const params = p.scratch.items[scratch_top..]; - res = switch (params.len) { - 0 => try p.addNode(.{ - .tag = if (comma) .call_one_comma else .call_one, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = 0, - }, - }), - 1 => try p.addNode(.{ - .tag = if (comma) .call_one_comma else .call_one, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = params[0], - }, - }), - else => try p.addNode(.{ - .tag = if (comma) .call_comma else .call, - .main_token = lparen, - .data = .{ - .lhs = res, - .rhs = try p.addExtra(try p.listToSpan(params)), - }, - }), - }; - } - } - - /// PrimaryTypeExpr - /// <- BUILTINIDENTIFIER FnCallArguments - /// / CHAR_LITERAL - /// / ContainerDecl - /// / DOT IDENTIFIER - /// / DOT InitList - /// / ErrorSetDecl - /// / FLOAT - /// / FnProto - /// / GroupedExpr - /// / LabeledTypeExpr - /// / IDENTIFIER - /// / IfTypeExpr - /// / INTEGER - /// / KEYWORD_comptime TypeExpr - /// / KEYWORD_error DOT IDENTIFIER - /// / KEYWORD_anyframe - /// / KEYWORD_unreachable - /// / STRINGLITERAL - /// / SwitchExpr - /// - /// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto - /// - /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE - /// - /// InitList - /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE - /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE - /// / LBRACE RBRACE - /// - /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE - /// - /// GroupedExpr <- LPAREN Expr RPAREN - /// - /// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? - /// - /// LabeledTypeExpr - /// <- BlockLabel Block - /// / BlockLabel? LoopTypeExpr - /// - /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) - fn parsePrimaryTypeExpr(p: *Parser) !Node.Index { - switch (p.token_tags[p.tok_i]) { - .char_literal => return p.addNode(.{ - .tag = .char_literal, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - .number_literal => return p.addNode(.{ - .tag = .number_literal, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - .keyword_unreachable => return p.addNode(.{ - .tag = .unreachable_literal, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - .keyword_anyframe => return p.addNode(.{ - .tag = .anyframe_literal, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - .string_literal => { - const main_token = p.nextToken(); - return p.addNode(.{ - .tag = .string_literal, - .main_token = main_token, - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }); - }, - - .builtin => return p.parseBuiltinCall(), - .keyword_fn => return p.parseFnProto(), - .keyword_if => return p.parseIf(expectTypeExpr), - .keyword_switch => return p.expectSwitchExpr(), - - .keyword_extern, - .keyword_packed, - => { - p.tok_i += 1; - return p.parseContainerDeclAuto(); - }, - - .keyword_struct, - .keyword_opaque, - .keyword_enum, - .keyword_union, - => return p.parseContainerDeclAuto(), - - .keyword_comptime => return p.addNode(.{ - .tag = .@"comptime", - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.expectTypeExpr(), - .rhs = undefined, - }, - }), - .multiline_string_literal_line => { - const first_line = p.nextToken(); - while (p.token_tags[p.tok_i] == .multiline_string_literal_line) { - p.tok_i += 1; - } - return p.addNode(.{ - .tag = .multiline_string_literal, - .main_token = first_line, - .data = .{ - .lhs = first_line, - .rhs = p.tok_i - 1, - }, - }); - }, - .identifier => switch (p.token_tags[p.tok_i + 1]) { - .colon => switch (p.token_tags[p.tok_i + 2]) { - .keyword_inline => { - p.tok_i += 3; - switch (p.token_tags[p.tok_i]) { - .keyword_for => return p.parseForTypeExpr(), - .keyword_while => return p.parseWhileTypeExpr(), - else => return p.fail(.expected_inlinable), - } - }, - .keyword_for => { - p.tok_i += 2; - return p.parseForTypeExpr(); - }, - .keyword_while => { - p.tok_i += 2; - return p.parseWhileTypeExpr(); - }, - .l_brace => { - p.tok_i += 2; - return p.parseBlock(); - }, - else => return p.addNode(.{ - .tag = .identifier, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - }, - else => return p.addNode(.{ - .tag = .identifier, - .main_token = p.nextToken(), - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }), - }, - .keyword_inline => { - p.tok_i += 1; - switch (p.token_tags[p.tok_i]) { - .keyword_for => return p.parseForTypeExpr(), - .keyword_while => return p.parseWhileTypeExpr(), - else => return p.fail(.expected_inlinable), - } - }, - .keyword_for => return p.parseForTypeExpr(), - .keyword_while => return p.parseWhileTypeExpr(), - .period => switch (p.token_tags[p.tok_i + 1]) { - .identifier => return p.addNode(.{ - .tag = .enum_literal, - .data = .{ - .lhs = p.nextToken(), // dot - .rhs = undefined, - }, - .main_token = p.nextToken(), // identifier - }), - .l_brace => { - const lbrace = p.tok_i + 1; - p.tok_i = lbrace + 1; - - // If there are 0, 1, or 2 items, we can use ArrayInitDotTwo/StructInitDotTwo; - // otherwise we use the full ArrayInitDot/StructInitDot. - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - const field_init = try p.parseFieldInit(); - if (field_init != 0) { - try p.scratch.append(p.gpa, field_init); - while (true) { - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_brace => { - p.tok_i += 1; - break; - }, - .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_initializer), - } - if (p.eatToken(.r_brace)) |_| break; - const next = try p.expectFieldInit(); - try p.scratch.append(p.gpa, next); - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const inits = p.scratch.items[scratch_top..]; - switch (inits.len) { - 0 => unreachable, - 1 => return p.addNode(.{ - .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, - .main_token = lbrace, - .data = .{ - .lhs = inits[0], - .rhs = 0, - }, - }), - 2 => return p.addNode(.{ - .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, - .main_token = lbrace, - .data = .{ - .lhs = inits[0], - .rhs = inits[1], - }, - }), - else => { - const span = try p.listToSpan(inits); - return p.addNode(.{ - .tag = if (comma) .struct_init_dot_comma else .struct_init_dot, - .main_token = lbrace, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - }, - } - } - - while (true) { - if (p.eatToken(.r_brace)) |_| break; - const elem_init = try p.expectExpr(); - try p.scratch.append(p.gpa, elem_init); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_brace => { - p.tok_i += 1; - break; - }, - .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_initializer), - } - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const inits = p.scratch.items[scratch_top..]; - switch (inits.len) { - 0 => return p.addNode(.{ - .tag = .struct_init_dot_two, - .main_token = lbrace, - .data = .{ - .lhs = 0, - .rhs = 0, - }, - }), - 1 => return p.addNode(.{ - .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, - .main_token = lbrace, - .data = .{ - .lhs = inits[0], - .rhs = 0, - }, - }), - 2 => return p.addNode(.{ - .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, - .main_token = lbrace, - .data = .{ - .lhs = inits[0], - .rhs = inits[1], - }, - }), - else => { - const span = try p.listToSpan(inits); - return p.addNode(.{ - .tag = if (comma) .array_init_dot_comma else .array_init_dot, - .main_token = lbrace, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - }, - } - }, - else => return null_node, - }, - .keyword_error => switch (p.token_tags[p.tok_i + 1]) { - .l_brace => { - const error_token = p.tok_i; - p.tok_i += 2; - while (true) { - if (p.eatToken(.r_brace)) |_| break; - _ = try p.eatDocComments(); - _ = try p.expectToken(.identifier); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_brace => { - p.tok_i += 1; - break; - }, - .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_field), - } - } - return p.addNode(.{ - .tag = .error_set_decl, - .main_token = error_token, - .data = .{ - .lhs = undefined, - .rhs = p.tok_i - 1, // rbrace - }, - }); - }, - else => { - const main_token = p.nextToken(); - const period = p.eatToken(.period); - if (period == null) try p.warnExpected(.period); - const identifier = p.eatToken(.identifier); - if (identifier == null) try p.warnExpected(.identifier); - return p.addNode(.{ - .tag = .error_value, - .main_token = main_token, - .data = .{ - .lhs = period orelse 0, - .rhs = identifier orelse 0, - }, - }); - }, - }, - .l_paren => return p.addNode(.{ - .tag = .grouped_expression, - .main_token = p.nextToken(), - .data = .{ - .lhs = try p.expectExpr(), - .rhs = try p.expectToken(.r_paren), - }, - }), - else => return null_node, - } - } - - fn expectPrimaryTypeExpr(p: *Parser) !Node.Index { - const node = try p.parsePrimaryTypeExpr(); - if (node == 0) { - return p.fail(.expected_primary_type_expr); - } - return node; - } - - /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload - /// - /// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)? - fn parseForTypeExpr(p: *Parser) !Node.Index { - const for_token = p.eatToken(.keyword_for) orelse return null_node; - _ = try p.expectToken(.l_paren); - const array_expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - const found_payload = try p.parsePtrIndexPayload(); - if (found_payload == 0) try p.warn(.expected_loop_payload); - - const then_expr = try p.expectTypeExpr(); - _ = p.eatToken(.keyword_else) orelse { - return p.addNode(.{ - .tag = .for_simple, - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = then_expr, - }, - }); - }; - const else_expr = try p.expectTypeExpr(); - return p.addNode(.{ - .tag = .@"for", - .main_token = for_token, - .data = .{ - .lhs = array_expr, - .rhs = try p.addExtra(Node.If{ - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? - /// - /// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? - fn parseWhileTypeExpr(p: *Parser) !Node.Index { - const while_token = p.eatToken(.keyword_while) orelse return null_node; - _ = try p.expectToken(.l_paren); - const condition = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.parsePtrPayload(); - const cont_expr = try p.parseWhileContinueExpr(); - - const then_expr = try p.expectTypeExpr(); - _ = p.eatToken(.keyword_else) orelse { - if (cont_expr == 0) { - return p.addNode(.{ - .tag = .while_simple, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = then_expr, - }, - }); - } else { - return p.addNode(.{ - .tag = .while_cont, - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.WhileCont{ - .cont_expr = cont_expr, - .then_expr = then_expr, - }), - }, - }); - } - }; - _ = try p.parsePayload(); - const else_expr = try p.expectTypeExpr(); - return p.addNode(.{ - .tag = .@"while", - .main_token = while_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.While{ - .cont_expr = cont_expr, - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE - fn expectSwitchExpr(p: *Parser) !Node.Index { - const switch_token = p.assertToken(.keyword_switch); - _ = try p.expectToken(.l_paren); - const expr_node = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.expectToken(.l_brace); - const cases = try p.parseSwitchProngList(); - const trailing_comma = p.token_tags[p.tok_i - 1] == .comma; - _ = try p.expectToken(.r_brace); - - return p.addNode(.{ - .tag = if (trailing_comma) .switch_comma else .@"switch", - .main_token = switch_token, - .data = .{ - .lhs = expr_node, - .rhs = try p.addExtra(Node.SubRange{ - .start = cases.start, - .end = cases.end, - }), - }, - }); - } - - /// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN - /// - /// AsmOutput <- COLON AsmOutputList AsmInput? - /// - /// AsmInput <- COLON AsmInputList AsmClobbers? - /// - /// AsmClobbers <- COLON StringList - /// - /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? - /// - /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? - /// - /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? - fn expectAsmExpr(p: *Parser) !Node.Index { - const asm_token = p.assertToken(.keyword_asm); - _ = p.eatToken(.keyword_volatile); - _ = try p.expectToken(.l_paren); - const template = try p.expectExpr(); - - if (p.eatToken(.r_paren)) |rparen| { - return p.addNode(.{ - .tag = .asm_simple, - .main_token = asm_token, - .data = .{ - .lhs = template, - .rhs = rparen, - }, - }); - } - - _ = try p.expectToken(.colon); - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - while (true) { - const output_item = try p.parseAsmOutputItem(); - if (output_item == 0) break; - try p.scratch.append(p.gpa, output_item); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - // All possible delimiters. - .colon, .r_paren, .r_brace, .r_bracket => break, - // Likely just a missing comma; give error but continue parsing. - else => try p.warnExpected(.comma), - } - } - if (p.eatToken(.colon)) |_| { - while (true) { - const input_item = try p.parseAsmInputItem(); - if (input_item == 0) break; - try p.scratch.append(p.gpa, input_item); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - // All possible delimiters. - .colon, .r_paren, .r_brace, .r_bracket => break, - // Likely just a missing comma; give error but continue parsing. - else => try p.warnExpected(.comma), - } - } - if (p.eatToken(.colon)) |_| { - while (p.eatToken(.string_literal)) |_| { - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .colon, .r_paren, .r_brace, .r_bracket => break, - // Likely just a missing comma; give error but continue parsing. - else => try p.warnExpected(.comma), - } - } - } - } - const rparen = try p.expectToken(.r_paren); - const span = try p.listToSpan(p.scratch.items[scratch_top..]); - return p.addNode(.{ - .tag = .@"asm", - .main_token = asm_token, - .data = .{ - .lhs = template, - .rhs = try p.addExtra(Node.Asm{ - .items_start = span.start, - .items_end = span.end, - .rparen = rparen, - }), - }, - }); - } - - /// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN - fn parseAsmOutputItem(p: *Parser) !Node.Index { - _ = p.eatToken(.l_bracket) orelse return null_node; - const identifier = try p.expectToken(.identifier); - _ = try p.expectToken(.r_bracket); - _ = try p.expectToken(.string_literal); - _ = try p.expectToken(.l_paren); - const type_expr: Node.Index = blk: { - if (p.eatToken(.arrow)) |_| { - break :blk try p.expectTypeExpr(); - } else { - _ = try p.expectToken(.identifier); - break :blk null_node; - } - }; - const rparen = try p.expectToken(.r_paren); - return p.addNode(.{ - .tag = .asm_output, - .main_token = identifier, - .data = .{ - .lhs = type_expr, - .rhs = rparen, - }, - }); - } - - /// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN - fn parseAsmInputItem(p: *Parser) !Node.Index { - _ = p.eatToken(.l_bracket) orelse return null_node; - const identifier = try p.expectToken(.identifier); - _ = try p.expectToken(.r_bracket); - _ = try p.expectToken(.string_literal); - _ = try p.expectToken(.l_paren); - const expr = try p.expectExpr(); - const rparen = try p.expectToken(.r_paren); - return p.addNode(.{ - .tag = .asm_input, - .main_token = identifier, - .data = .{ - .lhs = expr, - .rhs = rparen, - }, - }); - } - - /// BreakLabel <- COLON IDENTIFIER - fn parseBreakLabel(p: *Parser) !TokenIndex { - _ = p.eatToken(.colon) orelse return @as(TokenIndex, 0); - return p.expectToken(.identifier); - } - - /// BlockLabel <- IDENTIFIER COLON - fn parseBlockLabel(p: *Parser) TokenIndex { - if (p.token_tags[p.tok_i] == .identifier and - p.token_tags[p.tok_i + 1] == .colon) - { - const identifier = p.tok_i; - p.tok_i += 2; - return identifier; - } - return null_node; - } - - /// FieldInit <- DOT IDENTIFIER EQUAL Expr - fn parseFieldInit(p: *Parser) !Node.Index { - if (p.token_tags[p.tok_i + 0] == .period and - p.token_tags[p.tok_i + 1] == .identifier and - p.token_tags[p.tok_i + 2] == .equal) - { - p.tok_i += 3; - return p.expectExpr(); - } else { - return null_node; - } - } - - fn expectFieldInit(p: *Parser) !Node.Index { - if (p.token_tags[p.tok_i] != .period or - p.token_tags[p.tok_i + 1] != .identifier or - p.token_tags[p.tok_i + 2] != .equal) - return p.fail(.expected_initializer); - - p.tok_i += 3; - return p.expectExpr(); - } - - /// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN - fn parseWhileContinueExpr(p: *Parser) !Node.Index { - _ = p.eatToken(.colon) orelse { - if (p.token_tags[p.tok_i] == .l_paren and - p.tokensOnSameLine(p.tok_i - 1, p.tok_i)) - return p.fail(.expected_continue_expr); - return null_node; - }; - _ = try p.expectToken(.l_paren); - const node = try p.parseAssignExpr(); - if (node == 0) return p.fail(.expected_expr_or_assignment); - _ = try p.expectToken(.r_paren); - return node; - } - - /// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN - fn parseLinkSection(p: *Parser) !Node.Index { - _ = p.eatToken(.keyword_linksection) orelse return null_node; - _ = try p.expectToken(.l_paren); - const expr_node = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - return expr_node; - } - - /// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN - fn parseCallconv(p: *Parser) !Node.Index { - _ = p.eatToken(.keyword_callconv) orelse return null_node; - _ = try p.expectToken(.l_paren); - const expr_node = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - return expr_node; - } - - /// AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN - fn parseAddrSpace(p: *Parser) !Node.Index { - _ = p.eatToken(.keyword_addrspace) orelse return null_node; - _ = try p.expectToken(.l_paren); - const expr_node = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - return expr_node; - } - - /// This function can return null nodes and then still return nodes afterwards, - /// such as in the case of anytype and `...`. Caller must look for rparen to find - /// out when there are no more param decls left. - /// - /// ParamDecl - /// <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType - /// / DOT3 - /// - /// ParamType - /// <- KEYWORD_anytype - /// / TypeExpr - fn expectParamDecl(p: *Parser) !Node.Index { - _ = try p.eatDocComments(); - switch (p.token_tags[p.tok_i]) { - .keyword_noalias, .keyword_comptime => p.tok_i += 1, - .ellipsis3 => { - p.tok_i += 1; - return null_node; - }, - else => {}, - } - if (p.token_tags[p.tok_i] == .identifier and - p.token_tags[p.tok_i + 1] == .colon) - { - p.tok_i += 2; - } - switch (p.token_tags[p.tok_i]) { - .keyword_anytype => { - p.tok_i += 1; - return null_node; - }, - else => return p.expectTypeExpr(), - } - } - - /// Payload <- PIPE IDENTIFIER PIPE - fn parsePayload(p: *Parser) !TokenIndex { - _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); - const identifier = try p.expectToken(.identifier); - _ = try p.expectToken(.pipe); - return identifier; - } - - /// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE - fn parsePtrPayload(p: *Parser) !TokenIndex { - _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); - _ = p.eatToken(.asterisk); - const identifier = try p.expectToken(.identifier); - _ = try p.expectToken(.pipe); - return identifier; - } - - /// Returns the first identifier token, if any. - /// - /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE - fn parsePtrIndexPayload(p: *Parser) !TokenIndex { - _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); - _ = p.eatToken(.asterisk); - const identifier = try p.expectToken(.identifier); - if (p.eatToken(.comma) != null) { - _ = try p.expectToken(.identifier); - } - _ = try p.expectToken(.pipe); - return identifier; - } - - /// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? AssignExpr - /// - /// SwitchCase - /// <- SwitchItem (COMMA SwitchItem)* COMMA? - /// / KEYWORD_else - fn parseSwitchProng(p: *Parser) !Node.Index { - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - const is_inline = p.eatToken(.keyword_inline) != null; - - if (p.eatToken(.keyword_else) == null) { - while (true) { - const item = try p.parseSwitchItem(); - if (item == 0) break; - try p.scratch.append(p.gpa, item); - if (p.eatToken(.comma) == null) break; - } - if (scratch_top == p.scratch.items.len) { - if (is_inline) p.tok_i -= 1; - return null_node; - } - } - const arrow_token = try p.expectToken(.equal_angle_bracket_right); - _ = try p.parsePtrIndexPayload(); - - const items = p.scratch.items[scratch_top..]; - switch (items.len) { - 0 => return p.addNode(.{ - .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, - .main_token = arrow_token, - .data = .{ - .lhs = 0, - .rhs = try p.expectAssignExpr(), - }, - }), - 1 => return p.addNode(.{ - .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, - .main_token = arrow_token, - .data = .{ - .lhs = items[0], - .rhs = try p.expectAssignExpr(), - }, - }), - else => return p.addNode(.{ - .tag = if (is_inline) .switch_case_inline else .switch_case, - .main_token = arrow_token, - .data = .{ - .lhs = try p.addExtra(try p.listToSpan(items)), - .rhs = try p.expectAssignExpr(), - }, - }), - } - } - - /// SwitchItem <- Expr (DOT3 Expr)? - fn parseSwitchItem(p: *Parser) !Node.Index { - const expr = try p.parseExpr(); - if (expr == 0) return null_node; - - if (p.eatToken(.ellipsis3)) |token| { - return p.addNode(.{ - .tag = .switch_range, - .main_token = token, - .data = .{ - .lhs = expr, - .rhs = try p.expectExpr(), - }, - }); - } - return expr; - } - - const PtrModifiers = struct { - align_node: Node.Index, - addrspace_node: Node.Index, - bit_range_start: Node.Index, - bit_range_end: Node.Index, - }; - - fn parsePtrModifiers(p: *Parser) !PtrModifiers { - var result: PtrModifiers = .{ - .align_node = 0, - .addrspace_node = 0, - .bit_range_start = 0, - .bit_range_end = 0, - }; - var saw_const = false; - var saw_volatile = false; - var saw_allowzero = false; - var saw_addrspace = false; - while (true) { - switch (p.token_tags[p.tok_i]) { - .keyword_align => { - if (result.align_node != 0) { - try p.warn(.extra_align_qualifier); - } - p.tok_i += 1; - _ = try p.expectToken(.l_paren); - result.align_node = try p.expectExpr(); - - if (p.eatToken(.colon)) |_| { - result.bit_range_start = try p.expectExpr(); - _ = try p.expectToken(.colon); - result.bit_range_end = try p.expectExpr(); - } - - _ = try p.expectToken(.r_paren); - }, - .keyword_const => { - if (saw_const) { - try p.warn(.extra_const_qualifier); - } - p.tok_i += 1; - saw_const = true; - }, - .keyword_volatile => { - if (saw_volatile) { - try p.warn(.extra_volatile_qualifier); - } - p.tok_i += 1; - saw_volatile = true; - }, - .keyword_allowzero => { - if (saw_allowzero) { - try p.warn(.extra_allowzero_qualifier); - } - p.tok_i += 1; - saw_allowzero = true; - }, - .keyword_addrspace => { - if (saw_addrspace) { - try p.warn(.extra_addrspace_qualifier); - } - result.addrspace_node = try p.parseAddrSpace(); - }, - else => return result, - } - } - } - - /// SuffixOp - /// <- LBRACKET Expr (DOT2 (Expr? (COLON Expr)?)?)? RBRACKET - /// / DOT IDENTIFIER - /// / DOTASTERISK - /// / DOTQUESTIONMARK - fn parseSuffixOp(p: *Parser, lhs: Node.Index) !Node.Index { - switch (p.token_tags[p.tok_i]) { - .l_bracket => { - const lbracket = p.nextToken(); - const index_expr = try p.expectExpr(); - - if (p.eatToken(.ellipsis2)) |_| { - const end_expr = try p.parseExpr(); - if (p.eatToken(.colon)) |_| { - const sentinel = try p.expectExpr(); - _ = try p.expectToken(.r_bracket); - return p.addNode(.{ - .tag = .slice_sentinel, - .main_token = lbracket, - .data = .{ - .lhs = lhs, - .rhs = try p.addExtra(Node.SliceSentinel{ - .start = index_expr, - .end = end_expr, - .sentinel = sentinel, - }), - }, - }); - } - _ = try p.expectToken(.r_bracket); - if (end_expr == 0) { - return p.addNode(.{ - .tag = .slice_open, - .main_token = lbracket, - .data = .{ - .lhs = lhs, - .rhs = index_expr, - }, - }); - } - return p.addNode(.{ - .tag = .slice, - .main_token = lbracket, - .data = .{ - .lhs = lhs, - .rhs = try p.addExtra(Node.Slice{ - .start = index_expr, - .end = end_expr, - }), - }, - }); - } - _ = try p.expectToken(.r_bracket); - return p.addNode(.{ - .tag = .array_access, - .main_token = lbracket, - .data = .{ - .lhs = lhs, - .rhs = index_expr, - }, - }); - }, - .period_asterisk => return p.addNode(.{ - .tag = .deref, - .main_token = p.nextToken(), - .data = .{ - .lhs = lhs, - .rhs = undefined, - }, - }), - .invalid_periodasterisks => { - try p.warn(.asterisk_after_ptr_deref); - return p.addNode(.{ - .tag = .deref, - .main_token = p.nextToken(), - .data = .{ - .lhs = lhs, - .rhs = undefined, - }, - }); - }, - .period => switch (p.token_tags[p.tok_i + 1]) { - .identifier => return p.addNode(.{ - .tag = .field_access, - .main_token = p.nextToken(), - .data = .{ - .lhs = lhs, - .rhs = p.nextToken(), - }, - }), - .question_mark => return p.addNode(.{ - .tag = .unwrap_optional, - .main_token = p.nextToken(), - .data = .{ - .lhs = lhs, - .rhs = p.nextToken(), - }, - }), - .l_brace => { - // this a misplaced `.{`, handle the error somewhere else - return null_node; - }, - else => { - p.tok_i += 1; - try p.warn(.expected_suffix_op); - return null_node; - }, - }, - else => return null_node, - } - } - - /// Caller must have already verified the first token. - /// - /// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE - /// - /// ContainerDeclType - /// <- KEYWORD_struct (LPAREN Expr RPAREN)? - /// / KEYWORD_opaque - /// / KEYWORD_enum (LPAREN Expr RPAREN)? - /// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? - fn parseContainerDeclAuto(p: *Parser) !Node.Index { - const main_token = p.nextToken(); - const arg_expr = switch (p.token_tags[main_token]) { - .keyword_opaque => null_node, - .keyword_struct, .keyword_enum => blk: { - if (p.eatToken(.l_paren)) |_| { - const expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - break :blk expr; - } else { - break :blk null_node; - } - }, - .keyword_union => blk: { - if (p.eatToken(.l_paren)) |_| { - if (p.eatToken(.keyword_enum)) |_| { - if (p.eatToken(.l_paren)) |_| { - const enum_tag_expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.expectToken(.r_paren); - - _ = try p.expectToken(.l_brace); - const members = try p.parseContainerMembers(); - const members_span = try members.toSpan(p); - _ = try p.expectToken(.r_brace); - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .tagged_union_enum_tag_trailing, - false => .tagged_union_enum_tag, - }, - .main_token = main_token, - .data = .{ - .lhs = enum_tag_expr, - .rhs = try p.addExtra(members_span), - }, - }); - } else { - _ = try p.expectToken(.r_paren); - - _ = try p.expectToken(.l_brace); - const members = try p.parseContainerMembers(); - _ = try p.expectToken(.r_brace); - if (members.len <= 2) { - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .tagged_union_two_trailing, - false => .tagged_union_two, - }, - .main_token = main_token, - .data = .{ - .lhs = members.lhs, - .rhs = members.rhs, - }, - }); - } else { - const span = try members.toSpan(p); - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .tagged_union_trailing, - false => .tagged_union, - }, - .main_token = main_token, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - } - } - } else { - const expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - break :blk expr; - } - } else { - break :blk null_node; - } - }, - else => { - p.tok_i -= 1; - return p.fail(.expected_container); - }, - }; - _ = try p.expectToken(.l_brace); - const members = try p.parseContainerMembers(); - _ = try p.expectToken(.r_brace); - if (arg_expr == 0) { - if (members.len <= 2) { - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .container_decl_two_trailing, - false => .container_decl_two, - }, - .main_token = main_token, - .data = .{ - .lhs = members.lhs, - .rhs = members.rhs, - }, - }); - } else { - const span = try members.toSpan(p); - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .container_decl_trailing, - false => .container_decl, - }, - .main_token = main_token, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - } - } else { - const span = try members.toSpan(p); - return p.addNode(.{ - .tag = switch (members.trailing) { - true => .container_decl_arg_trailing, - false => .container_decl_arg, - }, - .main_token = main_token, - .data = .{ - .lhs = arg_expr, - .rhs = try p.addExtra(Node.SubRange{ - .start = span.start, - .end = span.end, - }), - }, - }); - } - } - - /// Give a helpful error message for those transitioning from - /// C's 'struct Foo {};' to Zig's 'const Foo = struct {};'. - fn parseCStyleContainer(p: *Parser) Error!bool { - const main_token = p.tok_i; - switch (p.token_tags[p.tok_i]) { - .keyword_enum, .keyword_union, .keyword_struct => {}, - else => return false, - } - const identifier = p.tok_i + 1; - if (p.token_tags[identifier] != .identifier) return false; - p.tok_i += 2; - - try p.warnMsg(.{ - .tag = .c_style_container, - .token = identifier, - .extra = .{ .expected_tag = p.token_tags[main_token] }, - }); - try p.warnMsg(.{ - .tag = .zig_style_container, - .is_note = true, - .token = identifier, - .extra = .{ .expected_tag = p.token_tags[main_token] }, - }); - - _ = try p.expectToken(.l_brace); - _ = try p.parseContainerMembers(); - _ = try p.expectToken(.r_brace); - try p.expectSemicolon(.expected_semi_after_decl, true); - return true; - } - - /// Holds temporary data until we are ready to construct the full ContainerDecl AST node. - /// - /// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN - fn parseByteAlign(p: *Parser) !Node.Index { - _ = p.eatToken(.keyword_align) orelse return null_node; - _ = try p.expectToken(.l_paren); - const expr = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - return expr; - } - - /// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? - fn parseSwitchProngList(p: *Parser) !Node.SubRange { - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - while (true) { - const item = try parseSwitchProng(p); - if (item == 0) break; - - try p.scratch.append(p.gpa, item); - - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - // All possible delimiters. - .colon, .r_paren, .r_brace, .r_bracket => break, - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_switch_prong), - } - } - return p.listToSpan(p.scratch.items[scratch_top..]); - } - - /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? - fn parseParamDeclList(p: *Parser) !SmallSpan { - _ = try p.expectToken(.l_paren); - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - var varargs: union(enum) { none, seen, nonfinal: TokenIndex } = .none; - while (true) { - if (p.eatToken(.r_paren)) |_| break; - if (varargs == .seen) varargs = .{ .nonfinal = p.tok_i }; - const param = try p.expectParamDecl(); - if (param != 0) { - try p.scratch.append(p.gpa, param); - } else if (p.token_tags[p.tok_i - 1] == .ellipsis3) { - if (varargs == .none) varargs = .seen; - } - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_paren => { - p.tok_i += 1; - break; - }, - .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_param), - } - } - if (varargs == .nonfinal) { - try p.warnMsg(.{ .tag = .varargs_nonfinal, .token = varargs.nonfinal }); - } - const params = p.scratch.items[scratch_top..]; - return switch (params.len) { - 0 => SmallSpan{ .zero_or_one = 0 }, - 1 => SmallSpan{ .zero_or_one = params[0] }, - else => SmallSpan{ .multi = try p.listToSpan(params) }, - }; - } - - /// FnCallArguments <- LPAREN ExprList RPAREN - /// - /// ExprList <- (Expr COMMA)* Expr? - fn parseBuiltinCall(p: *Parser) !Node.Index { - const builtin_token = p.assertToken(.builtin); - if (p.token_tags[p.nextToken()] != .l_paren) { - p.tok_i -= 1; - try p.warn(.expected_param_list); - // Pretend this was an identifier so we can continue parsing. - return p.addNode(.{ - .tag = .identifier, - .main_token = builtin_token, - .data = .{ - .lhs = undefined, - .rhs = undefined, - }, - }); - } - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - while (true) { - if (p.eatToken(.r_paren)) |_| break; - const param = try p.expectExpr(); - try p.scratch.append(p.gpa, param); - switch (p.token_tags[p.tok_i]) { - .comma => p.tok_i += 1, - .r_paren => { - p.tok_i += 1; - break; - }, - // Likely just a missing comma; give error but continue parsing. - else => try p.warn(.expected_comma_after_arg), - } - } - const comma = (p.token_tags[p.tok_i - 2] == .comma); - const params = p.scratch.items[scratch_top..]; - switch (params.len) { - 0 => return p.addNode(.{ - .tag = .builtin_call_two, - .main_token = builtin_token, - .data = .{ - .lhs = 0, - .rhs = 0, - }, - }), - 1 => return p.addNode(.{ - .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, - .main_token = builtin_token, - .data = .{ - .lhs = params[0], - .rhs = 0, - }, - }), - 2 => return p.addNode(.{ - .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, - .main_token = builtin_token, - .data = .{ - .lhs = params[0], - .rhs = params[1], - }, - }), - else => { - const span = try p.listToSpan(params); - return p.addNode(.{ - .tag = if (comma) .builtin_call_comma else .builtin_call, - .main_token = builtin_token, - .data = .{ - .lhs = span.start, - .rhs = span.end, - }, - }); - }, - } - } - - /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? - fn parseIf(p: *Parser, comptime bodyParseFn: fn (p: *Parser) Error!Node.Index) !Node.Index { - const if_token = p.eatToken(.keyword_if) orelse return null_node; - _ = try p.expectToken(.l_paren); - const condition = try p.expectExpr(); - _ = try p.expectToken(.r_paren); - _ = try p.parsePtrPayload(); - - const then_expr = try bodyParseFn(p); - assert(then_expr != 0); - - _ = p.eatToken(.keyword_else) orelse return p.addNode(.{ - .tag = .if_simple, - .main_token = if_token, - .data = .{ - .lhs = condition, - .rhs = then_expr, - }, - }); - _ = try p.parsePayload(); - const else_expr = try bodyParseFn(p); - assert(then_expr != 0); - - return p.addNode(.{ - .tag = .@"if", - .main_token = if_token, - .data = .{ - .lhs = condition, - .rhs = try p.addExtra(Node.If{ - .then_expr = then_expr, - .else_expr = else_expr, - }), - }, - }); - } - - /// Skips over doc comment tokens. Returns the first one, if any. - fn eatDocComments(p: *Parser) !?TokenIndex { - if (p.eatToken(.doc_comment)) |tok| { - var first_line = tok; - if (tok > 0 and tokensOnSameLine(p, tok - 1, tok)) { - try p.warnMsg(.{ - .tag = .same_line_doc_comment, - .token = tok, - }); - first_line = p.eatToken(.doc_comment) orelse return null; - } - while (p.eatToken(.doc_comment)) |_| {} - return first_line; - } - return null; - } - - fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { - return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null; - } - - fn eatToken(p: *Parser, tag: Token.Tag) ?TokenIndex { - return if (p.token_tags[p.tok_i] == tag) p.nextToken() else null; - } - - fn assertToken(p: *Parser, tag: Token.Tag) TokenIndex { - const token = p.nextToken(); - assert(p.token_tags[token] == tag); - return token; - } - - fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex { - if (p.token_tags[p.tok_i] != tag) { - return p.failMsg(.{ - .tag = .expected_token, - .token = p.tok_i, - .extra = .{ .expected_tag = tag }, - }); - } - return p.nextToken(); - } - - fn expectSemicolon(p: *Parser, error_tag: AstError.Tag, recoverable: bool) Error!void { - if (p.token_tags[p.tok_i] == .semicolon) { - _ = p.nextToken(); - return; - } - try p.warn(error_tag); - if (!recoverable) return error.ParseError; - } - - fn nextToken(p: *Parser) TokenIndex { - const result = p.tok_i; - p.tok_i += 1; - return result; - } -}; - -test { - _ = @import("parser_test.zig"); -} diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 49b0715695..3c44322ccc 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -6073,7 +6073,7 @@ var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: [:0]const u8, allocator: mem.Allocator, anything_changed: *bool) ![]u8 { const stderr = io.getStdErr().writer(); - var tree = try std.zig.parse(allocator, source); + var tree = try std.zig.Ast.parse(allocator, source, .zig); defer tree.deinit(allocator); for (tree.errors) |parse_error| { @@ -6124,7 +6124,7 @@ fn testCanonical(source: [:0]const u8) !void { const Error = std.zig.Ast.Error.Tag; fn testError(source: [:0]const u8, expected_errors: []const Error) !void { - var tree = try std.zig.parse(std.testing.allocator, source); + var tree = try std.zig.Ast.parse(std.testing.allocator, source, .zig); defer tree.deinit(std.testing.allocator); std.testing.expectEqual(expected_errors.len, tree.errors.len) catch |err| { diff --git a/lib/std/zig/perf_test.zig b/lib/std/zig/perf_test.zig index d3fc90eaea..58f7a67694 100644 --- a/lib/std/zig/perf_test.zig +++ b/lib/std/zig/perf_test.zig @@ -1,7 +1,6 @@ const std = @import("std"); const mem = std.mem; const Tokenizer = std.zig.Tokenizer; -const Parser = std.zig.Parser; const io = std.io; const fmtIntSizeBin = std.fmt.fmtIntSizeBin; @@ -34,6 +33,6 @@ pub fn main() !void { fn testOnce() usize { var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); var allocator = fixed_buf_alloc.allocator(); - _ = std.zig.parse(allocator, source) catch @panic("parse failure"); + _ = std.zig.Ast.parse(allocator, source, .zig) catch @panic("parse failure"); return fixed_buf_alloc.end_index; } diff --git a/src/Module.zig b/src/Module.zig index b395c0a950..3bb15e78c3 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -2057,7 +2057,7 @@ pub const File = struct { if (file.tree_loaded) return &file.tree; const source = try file.getSource(gpa); - file.tree = try std.zig.parse(gpa, source.bytes); + file.tree = try Ast.parse(gpa, source.bytes, .zig); file.tree_loaded = true; return &file.tree; } @@ -3662,7 +3662,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void { file.source = source; file.source_loaded = true; - file.tree = try std.zig.parse(gpa, source); + file.tree = try Ast.parse(gpa, source, .zig); defer if (!file.tree_loaded) file.tree.deinit(gpa); if (file.tree.errors.len != 0) { @@ -3977,7 +3977,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { else => |e| return e, } - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; assert(file.tree.errors.len == 0); // builtin.zig must parse diff --git a/src/main.zig b/src/main.zig index 72e7e094e6..06c36bad87 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4361,7 +4361,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void }; defer gpa.free(source_code); - var tree = std.zig.parse(gpa, source_code) catch |err| { + var tree = Ast.parse(gpa, source_code, .zig) catch |err| { fatal("error parsing stdin: {}", .{err}); }; defer tree.deinit(gpa); @@ -4566,7 +4566,7 @@ fn fmtPathFile( // Add to set after no longer possible to get error.IsDir. if (try fmt.seen.fetchPut(stat.inode, {})) |_| return; - var tree = try std.zig.parse(fmt.gpa, source_code); + var tree = try Ast.parse(fmt.gpa, source_code, .zig); defer tree.deinit(fmt.gpa); try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree.errors, tree, file_path, fmt.color); @@ -5312,7 +5312,7 @@ pub fn cmdAstCheck( file.pkg = try Package.create(gpa, "root", null, file.sub_file_path); defer file.pkg.destroy(gpa); - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; defer file.tree.deinit(gpa); @@ -5438,7 +5438,7 @@ pub fn cmdChangelist( file.source = source; file.source_loaded = true; - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; defer file.tree.deinit(gpa); @@ -5476,7 +5476,7 @@ pub fn cmdChangelist( if (new_amt != new_stat.size) return error.UnexpectedEndOfFile; - var new_tree = try std.zig.parse(gpa, new_source); + var new_tree = try Ast.parse(gpa, new_source, .zig); defer new_tree.deinit(gpa); try printErrsMsgToStdErr(gpa, arena, new_tree.errors, new_tree, new_source_file, .auto); -- cgit v1.2.3 From 81c27c74bc8ccc8087b75c5d4eb1b350ad907cd0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Feb 2023 23:45:23 -0700 Subject: use build.zig.zon instead of build.zig.ini for the manifest file * improve error message when build manifest file is missing * update std.zig.Ast to support ZON * Compilation.AllErrors.Message: make the notes field a const slice * move build manifest parsing logic into src/Manifest.zig and add more checks, and make the checks integrate into the standard error reporting code so that reported errors look sexy closes #14290 --- lib/std/Build.zig | 4 +- lib/std/array_hash_map.zig | 3 +- lib/std/zig/Ast.zig | 4 + lib/std/zig/Parse.zig | 13 +- src/Compilation.zig | 2 +- src/Manifest.zig | 499 +++++++++++++++++++++++++++++++++++++++++++++ src/Package.zig | 347 ++++++++++++------------------- src/main.zig | 17 +- 8 files changed, 665 insertions(+), 224 deletions(-) create mode 100644 src/Manifest.zig (limited to 'src/main.zig') diff --git a/lib/std/Build.zig b/lib/std/Build.zig index d695637fc3..6846007443 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1496,8 +1496,8 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { } } - const full_path = b.pathFromRoot("build.zig.ini"); - std.debug.print("no dependency named '{s}' in '{s}'\n", .{ name, full_path }); + const full_path = b.pathFromRoot("build.zig.zon"); + std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path }); std.process.exit(1); } diff --git a/lib/std/array_hash_map.zig b/lib/std/array_hash_map.zig index cf04a54116..57821d1b51 100644 --- a/lib/std/array_hash_map.zig +++ b/lib/std/array_hash_map.zig @@ -1145,7 +1145,8 @@ pub fn ArrayHashMapUnmanaged( } /// Create a copy of the hash map which can be modified separately. - /// The copy uses the same context and allocator as this instance. + /// The copy uses the same context as this instance, but is allocated + /// with the provided allocator. pub fn clone(self: Self, allocator: Allocator) !Self { if (@sizeOf(ByIndexContext) != 0) @compileError("Cannot infer context " ++ @typeName(Context) ++ ", call cloneContext instead."); diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index a9a02606eb..80dda052ab 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -1,4 +1,8 @@ //! Abstract Syntax Tree for Zig source code. +//! For Zig syntax, the root node is at nodes[0] and contains the list of +//! sub-nodes. +//! For Zon syntax, the root node is at nodes[0] and contains lhs as the node +//! index of the main expression. /// Reference to externally-owned data. source: [:0]const u8, diff --git a/lib/std/zig/Parse.zig b/lib/std/zig/Parse.zig index f599a08f55..d498366b34 100644 --- a/lib/std/zig/Parse.zig +++ b/lib/std/zig/Parse.zig @@ -181,17 +181,26 @@ pub fn parseRoot(p: *Parse) !void { /// TODO: set a flag in Parse struct, and honor that flag /// by emitting compilation errors when non-zon nodes are encountered. pub fn parseZon(p: *Parse) !void { - const node_index = p.parseExpr() catch |err| switch (err) { + // We must use index 0 so that 0 can be used as null elsewhere. + p.nodes.appendAssumeCapacity(.{ + .tag = .root, + .main_token = 0, + .data = undefined, + }); + const node_index = p.expectExpr() catch |err| switch (err) { error.ParseError => { assert(p.errors.items.len > 0); return; }, else => |e| return e, }; - assert(node_index == 0); if (p.token_tags[p.tok_i] != .eof) { try p.warnExpected(.eof); } + p.nodes.items(.data)[0] = .{ + .lhs = node_index, + .rhs = undefined, + }; } /// ContainerMembers <- ContainerDeclarations (ContainerField COMMA)* (ContainerField / ContainerDeclarations) diff --git a/src/Compilation.zig b/src/Compilation.zig index 7d42d3b610..e09b8f18ab 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -385,7 +385,7 @@ pub const AllErrors = struct { count: u32 = 1, /// Does not include the trailing newline. source_line: ?[]const u8, - notes: []Message = &.{}, + notes: []const Message = &.{}, reference_trace: []Message = &.{}, /// Splits the error message up into lines to properly indent them diff --git a/src/Manifest.zig b/src/Manifest.zig new file mode 100644 index 0000000000..c3f77aec98 --- /dev/null +++ b/src/Manifest.zig @@ -0,0 +1,499 @@ +pub const basename = "build.zig.zon"; +pub const Hash = std.crypto.hash.sha2.Sha256; + +pub const Dependency = struct { + url: []const u8, + url_tok: Ast.TokenIndex, + hash: ?[]const u8, + hash_tok: Ast.TokenIndex, +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@enumToInt(multihash_function) < 127); + assert(Hash.digest_length < 127); +} +pub const multihash_len = 1 + 1 + Hash.digest_length; + +name: []const u8, +version: std.SemanticVersion, +dependencies: std.StringArrayHashMapUnmanaged(Dependency), + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { + const node_tags = ast.nodes.items(.tag); + const node_datas = ast.nodes.items(.data); + assert(node_tags[0] == .root); + const main_node_index = node_datas[0].lhs; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast, + .arena = arena_instance.allocator(), + .errors = .{}, + + .name = undefined, + .version = undefined, + .dependencies = .{}, + .buf = .{}, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + + p.parseRoot(main_node_index) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .version = p.version, + .dependencies = try p.dependencies.clone(p.arena), + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +const hex_charset = "0123456789abcdef"; + +pub fn hex64(x: u64) [16]u8 { + var result: [16]u8 = undefined; + var i: usize = 0; + while (i < 8) : (i += 1) { + const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); + result[i * 2 + 0] = hex_charset[byte >> 4]; + result[i * 2 + 1] = hex_charset[byte & 15]; + } + return result; +} + +test hex64 { + const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; + try std.testing.expectEqualStrings("[00efcdab78563412]", s); +} + +pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { + var result: [multihash_len * 2]u8 = undefined; + + result[0] = hex_charset[@enumToInt(multihash_function) >> 4]; + result[1] = hex_charset[@enumToInt(multihash_function) & 15]; + + result[2] = hex_charset[Hash.digest_length >> 4]; + result[3] = hex_charset[Hash.digest_length & 15]; + + for (digest) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +const Parse = struct { + gpa: Allocator, + ast: std.zig.Ast, + arena: Allocator, + buf: std.ArrayListUnmanaged(u8), + errors: std.ArrayListUnmanaged(ErrorMessage), + + name: []const u8, + version: std.SemanticVersion, + dependencies: std.StringArrayHashMapUnmanaged(Dependency), + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const main_token = main_tokens[node]; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseString(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "version")) { + const version_text = try parseString(p, field_init); + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .url = undefined, + .url_tok = undefined, + .hash = null, + .hash_tok = undefined, + }; + var have_url = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + dep.url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.url_tok = main_tokens[field_init]; + have_url = true; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = main_tokens[field_init]; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_url) { + try appendError(p, main_tokens[node], "dependency is missing 'url' field", .{}); + } + + return dep; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const node_tags = ast.nodes.items(.tag); + const main_tokens = ast.nodes.items(.main_token); + if (node_tags[node] != .string_literal) { + return fail(p, main_tokens[node], "expected string literal", .{}); + } + const str_lit_token = main_tokens[node]; + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const tok = main_tokens[node]; + const h = try parseString(p, node); + + if (h.len >= 2) { + const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { + return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ + @errorName(err), + }); + }; + if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) { + return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); + } + } + + const hex_multihash_len = 2 * Manifest.multihash_len; + if (h.len != hex_multihash_len) { + return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ + hex_multihash_len, h.len, + }); + } + + return h; + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + const token_tags = ast.tokens.items(.tag); + assert(token_tags[token] == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayListUnmanaged(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + var buf_managed = buf.toManaged(p.gpa); + const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); + buf.* = buf_managed.moveToUnmanaged(); + switch (try result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "unicode escape does not correspond to a valid codepoint", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +const Manifest = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + \\ }, + \\ }, + \\} + ; + + var ast = try std.zig.Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].url, + ); + try testing.expectEqualStrings( + "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); +} diff --git a/src/Package.zig b/src/Package.zig index 35b1ff5056..401eef2121 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -6,8 +6,8 @@ const fs = std.fs; const mem = std.mem; const Allocator = mem.Allocator; const assert = std.debug.assert; -const Hash = std.crypto.hash.sha2.Sha256; const log = std.log.scoped(.package); +const main = @import("main.zig"); const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); @@ -15,6 +15,7 @@ const ThreadPool = @import("ThreadPool.zig"); const WaitGroup = @import("WaitGroup.zig"); const Cache = @import("Cache.zig"); const build_options = @import("build_options"); +const Manifest = @import("Manifest.zig"); pub const Table = std.StringHashMapUnmanaged(*Package); @@ -141,10 +142,10 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, child: *Package) !void { } pub const build_zig_basename = "build.zig"; -pub const ini_basename = build_zig_basename ++ ".ini"; pub fn fetchAndAddDependencies( pkg: *Package, + arena: Allocator, thread_pool: *ThreadPool, http_client: *std.http.Client, directory: Compilation.Directory, @@ -153,89 +154,77 @@ pub fn fetchAndAddDependencies( dependencies_source: *std.ArrayList(u8), build_roots_source: *std.ArrayList(u8), name_prefix: []const u8, + color: main.Color, ) !void { const max_bytes = 10 * 1024 * 1024; const gpa = thread_pool.allocator; - const build_zig_ini = directory.handle.readFileAlloc(gpa, ini_basename, max_bytes) catch |err| switch (err) { + const build_zig_zon_bytes = directory.handle.readFileAllocOptions( + arena, + Manifest.basename, + max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { error.FileNotFound => { // Handle the same as no dependencies. return; }, else => |e| return e, }; - defer gpa.free(build_zig_ini); - const ini: std.Ini = .{ .bytes = build_zig_ini }; - var any_error = false; - var it = ini.iterateSection("\n[dependency]\n"); - while (it.next()) |dep| { - var line_it = mem.split(u8, dep, "\n"); - var opt_name: ?[]const u8 = null; - var opt_url: ?[]const u8 = null; - var expected_hash: ?[]const u8 = null; - while (line_it.next()) |kv| { - const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; - const key = kv[0..eq_pos]; - const value = kv[eq_pos + 1 ..]; - if (mem.eql(u8, key, "name")) { - opt_name = value; - } else if (mem.eql(u8, key, "url")) { - opt_url = value; - } else if (mem.eql(u8, key, "hash")) { - expected_hash = value; - } else { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - key, - }); - } - } + var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon); + defer ast.deinit(gpa); - const name = opt_name orelse { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - }); - any_error = true; - continue; - }; + if (ast.errors.len > 0) { + const file_path = try directory.join(arena, &.{Manifest.basename}); + try main.printErrsMsgToStdErr(gpa, arena, ast, file_path, color); + return error.PackageFetchFailed; + } - const url = opt_url orelse { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - }); - any_error = true; - continue; + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + const ttyconf: std.debug.TTY.Config = switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, }; + const file_path = try directory.join(arena, &.{Manifest.basename}); + for (manifest.errors) |msg| { + Report.renderErrorMessage(ast, file_path, ttyconf, msg, &.{}); + } + return error.PackageFetchFailed; + } + + const report: Report = .{ + .ast = &ast, + .directory = directory, + .color = color, + .arena = arena, + }; + + var any_error = false; + const deps_list = manifest.dependencies.values(); + for (manifest.dependencies.keys()) |name, i| { + const dep = deps_list[i]; - const sub_prefix = try std.fmt.allocPrint(gpa, "{s}{s}.", .{ name_prefix, name }); - defer gpa.free(sub_prefix); + const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name }); const fqn = sub_prefix[0 .. sub_prefix.len - 1]; const sub_pkg = try fetchAndUnpack( thread_pool, http_client, global_cache_directory, - url, - expected_hash, - ini, - directory, + dep, + report, build_roots_source, fqn, ); try pkg.fetchAndAddDependencies( + arena, thread_pool, http_client, sub_pkg.root_src_directory, @@ -244,6 +233,7 @@ pub fn fetchAndAddDependencies( dependencies_source, build_roots_source, sub_prefix, + color, ); try addAndAdopt(pkg, gpa, sub_pkg); @@ -253,7 +243,7 @@ pub fn fetchAndAddDependencies( }); } - if (any_error) return error.InvalidBuildZigIniFile; + if (any_error) return error.InvalidBuildManifestFile; } pub fn createFilePkg( @@ -264,7 +254,7 @@ pub fn createFilePkg( contents: []const u8, ) !*Package { const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int); { var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); defer tmp_dir.close(); @@ -282,14 +272,73 @@ pub fn createFilePkg( return createWithDir(gpa, name, cache_directory, o_dir_sub_path, basename); } +const Report = struct { + ast: *const std.zig.Ast, + directory: Compilation.Directory, + color: main.Color, + arena: Allocator, + + fn fail( + report: Report, + tok: std.zig.Ast.TokenIndex, + comptime fmt_string: []const u8, + fmt_args: anytype, + ) error{ PackageFetchFailed, OutOfMemory } { + return failWithNotes(report, &.{}, tok, fmt_string, fmt_args); + } + + fn failWithNotes( + report: Report, + notes: []const Compilation.AllErrors.Message, + tok: std.zig.Ast.TokenIndex, + comptime fmt_string: []const u8, + fmt_args: anytype, + ) error{ PackageFetchFailed, OutOfMemory } { + const ttyconf: std.debug.TTY.Config = switch (report.color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + const file_path = try report.directory.join(report.arena, &.{Manifest.basename}); + renderErrorMessage(report.ast.*, file_path, ttyconf, .{ + .tok = tok, + .off = 0, + .msg = try std.fmt.allocPrint(report.arena, fmt_string, fmt_args), + }, notes); + return error.PackageFetchFailed; + } + + fn renderErrorMessage( + ast: std.zig.Ast, + file_path: []const u8, + ttyconf: std.debug.TTY.Config, + msg: Manifest.ErrorMessage, + notes: []const Compilation.AllErrors.Message, + ) void { + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, msg.tok); + Compilation.AllErrors.Message.renderToStdErr(.{ .src = .{ + .msg = msg.msg, + .src_path = file_path, + .line = @intCast(u32, start_loc.line), + .column = @intCast(u32, start_loc.column), + .span = .{ + .start = token_starts[msg.tok], + .end = @intCast(u32, token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), + .main = token_starts[msg.tok] + msg.off, + }, + .source_line = ast.source[start_loc.line_start..start_loc.line_end], + .notes = notes, + } }, ttyconf); + } +}; + fn fetchAndUnpack( thread_pool: *ThreadPool, http_client: *std.http.Client, global_cache_directory: Compilation.Directory, - url: []const u8, - expected_hash: ?[]const u8, - ini: std.Ini, - comp_directory: Compilation.Directory, + dep: Manifest.Dependency, + report: Report, build_roots_source: *std.ArrayList(u8), fqn: []const u8, ) !*Package { @@ -298,37 +347,8 @@ fn fetchAndUnpack( // Check if the expected_hash is already present in the global package // cache, and thereby avoid both fetching and unpacking. - if (expected_hash) |h| cached: { - const hex_multihash_len = 2 * multihash_len; - if (h.len >= 2) { - const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { - return reportError( - ini, - comp_directory, - h.ptr, - "invalid multihash value: unable to parse hash function: {s}", - .{@errorName(err)}, - ); - }; - if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) { - return reportError( - ini, - comp_directory, - h.ptr, - "unsupported hash function: only sha2-256 is supported", - .{}, - ); - } - } - if (h.len != hex_multihash_len) { - return reportError( - ini, - comp_directory, - h.ptr, - "wrong hash size. expected: {d}, found: {d}", - .{ hex_multihash_len, h.len }, - ); - } + if (dep.hash) |h| cached: { + const hex_multihash_len = 2 * Manifest.multihash_len; const hex_digest = h[0..hex_multihash_len]; const pkg_dir_sub_path = "p" ++ s ++ hex_digest; var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { @@ -366,10 +386,10 @@ fn fetchAndUnpack( return ptr; } - const uri = try std.Uri.parse(url); + const uri = try std.Uri.parse(dep.url); const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); const actual_hash = a: { var tmp_directory: Compilation.Directory = d: { @@ -398,13 +418,9 @@ fn fetchAndUnpack( // by default, so the same logic applies for buffering the reader as for gzip. try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz); } else { - return reportError( - ini, - comp_directory, - uri.path.ptr, - "unknown file extension for path '{s}'", - .{uri.path}, - ); + return report.fail(dep.url_tok, "unknown file extension for path '{s}'", .{ + uri.path, + }); } // TODO: delete files not included in the package prior to computing the package hash. @@ -415,28 +431,21 @@ fn fetchAndUnpack( break :a try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); }; - const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); + const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash); try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path); - const actual_hex = hexDigest(actual_hash); - if (expected_hash) |h| { + const actual_hex = Manifest.hexDigest(actual_hash); + if (dep.hash) |h| { if (!mem.eql(u8, h, &actual_hex)) { - return reportError( - ini, - comp_directory, - h.ptr, - "hash mismatch: expected: {s}, found: {s}", - .{ h, actual_hex }, - ); + return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ + h, actual_hex, + }); } } else { - return reportError( - ini, - comp_directory, - url.ptr, - "url field is missing corresponding hash field: hash={s}", - .{&actual_hex}, - ); + const notes: [1]Compilation.AllErrors.Message = .{.{ .plain = .{ + .msg = try std.fmt.allocPrint(report.arena, "expected .hash = \"{s}\",", .{&actual_hex}), + } }}; + return report.failWithNotes(¬es, dep.url_tok, "url field is missing corresponding hash field", .{}); } const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); @@ -471,29 +480,9 @@ fn unpackTarball( }); } -fn reportError( - ini: std.Ini, - comp_directory: Compilation.Directory, - src_ptr: [*]const u8, - comptime fmt_string: []const u8, - fmt_args: anytype, -) error{PackageFetchFailed} { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(src_ptr) - @ptrToInt(ini.bytes.ptr)); - if (comp_directory.path) |p| { - std.debug.print("{s}{c}{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ - p, fs.path.sep, ini_basename, loc.line + 1, loc.column + 1, - } ++ fmt_args); - } else { - std.debug.print("{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ - ini_basename, loc.line + 1, loc.column + 1, - } ++ fmt_args); - } - return error.PackageFetchFailed; -} - const HashedFile = struct { path: []const u8, - hash: [Hash.digest_length]u8, + hash: [Manifest.Hash.digest_length]u8, failure: Error!void, const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError; @@ -507,7 +496,7 @@ const HashedFile = struct { fn computePackageHash( thread_pool: *ThreadPool, pkg_dir: fs.IterableDir, -) ![Hash.digest_length]u8 { +) ![Manifest.Hash.digest_length]u8 { const gpa = thread_pool.allocator; // We'll use an arena allocator for the path name strings since they all @@ -550,7 +539,7 @@ fn computePackageHash( std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); - var hasher = Hash.init(.{}); + var hasher = Manifest.Hash.init(.{}); var any_failures = false; for (all_files.items) |hashed_file| { hashed_file.failure catch |err| { @@ -571,7 +560,7 @@ fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { var buf: [8000]u8 = undefined; var file = try dir.openFile(hashed_file.path, .{}); - var hasher = Hash.init(.{}); + var hasher = Manifest.Hash.init(.{}); hasher.update(hashed_file.path); hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) }); while (true) { @@ -595,52 +584,6 @@ fn isExecutable(file: fs.File) !bool { } } -const hex_charset = "0123456789abcdef"; - -fn hex64(x: u64) [16]u8 { - var result: [16]u8 = undefined; - var i: usize = 0; - while (i < 8) : (i += 1) { - const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); - result[i * 2 + 0] = hex_charset[byte >> 4]; - result[i * 2 + 1] = hex_charset[byte & 15]; - } - return result; -} - -test hex64 { - const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; - try std.testing.expectEqualStrings("[00efcdab78563412]", s); -} - -const multihash_function: MultihashFunction = switch (Hash) { - std.crypto.hash.sha2.Sha256 => .@"sha2-256", - else => @compileError("unreachable"), -}; -comptime { - // We avoid unnecessary uleb128 code in hexDigest by asserting here the - // values are small enough to be contained in the one-byte encoding. - assert(@enumToInt(multihash_function) < 127); - assert(Hash.digest_length < 127); -} -const multihash_len = 1 + 1 + Hash.digest_length; - -fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { - var result: [multihash_len * 2]u8 = undefined; - - result[0] = hex_charset[@enumToInt(multihash_function) >> 4]; - result[1] = hex_charset[@enumToInt(multihash_function) & 15]; - - result[2] = hex_charset[Hash.digest_length >> 4]; - result[3] = hex_charset[Hash.digest_length & 15]; - - for (digest) |byte, i| { - result[4 + i * 2] = hex_charset[byte >> 4]; - result[5 + i * 2] = hex_charset[byte & 15]; - } - return result; -} - fn renameTmpIntoCache( cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, @@ -669,21 +612,3 @@ fn renameTmpIntoCache( break; } } - -const MultihashFunction = enum(u16) { - identity = 0x00, - sha1 = 0x11, - @"sha2-256" = 0x12, - @"sha2-512" = 0x13, - @"sha3-512" = 0x14, - @"sha3-384" = 0x15, - @"sha3-256" = 0x16, - @"sha3-224" = 0x17, - @"sha2-384" = 0x20, - @"sha2-256-trunc254-padded" = 0x1012, - @"sha2-224" = 0x1013, - @"sha2-512-224" = 0x1014, - @"sha2-512-256" = 0x1015, - @"blake2b-256" = 0xb220, - _, -}; diff --git a/src/main.zig b/src/main.zig index 06c36bad87..f634c259ff 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3915,6 +3915,7 @@ pub const usage_build = ; pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + var color: Color = .auto; var prominent_compile_errors: bool = false; // We want to release all the locks before executing the child process, so we make a nice @@ -4117,6 +4118,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // Here we borrow main package's table and will replace it with a fresh // one after this process completes. main_pkg.fetchAndAddDependencies( + arena, &thread_pool, &http_client, build_directory, @@ -4125,6 +4127,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &dependencies_source, &build_roots_source, "", + color, ) catch |err| switch (err) { error.PackageFetchFailed => process.exit(1), else => |e| return e, @@ -4366,7 +4369,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void }; defer tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, tree.errors, tree, "", color); + try printErrsMsgToStdErr(gpa, arena, tree, "", color); var has_ast_error = false; if (check_ast_flag) { const Module = @import("Module.zig"); @@ -4569,7 +4572,7 @@ fn fmtPathFile( var tree = try Ast.parse(fmt.gpa, source_code, .zig); defer tree.deinit(fmt.gpa); - try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree.errors, tree, file_path, fmt.color); + try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree, file_path, fmt.color); if (tree.errors.len != 0) { fmt.any_error = true; return; @@ -4649,14 +4652,14 @@ fn fmtPathFile( } } -fn printErrsMsgToStdErr( +pub fn printErrsMsgToStdErr( gpa: mem.Allocator, arena: mem.Allocator, - parse_errors: []const Ast.Error, tree: Ast, path: []const u8, color: Color, ) !void { + const parse_errors: []const Ast.Error = tree.errors; var i: usize = 0; while (i < parse_errors.len) : (i += 1) { const parse_error = parse_errors[i]; @@ -5316,7 +5319,7 @@ pub fn cmdAstCheck( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree.errors, file.tree, file.sub_file_path, color); + try printErrsMsgToStdErr(gpa, arena, file.tree, file.sub_file_path, color); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5442,7 +5445,7 @@ pub fn cmdChangelist( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree.errors, file.tree, old_source_file, .auto); + try printErrsMsgToStdErr(gpa, arena, file.tree, old_source_file, .auto); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5479,7 +5482,7 @@ pub fn cmdChangelist( var new_tree = try Ast.parse(gpa, new_source, .zig); defer new_tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, new_tree.errors, new_tree, new_source_file, .auto); + try printErrsMsgToStdErr(gpa, arena, new_tree, new_source_file, .auto); if (new_tree.errors.len != 0) { process.exit(1); } -- cgit v1.2.3