From 3010ccfca5f65a8ee0d70559a38d39c2c56f7367 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Thu, 20 Jan 2022 16:08:50 +0100 Subject: astgen saves decl doc comments in zir The field is saved in `extra` unconditionally for each decl. --- src/AstGen.zig | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- src/Module.zig | 2 +- src/Zir.zig | 2 +- src/print_zir.zig | 13 +++++++++++ 4 files changed, 77 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/AstGen.zig b/src/AstGen.zig index 750c7f53a9..eda06d0441 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3080,9 +3080,9 @@ const WipMembers = struct { /// struct, union, enum, and opaque decls all use same 4 bits per decl const bits_per_decl = 4; const decls_per_u32 = 32 / bits_per_decl; - /// struct, union, enum, and opaque decls all have maximum size of 10 u32 slots - /// (4 for src_hash + line + name + value + align + link_section + address_space) - const max_decl_size = 10; + /// struct, union, enum, and opaque decls all have maximum size of 11 u32 slots + /// (4 for src_hash + line + name + value + doc_comment + align + link_section + address_space ) + const max_decl_size = 11; pub fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self { const payload_top = @intCast(u32, payload.items.len); @@ -3193,6 +3193,7 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); + const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3445,6 +3446,7 @@ fn fnDecl( } wip_members.appendToDecl(fn_name_str_index); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(doc_comment_index); if (align_inst != .none) { wip_members.appendToDecl(@enumToInt(align_inst)); } @@ -3472,6 +3474,7 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); + const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3594,6 +3597,7 @@ fn globalVarDecl( } wip_members.appendToDecl(name_str_index); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(doc_comment_index); // doc_comment wip if (align_inst != .none) { wip_members.appendToDecl(@enumToInt(align_inst)); } @@ -3648,6 +3652,7 @@ fn comptimeDecl( } wip_members.appendToDecl(0); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on comptime decls } fn usingnamespaceDecl( @@ -3699,6 +3704,7 @@ fn usingnamespaceDecl( } wip_members.appendToDecl(0); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on usingnamespace decls } fn testDecl( @@ -3802,6 +3808,7 @@ fn testDecl( } wip_members.appendToDecl(test_name); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on test decls } fn structDeclInner( @@ -8784,6 +8791,58 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 { } } +/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. +/// `end_token` must point at the first token after the last doc coment line. +/// Returns 0 if no doc comment is present. +fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 { + const gpa = astgen.gpa; + const string_bytes = &astgen.string_bytes; + const str_index = @intCast(u32, string_bytes.items.len); + const token_tags = astgen.tree.tokens.items(.tag); + const token_starts = astgen.tree.tokens.items(.start); + + if (end_token == 0) return 0; + const start_token: u32 = blk: { + var tok = end_token - 1; + while (token_tags[tok] == .doc_comment) { + if (tok == 0) break; + tok -= 1; + } else { + tok += 1; + } + break :blk tok; + }; + if (start_token == end_token) return 0; + + const total_bytes = token_starts[end_token] - token_starts[start_token]; + try string_bytes.ensureUnusedCapacity(gpa, total_bytes); + + var current_token = start_token; + while (current_token < end_token) : (current_token += 1) { + const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; + string_bytes.appendSliceAssumeCapacity(tok_bytes); + if (current_token != end_token - 1) { + string_bytes.appendAssumeCapacity('\n'); + } + } + + const key = string_bytes.items[str_index..]; + const gop = try astgen.string_table.getOrPutContextAdapted(gpa, @as([]const u8, key), StringIndexAdapter{ + .bytes = string_bytes, + }, StringIndexContext{ + .bytes = string_bytes, + }); + + if (gop.found_existing) { + string_bytes.shrinkRetainingCapacity(str_index); + return gop.key_ptr.*; + } else { + gop.key_ptr.* = str_index; + try string_bytes.append(gpa, 0); + return str_index; + } +} + const IndexSlice = struct { index: u32, len: u32 }; fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice { diff --git a/src/Module.zig b/src/Module.zig index 1cb890b886..ecfc15a319 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4011,7 +4011,7 @@ pub fn scanNamespace( cur_bit_bag >>= 4; const decl_sub_index = extra_index; - extra_index += 7; // src_hash(4) + line(1) + name(1) + value(1) + extra_index += 8; // src_hash(4) + line(1) + name(1) + value(1) + doc_comment(1) extra_index += @truncate(u1, flags >> 2); // Align extra_index += @as(u2, @truncate(u1, flags >> 3)) * 2; // Link section or address space, consists of 2 Refs diff --git a/src/Zir.zig b/src/Zir.zig index 96f85b73df..74f85aa95a 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -3001,7 +3001,7 @@ pub const DeclIterator = struct { const sub_index = @intCast(u32, it.extra_index); it.extra_index += 5; // src_hash(4) + line(1) const name = it.zir.nullTerminatedString(it.zir.extra[it.extra_index]); - it.extra_index += 2; // name(1) + value(1) + it.extra_index += 3; // name(1) + value(1) + doc_comment(1) it.extra_index += @truncate(u1, flags >> 2); it.extra_index += @truncate(u1, flags >> 3); diff --git a/src/print_zir.zig b/src/print_zir.zig index 6f580abab5..7a82d84bbd 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -1398,6 +1398,9 @@ const Writer = struct { extra_index += 1; const decl_index = self.code.extra[extra_index]; extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + const align_inst: Zir.Inst.Ref = if (!has_align) .none else inst: { const inst = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]); extra_index += 1; @@ -1431,6 +1434,16 @@ const Writer = struct { raw_decl_name; const test_str = if (raw_decl_name.len == 0) "test " else ""; const export_str = if (is_exported) "export " else ""; + + if (doc_comment_index != 0) { + const doc_comment = self.code.nullTerminatedString(doc_comment_index); + var it = std.mem.tokenize(u8, doc_comment, "\n"); + while (it.next()) |doc_line| { + try stream.print("///{s}\n", .{doc_line}); + try stream.writeByteNTimes(' ', self.indent); + } + } + try stream.print("[{d}] {s}{s}{s}{}", .{ sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name), }); -- cgit v1.2.3 From 98fddd1c54171b080eb254527203b6f4e4e2f6ca Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Thu, 20 Jan 2022 19:59:40 +0100 Subject: add field doc comments to zir Doc comment information is stored in `extra` unconditionally for each field. This commmit covers Structs, Enums, Unions, and ErrSets. --- src/AstGen.zig | 27 +++++++++++++++++++++------ src/Sema.zig | 18 +++++++++++++++--- src/Zir.zig | 7 +++++++ src/print_zir.zig | 47 ++++++++++++++++++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/AstGen.zig b/src/AstGen.zig index eda06d0441..142806e74c 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3193,7 +3193,7 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); - const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1); + const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3474,7 +3474,7 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); - const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token); + const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3864,7 +3864,7 @@ fn structDeclInner( const field_count = @intCast(u32, container_decl.ast.members.len - decl_count); const bits_per_field = 4; - const max_field_size = 4; + const max_field_size = 5; var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size); defer wip_members.deinit(); @@ -3888,6 +3888,9 @@ fn structDeclInner( try typeExpr(&block_scope, &namespace.base, member.ast.type_expr); wip_members.appendToField(@enumToInt(field_type)); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr); const have_align = member.ast.align_expr != 0; @@ -3986,7 +3989,7 @@ fn unionDeclInner( .none; const bits_per_field = 4; - const max_field_size = 4; + const max_field_size = 5; var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size); defer wip_members.deinit(); @@ -4002,6 +4005,9 @@ fn unionDeclInner( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + const have_type = member.ast.type_expr != 0; const have_align = member.ast.align_expr != 0; const have_value = member.ast.value_expr != 0; @@ -4265,7 +4271,7 @@ fn containerDecl( .none; const bits_per_field = 1; - const max_field_size = 2; + const max_field_size = 3; var wip_members = try WipMembers.init(gpa, &astgen.scratch, @intCast(u32, counts.decls), @intCast(u32, counts.total_fields), bits_per_field, max_field_size); defer wip_members.deinit(); @@ -4283,6 +4289,9 @@ fn containerDecl( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + const have_value = member.ast.value_expr != 0; wip_members.nextField(bits_per_field, .{have_value}); @@ -4513,8 +4522,14 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir switch (token_tags[tok_i]) { .doc_comment, .comma => {}, .identifier => { + // TODO: maybe consider not using `docCommentAsString` + // since we're already visiting the first doc comment + // token. + try astgen.extra.ensureUnusedCapacity(gpa, 2); const str_index = try astgen.identAsString(tok_i); - try astgen.extra.append(gpa, str_index); + astgen.extra.appendAssumeCapacity(str_index); + const doc_comment_index = try astgen.docCommentAsString(tok_i); + astgen.extra.appendAssumeCapacity(doc_comment_index); fields_len += 1; }, .r_brace => break, diff --git a/src/Sema.zig b/src/Sema.zig index d3802a1784..6a2085cf3f 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -1916,6 +1916,9 @@ fn zirEnumDecl( const field_name_zir = sema.code.nullTerminatedString(sema.code.extra[extra_index]); extra_index += 1; + // doc comment + extra_index += 1; + // This string needs to outlive the ZIR code. const field_name = try new_decl_arena_allocator.dupe(u8, field_name_zir); @@ -2103,7 +2106,6 @@ fn zirErrorSetDecl( const inst_data = sema.code.instructions.items(.data)[inst].pl_node; const src = inst_data.src(); const extra = sema.code.extraData(Zir.Inst.ErrorSetDecl, inst_data.payload_index); - const fields = sema.code.extra[extra.end..][0..extra.data.fields_len]; var new_decl_arena = std.heap.ArenaAllocator.init(gpa); errdefer new_decl_arena.deinit(); @@ -2121,8 +2123,12 @@ fn zirErrorSetDecl( errdefer sema.mod.abortAnonDecl(new_decl); var names = Module.ErrorSet.NameMap{}; - try names.ensureUnusedCapacity(new_decl_arena_allocator, fields.len); - for (fields) |str_index| { + try names.ensureUnusedCapacity(new_decl_arena_allocator, extra.data.fields_len); + + var extra_index = @intCast(u32, extra.end); + const extra_index_end = extra_index + (extra.data.fields_len * 2); + while (extra_index < extra_index_end) : (extra_index += 2) { // +2 to skip over doc_string + const str_index = sema.code.extra[extra_index]; const name = try new_decl_arena_allocator.dupe(u8, sema.code.nullTerminatedString(str_index)); // TODO: This check should be performed in AstGen instead. @@ -16313,6 +16319,9 @@ fn semaStructFields( const field_type_ref = @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); extra_index += 1; + // doc_comment + extra_index += 1; + // This string needs to outlive the ZIR code. const field_name = try decl_arena_allocator.dupe(u8, field_name_zir); const field_ty: Type = if (field_type_ref == .none) @@ -16502,6 +16511,9 @@ fn semaUnionFields(mod: *Module, union_obj: *Module.Union) CompileError!void { const field_name_zir = zir.nullTerminatedString(zir.extra[extra_index]); extra_index += 1; + // doc_comment + extra_index += 1; + const field_type_ref: Zir.Inst.Ref = if (has_type) blk: { const field_type_ref = @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); extra_index += 1; diff --git a/src/Zir.zig b/src/Zir.zig index 74f85aa95a..b50a63163f 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2571,6 +2571,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2588,6 +2589,7 @@ pub const Inst = struct { /// field_name: u32, /// field_type: Ref, /// - if none, means `anytype`. + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// default_value: Ref, // if corresponding bit is set /// } @@ -2638,6 +2640,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc_comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2649,6 +2652,7 @@ pub const Inst = struct { /// - the bit is whether corresponding field has an value expression /// 9. fields: { // for every fields_len /// field_name: u32, + /// doc_comment: u32, // 0 if no doc_comment /// value: Ref, // if corresponding bit is set /// } pub const EnumDecl = struct { @@ -2686,6 +2690,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2701,6 +2706,7 @@ pub const Inst = struct { /// 0bX000: unused /// 9. fields: { // for every fields_len /// field_name: u32, // null terminated string index + /// doc_comment: u32, // 0 if no doc comment /// field_type: Ref, // if corresponding bit is set /// - if none, means `anytype`. /// align: Ref, // if corresponding bit is set @@ -2745,6 +2751,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment, /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, diff --git a/src/print_zir.zig b/src/print_zir.zig index 7a82d84bbd..9fd6b61231 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -1207,6 +1207,10 @@ const Writer = struct { extra_index += 1; const field_type = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try self.writeFlag(stream, "comptime ", is_comptime); @@ -1332,6 +1336,10 @@ const Writer = struct { const field_name = self.code.nullTerminatedString(self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try stream.print("{}", .{std.zig.fmtId(field_name)}); @@ -1419,12 +1427,13 @@ const Writer = struct { const pub_str = if (is_pub) "pub " else ""; const hash_bytes = @bitCast([16]u8, hash_u32s.*); - try stream.writeByteNTimes(' ', self.indent); if (decl_name_index == 0) { + try stream.writeByteNTimes(' ', self.indent); const name = if (is_exported) "usingnamespace" else "comptime"; try stream.writeAll(pub_str); try stream.writeAll(name); } else if (decl_name_index == 1) { + try stream.writeByteNTimes(' ', self.indent); try stream.writeAll("test"); } else { const raw_decl_name = self.code.nullTerminatedString(decl_name_index); @@ -1435,15 +1444,9 @@ const Writer = struct { const test_str = if (raw_decl_name.len == 0) "test " else ""; const export_str = if (is_exported) "export " else ""; - if (doc_comment_index != 0) { - const doc_comment = self.code.nullTerminatedString(doc_comment_index); - var it = std.mem.tokenize(u8, doc_comment, "\n"); - while (it.next()) |doc_line| { - try stream.print("///{s}\n", .{doc_line}); - try stream.writeByteNTimes(' ', self.indent); - } - } + try self.writeDocComment(stream, doc_comment_index); + try stream.writeByteNTimes(' ', self.indent); try stream.print("[{d}] {s}{s}{s}{}", .{ sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name), }); @@ -1569,6 +1572,11 @@ const Writer = struct { const field_name = self.code.nullTerminatedString(self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); + try stream.writeByteNTimes(' ', self.indent); try stream.print("{}", .{std.zig.fmtId(field_name)}); @@ -1632,17 +1640,23 @@ const Writer = struct { ) !void { const inst_data = self.code.instructions.items(.data)[inst].pl_node; const extra = self.code.extraData(Zir.Inst.ErrorSetDecl, inst_data.payload_index); - const fields = self.code.extra[extra.end..][0..extra.data.fields_len]; try stream.print("{s}, ", .{@tagName(name_strategy)}); try stream.writeAll("{\n"); self.indent += 2; - for (fields) |str_index| { + + var extra_index = @intCast(u32, extra.end); + const extra_index_end = extra_index + (extra.data.fields_len * 2); + while (extra_index < extra_index_end) : (extra_index += 2) { + const str_index = self.code.extra[extra_index]; const name = self.code.nullTerminatedString(str_index); + const doc_comment_index = self.code.extra[extra_index + 1]; + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try stream.print("{},\n", .{std.zig.fmtId(name)}); } + self.indent -= 2; try stream.writeByteNTimes(' ', self.indent); try stream.writeAll("}) "); @@ -2134,6 +2148,17 @@ const Writer = struct { } } + fn writeDocComment(self: *Writer, stream: anytype, doc_comment_index: u32) !void { + if (doc_comment_index != 0) { + const doc_comment = self.code.nullTerminatedString(doc_comment_index); + var it = std.mem.tokenize(u8, doc_comment, "\n"); + while (it.next()) |doc_line| { + try stream.writeByteNTimes(' ', self.indent); + try stream.print("///{s}\n", .{doc_line}); + } + } + } + fn writeBody(self: *Writer, stream: anytype, body: []const Zir.Inst.Index) !void { for (body) |inst| { try stream.writeByteNTimes(' ', self.indent); -- cgit v1.2.3 From 1f56ff8343ef07d2d04d4a596b3f03f61230a753 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 17:42:48 +0100 Subject: add support for more decl attributes in doc comment zir The previous commit that implemented doc comment zir support for decls did not properly account for all the possible attribute keyword combinations (threadlocal, extern, and such). --- src/AstGen.zig | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/AstGen.zig b/src/AstGen.zig index 142806e74c..c6b4e6e866 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3193,7 +3193,6 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); - const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3237,6 +3236,13 @@ fn fnDecl( const maybe_inline_token = fn_proto.extern_export_inline_token orelse break :blk false; break :blk token_tags[maybe_inline_token] == .keyword_inline; }; + + const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 - + @boolToInt(is_pub) - + @boolToInt(is_export) - + @boolToInt(is_extern) - + @boolToInt(has_inline_keyword)); // TODO subtract noinline too + const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0; wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace); @@ -3474,7 +3480,6 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); - const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3522,6 +3527,12 @@ fn globalVarDecl( break :blk lib_name_str.index; } else 0; + const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token - + @boolToInt(is_pub) - + @boolToInt(is_export) - + @boolToInt(lib_name != 0) - + @boolToInt(is_threadlocal)); + assert(var_decl.comptime_token == null); // handled by parser const var_inst: Zir.Inst.Ref = if (var_decl.ast.init_node != 0) vi: { -- cgit v1.2.3 From ba55e32ef2d8ed5875bff8524fbbb05c6d8afd40 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 18:52:42 +0100 Subject: add function param doc comment info in zir --- src/AstGen.zig | 4 ++++ src/Zir.zig | 2 ++ src/print_zir.zig | 6 ++++++ 3 files changed, 12 insertions(+) (limited to 'src') diff --git a/src/AstGen.zig b/src/AstGen.zig index c6b4e6e866..5a05209f4c 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -9721,8 +9721,12 @@ const GenZir = struct { try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + param_body.len); + const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index - + @boolToInt(tag == .param_comptime)); + const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{ .name = name, + .doc_comment = doc_comment_index, .body_len = @intCast(u32, param_body.len), }); gz.astgen.extra.appendSliceAssumeCapacity(param_body); diff --git a/src/Zir.zig b/src/Zir.zig index b50a63163f..05c4f2c5fb 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2906,6 +2906,8 @@ pub const Inst = struct { pub const Param = struct { /// Null-terminated string index. name: u32, + /// 0 if no doc comment + doc_comment: u32, /// The body contains the type of the parameter. body_len: u32, }; diff --git a/src/print_zir.zig b/src/print_zir.zig index 9fd6b61231..7ce459568b 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -785,6 +785,12 @@ const Writer = struct { try stream.print("\"{}\", ", .{ std.zig.fmtEscapes(self.code.nullTerminatedString(extra.data.name)), }); + + if (extra.data.doc_comment != 0) { + try stream.writeAll("\n"); + try self.writeDocComment(stream, extra.data.doc_comment); + try stream.writeByteNTimes(' ', self.indent); + } try self.writeBracedBody(stream, body); try stream.writeAll(") "); try self.writeSrc(stream, inst_data.src()); -- cgit v1.2.3 From 8c23b81e8640df490e2168b9a623bbb57a7d353e Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 22:46:12 +0100 Subject: fix incorrect zir offset loading in sema --- src/Module.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/Module.zig b/src/Module.zig index ecfc15a319..3892b35aea 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -558,14 +558,14 @@ pub const Decl = struct { if (!decl.has_align) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - return @intToEnum(Zir.Inst.Ref, zir.extra[decl.zir_decl_index + 7]); + return @intToEnum(Zir.Inst.Ref, zir.extra[decl.zir_decl_index + 8]); } pub fn zirLinksectionRef(decl: Decl) Zir.Inst.Ref { if (!decl.has_linksection_or_addrspace) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - const extra_index = decl.zir_decl_index + 7 + @boolToInt(decl.has_align); + const extra_index = decl.zir_decl_index + 8 + @boolToInt(decl.has_align); return @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); } @@ -573,7 +573,7 @@ pub const Decl = struct { if (!decl.has_linksection_or_addrspace) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - const extra_index = decl.zir_decl_index + 7 + @boolToInt(decl.has_align) + 1; + const extra_index = decl.zir_decl_index + 8 + @boolToInt(decl.has_align) + 1; return @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); } -- cgit v1.2.3 From 8a697262099d64b2ff222dc9271493fe3b5f2b76 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 23 Jan 2022 16:21:26 -0700 Subject: AstGen: doc comment fixups * AstGen: use Ast.zig helper methods to avoid copy pasting token counting logic - take advantage of the `first_doc_comment` field we already have for param AST nodes * Add missing ZIR docs --- lib/std/zig/Ast.zig | 18 ++++++++++++ src/AstGen.zig | 84 ++++++++++++++++++++++++++++------------------------- src/Zir.zig | 8 +++-- 3 files changed, 69 insertions(+), 41 deletions(-) (limited to 'src') diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index 65772c87a8..96007d564f 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -2120,6 +2120,14 @@ pub const full = struct { section_node: Node.Index, init_node: Node.Index, }; + + pub fn firstToken(var_decl: VarDecl) TokenIndex { + return var_decl.visib_token orelse + var_decl.extern_export_token orelse + var_decl.threadlocal_token orelse + var_decl.comptime_token orelse + var_decl.ast.mut_token; + } }; pub const If = struct { @@ -2168,6 +2176,10 @@ pub const full = struct { value_expr: Node.Index, align_expr: Node.Index, }; + + pub fn firstToken(cf: ContainerField) TokenIndex { + return cf.comptime_token orelse cf.ast.name_token; + } }; pub const FnProto = struct { @@ -2197,6 +2209,12 @@ pub const full = struct { type_expr: Node.Index, }; + pub fn firstToken(fn_proto: FnProto) TokenIndex { + return fn_proto.visib_token orelse + fn_proto.extern_export_inline_token orelse + fn_proto.ast.fn_token; + } + /// Abstracts over the fact that anytype and ... are not included /// in the params slice, since they are simple identifiers and /// not sub-expressions. diff --git a/src/AstGen.zig b/src/AstGen.zig index 5a05209f4c..114cd5f505 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -1171,7 +1171,7 @@ fn fnProtoExpr( const main_tokens = tree.nodes.items(.main_token); const name_token = param.name_token orelse main_tokens[param_type_node]; const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param; - const param_inst = try block_scope.addParam(¶m_gz, tag, name_token, param_name); + const param_inst = try block_scope.addParam(¶m_gz, tag, name_token, param_name, param.first_doc_comment); assert(param_inst_expected == param_inst); } } @@ -3237,11 +3237,7 @@ fn fnDecl( break :blk token_tags[maybe_inline_token] == .keyword_inline; }; - const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 - - @boolToInt(is_pub) - - @boolToInt(is_export) - - @boolToInt(is_extern) - - @boolToInt(has_inline_keyword)); // TODO subtract noinline too + const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken()); const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0; wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace); @@ -3301,7 +3297,7 @@ fn fnDecl( const main_tokens = tree.nodes.items(.main_token); const name_token = param.name_token orelse main_tokens[param_type_node]; const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param; - const param_inst = try decl_gz.addParam(¶m_gz, tag, name_token, param_name); + const param_inst = try decl_gz.addParam(¶m_gz, tag, name_token, param_name, param.first_doc_comment); assert(param_inst_expected == param_inst); break :param indexToRef(param_inst); }; @@ -3527,11 +3523,7 @@ fn globalVarDecl( break :blk lib_name_str.index; } else 0; - const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token - - @boolToInt(is_pub) - - @boolToInt(is_export) - - @boolToInt(lib_name != 0) - - @boolToInt(is_threadlocal)); + const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken()); assert(var_decl.comptime_token == null); // handled by parser @@ -3899,7 +3891,7 @@ fn structDeclInner( try typeExpr(&block_scope, &namespace.base, member.ast.type_expr); wip_members.appendToField(@enumToInt(field_type)); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr); @@ -4016,7 +4008,7 @@ fn unionDeclInner( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); const have_type = member.ast.type_expr != 0; @@ -4300,7 +4292,7 @@ fn containerDecl( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); const have_value = member.ast.value_expr != 0; @@ -4533,9 +4525,6 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir switch (token_tags[tok_i]) { .doc_comment, .comma => {}, .identifier => { - // TODO: maybe consider not using `docCommentAsString` - // since we're already visiting the first doc comment - // token. try astgen.extra.ensureUnusedCapacity(gpa, 2); const str_index = try astgen.identAsString(tok_i); astgen.extra.appendAssumeCapacity(str_index); @@ -8817,38 +8806,52 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 { } } -/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. +/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. /// `end_token` must point at the first token after the last doc coment line. /// Returns 0 if no doc comment is present. fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 { + if (end_token == 0) return @as(u32, 0); + + const token_tags = astgen.tree.tokens.items(.tag); + + var tok = end_token - 1; + while (token_tags[tok] == .doc_comment) { + if (tok == 0) break; + tok -= 1; + } else { + tok += 1; + } + return docCommentAsStringFromFirst(astgen, end_token, tok); +} + +/// end_token must be > the index of the last doc comment. +fn docCommentAsStringFromFirst( + astgen: *AstGen, + end_token: Ast.TokenIndex, + start_token: Ast.TokenIndex, +) !u32 { + if (start_token == end_token) return 0; + const gpa = astgen.gpa; const string_bytes = &astgen.string_bytes; const str_index = @intCast(u32, string_bytes.items.len); - const token_tags = astgen.tree.tokens.items(.tag); const token_starts = astgen.tree.tokens.items(.start); - - if (end_token == 0) return 0; - const start_token: u32 = blk: { - var tok = end_token - 1; - while (token_tags[tok] == .doc_comment) { - if (tok == 0) break; - tok -= 1; - } else { - tok += 1; - } - break :blk tok; - }; - if (start_token == end_token) return 0; + const token_tags = astgen.tree.tokens.items(.tag); const total_bytes = token_starts[end_token] - token_starts[start_token]; try string_bytes.ensureUnusedCapacity(gpa, total_bytes); var current_token = start_token; while (current_token < end_token) : (current_token += 1) { - const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; - string_bytes.appendSliceAssumeCapacity(tok_bytes); - if (current_token != end_token - 1) { - string_bytes.appendAssumeCapacity('\n'); + switch (token_tags[current_token]) { + .doc_comment => { + const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; + string_bytes.appendSliceAssumeCapacity(tok_bytes); + if (current_token != end_token - 1) { + string_bytes.appendAssumeCapacity('\n'); + } + }, + else => break, } } @@ -9714,6 +9717,7 @@ const GenZir = struct { /// Absolute token index. This function does the conversion to Decl offset. abs_tok_index: Ast.TokenIndex, name: u32, + first_doc_comment: ?Ast.TokenIndex, ) !Zir.Inst.Index { const gpa = gz.astgen.gpa; const param_body = param_gz.instructionsSlice(); @@ -9721,8 +9725,10 @@ const GenZir = struct { try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + param_body.len); - const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index - - @boolToInt(tag == .param_comptime)); + const doc_comment_index = if (first_doc_comment) |first| + try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first) + else + 0; const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{ .name = name, diff --git a/src/Zir.zig b/src/Zir.zig index 05c4f2c5fb..f38cb2d5f5 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2767,7 +2767,11 @@ pub const Inst = struct { }; }; - /// Trailing: field_name: u32 // for every field: null terminated string index + /// Trailing: + /// { // for every fields_len + /// field_name: u32 // null terminated string index + /// doc_comment: u32 // null terminated string index + /// } pub const ErrorSetDecl = struct { fields_len: u32, }; @@ -2906,7 +2910,7 @@ pub const Inst = struct { pub const Param = struct { /// Null-terminated string index. name: u32, - /// 0 if no doc comment + /// 0 if no doc comment doc_comment: u32, /// The body contains the type of the parameter. body_len: u32, -- cgit v1.2.3