From de30438ed2efb909538f3177cced441b57eb1a5d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 6 Nov 2019 16:15:12 -0500 Subject: stage1 parser code for anon container lit --- src/parser.cpp | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index a1ece3ed10..484e145cfa 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -81,7 +81,7 @@ static AstNode *ast_parse_for_type_expr(ParseContext *pc); static AstNode *ast_parse_while_type_expr(ParseContext *pc); static AstNode *ast_parse_switch_expr(ParseContext *pc); static AstNode *ast_parse_asm_expr(ParseContext *pc); -static AstNode *ast_parse_enum_lit(ParseContext *pc); +static AstNode *ast_parse_anon_lit(ParseContext *pc); static AstNode *ast_parse_asm_output(ParseContext *pc); static AsmOutput *ast_parse_asm_output_item(ParseContext *pc); static AstNode *ast_parse_asm_input(ParseContext *pc); @@ -1600,9 +1600,9 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) { if (container_decl != nullptr) return container_decl; - AstNode *enum_lit = ast_parse_enum_lit(pc); - if (enum_lit != nullptr) - return enum_lit; + AstNode *anon_lit = ast_parse_anon_lit(pc); + if (anon_lit != nullptr) + return anon_lit; AstNode *error_set_decl = ast_parse_error_set_decl(pc); if (error_set_decl != nullptr) @@ -1876,16 +1876,22 @@ static AstNode *ast_parse_asm_expr(ParseContext *pc) { return res; } -static AstNode *ast_parse_enum_lit(ParseContext *pc) { +static AstNode *ast_parse_anon_lit(ParseContext *pc) { Token *period = eat_token_if(pc, TokenIdDot); if (period == nullptr) return nullptr; - Token *identifier = expect_token(pc, TokenIdSymbol); - AstNode *res = ast_create_node(pc, NodeTypeEnumLiteral, period); - res->data.enum_literal.period = period; - res->data.enum_literal.identifier = identifier; - return res; + // anon enum literal + Token *identifier = eat_token_if(pc, TokenIdSymbol); + if (identifier != nullptr) { + AstNode *res = ast_create_node(pc, NodeTypeEnumLiteral, period); + res->data.enum_literal.period = period; + res->data.enum_literal.identifier = identifier; + return res; + } + + // anon container literal + return ast_parse_init_list(pc); } // AsmOutput <- COLON AsmOutputList AsmInput? -- cgit v1.2.3 From e509d21f39af726da3c6001b0c20f2c765be07a5 Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Fri, 15 Nov 2019 14:12:14 +0200 Subject: implemented container doc comments in stage 1 --- src/all_types.hpp | 1 + src/parser.cpp | 23 ++++++++++++++++++++++- src/tokenizer.cpp | 32 ++++++++++++++++++++++++++++++++ src/tokenizer.hpp | 1 + 4 files changed, 56 insertions(+), 1 deletion(-) (limited to 'src/parser.cpp') diff --git a/src/all_types.hpp b/src/all_types.hpp index a1e9d76be7..c4178cb130 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -968,6 +968,7 @@ struct AstNodeContainerDecl { AstNode *init_arg_expr; // enum(T), struct(endianness), or union(T), or union(enum(T)) ZigList fields; ZigList decls; + Buf doc_comments; ContainerKind kind; ContainerLayout layout; diff --git a/src/parser.cpp b/src/parser.cpp index 484e145cfa..eef832e32c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -493,6 +493,7 @@ static AstNode *ast_parse_root(ParseContext *pc) { node->data.container_decl.layout = ContainerLayoutAuto; node->data.container_decl.kind = ContainerKindStruct; node->data.container_decl.is_root = true; + node->data.container_decl.doc_comments = members.doc_comments; return node; } @@ -514,6 +515,21 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { return first_doc_token; } +static void ast_parse_container_doc_comments(ParseContext *pc, Buf *buf) { + if (buf_len(buf) != 0 && peek_token(pc)->id == TokenIdContainerDocComment) { + buf_append_char(buf, '\n'); + } + Token *doc_token = nullptr; + while ((doc_token = eat_token_if(pc, TokenIdContainerDocComment))) { + if (buf->list.length == 0) { + buf_resize(buf, 0); + } + // chops off '//!' but leaves '\n' + buf_append_mem(buf, buf_ptr(pc->buf) + doc_token->start_pos + 3, + doc_token->end_pos - doc_token->start_pos - 3); + } +} + // ContainerMembers // <- TestDecl ContainerMembers // / TopLevelComptime ContainerMembers @@ -523,7 +539,11 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { // / static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { AstNodeContainerDecl res = {}; + Buf tld_doc_comment_buf = BUF_INIT; + buf_resize(&tld_doc_comment_buf, 0); for (;;) { + ast_parse_container_doc_comments(pc, &tld_doc_comment_buf); + AstNode *test_decl = ast_parse_test_decl(pc); if (test_decl != nullptr) { res.decls.append(test_decl); @@ -566,7 +586,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { break; } - + res.doc_comments = tld_doc_comment_buf; return res; } @@ -2797,6 +2817,7 @@ static AstNode *ast_parse_container_decl_auto(ParseContext *pc) { res->data.container_decl.fields = members.fields; res->data.container_decl.decls = members.decls; + res->data.container_decl.doc_comments = members.doc_comments; return res; } diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 8b301f85ac..7ece5ff3fe 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -198,6 +198,7 @@ enum TokenizeState { TokenizeStateSawSlash, TokenizeStateSawSlash2, TokenizeStateSawSlash3, + TokenizeStateSawSlashBang, TokenizeStateSawBackslash, TokenizeStateSawPercent, TokenizeStateSawPlus, @@ -209,6 +210,7 @@ enum TokenizeState { TokenizeStateSawBar, TokenizeStateSawBarBar, TokenizeStateDocComment, + TokenizeStateContainerDocComment, TokenizeStateLineComment, TokenizeStateLineString, TokenizeStateLineStringEnd, @@ -938,6 +940,9 @@ void tokenize(Buf *buf, Tokenization *out) { case '/': t.state = TokenizeStateSawSlash3; break; + case '!': + t.state = TokenizeStateSawSlashBang; + break; case '\n': cancel_token(&t); t.state = TokenizeStateStart; @@ -965,6 +970,19 @@ void tokenize(Buf *buf, Tokenization *out) { break; } break; + case TokenizeStateSawSlashBang: + switch (c) { + case '\n': + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + end_token(&t); + t.state = TokenizeStateStart; + break; + default: + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + t.state = TokenizeStateContainerDocComment; + break; + } + break; case TokenizeStateSawBackslash: switch (c) { case '\\': @@ -1055,6 +1073,17 @@ void tokenize(Buf *buf, Tokenization *out) { break; } break; + case TokenizeStateContainerDocComment: + switch (c) { + case '\n': + end_token(&t); + t.state = TokenizeStateStart; + break; + default: + // do nothing + break; + } + break; case TokenizeStateSymbolFirstC: switch (c) { case '"': @@ -1545,6 +1574,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateSawBarBar: case TokenizeStateLBracket: case TokenizeStateDocComment: + case TokenizeStateContainerDocComment: end_token(&t); break; case TokenizeStateSawDotDot: @@ -1559,6 +1589,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateLineComment: case TokenizeStateSawSlash2: case TokenizeStateSawSlash3: + case TokenizeStateSawSlashBang: break; } if (t.state != TokenizeStateError) { @@ -1606,6 +1637,7 @@ const char * token_name(TokenId id) { case TokenIdDash: return "-"; case TokenIdDivEq: return "/="; case TokenIdDocComment: return "DocComment"; + case TokenIdContainerDocComment: return "ContainerDocComment"; case TokenIdDot: return "."; case TokenIdDotStar: return ".*"; case TokenIdEllipsis2: return ".."; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index ee336123bb..149e58b6a7 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -43,6 +43,7 @@ enum TokenId { TokenIdDash, TokenIdDivEq, TokenIdDocComment, + TokenIdContainerDocComment, TokenIdDot, TokenIdDotStar, TokenIdEllipsis2, -- cgit v1.2.3 From 977b6138817c4b09d6d1e47fa3562d9f9e0268a8 Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Fri, 15 Nov 2019 15:03:28 +0200 Subject: add container doc comments to generated docs --- lib/std/special/docs/index.html | 2 +- lib/std/special/docs/main.js | 31 +++++++++++++++++++------------ lib/std/zig/ast.zig | 1 - src/dump_analysis.cpp | 1 + src/parser.cpp | 8 ++++++-- 5 files changed, 27 insertions(+), 16 deletions(-) (limited to 'src/parser.cpp') diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index 81df8f30f1..b170ad4a14 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -484,7 +484,7 @@ doc comments.

- +