diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/link/tapi/Tokenizer.zig | 561 | ||||
| -rw-r--r-- | src/link/tapi/parse.zig | 701 | ||||
| -rw-r--r-- | src/link/tapi/parse/test.zig | 564 | ||||
| -rw-r--r-- | src/link/tapi/yaml.zig | 572 | ||||
| -rw-r--r-- | src/link/tapi/yaml/test.zig | 475 |
5 files changed, 1752 insertions, 1121 deletions
diff --git a/src/link/tapi/Tokenizer.zig b/src/link/tapi/Tokenizer.zig index d28700a02c..df46bb7d83 100644 --- a/src/link/tapi/Tokenizer.zig +++ b/src/link/tapi/Tokenizer.zig @@ -1,7 +1,7 @@ const Tokenizer = @This(); const std = @import("std"); -const log = std.log.scoped(.tapi); +const log = std.log.scoped(.yaml); const testing = std.testing; buffer: []const u8, @@ -13,29 +13,31 @@ pub const Token = struct { end: usize, pub const Id = enum { - Eof, - - NewLine, - DocStart, // --- - DocEnd, // ... - SeqItemInd, // - - MapValueInd, // : - FlowMapStart, // { - FlowMapEnd, // } - FlowSeqStart, // [ - FlowSeqEnd, // ] - - Comma, - Space, - Tab, - Comment, // # - Alias, // * - Anchor, // & - Tag, // ! - SingleQuote, // ' - DoubleQuote, // " - - Literal, + // zig fmt: off + eof, + + new_line, + doc_start, // --- + doc_end, // ... + seq_item_ind, // - + map_value_ind, // : + flow_map_start, // { + flow_map_end, // } + flow_seq_start, // [ + flow_seq_end, // ] + + comma, + space, + tab, + comment, // # + alias, // * + anchor, // & + tag, // ! + + single_quoted, // '...' + double_quoted, // "..." + literal, + // zig fmt: on }; }; @@ -45,8 +47,8 @@ pub const TokenIterator = struct { buffer: []const Token, pos: TokenIndex = 0, - pub fn next(self: *TokenIterator) Token { - const token = self.buffer[self.pos]; + pub fn next(self: *TokenIterator) ?Token { + const token = self.peek() orelse return null; self.pos += 1; return token; } @@ -74,180 +76,212 @@ pub const TokenIterator = struct { } }; +fn stringMatchesPattern(comptime pattern: []const u8, slice: []const u8) bool { + comptime var count: usize = 0; + inline while (count < pattern.len) : (count += 1) { + if (count >= slice.len) return false; + const c = slice[count]; + if (pattern[count] != c) return false; + } + return true; +} + +fn matchesPattern(self: Tokenizer, comptime pattern: []const u8) bool { + return stringMatchesPattern(pattern, self.buffer[self.index..]); +} + pub fn next(self: *Tokenizer) Token { var result = Token{ - .id = .Eof, + .id = .eof, .start = self.index, .end = undefined, }; - var state: union(enum) { - Start, - NewLine, - Space, - Tab, - Hyphen: usize, - Dot: usize, - Literal, - } = .Start; + var state: enum { + start, + new_line, + space, + tab, + comment, + single_quoted, + double_quoted, + literal, + } = .start; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (state) { - .Start => switch (c) { + .start => switch (c) { ' ' => { - state = .Space; + state = .space; }, '\t' => { - state = .Tab; + state = .tab; }, '\n' => { - result.id = .NewLine; + result.id = .new_line; self.index += 1; break; }, '\r' => { - state = .NewLine; + state = .new_line; }, - '-' => { - state = .{ .Hyphen = 1 }; + + '-' => if (self.matchesPattern("---")) { + result.id = .doc_start; + self.index += "---".len; + break; + } else if (self.matchesPattern("- ")) { + result.id = .seq_item_ind; + self.index += "- ".len; + break; + } else { + state = .literal; }, - '.' => { - state = .{ .Dot = 1 }; + + '.' => if (self.matchesPattern("...")) { + result.id = .doc_end; + self.index += "...".len; + break; + } else { + state = .literal; }, + ',' => { - result.id = .Comma; + result.id = .comma; self.index += 1; break; }, '#' => { - result.id = .Comment; - self.index += 1; - break; + state = .comment; }, '*' => { - result.id = .Alias; + result.id = .alias; self.index += 1; break; }, '&' => { - result.id = .Anchor; + result.id = .anchor; self.index += 1; break; }, '!' => { - result.id = .Tag; - self.index += 1; - break; - }, - '\'' => { - result.id = .SingleQuote; - self.index += 1; - break; - }, - '"' => { - result.id = .DoubleQuote; + result.id = .tag; self.index += 1; break; }, '[' => { - result.id = .FlowSeqStart; + result.id = .flow_seq_start; self.index += 1; break; }, ']' => { - result.id = .FlowSeqEnd; + result.id = .flow_seq_end; self.index += 1; break; }, ':' => { - result.id = .MapValueInd; + result.id = .map_value_ind; self.index += 1; break; }, '{' => { - result.id = .FlowMapStart; + result.id = .flow_map_start; self.index += 1; break; }, '}' => { - result.id = .FlowMapEnd; + result.id = .flow_map_end; self.index += 1; break; }, + '\'' => { + state = .single_quoted; + }, + '"' => { + state = .double_quoted; + }, else => { - state = .Literal; + state = .literal; + }, + }, + + .comment => switch (c) { + '\r', '\n' => { + result.id = .comment; + break; }, + else => {}, }, - .Space => switch (c) { + + .space => switch (c) { ' ' => {}, else => { - result.id = .Space; + result.id = .space; break; }, }, - .Tab => switch (c) { + + .tab => switch (c) { '\t' => {}, else => { - result.id = .Tab; + result.id = .tab; break; }, }, - .NewLine => switch (c) { + + .new_line => switch (c) { '\n' => { - result.id = .NewLine; + result.id = .new_line; self.index += 1; break; }, else => {}, // TODO this should be an error condition }, - .Hyphen => |*count| switch (c) { - ' ' => { - result.id = .SeqItemInd; + + .single_quoted => switch (c) { + '\'' => if (!self.matchesPattern("''")) { + result.id = .single_quoted; self.index += 1; break; + } else { + self.index += "''".len - 1; }, - '-' => { - count.* += 1; - - if (count.* == 3) { - result.id = .DocStart; - self.index += 1; - break; - } - }, - else => { - state = .Literal; - }, + else => {}, }, - .Dot => |*count| switch (c) { - '.' => { - count.* += 1; - if (count.* == 3) { - result.id = .DocEnd; + .double_quoted => switch (c) { + '"' => { + if (stringMatchesPattern("\\", self.buffer[self.index - 1 ..])) { + self.index += 1; + } else { + result.id = .double_quoted; self.index += 1; break; } }, - else => { - state = .Literal; - }, + else => {}, }, - .Literal => switch (c) { + + .literal => switch (c) { '\r', '\n', ' ', '\'', '"', ',', ':', ']', '}' => { - result.id = .Literal; + result.id = .literal; break; }, else => { - result.id = .Literal; + result.id = .literal; }, }, } } - if (state == .Literal and result.id == .Eof) { - result.id = .Literal; + if (self.index >= self.buffer.len) { + switch (state) { + .literal => { + result.id = .literal; + }, + else => {}, + } } result.end = self.index; @@ -263,22 +297,24 @@ fn testExpected(source: []const u8, expected: []const Token.Id) !void { .buffer = source, }; - var token_len: usize = 0; - for (expected) |exp| { - token_len += 1; + var given = std.ArrayList(Token.Id).init(testing.allocator); + defer given.deinit(); + + while (true) { const token = tokenizer.next(); - try testing.expectEqual(exp, token.id); + try given.append(token.id); + if (token.id == .eof) break; } - while (tokenizer.next().id != .Eof) { - token_len += 1; // consume all tokens - } + try testing.expectEqualSlices(Token.Id, expected, given.items); +} - try testing.expectEqual(expected.len, token_len); +test { + std.testing.refAllDecls(@This()); } test "empty doc" { - try testExpected("", &[_]Token.Id{.Eof}); + try testExpected("", &[_]Token.Id{.eof}); } test "empty doc with explicit markers" { @@ -286,7 +322,22 @@ test "empty doc with explicit markers" { \\--- \\... , &[_]Token.Id{ - .DocStart, .NewLine, .DocEnd, .Eof, + .doc_start, .new_line, .doc_end, .eof, + }); +} + +test "empty doc with explicit markers and a directive" { + try testExpected( + \\--- !tbd-v1 + \\... + , &[_]Token.Id{ + .doc_start, + .space, + .tag, + .literal, + .new_line, + .doc_end, + .eof, }); } @@ -296,15 +347,15 @@ test "sequence of values" { \\- 1 \\- 2 , &[_]Token.Id{ - .SeqItemInd, - .Literal, - .NewLine, - .SeqItemInd, - .Literal, - .NewLine, - .SeqItemInd, - .Literal, - .Eof, + .seq_item_ind, + .literal, + .new_line, + .seq_item_ind, + .literal, + .new_line, + .seq_item_ind, + .literal, + .eof, }); } @@ -313,24 +364,24 @@ test "sequence of sequences" { \\- [ val1, val2] \\- [val3, val4 ] , &[_]Token.Id{ - .SeqItemInd, - .FlowSeqStart, - .Space, - .Literal, - .Comma, - .Space, - .Literal, - .FlowSeqEnd, - .NewLine, - .SeqItemInd, - .FlowSeqStart, - .Literal, - .Comma, - .Space, - .Literal, - .Space, - .FlowSeqEnd, - .Eof, + .seq_item_ind, + .flow_seq_start, + .space, + .literal, + .comma, + .space, + .literal, + .flow_seq_end, + .new_line, + .seq_item_ind, + .flow_seq_start, + .literal, + .comma, + .space, + .literal, + .space, + .flow_seq_end, + .eof, }); } @@ -339,16 +390,16 @@ test "mappings" { \\key1: value1 \\key2: value2 , &[_]Token.Id{ - .Literal, - .MapValueInd, - .Space, - .Literal, - .NewLine, - .Literal, - .MapValueInd, - .Space, - .Literal, - .Eof, + .literal, + .map_value_ind, + .space, + .literal, + .new_line, + .literal, + .map_value_ind, + .space, + .literal, + .eof, }); } @@ -357,21 +408,21 @@ test "inline mapped sequence of values" { \\key : [ val1, \\ val2 ] , &[_]Token.Id{ - .Literal, - .Space, - .MapValueInd, - .Space, - .FlowSeqStart, - .Space, - .Literal, - .Comma, - .Space, - .NewLine, - .Space, - .Literal, - .Space, - .FlowSeqEnd, - .Eof, + .literal, + .space, + .map_value_ind, + .space, + .flow_seq_start, + .space, + .literal, + .comma, + .space, + .new_line, + .space, + .literal, + .space, + .flow_seq_end, + .eof, }); } @@ -388,52 +439,50 @@ test "part of tbd" { \\install-name: '/usr/lib/libSystem.B.dylib' \\... , &[_]Token.Id{ - .DocStart, - .Space, - .Tag, - .Literal, - .NewLine, - .Literal, - .MapValueInd, - .Space, - .Literal, - .NewLine, - .Literal, - .MapValueInd, - .Space, - .FlowSeqStart, - .Space, - .Literal, - .Space, - .FlowSeqEnd, - .NewLine, - .NewLine, - .Literal, - .MapValueInd, - .NewLine, - .Space, - .SeqItemInd, - .Literal, - .MapValueInd, - .Space, - .Literal, - .NewLine, - .Space, - .Literal, - .MapValueInd, - .Space, - .Literal, - .NewLine, - .NewLine, - .Literal, - .MapValueInd, - .Space, - .SingleQuote, - .Literal, - .SingleQuote, - .NewLine, - .DocEnd, - .Eof, + .doc_start, + .space, + .tag, + .literal, + .new_line, + .literal, + .map_value_ind, + .space, + .literal, + .new_line, + .literal, + .map_value_ind, + .space, + .flow_seq_start, + .space, + .literal, + .space, + .flow_seq_end, + .new_line, + .new_line, + .literal, + .map_value_ind, + .new_line, + .space, + .seq_item_ind, + .literal, + .map_value_ind, + .space, + .literal, + .new_line, + .space, + .literal, + .map_value_ind, + .space, + .literal, + .new_line, + .new_line, + .literal, + .map_value_ind, + .space, + .single_quoted, + .new_line, + .doc_end, + .eof, }); } @@ -443,18 +492,84 @@ test "Unindented list" { \\- foo: 1 \\c: 1 , &[_]Token.Id{ - .Literal, - .MapValueInd, - .NewLine, - .SeqItemInd, - .Literal, - .MapValueInd, - .Space, - .Literal, - .NewLine, - .Literal, - .MapValueInd, - .Space, - .Literal, + .literal, + .map_value_ind, + .new_line, + .seq_item_ind, + .literal, + .map_value_ind, + .space, + .literal, + .new_line, + .literal, + .map_value_ind, + .space, + .literal, + .eof, + }); +} + +test "escape sequences" { + try testExpected( + \\a: 'here''s an apostrophe' + \\b: "a newline\nand a\ttab" + \\c: "\"here\" and there" + , &[_]Token.Id{ + .literal, + .map_value_ind, + .space, + .single_quoted, + .new_line, + .literal, + .map_value_ind, + .space, + .double_quoted, + .new_line, + .literal, + .map_value_ind, + .space, + .double_quoted, + .eof, + }); +} + +test "comments" { + try testExpected( + \\key: # some comment about the key + \\# first value + \\- val1 + \\# second value + \\- val2 + , &[_]Token.Id{ + .literal, + .map_value_ind, + .space, + .comment, + .new_line, + .comment, + .new_line, + .seq_item_ind, + .literal, + .new_line, + .comment, + .new_line, + .seq_item_ind, + .literal, + .eof, + }); +} + +test "quoted literals" { + try testExpected( + \\'#000000' + \\'[000000' + \\"&someString" + , &[_]Token.Id{ + .single_quoted, + .new_line, + .single_quoted, + .new_line, + .double_quoted, + .eof, }); } diff --git a/src/link/tapi/parse.zig b/src/link/tapi/parse.zig index eb7bb2a0cf..09774cf00a 100644 --- a/src/link/tapi/parse.zig +++ b/src/link/tapi/parse.zig @@ -1,8 +1,7 @@ const std = @import("std"); const assert = std.debug.assert; -const log = std.log.scoped(.tapi); +const log = std.log.scoped(.yaml); const mem = std.mem; -const testing = std.testing; const Allocator = mem.Allocator; const Tokenizer = @import("Tokenizer.zig"); @@ -11,9 +10,9 @@ const TokenIndex = Tokenizer.TokenIndex; const TokenIterator = Tokenizer.TokenIterator; pub const ParseError = error{ + InvalidEscapeSequence, MalformedYaml, NestedDocuments, - UnexpectedTag, UnexpectedEof, UnexpectedToken, Unhandled, @@ -22,6 +21,8 @@ pub const ParseError = error{ pub const Node = struct { tag: Tag, tree: *const Tree, + start: TokenIndex, + end: TokenIndex, pub const Tag = enum { doc, @@ -61,9 +62,12 @@ pub const Node = struct { } pub const Doc = struct { - base: Node = Node{ .tag = Tag.doc, .tree = undefined }, - start: ?TokenIndex = null, - end: ?TokenIndex = null, + base: Node = Node{ + .tag = Tag.doc, + .tree = undefined, + .start = undefined, + .end = undefined, + }, directive: ?TokenIndex = null, value: ?*Node = null, @@ -86,10 +90,8 @@ pub const Node = struct { _ = fmt; if (self.directive) |id| { try std.fmt.format(writer, "{{ ", .{}); - const directive = self.base.tree.tokens[id]; - try std.fmt.format(writer, ".directive = {s}, ", .{ - self.base.tree.source[directive.start..directive.end], - }); + const directive = self.base.tree.getRaw(id, id); + try std.fmt.format(writer, ".directive = {s}, ", .{directive}); } if (self.value) |node| { try std.fmt.format(writer, "{}", .{node}); @@ -101,22 +103,27 @@ pub const Node = struct { }; pub const Map = struct { - base: Node = Node{ .tag = Tag.map, .tree = undefined }, - start: ?TokenIndex = null, - end: ?TokenIndex = null, + base: Node = Node{ + .tag = Tag.map, + .tree = undefined, + .start = undefined, + .end = undefined, + }, values: std.ArrayListUnmanaged(Entry) = .{}, pub const base_tag: Node.Tag = .map; pub const Entry = struct { key: TokenIndex, - value: *Node, + value: ?*Node, }; pub fn deinit(self: *Map, allocator: Allocator) void { for (self.values.items) |entry| { - entry.value.deinit(allocator); - allocator.destroy(entry.value); + if (entry.value) |value| { + value.deinit(allocator); + allocator.destroy(value); + } } self.values.deinit(allocator); } @@ -131,20 +138,24 @@ pub const Node = struct { _ = fmt; try std.fmt.format(writer, "{{ ", .{}); for (self.values.items) |entry| { - const key = self.base.tree.tokens[entry.key]; - try std.fmt.format(writer, "{s} => {}, ", .{ - self.base.tree.source[key.start..key.end], - entry.value, - }); + const key = self.base.tree.getRaw(entry.key, entry.key); + if (entry.value) |value| { + try std.fmt.format(writer, "{s} => {}, ", .{ key, value }); + } else { + try std.fmt.format(writer, "{s} => null, ", .{key}); + } } return std.fmt.format(writer, " }}", .{}); } }; pub const List = struct { - base: Node = Node{ .tag = Tag.list, .tree = undefined }, - start: ?TokenIndex = null, - end: ?TokenIndex = null, + base: Node = Node{ + .tag = Tag.list, + .tree = undefined, + .start = undefined, + .end = undefined, + }, values: std.ArrayListUnmanaged(*Node) = .{}, pub const base_tag: Node.Tag = .list; @@ -174,15 +185,18 @@ pub const Node = struct { }; pub const Value = struct { - base: Node = Node{ .tag = Tag.value, .tree = undefined }, - start: ?TokenIndex = null, - end: ?TokenIndex = null, + base: Node = Node{ + .tag = Tag.value, + .tree = undefined, + .start = undefined, + .end = undefined, + }, + string_value: std.ArrayListUnmanaged(u8) = .{}, pub const base_tag: Node.Tag = .value; pub fn deinit(self: *Value, allocator: Allocator) void { - _ = self; - _ = allocator; + self.string_value.deinit(allocator); } pub fn format( @@ -193,11 +207,8 @@ pub const Node = struct { ) !void { _ = options; _ = fmt; - const start = self.base.tree.tokens[self.start.?]; - const end = self.base.tree.tokens[self.end.?]; - return std.fmt.format(writer, "{s}", .{ - self.base.tree.source[start.start..end.end], - }); + const raw = self.base.tree.getRaw(self.base.start, self.base.end); + return std.fmt.format(writer, "{s}", .{raw}); } }; }; @@ -233,6 +244,21 @@ pub const Tree = struct { self.docs.deinit(self.allocator); } + pub fn getDirective(self: Tree, doc_index: usize) ?[]const u8 { + assert(doc_index < self.docs.items.len); + const doc = self.docs.items[doc_index].cast(Node.Doc) orelse return null; + const id = doc.directive orelse return null; + return self.getRaw(id, id); + } + + pub fn getRaw(self: Tree, start: TokenIndex, end: TokenIndex) []const u8 { + assert(start <= end); + assert(start < self.tokens.len and end < self.tokens.len); + const start_token = self.tokens[start]; + const end_token = self.tokens[end]; + return self.source[start_token.start..end_token.end]; + } + pub fn parse(self: *Tree, source: []const u8) !void { var tokenizer = Tokenizer{ .buffer = source }; var tokens = std.ArrayList(Token).init(self.allocator); @@ -252,8 +278,8 @@ pub const Tree = struct { }); switch (token.id) { - .Eof => break, - .NewLine => { + .eof => break, + .new_line => { line += 1; prev_line_last_col = token.end; }, @@ -272,20 +298,20 @@ pub const Tree = struct { .line_cols = &self.line_cols, }; - while (true) { - if (parser.token_it.peek() == null) return; + parser.eatCommentsAndSpace(&.{}); - const pos = parser.token_it.pos; - const token = parser.token_it.next(); + while (true) { + parser.eatCommentsAndSpace(&.{}); + const token = parser.token_it.next() orelse break; - log.debug("Next token: {}, {}", .{ pos, token }); + log.debug("(main) next {s}@{d}", .{ @tagName(token.id), parser.token_it.pos - 1 }); switch (token.id) { - .Space, .Comment, .NewLine => {}, - .Eof => break, + .eof => break, else => { - const doc = try parser.doc(pos); - try self.docs.append(self.allocator, &doc.base); + parser.token_it.seekBy(-1); + const doc = try parser.doc(); + try self.docs.append(self.allocator, doc); }, } } @@ -298,355 +324,308 @@ const Parser = struct { token_it: *TokenIterator, line_cols: *const std.AutoHashMap(TokenIndex, LineCol), - fn doc(self: *Parser, start: TokenIndex) ParseError!*Node.Doc { + fn value(self: *Parser) ParseError!?*Node { + self.eatCommentsAndSpace(&.{}); + + const pos = self.token_it.pos; + const token = self.token_it.next() orelse return error.UnexpectedEof; + + log.debug(" next {s}@{d}", .{ @tagName(token.id), pos }); + + switch (token.id) { + .literal => if (self.eatToken(.map_value_ind, &.{ .new_line, .comment })) |_| { + // map + self.token_it.seekTo(pos); + return self.map(); + } else { + // leaf value + self.token_it.seekTo(pos); + return self.leaf_value(); + }, + .single_quoted, .double_quoted => { + // leaf value + self.token_it.seekBy(-1); + return self.leaf_value(); + }, + .seq_item_ind => { + // list + self.token_it.seekBy(-1); + return self.list(); + }, + .flow_seq_start => { + // list + self.token_it.seekBy(-1); + return self.list_bracketed(); + }, + else => return null, + } + } + + fn doc(self: *Parser) ParseError!*Node { const node = try self.allocator.create(Node.Doc); errdefer self.allocator.destroy(node); - node.* = .{ .start = start }; + node.* = .{}; node.base.tree = self.tree; + node.base.start = self.token_it.pos; - self.token_it.seekTo(start); - - log.debug("Doc start: {}, {}", .{ start, self.tree.tokens[start] }); + log.debug("(doc) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start }); - const explicit_doc: bool = if (self.eatToken(.DocStart)) |_| explicit_doc: { - if (self.eatToken(.Tag)) |_| { - node.directive = try self.expectToken(.Literal); + // Parse header + const explicit_doc: bool = if (self.eatToken(.doc_start, &.{})) |doc_pos| explicit_doc: { + if (self.getCol(doc_pos) > 0) return error.MalformedYaml; + if (self.eatToken(.tag, &.{ .new_line, .comment })) |_| { + node.directive = try self.expectToken(.literal, &.{ .new_line, .comment }); } - _ = try self.expectToken(.NewLine); break :explicit_doc true; } else false; - while (true) { - const pos = self.token_it.pos; - const token = self.token_it.next(); - - log.debug("Next token: {}, {}", .{ pos, token }); + // Parse value + node.value = try self.value(); + if (node.value == null) { + self.token_it.seekBy(-1); + } + errdefer if (node.value) |val| { + val.deinit(self.allocator); + self.allocator.destroy(val); + }; - switch (token.id) { - .Tag => { - return error.UnexpectedTag; - }, - .Literal, .SingleQuote, .DoubleQuote => { - _ = try self.expectToken(.MapValueInd); - const map_node = try self.map(pos); - node.value = &map_node.base; - }, - .SeqItemInd => { - const list_node = try self.list(pos); - node.value = &list_node.base; - }, - .FlowSeqStart => { - const list_node = try self.list_bracketed(pos); - node.value = &list_node.base; - }, - .DocEnd => { - if (explicit_doc) break; - return error.UnexpectedToken; - }, - .DocStart, .Eof => { - self.token_it.seekBy(-1); - break; - }, - else => { - return error.UnexpectedToken; - }, + // Parse footer + footer: { + if (self.eatToken(.doc_end, &.{})) |pos| { + if (!explicit_doc) return error.UnexpectedToken; + if (self.getCol(pos) > 0) return error.MalformedYaml; + node.base.end = pos; + break :footer; + } + if (self.eatToken(.doc_start, &.{})) |pos| { + if (!explicit_doc) return error.UnexpectedToken; + if (self.getCol(pos) > 0) return error.MalformedYaml; + self.token_it.seekBy(-1); + node.base.end = pos - 1; + break :footer; + } + if (self.eatToken(.eof, &.{})) |pos| { + node.base.end = pos - 1; + break :footer; } + return error.UnexpectedToken; } - node.end = self.token_it.pos - 1; + log.debug("(doc) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end }); - log.debug("Doc end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] }); - - return node; + return &node.base; } - fn map(self: *Parser, start: TokenIndex) ParseError!*Node.Map { + fn map(self: *Parser) ParseError!*Node { const node = try self.allocator.create(Node.Map); errdefer self.allocator.destroy(node); - node.* = .{ .start = start }; + node.* = .{}; node.base.tree = self.tree; + node.base.start = self.token_it.pos; + errdefer { + for (node.values.items) |entry| { + if (entry.value) |val| { + val.deinit(self.allocator); + self.allocator.destroy(val); + } + } + node.values.deinit(self.allocator); + } - self.token_it.seekTo(start); - - log.debug("Map start: {}, {}", .{ start, self.tree.tokens[start] }); + log.debug("(map) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start }); - const col = self.getCol(start); + const col = self.getCol(node.base.start); while (true) { - self.eatCommentsAndSpace(); + self.eatCommentsAndSpace(&.{}); - // Parse key. + // Parse key const key_pos = self.token_it.pos; - if (self.getCol(key_pos) != col) { + if (self.getCol(key_pos) < col) { break; } - const key = self.token_it.next(); + const key = self.token_it.next() orelse return error.UnexpectedEof; switch (key.id) { - .Literal => {}, - else => { + .literal => {}, + .doc_start, .doc_end, .eof => { self.token_it.seekBy(-1); break; }, + else => { + // TODO key not being a literal + return error.Unhandled; + }, } - log.debug("Map key: {}, '{s}'", .{ key, self.tree.source[key.start..key.end] }); + log.debug("(map) key {s}@{d}", .{ self.tree.getRaw(key_pos, key_pos), key_pos }); // Separator - _ = try self.expectToken(.MapValueInd); - - // Parse value. - const value: *Node = value: { - if (self.eatToken(.NewLine)) |_| { - self.eatCommentsAndSpace(); - - // Explicit, complex value such as list or map. - const value_pos = self.token_it.pos; - const value = self.token_it.next(); - switch (value.id) { - .Literal, .SingleQuote, .DoubleQuote => { - // Assume nested map. - const map_node = try self.map(value_pos); - break :value &map_node.base; - }, - .SeqItemInd => { - // Assume list of values. - const list_node = try self.list(value_pos); - break :value &list_node.base; - }, - else => { - log.err("{}", .{key}); - return error.Unhandled; - }, - } - } else { - self.eatCommentsAndSpace(); - - const value_pos = self.token_it.pos; - const value = self.token_it.next(); - switch (value.id) { - .Literal, .SingleQuote, .DoubleQuote => { - // Assume leaf value. - const leaf_node = try self.leaf_value(value_pos); - break :value &leaf_node.base; - }, - .FlowSeqStart => { - const list_node = try self.list_bracketed(value_pos); - break :value &list_node.base; - }, - else => { - log.err("{}", .{key}); - return error.Unhandled; - }, + _ = try self.expectToken(.map_value_ind, &.{ .new_line, .comment }); + + // Parse value + const val = try self.value(); + errdefer if (val) |v| { + v.deinit(self.allocator); + self.allocator.destroy(v); + }; + + if (val) |v| { + if (self.getCol(v.start) < self.getCol(key_pos)) { + return error.MalformedYaml; + } + if (v.cast(Node.Value)) |_| { + if (self.getCol(v.start) == self.getCol(key_pos)) { + return error.MalformedYaml; } } - }; - log.debug("Map value: {}", .{value}); + } try node.values.append(self.allocator, .{ .key = key_pos, - .value = value, + .value = val, }); - - _ = self.eatToken(.NewLine); } - node.end = self.token_it.pos - 1; + node.base.end = self.token_it.pos - 1; - log.debug("Map end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] }); + log.debug("(map) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end }); - return node; + return &node.base; } - fn list(self: *Parser, start: TokenIndex) ParseError!*Node.List { + fn list(self: *Parser) ParseError!*Node { const node = try self.allocator.create(Node.List); errdefer self.allocator.destroy(node); - node.* = .{ - .start = start, - }; + node.* = .{}; node.base.tree = self.tree; + node.base.start = self.token_it.pos; + errdefer { + for (node.values.items) |val| { + val.deinit(self.allocator); + self.allocator.destroy(val); + } + node.values.deinit(self.allocator); + } - self.token_it.seekTo(start); - - log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); - - const col = self.getCol(start); + log.debug("(list) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start }); while (true) { - self.eatCommentsAndSpace(); + self.eatCommentsAndSpace(&.{}); - if (self.getCol(self.token_it.pos) != col) { - break; - } - _ = self.eatToken(.SeqItemInd) orelse { - break; - }; + _ = self.eatToken(.seq_item_ind, &.{}) orelse break; - const pos = self.token_it.pos; - const token = self.token_it.next(); - const value: *Node = value: { - switch (token.id) { - .Literal, .SingleQuote, .DoubleQuote => { - if (self.eatToken(.MapValueInd)) |_| { - // nested map - const map_node = try self.map(pos); - break :value &map_node.base; - } else { - // standalone (leaf) value - const leaf_node = try self.leaf_value(pos); - break :value &leaf_node.base; - } - }, - .FlowSeqStart => { - const list_node = try self.list_bracketed(pos); - break :value &list_node.base; - }, - else => { - log.err("{}", .{token}); - return error.Unhandled; - }, - } - }; - try node.values.append(self.allocator, value); - - _ = self.eatToken(.NewLine); + const val = (try self.value()) orelse return error.MalformedYaml; + try node.values.append(self.allocator, val); } - node.end = self.token_it.pos - 1; + node.base.end = self.token_it.pos - 1; - log.debug("List end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] }); + log.debug("(list) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end }); - return node; + return &node.base; } - fn list_bracketed(self: *Parser, start: TokenIndex) ParseError!*Node.List { + fn list_bracketed(self: *Parser) ParseError!*Node { const node = try self.allocator.create(Node.List); errdefer self.allocator.destroy(node); - node.* = .{ .start = start }; + node.* = .{}; node.base.tree = self.tree; + node.base.start = self.token_it.pos; + errdefer { + for (node.values.items) |val| { + val.deinit(self.allocator); + self.allocator.destroy(val); + } + node.values.deinit(self.allocator); + } - self.token_it.seekTo(start); - - log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); + log.debug("(list) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start }); - _ = try self.expectToken(.FlowSeqStart); + _ = try self.expectToken(.flow_seq_start, &.{}); while (true) { - _ = self.eatToken(.NewLine); - self.eatCommentsAndSpace(); + self.eatCommentsAndSpace(&.{.comment}); - const pos = self.token_it.pos; - const token = self.token_it.next(); - - log.debug("Next token: {}, {}", .{ pos, token }); + if (self.eatToken(.flow_seq_end, &.{.comment})) |pos| { + node.base.end = pos; + break; + } + _ = self.eatToken(.comma, &.{.comment}); - const value: *Node = value: { - switch (token.id) { - .FlowSeqStart => { - const list_node = try self.list_bracketed(pos); - break :value &list_node.base; - }, - .FlowSeqEnd => { - break; - }, - .Literal, .SingleQuote, .DoubleQuote => { - const leaf_node = try self.leaf_value(pos); - _ = self.eatToken(.Comma); - // TODO newline - break :value &leaf_node.base; - }, - else => { - log.err("{}", .{token}); - return error.Unhandled; - }, - } - }; - try node.values.append(self.allocator, value); + const val = (try self.value()) orelse return error.MalformedYaml; + try node.values.append(self.allocator, val); } - node.end = self.token_it.pos - 1; - - log.debug("List end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] }); + log.debug("(list) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end }); - return node; + return &node.base; } - fn leaf_value(self: *Parser, start: TokenIndex) ParseError!*Node.Value { + fn leaf_value(self: *Parser) ParseError!*Node { const node = try self.allocator.create(Node.Value); errdefer self.allocator.destroy(node); - node.* = .{ .start = start }; + node.* = .{ .string_value = .{} }; node.base.tree = self.tree; - - self.token_it.seekTo(start); - - log.debug("Leaf start: {}, {}", .{ node.start.?, self.tree.tokens[node.start.?] }); - - parse: { - if (self.eatToken(.SingleQuote)) |_| { - node.start = node.start.? + 1; - while (true) { - const tok = self.token_it.next(); - switch (tok.id) { - .SingleQuote => { - node.end = self.token_it.pos - 2; - break :parse; - }, - .NewLine => return error.UnexpectedToken, - else => {}, - } - } - } - - if (self.eatToken(.DoubleQuote)) |_| { - node.start = node.start.? + 1; - while (true) { - const tok = self.token_it.next(); - switch (tok.id) { - .DoubleQuote => { - node.end = self.token_it.pos - 2; - break :parse; - }, - .NewLine => return error.UnexpectedToken, - else => {}, - } - } - } - - // TODO handle multiline strings in new block scope - while (true) { - const tok = self.token_it.next(); - switch (tok.id) { - .Literal => {}, - .Space => { - const trailing = self.token_it.pos - 2; - self.eatCommentsAndSpace(); - if (self.token_it.peek()) |peek| { - if (peek.id != .Literal) { - node.end = trailing; - break; - } + node.base.start = self.token_it.pos; + errdefer node.string_value.deinit(self.allocator); + + // TODO handle multiline strings in new block scope + while (self.token_it.next()) |tok| { + switch (tok.id) { + .single_quoted => { + node.base.end = self.token_it.pos - 1; + const raw = self.tree.getRaw(node.base.start, node.base.end); + try self.parseSingleQuoted(node, raw); + break; + }, + .double_quoted => { + node.base.end = self.token_it.pos - 1; + const raw = self.tree.getRaw(node.base.start, node.base.end); + try self.parseDoubleQuoted(node, raw); + break; + }, + .literal => {}, + .space => { + const trailing = self.token_it.pos - 2; + self.eatCommentsAndSpace(&.{}); + if (self.token_it.peek()) |peek| { + if (peek.id != .literal) { + node.base.end = trailing; + const raw = self.tree.getRaw(node.base.start, node.base.end); + try node.string_value.appendSlice(self.allocator, raw); + break; } - }, - else => { - self.token_it.seekBy(-1); - node.end = self.token_it.pos - 1; - break; - }, - } + } + }, + else => { + self.token_it.seekBy(-1); + node.base.end = self.token_it.pos - 1; + const raw = self.tree.getRaw(node.base.start, node.base.end); + try node.string_value.appendSlice(self.allocator, raw); + break; + }, } } - log.debug("Leaf end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] }); + log.debug("(leaf) {s}", .{self.tree.getRaw(node.base.start, node.base.end)}); - return node; + return &node.base; } - fn eatCommentsAndSpace(self: *Parser) void { - while (true) { - _ = self.token_it.peek() orelse return; - const token = self.token_it.next(); + fn eatCommentsAndSpace(self: *Parser, comptime exclusions: []const Token.Id) void { + log.debug("eatCommentsAndSpace", .{}); + outer: while (self.token_it.next()) |token| { + log.debug(" (token '{s}')", .{@tagName(token.id)}); switch (token.id) { - .Comment, .Space => {}, + .comment, .space, .new_line => |space| { + inline for (exclusions) |excl| { + if (excl == space) { + self.token_it.seekBy(-1); + break :outer; + } + } else continue; + }, else => { self.token_it.seekBy(-1); break; @@ -655,25 +634,24 @@ const Parser = struct { } } - fn eatToken(self: *Parser, id: Token.Id) ?TokenIndex { - while (true) { - const pos = self.token_it.pos; - _ = self.token_it.peek() orelse return null; - const token = self.token_it.next(); - switch (token.id) { - .Comment, .Space => continue, - else => |next_id| if (next_id == id) { - return pos; - } else { - self.token_it.seekTo(pos); - return null; - }, - } + fn eatToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ?TokenIndex { + log.debug("eatToken('{s}')", .{@tagName(id)}); + self.eatCommentsAndSpace(exclusions); + const pos = self.token_it.pos; + const token = self.token_it.next() orelse return null; + if (token.id == id) { + log.debug(" (found at {d})", .{pos}); + return pos; + } else { + log.debug(" (not found)", .{}); + self.token_it.seekBy(-1); + return null; } } - fn expectToken(self: *Parser, id: Token.Id) ParseError!TokenIndex { - return self.eatToken(id) orelse error.UnexpectedToken; + fn expectToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ParseError!TokenIndex { + log.debug("expectToken('{s}')", .{@tagName(id)}); + return self.eatToken(id, exclusions) orelse error.UnexpectedToken; } fn getLine(self: *Parser, index: TokenIndex) usize { @@ -683,8 +661,85 @@ const Parser = struct { fn getCol(self: *Parser, index: TokenIndex) usize { return self.line_cols.get(index).?.col; } + + fn parseSingleQuoted(self: *Parser, node: *Node.Value, raw: []const u8) ParseError!void { + assert(raw[0] == '\'' and raw[raw.len - 1] == '\''); + + const raw_no_quotes = raw[1 .. raw.len - 1]; + try node.string_value.ensureTotalCapacity(self.allocator, raw_no_quotes.len); + + var state: enum { + start, + escape, + } = .start; + var index: usize = 0; + + while (index < raw_no_quotes.len) : (index += 1) { + const c = raw_no_quotes[index]; + switch (state) { + .start => switch (c) { + '\'' => { + state = .escape; + }, + else => { + node.string_value.appendAssumeCapacity(c); + }, + }, + .escape => switch (c) { + '\'' => { + state = .start; + node.string_value.appendAssumeCapacity(c); + }, + else => return error.InvalidEscapeSequence, + }, + } + } + } + + fn parseDoubleQuoted(self: *Parser, node: *Node.Value, raw: []const u8) ParseError!void { + assert(raw[0] == '"' and raw[raw.len - 1] == '"'); + + const raw_no_quotes = raw[1 .. raw.len - 1]; + try node.string_value.ensureTotalCapacity(self.allocator, raw_no_quotes.len); + + var state: enum { + start, + escape, + } = .start; + + var index: usize = 0; + while (index < raw_no_quotes.len) : (index += 1) { + const c = raw_no_quotes[index]; + switch (state) { + .start => switch (c) { + '\\' => { + state = .escape; + }, + else => { + node.string_value.appendAssumeCapacity(c); + }, + }, + .escape => switch (c) { + 'n' => { + state = .start; + node.string_value.appendAssumeCapacity('\n'); + }, + 't' => { + state = .start; + node.string_value.appendAssumeCapacity('\t'); + }, + '"' => { + state = .start; + node.string_value.appendAssumeCapacity('"'); + }, + else => return error.InvalidEscapeSequence, + }, + } + } + } }; test { + std.testing.refAllDecls(@This()); _ = @import("parse/test.zig"); } diff --git a/src/link/tapi/parse/test.zig b/src/link/tapi/parse/test.zig index b310a5c0bd..2906801f23 100644 --- a/src/link/tapi/parse/test.zig +++ b/src/link/tapi/parse/test.zig @@ -21,45 +21,45 @@ test "explicit doc" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); const directive = tree.tokens[doc.directive.?]; - try testing.expectEqual(directive.id, .Literal); - try testing.expect(mem.eql(u8, "tapi-tbd", tree.source[directive.start..directive.end])); + try testing.expectEqual(directive.id, .literal); + try testing.expectEqualStrings("tapi-tbd", tree.source[directive.start..directive.end]); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 5); - try testing.expectEqual(map.end.?, 14); + try testing.expectEqual(map.base.start, 5); + try testing.expectEqual(map.base.end, 14); try testing.expectEqual(map.values.items.len, 2); { const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "tbd-version", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("tbd-version", tree.source[key.start..key.end]); - const value = entry.value.cast(Node.Value).?; - const value_tok = tree.tokens[value.start.?]; - try testing.expectEqual(value_tok.id, .Literal); - try testing.expect(mem.eql(u8, "4", tree.source[value_tok.start..value_tok.end])); + const value = entry.value.?.cast(Node.Value).?; + const value_tok = tree.tokens[value.base.start]; + try testing.expectEqual(value_tok.id, .literal); + try testing.expectEqualStrings("4", tree.source[value_tok.start..value_tok.end]); } { const entry = map.values.items[1]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "abc-version", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("abc-version", tree.source[key.start..key.end]); - const value = entry.value.cast(Node.Value).?; - const value_tok = tree.tokens[value.start.?]; - try testing.expectEqual(value_tok.id, .Literal); - try testing.expect(mem.eql(u8, "5", tree.source[value_tok.start..value_tok.end])); + const value = entry.value.?.cast(Node.Value).?; + const value_tok = tree.tokens[value.base.start]; + try testing.expectEqual(value_tok.id, .literal); + try testing.expectEqualStrings("5", tree.source[value_tok.start..value_tok.end]); } } @@ -77,39 +77,31 @@ test "leaf in quotes" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.directive == null); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 0); - try testing.expectEqual(map.end.?, tree.tokens.len - 2); + try testing.expectEqual(map.base.start, 0); + try testing.expectEqual(map.base.end, tree.tokens.len - 2); try testing.expectEqual(map.values.items.len, 3); { const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql( - u8, - "key1", - tree.source[key.start..key.end], - )); - - const value = entry.value.cast(Node.Value).?; - const start = tree.tokens[value.start.?]; - const end = tree.tokens[value.end.?]; - try testing.expectEqual(start.id, .Literal); - try testing.expectEqual(end.id, .Literal); - try testing.expect(mem.eql( - u8, - "no quotes", - tree.source[start.start..end.end], - )); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("key1", tree.source[key.start..key.end]); + + const value = entry.value.?.cast(Node.Value).?; + const start = tree.tokens[value.base.start]; + const end = tree.tokens[value.base.end]; + try testing.expectEqual(start.id, .literal); + try testing.expectEqual(end.id, .literal); + try testing.expectEqualStrings("no quotes", tree.source[start.start..end.end]); } } @@ -128,70 +120,60 @@ test "nested maps" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.directive == null); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 0); - try testing.expectEqual(map.end.?, tree.tokens.len - 2); + try testing.expectEqual(map.base.start, 0); + try testing.expectEqual(map.base.end, tree.tokens.len - 2); try testing.expectEqual(map.values.items.len, 2); { const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "key1", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("key1", tree.source[key.start..key.end]); - const nested_map = entry.value.cast(Node.Map).?; - try testing.expectEqual(nested_map.start.?, 4); - try testing.expectEqual(nested_map.end.?, 16); + const nested_map = entry.value.?.cast(Node.Map).?; + try testing.expectEqual(nested_map.base.start, 4); + try testing.expectEqual(nested_map.base.end, 16); try testing.expectEqual(nested_map.values.items.len, 2); { const nested_entry = nested_map.values.items[0]; const nested_key = tree.tokens[nested_entry.key]; - try testing.expectEqual(nested_key.id, .Literal); - try testing.expect(mem.eql( - u8, - "key1_1", - tree.source[nested_key.start..nested_key.end], - )); - - const nested_value = nested_entry.value.cast(Node.Value).?; - const nested_value_tok = tree.tokens[nested_value.start.?]; - try testing.expectEqual(nested_value_tok.id, .Literal); - try testing.expect(mem.eql( - u8, + try testing.expectEqual(nested_key.id, .literal); + try testing.expectEqualStrings("key1_1", tree.source[nested_key.start..nested_key.end]); + + const nested_value = nested_entry.value.?.cast(Node.Value).?; + const nested_value_tok = tree.tokens[nested_value.base.start]; + try testing.expectEqual(nested_value_tok.id, .literal); + try testing.expectEqualStrings( "value1_1", tree.source[nested_value_tok.start..nested_value_tok.end], - )); + ); } { const nested_entry = nested_map.values.items[1]; const nested_key = tree.tokens[nested_entry.key]; - try testing.expectEqual(nested_key.id, .Literal); - try testing.expect(mem.eql( - u8, - "key1_2", - tree.source[nested_key.start..nested_key.end], - )); - - const nested_value = nested_entry.value.cast(Node.Value).?; - const nested_value_tok = tree.tokens[nested_value.start.?]; - try testing.expectEqual(nested_value_tok.id, .Literal); - try testing.expect(mem.eql( - u8, + try testing.expectEqual(nested_key.id, .literal); + try testing.expectEqualStrings("key1_2", tree.source[nested_key.start..nested_key.end]); + + const nested_value = nested_entry.value.?.cast(Node.Value).?; + const nested_value_tok = tree.tokens[nested_value.base.start]; + try testing.expectEqual(nested_value_tok.id, .literal); + try testing.expectEqualStrings( "value1_2", tree.source[nested_value_tok.start..nested_value_tok.end], - )); + ); } } @@ -199,17 +181,13 @@ test "nested maps" { const entry = map.values.items[1]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "key2", tree.source[key.start..key.end])); - - const value = entry.value.cast(Node.Value).?; - const value_tok = tree.tokens[value.start.?]; - try testing.expectEqual(value_tok.id, .Literal); - try testing.expect(mem.eql( - u8, - "value2", - tree.source[value_tok.start..value_tok.end], - )); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("key2", tree.source[key.start..key.end]); + + const value = entry.value.?.cast(Node.Value).?; + const value_tok = tree.tokens[value.base.start]; + try testing.expectEqual(value_tok.id, .literal); + try testing.expectEqualStrings("value2", tree.source[value_tok.start..value_tok.end]); } } @@ -227,46 +205,46 @@ test "map of list of values" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 0); - try testing.expectEqual(map.end.?, tree.tokens.len - 2); + try testing.expectEqual(map.base.start, 0); + try testing.expectEqual(map.base.end, tree.tokens.len - 2); try testing.expectEqual(map.values.items.len, 1); const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "ints", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("ints", tree.source[key.start..key.end]); - const value = entry.value.cast(Node.List).?; - try testing.expectEqual(value.start.?, 4); - try testing.expectEqual(value.end.?, tree.tokens.len - 2); + const value = entry.value.?.cast(Node.List).?; + try testing.expectEqual(value.base.start, 4); + try testing.expectEqual(value.base.end, tree.tokens.len - 2); try testing.expectEqual(value.values.items.len, 3); { const elem = value.values.items[0].cast(Node.Value).?; - const leaf = tree.tokens[elem.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "0", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[elem.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("0", tree.source[leaf.start..leaf.end]); } { const elem = value.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[elem.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "1", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[elem.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("1", tree.source[leaf.start..leaf.end]); } { const elem = value.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[elem.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "2", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[elem.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("2", tree.source[leaf.start..leaf.end]); } } @@ -285,64 +263,64 @@ test "map of list of maps" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 0); - try testing.expectEqual(map.end.?, tree.tokens.len - 2); + try testing.expectEqual(map.base.start, 0); + try testing.expectEqual(map.base.end, tree.tokens.len - 2); try testing.expectEqual(map.values.items.len, 1); const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "key1", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("key1", tree.source[key.start..key.end]); - const value = entry.value.cast(Node.List).?; - try testing.expectEqual(value.start.?, 3); - try testing.expectEqual(value.end.?, tree.tokens.len - 2); + const value = entry.value.?.cast(Node.List).?; + try testing.expectEqual(value.base.start, 3); + try testing.expectEqual(value.base.end, tree.tokens.len - 2); try testing.expectEqual(value.values.items.len, 3); { const elem = value.values.items[0].cast(Node.Map).?; const nested = elem.values.items[0]; const nested_key = tree.tokens[nested.key]; - try testing.expectEqual(nested_key.id, .Literal); - try testing.expect(mem.eql(u8, "key2", tree.source[nested_key.start..nested_key.end])); + try testing.expectEqual(nested_key.id, .literal); + try testing.expectEqualStrings("key2", tree.source[nested_key.start..nested_key.end]); - const nested_v = nested.value.cast(Node.Value).?; - const leaf = tree.tokens[nested_v.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "value2", tree.source[leaf.start..leaf.end])); + const nested_v = nested.value.?.cast(Node.Value).?; + const leaf = tree.tokens[nested_v.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("value2", tree.source[leaf.start..leaf.end]); } { const elem = value.values.items[1].cast(Node.Map).?; const nested = elem.values.items[0]; const nested_key = tree.tokens[nested.key]; - try testing.expectEqual(nested_key.id, .Literal); - try testing.expect(mem.eql(u8, "key3", tree.source[nested_key.start..nested_key.end])); + try testing.expectEqual(nested_key.id, .literal); + try testing.expectEqualStrings("key3", tree.source[nested_key.start..nested_key.end]); - const nested_v = nested.value.cast(Node.Value).?; - const leaf = tree.tokens[nested_v.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "value3", tree.source[leaf.start..leaf.end])); + const nested_v = nested.value.?.cast(Node.Value).?; + const leaf = tree.tokens[nested_v.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("value3", tree.source[leaf.start..leaf.end]); } { const elem = value.values.items[2].cast(Node.Map).?; const nested = elem.values.items[0]; const nested_key = tree.tokens[nested.key]; - try testing.expectEqual(nested_key.id, .Literal); - try testing.expect(mem.eql(u8, "key4", tree.source[nested_key.start..nested_key.end])); + try testing.expectEqual(nested_key.id, .literal); + try testing.expectEqualStrings("key4", tree.source[nested_key.start..nested_key.end]); - const nested_v = nested.value.cast(Node.Value).?; - const leaf = tree.tokens[nested_v.start.?]; - try testing.expectEqual(leaf.id, .Literal); - try testing.expect(mem.eql(u8, "value4", tree.source[leaf.start..leaf.end])); + const nested_v = nested.value.?.cast(Node.Value).?; + const leaf = tree.tokens[nested_v.base.start]; + try testing.expectEqual(leaf.id, .literal); + try testing.expectEqualStrings("value4", tree.source[leaf.start..leaf.end]); } } @@ -360,15 +338,15 @@ test "list of lists" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .list); const list = doc.value.?.cast(Node.List).?; - try testing.expectEqual(list.start.?, 0); - try testing.expectEqual(list.end.?, tree.tokens.len - 2); + try testing.expectEqual(list.base.start, 0); + try testing.expectEqual(list.base.end, tree.tokens.len - 2); try testing.expectEqual(list.values.items.len, 3); { @@ -379,22 +357,22 @@ test "list of lists" { { try testing.expectEqual(nested.values.items[0].tag, .value); const value = nested.values.items[0].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(nested.values.items[1].tag, .value); const value = nested.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(nested.values.items[2].tag, .value); const value = nested.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]); } } @@ -406,23 +384,23 @@ test "list of lists" { { try testing.expectEqual(nested.values.items[0].tag, .value); const value = nested.values.items[0].cast(Node.Value).?; - const start = tree.tokens[value.start.?]; - const end = tree.tokens[value.end.?]; - try testing.expect(mem.eql(u8, "Mark McGwire", tree.source[start.start..end.end])); + const start = tree.tokens[value.base.start]; + const end = tree.tokens[value.base.end]; + try testing.expectEqualStrings("Mark McGwire", tree.source[start.start..end.end]); } { try testing.expectEqual(nested.values.items[1].tag, .value); const value = nested.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "65", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("65", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(nested.values.items[2].tag, .value); const value = nested.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "0.278", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("0.278", tree.source[leaf.start..leaf.end]); } } @@ -434,23 +412,23 @@ test "list of lists" { { try testing.expectEqual(nested.values.items[0].tag, .value); const value = nested.values.items[0].cast(Node.Value).?; - const start = tree.tokens[value.start.?]; - const end = tree.tokens[value.end.?]; - try testing.expect(mem.eql(u8, "Sammy Sosa", tree.source[start.start..end.end])); + const start = tree.tokens[value.base.start]; + const end = tree.tokens[value.base.end]; + try testing.expectEqualStrings("Sammy Sosa", tree.source[start.start..end.end]); } { try testing.expectEqual(nested.values.items[1].tag, .value); const value = nested.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "63", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("63", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(nested.values.items[2].tag, .value); const value = nested.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "0.288", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("0.288", tree.source[leaf.start..leaf.end]); } } } @@ -467,36 +445,36 @@ test "inline list" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .list); const list = doc.value.?.cast(Node.List).?; - try testing.expectEqual(list.start.?, 0); - try testing.expectEqual(list.end.?, tree.tokens.len - 2); + try testing.expectEqual(list.base.start, 0); + try testing.expectEqual(list.base.end, tree.tokens.len - 2); try testing.expectEqual(list.values.items.len, 3); { try testing.expectEqual(list.values.items[0].tag, .value); const value = list.values.items[0].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(list.values.items[1].tag, .value); const value = list.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(list.values.items[2].tag, .value); const value = list.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]); } } @@ -514,45 +492,273 @@ test "inline list as mapping value" { try testing.expectEqual(tree.docs.items.len, 1); const doc = tree.docs.items[0].cast(Node.Doc).?; - try testing.expectEqual(doc.start.?, 0); - try testing.expectEqual(doc.end.?, tree.tokens.len - 2); + try testing.expectEqual(doc.base.start, 0); + try testing.expectEqual(doc.base.end, tree.tokens.len - 2); try testing.expect(doc.value != null); try testing.expectEqual(doc.value.?.tag, .map); const map = doc.value.?.cast(Node.Map).?; - try testing.expectEqual(map.start.?, 0); - try testing.expectEqual(map.end.?, tree.tokens.len - 2); + try testing.expectEqual(map.base.start, 0); + try testing.expectEqual(map.base.end, tree.tokens.len - 2); try testing.expectEqual(map.values.items.len, 1); const entry = map.values.items[0]; const key = tree.tokens[entry.key]; - try testing.expectEqual(key.id, .Literal); - try testing.expect(mem.eql(u8, "key", tree.source[key.start..key.end])); + try testing.expectEqual(key.id, .literal); + try testing.expectEqualStrings("key", tree.source[key.start..key.end]); - const list = entry.value.cast(Node.List).?; - try testing.expectEqual(list.start.?, 4); - try testing.expectEqual(list.end.?, tree.tokens.len - 2); + const list = entry.value.?.cast(Node.List).?; + try testing.expectEqual(list.base.start, 4); + try testing.expectEqual(list.base.end, tree.tokens.len - 2); try testing.expectEqual(list.values.items.len, 3); { try testing.expectEqual(list.values.items[0].tag, .value); const value = list.values.items[0].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(list.values.items[1].tag, .value); const value = list.values.items[1].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]); } { try testing.expectEqual(list.values.items[2].tag, .value); const value = list.values.items[2].cast(Node.Value).?; - const leaf = tree.tokens[value.start.?]; - try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end])); + const leaf = tree.tokens[value.base.start]; + try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]); } } + +fn parseSuccess(comptime source: []const u8) !void { + var tree = Tree.init(testing.allocator); + defer tree.deinit(); + try tree.parse(source); +} + +fn parseError(comptime source: []const u8, err: parse.ParseError) !void { + var tree = Tree.init(testing.allocator); + defer tree.deinit(); + try testing.expectError(err, tree.parse(source)); +} + +test "empty doc with spaces and comments" { + try parseSuccess( + \\ + \\ + \\ # this is a comment in a weird place + \\# and this one is too + ); +} + +test "comment between --- and ! in document start" { + try parseError( + \\--- # what is it? + \\! + , error.UnexpectedToken); +} + +test "correct doc start with tag" { + try parseSuccess( + \\--- !some-tag + \\ + ); +} + +test "doc close without explicit doc open" { + try parseError( + \\ + \\ + \\# something cool + \\... + , error.UnexpectedToken); +} + +test "doc open and close are ok" { + try parseSuccess( + \\--- + \\# first doc + \\ + \\ + \\--- + \\# second doc + \\ + \\ + \\... + ); +} + +test "doc with a single string is ok" { + try parseSuccess( + \\a string of some sort + \\ + ); +} + +test "explicit doc with a single string is ok" { + try parseSuccess( + \\--- !anchor + \\# nothing to see here except one string + \\ # not a lot to go on with + \\a single string + \\... + ); +} + +test "doc with two string is bad" { + try parseError( + \\first + \\second + \\# this should fail already + , error.UnexpectedToken); +} + +test "single quote string can have new lines" { + try parseSuccess( + \\'what is this + \\ thing?' + ); +} + +test "single quote string on one line is fine" { + try parseSuccess( + \\'here''s an apostrophe' + ); +} + +test "double quote string can have new lines" { + try parseSuccess( + \\"what is this + \\ thing?" + ); +} + +test "double quote string on one line is fine" { + try parseSuccess( + \\"a newline\nand a\ttab" + ); +} + +test "map with key and value literals" { + try parseSuccess( + \\key1: val1 + \\key2 : val2 + ); +} + +test "map of maps" { + try parseSuccess( + \\ + \\# the first key + \\key1: + \\ # the first subkey + \\ key1_1: 0 + \\ key1_2: 1 + \\# the second key + \\key2: + \\ key2_1: -1 + \\ key2_2: -2 + \\# the end of map + ); +} + +test "map value indicator needs to be on the same line" { + try parseError( + \\a + \\ : b + , error.UnexpectedToken); +} + +test "value needs to be indented" { + try parseError( + \\a: + \\b + , error.MalformedYaml); +} + +test "comment between a key and a value is fine" { + try parseSuccess( + \\a: + \\ # this is a value + \\ b + ); +} + +test "simple list" { + try parseSuccess( + \\# first el + \\- a + \\# second el + \\- b + \\# third el + \\- c + ); +} + +test "list indentation matters" { + try parseSuccess( + \\ - a + \\- b + ); + + try parseSuccess( + \\- a + \\ - b + ); +} + +test "unindented list is fine too" { + try parseSuccess( + \\a: + \\- 0 + \\- 1 + ); +} + +test "empty values in a map" { + try parseSuccess( + \\a: + \\b: + \\- 0 + ); +} + +test "weirdly nested map of maps of lists" { + try parseSuccess( + \\a: + \\ b: + \\ - 0 + \\ - 1 + ); +} + +test "square brackets denote a list" { + try parseSuccess( + \\[ a, + \\ b, c ] + ); +} + +test "empty list" { + try parseSuccess( + \\[ ] + ); +} + +test "comment within a bracketed list is an error" { + try parseError( + \\[ # something + \\] + , error.MalformedYaml); +} + +test "mixed ints with floats in a list" { + try parseSuccess( + \\[0, 1.0] + ); +} diff --git a/src/link/tapi/yaml.zig b/src/link/tapi/yaml.zig index d4136b35d3..0fc165003a 100644 --- a/src/link/tapi/yaml.zig +++ b/src/link/tapi/yaml.zig @@ -2,8 +2,7 @@ const std = @import("std"); const assert = std.debug.assert; const math = std.math; const mem = std.mem; -const testing = std.testing; -const log = std.log.scoped(.tapi); +const log = std.log.scoped(.yaml); const Allocator = mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; @@ -17,22 +16,15 @@ const ParseError = parse.ParseError; pub const YamlError = error{ UnexpectedNodeType, + DuplicateMapKey, OutOfMemory, + CannotEncodeValue, } || ParseError || std.fmt.ParseIntError; -pub const ValueType = enum { - empty, - int, - float, - string, - list, - map, -}; - pub const List = []Value; -pub const Map = std.StringArrayHashMap(Value); +pub const Map = std.StringHashMap(Value); -pub const Value = union(ValueType) { +pub const Value = union(enum) { empty, int: i64, float: f64, @@ -70,9 +62,7 @@ pub const Value = union(ValueType) { should_inline_first_key: bool = false, }; - pub const StringifyError = std.os.WriteError; - - pub fn stringify(self: Value, writer: anytype, args: StringifyArgs) StringifyError!void { + pub fn stringify(self: Value, writer: anytype, args: StringifyArgs) anyerror!void { switch (self) { .empty => return, .int => |int| return writer.print("{}", .{int}), @@ -83,7 +73,7 @@ pub const Value = union(ValueType) { if (len == 0) return; const first = list[0]; - if (first.is_compound()) { + if (first.isCompound()) { for (list, 0..) |elem, i| { try writer.writeByteNTimes(' ', args.indentation); try writer.writeAll("- "); @@ -108,20 +98,23 @@ pub const Value = union(ValueType) { try writer.writeAll(" ]"); }, .map => |map| { - const keys = map.keys(); - const len = keys.len; + const len = map.count(); if (len == 0) return; - for (keys, 0..) |key, i| { + var i: usize = 0; + var it = map.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + if (!args.should_inline_first_key or i != 0) { try writer.writeByteNTimes(' ', args.indentation); } try writer.print("{s}: ", .{key}); - const value = map.get(key) orelse unreachable; const should_inline = blk: { - if (!value.is_compound()) break :blk true; - if (value == .list and value.list.len > 0 and !value.list[0].is_compound()) break :blk true; + if (!value.isCompound()) break :blk true; + if (value == .list and value.list.len > 0 and !value.list[0].isCompound()) break :blk true; break :blk false; }; @@ -137,35 +130,44 @@ pub const Value = union(ValueType) { if (i < len - 1) { try writer.writeByte('\n'); } + + i += 1; } }, } } - fn is_compound(self: Value) bool { + fn isCompound(self: Value) bool { return switch (self) { .list, .map => true, else => false, }; } - fn fromNode(arena: Allocator, tree: *const Tree, node: *const Node, type_hint: ?ValueType) YamlError!Value { + fn fromNode(arena: Allocator, tree: *const Tree, node: *const Node) YamlError!Value { if (node.cast(Node.Doc)) |doc| { const inner = doc.value orelse { // empty doc return Value{ .empty = {} }; }; - return Value.fromNode(arena, tree, inner, null); + return Value.fromNode(arena, tree, inner); } else if (node.cast(Node.Map)) |map| { - var out_map = std.StringArrayHashMap(Value).init(arena); - try out_map.ensureUnusedCapacity(map.values.items.len); + // TODO use ContextAdapted HashMap and do not duplicate keys, intern + // in a contiguous string buffer. + var out_map = std.StringHashMap(Value).init(arena); + try out_map.ensureUnusedCapacity(math.cast(u32, map.values.items.len) orelse return error.Overflow); for (map.values.items) |entry| { - const key_tok = tree.tokens[entry.key]; - const key = try arena.dupe(u8, tree.source[key_tok.start..key_tok.end]); - const value = try Value.fromNode(arena, tree, entry.value, null); - - out_map.putAssumeCapacityNoClobber(key, value); + const key = try arena.dupe(u8, tree.getRaw(entry.key, entry.key)); + const gop = out_map.getOrPutAssumeCapacity(key); + if (gop.found_existing) { + return error.DuplicateMapKey; + } + const value = if (entry.value) |value| + try Value.fromNode(arena, tree, value) + else + .empty; + gop.value_ptr.* = value; } return Value{ .map = out_map }; @@ -173,56 +175,124 @@ pub const Value = union(ValueType) { var out_list = std.ArrayList(Value).init(arena); try out_list.ensureUnusedCapacity(list.values.items.len); - if (list.values.items.len > 0) { - const hint = if (list.values.items[0].cast(Node.Value)) |value| hint: { - const start = tree.tokens[value.start.?]; - const end = tree.tokens[value.end.?]; - const raw = tree.source[start.start..end.end]; - _ = std.fmt.parseInt(i64, raw, 10) catch { - _ = std.fmt.parseFloat(f64, raw) catch { - break :hint ValueType.string; - }; - break :hint ValueType.float; - }; - break :hint ValueType.int; - } else null; - - for (list.values.items) |elem| { - const value = try Value.fromNode(arena, tree, elem, hint); - out_list.appendAssumeCapacity(value); - } + for (list.values.items) |elem| { + const value = try Value.fromNode(arena, tree, elem); + out_list.appendAssumeCapacity(value); } return Value{ .list = try out_list.toOwnedSlice() }; } else if (node.cast(Node.Value)) |value| { - const start = tree.tokens[value.start.?]; - const end = tree.tokens[value.end.?]; - const raw = tree.source[start.start..end.end]; - - if (type_hint) |hint| { - return switch (hint) { - .int => Value{ .int = try std.fmt.parseInt(i64, raw, 10) }, - .float => Value{ .float = try std.fmt.parseFloat(f64, raw) }, - .string => Value{ .string = try arena.dupe(u8, raw) }, - else => unreachable, - }; - } + const raw = tree.getRaw(node.start, node.end); try_int: { // TODO infer base for int const int = std.fmt.parseInt(i64, raw, 10) catch break :try_int; return Value{ .int = int }; } + try_float: { const float = std.fmt.parseFloat(f64, raw) catch break :try_float; return Value{ .float = float }; } - return Value{ .string = try arena.dupe(u8, raw) }; + + return Value{ .string = try arena.dupe(u8, value.string_value.items) }; } else { log.err("Unexpected node type: {}", .{node.tag}); return error.UnexpectedNodeType; } } + + fn encode(arena: Allocator, input: anytype) YamlError!?Value { + switch (@typeInfo(@TypeOf(input))) { + .ComptimeInt, + .Int, + => return Value{ .int = math.cast(i64, input) orelse return error.Overflow }, + + .Float => return Value{ .float = math.lossyCast(f64, input) }, + + .Struct => |info| if (info.is_tuple) { + var list = std.ArrayList(Value).init(arena); + errdefer list.deinit(); + try list.ensureTotalCapacityPrecise(info.fields.len); + + inline for (info.fields) |field| { + if (try encode(arena, @field(input, field.name))) |value| { + list.appendAssumeCapacity(value); + } + } + + return Value{ .list = try list.toOwnedSlice() }; + } else { + var map = Map.init(arena); + errdefer map.deinit(); + try map.ensureTotalCapacity(info.fields.len); + + inline for (info.fields) |field| { + if (try encode(arena, @field(input, field.name))) |value| { + const key = try arena.dupe(u8, field.name); + map.putAssumeCapacityNoClobber(key, value); + } + } + + return Value{ .map = map }; + }, + + .Union => |info| if (info.tag_type) |tag_type| { + inline for (info.fields) |field| { + if (@field(tag_type, field.name) == input) { + return try encode(arena, @field(input, field.name)); + } + } else unreachable; + } else return error.UntaggedUnion, + + .Array => return encode(arena, &input), + + .Pointer => |info| switch (info.size) { + .One => switch (@typeInfo(info.child)) { + .Array => |child_info| { + const Slice = []const child_info.child; + return encode(arena, @as(Slice, input)); + }, + else => { + @compileError("Unhandled type: {s}" ++ @typeName(info.child)); + }, + }, + .Slice => { + if (info.child == u8) { + return Value{ .string = try arena.dupe(u8, input) }; + } + + var list = std.ArrayList(Value).init(arena); + errdefer list.deinit(); + try list.ensureTotalCapacityPrecise(input.len); + + for (input) |elem| { + if (try encode(arena, elem)) |value| { + list.appendAssumeCapacity(value); + } else { + log.err("Could not encode value in a list: {any}", .{elem}); + return error.CannotEncodeValue; + } + } + + return Value{ .list = try list.toOwnedSlice() }; + }, + else => { + @compileError("Unhandled type: {s}" ++ @typeName(@TypeOf(input))); + }, + }, + + // TODO we should probably have an option to encode `null` and also + // allow for some default value too. + .Optional => return if (input) |val| encode(arena, val) else null, + + .Null => return null, + + else => { + @compileError("Unhandled type: {s}" ++ @typeName(@TypeOf(input))); + }, + } + } }; pub const Yaml = struct { @@ -234,30 +304,18 @@ pub const Yaml = struct { self.arena.deinit(); } - pub fn stringify(self: Yaml, writer: anytype) !void { - for (self.docs.items) |doc| { - // if (doc.directive) |directive| { - // try writer.print("--- !{s}\n", .{directive}); - // } - try doc.stringify(writer, .{}); - // if (doc.directive != null) { - // try writer.writeAll("...\n"); - // } - } - } - pub fn load(allocator: Allocator, source: []const u8) !Yaml { var arena = ArenaAllocator.init(allocator); - const arena_allocator = arena.allocator(); + errdefer arena.deinit(); - var tree = Tree.init(arena_allocator); + var tree = Tree.init(arena.allocator()); try tree.parse(source); - var docs = std.ArrayList(Value).init(arena_allocator); - try docs.ensureUnusedCapacity(tree.docs.items.len); + var docs = std.ArrayList(Value).init(arena.allocator()); + try docs.ensureTotalCapacityPrecise(tree.docs.items.len); for (tree.docs.items) |node| { - const value = try Value.fromNode(arena_allocator, &tree, node, null); + const value = try Value.fromNode(arena.allocator(), &tree, node); docs.appendAssumeCapacity(value); } @@ -316,17 +374,19 @@ pub const Yaml = struct { fn parseValue(self: *Yaml, comptime T: type, value: Value) Error!T { return switch (@typeInfo(T)) { - .Int => math.cast(T, try value.asInt()) orelse error.Overflow, - .Float => math.lossyCast(T, try value.asFloat()), + .Int => math.cast(T, try value.asInt()) orelse return error.Overflow, + .Float => if (value.asFloat()) |float| { + return math.lossyCast(T, float); + } else |_| { + return math.lossyCast(T, try value.asInt()); + }, .Struct => self.parseStruct(T, try value.asMap()), .Union => self.parseUnion(T, value), .Array => self.parseArray(T, try value.asList()), - .Pointer => { - if (value.asList()) |list| { - return self.parsePointer(T, .{ .list = list }); - } else |_| { - return self.parsePointer(T, .{ .string = try value.asString() }); - } + .Pointer => if (value.asList()) |list| { + return self.parsePointer(T, .{ .list = list }); + } else |_| { + return self.parsePointer(T, .{ .string = try value.asString() }); }, .Void => error.TypeMismatch, .Optional => unreachable, @@ -372,7 +432,7 @@ pub const Yaml = struct { } const unwrapped = value orelse { - log.debug("missing struct field: {s}: {s}", .{ field.name, @typeName(field.type) }); + log.err("missing struct field: {s}: {s}", .{ field.name, @typeName(field.type) }); return error.StructFieldMissing; }; @field(parsed, field.name) = try self.parseValue(field.type, unwrapped); @@ -387,8 +447,7 @@ pub const Yaml = struct { switch (ptr_info.size) { .Slice => { - const child_info = @typeInfo(ptr_info.child); - if (child_info == .Int and child_info.Int.bits == 8) { + if (ptr_info.child == u8) { return value.asString(); } @@ -413,315 +472,36 @@ pub const Yaml = struct { return parsed; } -}; - -test { - testing.refAllDecls(@This()); -} - -test "simple list" { - const source = - \\- a - \\- b - \\- c - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - try testing.expectEqual(yaml.docs.items.len, 1); - - const list = yaml.docs.items[0].list; - try testing.expectEqual(list.len, 3); - - try testing.expect(mem.eql(u8, list[0].string, "a")); - try testing.expect(mem.eql(u8, list[1].string, "b")); - try testing.expect(mem.eql(u8, list[2].string, "c")); -} - -test "simple list typed as array of strings" { - const source = - \\- a - \\- b - \\- c - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const arr = try yaml.parse([3][]const u8); - try testing.expectEqual(arr.len, 3); - try testing.expect(mem.eql(u8, arr[0], "a")); - try testing.expect(mem.eql(u8, arr[1], "b")); - try testing.expect(mem.eql(u8, arr[2], "c")); -} - -test "simple list typed as array of ints" { - const source = - \\- 0 - \\- 1 - \\- 2 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const arr = try yaml.parse([3]u8); - try testing.expectEqual(arr.len, 3); - try testing.expectEqual(arr[0], 0); - try testing.expectEqual(arr[1], 1); - try testing.expectEqual(arr[2], 2); -} - -test "list of mixed sign integer" { - const source = - \\- 0 - \\- -1 - \\- 2 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const arr = try yaml.parse([3]i8); - try testing.expectEqual(arr.len, 3); - try testing.expectEqual(arr[0], 0); - try testing.expectEqual(arr[1], -1); - try testing.expectEqual(arr[2], 2); -} - -test "simple map untyped" { - const source = - \\a: 0 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const map = yaml.docs.items[0].map; - try testing.expect(map.contains("a")); - try testing.expectEqual(map.get("a").?.int, 0); -} - -test "simple map untyped with a list of maps" { - const source = - \\a: 0 - \\b: - \\ - foo: 1 - \\ bar: 2 - \\ - foo: 3 - \\ bar: 4 - \\c: 1 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const map = yaml.docs.items[0].map; - try testing.expect(map.contains("a")); - try testing.expect(map.contains("b")); - try testing.expect(map.contains("c")); - try testing.expectEqual(map.get("a").?.int, 0); - try testing.expectEqual(map.get("c").?.int, 1); - try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1); - try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2); - try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3); - try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4); -} - -test "simple map untyped with a list of maps. no indent" { - const source = - \\b: - \\- foo: 1 - \\c: 1 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const map = yaml.docs.items[0].map; - try testing.expect(map.contains("b")); - try testing.expect(map.contains("c")); - try testing.expectEqual(map.get("c").?.int, 1); - try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1); -} - -test "simple map untyped with a list of maps. no indent 2" { - const source = - \\a: 0 - \\b: - \\- foo: 1 - \\ bar: 2 - \\- foo: 3 - \\ bar: 4 - \\c: 1 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectEqual(yaml.docs.items.len, 1); - - const map = yaml.docs.items[0].map; - try testing.expect(map.contains("a")); - try testing.expect(map.contains("b")); - try testing.expect(map.contains("c")); - try testing.expectEqual(map.get("a").?.int, 0); - try testing.expectEqual(map.get("c").?.int, 1); - try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1); - try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2); - try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3); - try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4); -} - -test "simple map typed" { - const source = - \\a: 0 - \\b: hello there - \\c: 'wait, what?' - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - const simple = try yaml.parse(struct { a: usize, b: []const u8, c: []const u8 }); - try testing.expectEqual(simple.a, 0); - try testing.expect(mem.eql(u8, simple.b, "hello there")); - try testing.expect(mem.eql(u8, simple.c, "wait, what?")); -} - -test "typed nested structs" { - const source = - \\a: - \\ b: hello there - \\ c: 'wait, what?' - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - const simple = try yaml.parse(struct { - a: struct { - b: []const u8, - c: []const u8, - }, - }); - try testing.expect(mem.eql(u8, simple.a.b, "hello there")); - try testing.expect(mem.eql(u8, simple.a.c, "wait, what?")); -} - -test "multidoc typed as a slice of structs" { - const source = - \\--- - \\a: 0 - \\--- - \\a: 1 - \\... - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - { - const result = try yaml.parse([2]struct { a: usize }); - try testing.expectEqual(result.len, 2); - try testing.expectEqual(result[0].a, 0); - try testing.expectEqual(result[1].a, 1); - } - - { - const result = try yaml.parse([]struct { a: usize }); - try testing.expectEqual(result.len, 2); - try testing.expectEqual(result[0].a, 0); - try testing.expectEqual(result[1].a, 1); + pub fn stringify(self: Yaml, writer: anytype) !void { + for (self.docs.items, 0..) |doc, i| { + try writer.writeAll("---"); + if (self.tree.?.getDirective(i)) |directive| { + try writer.print(" !{s}", .{directive}); + } + try writer.writeByte('\n'); + try doc.stringify(writer, .{}); + try writer.writeByte('\n'); + } + try writer.writeAll("...\n"); } -} - -test "multidoc typed as a struct is an error" { - const source = - \\--- - \\a: 0 - \\--- - \\b: 1 - \\... - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize })); - try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { b: usize })); - try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize, b: usize })); -} - -test "multidoc typed as a slice of structs with optionals" { - const source = - \\--- - \\a: 0 - \\c: 1.0 - \\--- - \\a: 1 - \\b: different field - \\... - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - const result = try yaml.parse([]struct { a: usize, b: ?[]const u8, c: ?f16 }); - try testing.expectEqual(result.len, 2); - - try testing.expectEqual(result[0].a, 0); - try testing.expect(result[0].b == null); - try testing.expect(result[0].c != null); - try testing.expectEqual(result[0].c.?, 1.0); - - try testing.expectEqual(result[1].a, 1); - try testing.expect(result[1].b != null); - try testing.expect(mem.eql(u8, result[1].b.?, "different field")); - try testing.expect(result[1].c == null); -} - -test "empty yaml can be represented as void" { - const source = ""; - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - const result = try yaml.parse(void); - try testing.expect(@TypeOf(result) == void); -} +}; -test "nonempty yaml cannot be represented as void" { - const source = - \\a: b - ; +pub fn stringify(allocator: Allocator, input: anytype, writer: anytype) !void { + var arena = ArenaAllocator.init(allocator); + defer arena.deinit(); - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); + var maybe_value = try Value.encode(arena.allocator(), input); - try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(void)); + if (maybe_value) |value| { + // TODO should we output as an explicit doc? + // How can allow the user to specify? + try value.stringify(writer, .{}); + } } -test "typed array size mismatch" { - const source = - \\- 0 - \\- 0 - ; - - var yaml = try Yaml.load(testing.allocator, source); - defer yaml.deinit(); - - try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([1]usize)); - try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([5]usize)); +test { + std.testing.refAllDecls(Tokenizer); + std.testing.refAllDecls(parse); + _ = @import("yaml/test.zig"); } diff --git a/src/link/tapi/yaml/test.zig b/src/link/tapi/yaml/test.zig new file mode 100644 index 0000000000..8db9435885 --- /dev/null +++ b/src/link/tapi/yaml/test.zig @@ -0,0 +1,475 @@ +const std = @import("std"); +const mem = std.mem; +const testing = std.testing; + +const yaml_mod = @import("../yaml.zig"); +const Yaml = yaml_mod.Yaml; + +test "simple list" { + const source = + \\- a + \\- b + \\- c + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const list = yaml.docs.items[0].list; + try testing.expectEqual(list.len, 3); + + try testing.expectEqualStrings("a", list[0].string); + try testing.expectEqualStrings("b", list[1].string); + try testing.expectEqualStrings("c", list[2].string); +} + +test "simple list typed as array of strings" { + const source = + \\- a + \\- b + \\- c + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const arr = try yaml.parse([3][]const u8); + try testing.expectEqual(3, arr.len); + try testing.expectEqualStrings("a", arr[0]); + try testing.expectEqualStrings("b", arr[1]); + try testing.expectEqualStrings("c", arr[2]); +} + +test "simple list typed as array of ints" { + const source = + \\- 0 + \\- 1 + \\- 2 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const arr = try yaml.parse([3]u8); + try testing.expectEqualSlices(u8, &[_]u8{ 0, 1, 2 }, &arr); +} + +test "list of mixed sign integer" { + const source = + \\- 0 + \\- -1 + \\- 2 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const arr = try yaml.parse([3]i8); + try testing.expectEqualSlices(i8, &[_]i8{ 0, -1, 2 }, &arr); +} + +test "simple map untyped" { + const source = + \\a: 0 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const map = yaml.docs.items[0].map; + try testing.expect(map.contains("a")); + try testing.expectEqual(@as(i64, 0), map.get("a").?.int); +} + +test "simple map untyped with a list of maps" { + const source = + \\a: 0 + \\b: + \\ - foo: 1 + \\ bar: 2 + \\ - foo: 3 + \\ bar: 4 + \\c: 1 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const map = yaml.docs.items[0].map; + try testing.expect(map.contains("a")); + try testing.expect(map.contains("b")); + try testing.expect(map.contains("c")); + try testing.expectEqual(@as(i64, 0), map.get("a").?.int); + try testing.expectEqual(@as(i64, 1), map.get("c").?.int); + try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int); + try testing.expectEqual(@as(i64, 2), map.get("b").?.list[0].map.get("bar").?.int); + try testing.expectEqual(@as(i64, 3), map.get("b").?.list[1].map.get("foo").?.int); + try testing.expectEqual(@as(i64, 4), map.get("b").?.list[1].map.get("bar").?.int); +} + +test "simple map untyped with a list of maps. no indent" { + const source = + \\b: + \\- foo: 1 + \\c: 1 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const map = yaml.docs.items[0].map; + try testing.expect(map.contains("b")); + try testing.expect(map.contains("c")); + try testing.expectEqual(@as(i64, 1), map.get("c").?.int); + try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int); +} + +test "simple map untyped with a list of maps. no indent 2" { + const source = + \\a: 0 + \\b: + \\- foo: 1 + \\ bar: 2 + \\- foo: 3 + \\ bar: 4 + \\c: 1 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectEqual(yaml.docs.items.len, 1); + + const map = yaml.docs.items[0].map; + try testing.expect(map.contains("a")); + try testing.expect(map.contains("b")); + try testing.expect(map.contains("c")); + try testing.expectEqual(@as(i64, 0), map.get("a").?.int); + try testing.expectEqual(@as(i64, 1), map.get("c").?.int); + try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int); + try testing.expectEqual(@as(i64, 2), map.get("b").?.list[0].map.get("bar").?.int); + try testing.expectEqual(@as(i64, 3), map.get("b").?.list[1].map.get("foo").?.int); + try testing.expectEqual(@as(i64, 4), map.get("b").?.list[1].map.get("bar").?.int); +} + +test "simple map typed" { + const source = + \\a: 0 + \\b: hello there + \\c: 'wait, what?' + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const simple = try yaml.parse(struct { a: usize, b: []const u8, c: []const u8 }); + try testing.expectEqual(@as(usize, 0), simple.a); + try testing.expectEqualStrings("hello there", simple.b); + try testing.expectEqualStrings("wait, what?", simple.c); +} + +test "typed nested structs" { + const source = + \\a: + \\ b: hello there + \\ c: 'wait, what?' + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const simple = try yaml.parse(struct { + a: struct { + b: []const u8, + c: []const u8, + }, + }); + try testing.expectEqualStrings("hello there", simple.a.b); + try testing.expectEqualStrings("wait, what?", simple.a.c); +} + +test "single quoted string" { + const source = + \\- 'hello' + \\- 'here''s an escaped quote' + \\- 'newlines and tabs\nare not\tsupported' + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const arr = try yaml.parse([3][]const u8); + try testing.expectEqual(arr.len, 3); + try testing.expectEqualStrings("hello", arr[0]); + try testing.expectEqualStrings("here's an escaped quote", arr[1]); + try testing.expectEqualStrings("newlines and tabs\\nare not\\tsupported", arr[2]); +} + +test "double quoted string" { + const source = + \\- "hello" + \\- "\"here\" are some escaped quotes" + \\- "newlines and tabs\nare\tsupported" + \\- "let's have + \\some fun!" + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const arr = try yaml.parse([4][]const u8); + try testing.expectEqual(arr.len, 4); + try testing.expectEqualStrings("hello", arr[0]); + try testing.expectEqualStrings( + \\"here" are some escaped quotes + , arr[1]); + try testing.expectEqualStrings( + \\newlines and tabs + \\are supported + , arr[2]); + try testing.expectEqualStrings( + \\let's have + \\some fun! + , arr[3]); +} + +test "multidoc typed as a slice of structs" { + const source = + \\--- + \\a: 0 + \\--- + \\a: 1 + \\... + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + { + const result = try yaml.parse([2]struct { a: usize }); + try testing.expectEqual(result.len, 2); + try testing.expectEqual(result[0].a, 0); + try testing.expectEqual(result[1].a, 1); + } + + { + const result = try yaml.parse([]struct { a: usize }); + try testing.expectEqual(result.len, 2); + try testing.expectEqual(result[0].a, 0); + try testing.expectEqual(result[1].a, 1); + } +} + +test "multidoc typed as a struct is an error" { + const source = + \\--- + \\a: 0 + \\--- + \\b: 1 + \\... + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize })); + try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { b: usize })); + try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize, b: usize })); +} + +test "multidoc typed as a slice of structs with optionals" { + const source = + \\--- + \\a: 0 + \\c: 1.0 + \\--- + \\a: 1 + \\b: different field + \\... + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const result = try yaml.parse([]struct { a: usize, b: ?[]const u8, c: ?f16 }); + try testing.expectEqual(result.len, 2); + + try testing.expectEqual(result[0].a, 0); + try testing.expect(result[0].b == null); + try testing.expect(result[0].c != null); + try testing.expectEqual(result[0].c.?, 1.0); + + try testing.expectEqual(result[1].a, 1); + try testing.expect(result[1].b != null); + try testing.expectEqualStrings("different field", result[1].b.?); + try testing.expect(result[1].c == null); +} + +test "empty yaml can be represented as void" { + const source = ""; + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + const result = try yaml.parse(void); + try testing.expect(@TypeOf(result) == void); +} + +test "nonempty yaml cannot be represented as void" { + const source = + \\a: b + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(void)); +} + +test "typed array size mismatch" { + const source = + \\- 0 + \\- 0 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([1]usize)); + try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([5]usize)); +} + +test "comments" { + const source = + \\ + \\key: # this is the key + \\# first value + \\ + \\- val1 + \\ + \\# second value + \\- val2 + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const simple = try yaml.parse(struct { + key: []const []const u8, + }); + try testing.expect(simple.key.len == 2); + try testing.expectEqualStrings("val1", simple.key[0]); + try testing.expectEqualStrings("val2", simple.key[1]); +} + +test "promote ints to floats in a list mixed numeric types" { + const source = + \\a_list: [0, 1.0] + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + const simple = try yaml.parse(struct { + a_list: []const f64, + }); + try testing.expectEqualSlices(f64, &[_]f64{ 0.0, 1.0 }, simple.a_list); +} + +test "demoting floats to ints in a list is an error" { + const source = + \\a_list: [0, 1.0] + ; + + var yaml = try Yaml.load(testing.allocator, source); + defer yaml.deinit(); + + try testing.expectError(error.TypeMismatch, yaml.parse(struct { + a_list: []const u64, + })); +} + +test "duplicate map keys" { + const source = + \\a: b + \\a: c + ; + try testing.expectError(error.DuplicateMapKey, Yaml.load(testing.allocator, source)); +} + +fn testStringify(expected: []const u8, input: anytype) !void { + var output = std.ArrayList(u8).init(testing.allocator); + defer output.deinit(); + + try yaml_mod.stringify(testing.allocator, input, output.writer()); + try testing.expectEqualStrings(expected, output.items); +} + +test "stringify an int" { + try testStringify("128", @as(u32, 128)); +} + +test "stringify a simple struct" { + try testStringify( + \\a: 1 + \\b: 2 + \\c: 2.5 + , struct { a: i64, b: f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 }); +} + +test "stringify a struct with an optional" { + try testStringify( + \\a: 1 + \\b: 2 + \\c: 2.5 + , struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 }); + + try testStringify( + \\a: 1 + \\c: 2.5 + , struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = null, .c = 2.5 }); +} + +test "stringify a struct with all optionals" { + try testStringify("", struct { a: ?i64, b: ?f64 }{ .a = null, .b = null }); +} + +test "stringify an optional" { + try testStringify("", null); + try testStringify("", @as(?u64, null)); +} + +test "stringify a union" { + const Dummy = union(enum) { + x: u64, + y: f64, + }; + try testStringify("a: 1", struct { a: Dummy }{ .a = .{ .x = 1 } }); + try testStringify("a: 2.1", struct { a: Dummy }{ .a = .{ .y = 2.1 } }); +} + +test "stringify a string" { + try testStringify("a: name", struct { a: []const u8 }{ .a = "name" }); + try testStringify("name", "name"); +} + +test "stringify a list" { + try testStringify("[ 1, 2, 3 ]", @as([]const u64, &.{ 1, 2, 3 })); + try testStringify("[ 1, 2, 3 ]", .{ @as(i64, 1), 2, 3 }); + try testStringify("[ 1, name, 3 ]", .{ 1, "name", 3 }); + + const arr: [3]i64 = .{ 1, 2, 3 }; + try testStringify("[ 1, 2, 3 ]", arr); +} |