diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Compilation.zig | 2 | ||||
| -rw-r--r-- | src/Manifest.zig | 499 | ||||
| -rw-r--r-- | src/Module.zig | 6 | ||||
| -rw-r--r-- | src/Package.zig | 347 | ||||
| -rw-r--r-- | src/main.zig | 27 |
5 files changed, 654 insertions, 227 deletions
diff --git a/src/Compilation.zig b/src/Compilation.zig index 7d42d3b610..e09b8f18ab 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -385,7 +385,7 @@ pub const AllErrors = struct { count: u32 = 1, /// Does not include the trailing newline. source_line: ?[]const u8, - notes: []Message = &.{}, + notes: []const Message = &.{}, reference_trace: []Message = &.{}, /// Splits the error message up into lines to properly indent them diff --git a/src/Manifest.zig b/src/Manifest.zig new file mode 100644 index 0000000000..c3f77aec98 --- /dev/null +++ b/src/Manifest.zig @@ -0,0 +1,499 @@ +pub const basename = "build.zig.zon"; +pub const Hash = std.crypto.hash.sha2.Sha256; + +pub const Dependency = struct { + url: []const u8, + url_tok: Ast.TokenIndex, + hash: ?[]const u8, + hash_tok: Ast.TokenIndex, +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@enumToInt(multihash_function) < 127); + assert(Hash.digest_length < 127); +} +pub const multihash_len = 1 + 1 + Hash.digest_length; + +name: []const u8, +version: std.SemanticVersion, +dependencies: std.StringArrayHashMapUnmanaged(Dependency), + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest { + const node_tags = ast.nodes.items(.tag); + const node_datas = ast.nodes.items(.data); + assert(node_tags[0] == .root); + const main_node_index = node_datas[0].lhs; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast, + .arena = arena_instance.allocator(), + .errors = .{}, + + .name = undefined, + .version = undefined, + .dependencies = .{}, + .buf = .{}, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + + p.parseRoot(main_node_index) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .version = p.version, + .dependencies = try p.dependencies.clone(p.arena), + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +const hex_charset = "0123456789abcdef"; + +pub fn hex64(x: u64) [16]u8 { + var result: [16]u8 = undefined; + var i: usize = 0; + while (i < 8) : (i += 1) { + const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); + result[i * 2 + 0] = hex_charset[byte >> 4]; + result[i * 2 + 1] = hex_charset[byte & 15]; + } + return result; +} + +test hex64 { + const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; + try std.testing.expectEqualStrings("[00efcdab78563412]", s); +} + +pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { + var result: [multihash_len * 2]u8 = undefined; + + result[0] = hex_charset[@enumToInt(multihash_function) >> 4]; + result[1] = hex_charset[@enumToInt(multihash_function) & 15]; + + result[2] = hex_charset[Hash.digest_length >> 4]; + result[3] = hex_charset[Hash.digest_length & 15]; + + for (digest) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +const Parse = struct { + gpa: Allocator, + ast: std.zig.Ast, + arena: Allocator, + buf: std.ArrayListUnmanaged(u8), + errors: std.ArrayListUnmanaged(ErrorMessage), + + name: []const u8, + version: std.SemanticVersion, + dependencies: std.StringArrayHashMapUnmanaged(Dependency), + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const main_token = main_tokens[node]; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseString(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "version")) { + const version_text = try parseString(p, field_init); + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = main_tokens[node]; + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .url = undefined, + .url_tok = undefined, + .hash = null, + .hash_tok = undefined, + }; + var have_url = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + dep.url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.url_tok = main_tokens[field_init]; + have_url = true; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = main_tokens[field_init]; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_url) { + try appendError(p, main_tokens[node], "dependency is missing 'url' field", .{}); + } + + return dep; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const node_tags = ast.nodes.items(.tag); + const main_tokens = ast.nodes.items(.main_token); + if (node_tags[node] != .string_literal) { + return fail(p, main_tokens[node], "expected string literal", .{}); + } + const str_lit_token = main_tokens[node]; + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_tokens = ast.nodes.items(.main_token); + const tok = main_tokens[node]; + const h = try parseString(p, node); + + if (h.len >= 2) { + const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { + return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ + @errorName(err), + }); + }; + if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) { + return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); + } + } + + const hex_multihash_len = 2 * Manifest.multihash_len; + if (h.len != hex_multihash_len) { + return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ + hex_multihash_len, h.len, + }); + } + + return h; + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + const token_tags = ast.tokens.items(.tag); + assert(token_tags[token] == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayListUnmanaged(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + var buf_managed = buf.toManaged(p.gpa); + const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string); + buf.* = buf_managed.moveToUnmanaged(); + switch (try result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "unicode escape does not correspond to a valid codepoint", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @intCast(u32, bad_index), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +const Manifest = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + \\ }, + \\ }, + \\} + ; + + var ast = try std.zig.Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].url, + ); + try testing.expectEqualStrings( + "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); +} diff --git a/src/Module.zig b/src/Module.zig index b395c0a950..3bb15e78c3 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -2057,7 +2057,7 @@ pub const File = struct { if (file.tree_loaded) return &file.tree; const source = try file.getSource(gpa); - file.tree = try std.zig.parse(gpa, source.bytes); + file.tree = try Ast.parse(gpa, source.bytes, .zig); file.tree_loaded = true; return &file.tree; } @@ -3662,7 +3662,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void { file.source = source; file.source_loaded = true; - file.tree = try std.zig.parse(gpa, source); + file.tree = try Ast.parse(gpa, source, .zig); defer if (!file.tree_loaded) file.tree.deinit(gpa); if (file.tree.errors.len != 0) { @@ -3977,7 +3977,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { else => |e| return e, } - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; assert(file.tree.errors.len == 0); // builtin.zig must parse diff --git a/src/Package.zig b/src/Package.zig index 35b1ff5056..401eef2121 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -6,8 +6,8 @@ const fs = std.fs; const mem = std.mem; const Allocator = mem.Allocator; const assert = std.debug.assert; -const Hash = std.crypto.hash.sha2.Sha256; const log = std.log.scoped(.package); +const main = @import("main.zig"); const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); @@ -15,6 +15,7 @@ const ThreadPool = @import("ThreadPool.zig"); const WaitGroup = @import("WaitGroup.zig"); const Cache = @import("Cache.zig"); const build_options = @import("build_options"); +const Manifest = @import("Manifest.zig"); pub const Table = std.StringHashMapUnmanaged(*Package); @@ -141,10 +142,10 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, child: *Package) !void { } pub const build_zig_basename = "build.zig"; -pub const ini_basename = build_zig_basename ++ ".ini"; pub fn fetchAndAddDependencies( pkg: *Package, + arena: Allocator, thread_pool: *ThreadPool, http_client: *std.http.Client, directory: Compilation.Directory, @@ -153,89 +154,77 @@ pub fn fetchAndAddDependencies( dependencies_source: *std.ArrayList(u8), build_roots_source: *std.ArrayList(u8), name_prefix: []const u8, + color: main.Color, ) !void { const max_bytes = 10 * 1024 * 1024; const gpa = thread_pool.allocator; - const build_zig_ini = directory.handle.readFileAlloc(gpa, ini_basename, max_bytes) catch |err| switch (err) { + const build_zig_zon_bytes = directory.handle.readFileAllocOptions( + arena, + Manifest.basename, + max_bytes, + null, + 1, + 0, + ) catch |err| switch (err) { error.FileNotFound => { // Handle the same as no dependencies. return; }, else => |e| return e, }; - defer gpa.free(build_zig_ini); - const ini: std.Ini = .{ .bytes = build_zig_ini }; - var any_error = false; - var it = ini.iterateSection("\n[dependency]\n"); - while (it.next()) |dep| { - var line_it = mem.split(u8, dep, "\n"); - var opt_name: ?[]const u8 = null; - var opt_url: ?[]const u8 = null; - var expected_hash: ?[]const u8 = null; - while (line_it.next()) |kv| { - const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; - const key = kv[0..eq_pos]; - const value = kv[eq_pos + 1 ..]; - if (mem.eql(u8, key, "name")) { - opt_name = value; - } else if (mem.eql(u8, key, "url")) { - opt_url = value; - } else if (mem.eql(u8, key, "hash")) { - expected_hash = value; - } else { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - key, - }); - } - } + var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon); + defer ast.deinit(gpa); - const name = opt_name orelse { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - }); - any_error = true; - continue; - }; + if (ast.errors.len > 0) { + const file_path = try directory.join(arena, &.{Manifest.basename}); + try main.printErrsMsgToStdErr(gpa, arena, ast, file_path, color); + return error.PackageFetchFailed; + } - const url = opt_url orelse { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); - std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ - directory.path orelse ".", - "build.zig.ini", - loc.line, - loc.column, - }); - any_error = true; - continue; + var manifest = try Manifest.parse(gpa, ast); + defer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + const ttyconf: std.debug.TTY.Config = switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, }; + const file_path = try directory.join(arena, &.{Manifest.basename}); + for (manifest.errors) |msg| { + Report.renderErrorMessage(ast, file_path, ttyconf, msg, &.{}); + } + return error.PackageFetchFailed; + } + + const report: Report = .{ + .ast = &ast, + .directory = directory, + .color = color, + .arena = arena, + }; + + var any_error = false; + const deps_list = manifest.dependencies.values(); + for (manifest.dependencies.keys()) |name, i| { + const dep = deps_list[i]; - const sub_prefix = try std.fmt.allocPrint(gpa, "{s}{s}.", .{ name_prefix, name }); - defer gpa.free(sub_prefix); + const sub_prefix = try std.fmt.allocPrint(arena, "{s}{s}.", .{ name_prefix, name }); const fqn = sub_prefix[0 .. sub_prefix.len - 1]; const sub_pkg = try fetchAndUnpack( thread_pool, http_client, global_cache_directory, - url, - expected_hash, - ini, - directory, + dep, + report, build_roots_source, fqn, ); try pkg.fetchAndAddDependencies( + arena, thread_pool, http_client, sub_pkg.root_src_directory, @@ -244,6 +233,7 @@ pub fn fetchAndAddDependencies( dependencies_source, build_roots_source, sub_prefix, + color, ); try addAndAdopt(pkg, gpa, sub_pkg); @@ -253,7 +243,7 @@ pub fn fetchAndAddDependencies( }); } - if (any_error) return error.InvalidBuildZigIniFile; + if (any_error) return error.InvalidBuildManifestFile; } pub fn createFilePkg( @@ -264,7 +254,7 @@ pub fn createFilePkg( contents: []const u8, ) !*Package { const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int); { var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); defer tmp_dir.close(); @@ -282,14 +272,73 @@ pub fn createFilePkg( return createWithDir(gpa, name, cache_directory, o_dir_sub_path, basename); } +const Report = struct { + ast: *const std.zig.Ast, + directory: Compilation.Directory, + color: main.Color, + arena: Allocator, + + fn fail( + report: Report, + tok: std.zig.Ast.TokenIndex, + comptime fmt_string: []const u8, + fmt_args: anytype, + ) error{ PackageFetchFailed, OutOfMemory } { + return failWithNotes(report, &.{}, tok, fmt_string, fmt_args); + } + + fn failWithNotes( + report: Report, + notes: []const Compilation.AllErrors.Message, + tok: std.zig.Ast.TokenIndex, + comptime fmt_string: []const u8, + fmt_args: anytype, + ) error{ PackageFetchFailed, OutOfMemory } { + const ttyconf: std.debug.TTY.Config = switch (report.color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + const file_path = try report.directory.join(report.arena, &.{Manifest.basename}); + renderErrorMessage(report.ast.*, file_path, ttyconf, .{ + .tok = tok, + .off = 0, + .msg = try std.fmt.allocPrint(report.arena, fmt_string, fmt_args), + }, notes); + return error.PackageFetchFailed; + } + + fn renderErrorMessage( + ast: std.zig.Ast, + file_path: []const u8, + ttyconf: std.debug.TTY.Config, + msg: Manifest.ErrorMessage, + notes: []const Compilation.AllErrors.Message, + ) void { + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, msg.tok); + Compilation.AllErrors.Message.renderToStdErr(.{ .src = .{ + .msg = msg.msg, + .src_path = file_path, + .line = @intCast(u32, start_loc.line), + .column = @intCast(u32, start_loc.column), + .span = .{ + .start = token_starts[msg.tok], + .end = @intCast(u32, token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), + .main = token_starts[msg.tok] + msg.off, + }, + .source_line = ast.source[start_loc.line_start..start_loc.line_end], + .notes = notes, + } }, ttyconf); + } +}; + fn fetchAndUnpack( thread_pool: *ThreadPool, http_client: *std.http.Client, global_cache_directory: Compilation.Directory, - url: []const u8, - expected_hash: ?[]const u8, - ini: std.Ini, - comp_directory: Compilation.Directory, + dep: Manifest.Dependency, + report: Report, build_roots_source: *std.ArrayList(u8), fqn: []const u8, ) !*Package { @@ -298,37 +347,8 @@ fn fetchAndUnpack( // Check if the expected_hash is already present in the global package // cache, and thereby avoid both fetching and unpacking. - if (expected_hash) |h| cached: { - const hex_multihash_len = 2 * multihash_len; - if (h.len >= 2) { - const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { - return reportError( - ini, - comp_directory, - h.ptr, - "invalid multihash value: unable to parse hash function: {s}", - .{@errorName(err)}, - ); - }; - if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) { - return reportError( - ini, - comp_directory, - h.ptr, - "unsupported hash function: only sha2-256 is supported", - .{}, - ); - } - } - if (h.len != hex_multihash_len) { - return reportError( - ini, - comp_directory, - h.ptr, - "wrong hash size. expected: {d}, found: {d}", - .{ hex_multihash_len, h.len }, - ); - } + if (dep.hash) |h| cached: { + const hex_multihash_len = 2 * Manifest.multihash_len; const hex_digest = h[0..hex_multihash_len]; const pkg_dir_sub_path = "p" ++ s ++ hex_digest; var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { @@ -366,10 +386,10 @@ fn fetchAndUnpack( return ptr; } - const uri = try std.Uri.parse(url); + const uri = try std.Uri.parse(dep.url); const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); const actual_hash = a: { var tmp_directory: Compilation.Directory = d: { @@ -398,13 +418,9 @@ fn fetchAndUnpack( // by default, so the same logic applies for buffering the reader as for gzip. try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz); } else { - return reportError( - ini, - comp_directory, - uri.path.ptr, - "unknown file extension for path '{s}'", - .{uri.path}, - ); + return report.fail(dep.url_tok, "unknown file extension for path '{s}'", .{ + uri.path, + }); } // TODO: delete files not included in the package prior to computing the package hash. @@ -415,28 +431,21 @@ fn fetchAndUnpack( break :a try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); }; - const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); + const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash); try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path); - const actual_hex = hexDigest(actual_hash); - if (expected_hash) |h| { + const actual_hex = Manifest.hexDigest(actual_hash); + if (dep.hash) |h| { if (!mem.eql(u8, h, &actual_hex)) { - return reportError( - ini, - comp_directory, - h.ptr, - "hash mismatch: expected: {s}, found: {s}", - .{ h, actual_hex }, - ); + return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ + h, actual_hex, + }); } } else { - return reportError( - ini, - comp_directory, - url.ptr, - "url field is missing corresponding hash field: hash={s}", - .{&actual_hex}, - ); + const notes: [1]Compilation.AllErrors.Message = .{.{ .plain = .{ + .msg = try std.fmt.allocPrint(report.arena, "expected .hash = \"{s}\",", .{&actual_hex}), + } }}; + return report.failWithNotes(¬es, dep.url_tok, "url field is missing corresponding hash field", .{}); } const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); @@ -471,29 +480,9 @@ fn unpackTarball( }); } -fn reportError( - ini: std.Ini, - comp_directory: Compilation.Directory, - src_ptr: [*]const u8, - comptime fmt_string: []const u8, - fmt_args: anytype, -) error{PackageFetchFailed} { - const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(src_ptr) - @ptrToInt(ini.bytes.ptr)); - if (comp_directory.path) |p| { - std.debug.print("{s}{c}{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ - p, fs.path.sep, ini_basename, loc.line + 1, loc.column + 1, - } ++ fmt_args); - } else { - std.debug.print("{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ - ini_basename, loc.line + 1, loc.column + 1, - } ++ fmt_args); - } - return error.PackageFetchFailed; -} - const HashedFile = struct { path: []const u8, - hash: [Hash.digest_length]u8, + hash: [Manifest.Hash.digest_length]u8, failure: Error!void, const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError; @@ -507,7 +496,7 @@ const HashedFile = struct { fn computePackageHash( thread_pool: *ThreadPool, pkg_dir: fs.IterableDir, -) ![Hash.digest_length]u8 { +) ![Manifest.Hash.digest_length]u8 { const gpa = thread_pool.allocator; // We'll use an arena allocator for the path name strings since they all @@ -550,7 +539,7 @@ fn computePackageHash( std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); - var hasher = Hash.init(.{}); + var hasher = Manifest.Hash.init(.{}); var any_failures = false; for (all_files.items) |hashed_file| { hashed_file.failure catch |err| { @@ -571,7 +560,7 @@ fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { var buf: [8000]u8 = undefined; var file = try dir.openFile(hashed_file.path, .{}); - var hasher = Hash.init(.{}); + var hasher = Manifest.Hash.init(.{}); hasher.update(hashed_file.path); hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) }); while (true) { @@ -595,52 +584,6 @@ fn isExecutable(file: fs.File) !bool { } } -const hex_charset = "0123456789abcdef"; - -fn hex64(x: u64) [16]u8 { - var result: [16]u8 = undefined; - var i: usize = 0; - while (i < 8) : (i += 1) { - const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); - result[i * 2 + 0] = hex_charset[byte >> 4]; - result[i * 2 + 1] = hex_charset[byte & 15]; - } - return result; -} - -test hex64 { - const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; - try std.testing.expectEqualStrings("[00efcdab78563412]", s); -} - -const multihash_function: MultihashFunction = switch (Hash) { - std.crypto.hash.sha2.Sha256 => .@"sha2-256", - else => @compileError("unreachable"), -}; -comptime { - // We avoid unnecessary uleb128 code in hexDigest by asserting here the - // values are small enough to be contained in the one-byte encoding. - assert(@enumToInt(multihash_function) < 127); - assert(Hash.digest_length < 127); -} -const multihash_len = 1 + 1 + Hash.digest_length; - -fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { - var result: [multihash_len * 2]u8 = undefined; - - result[0] = hex_charset[@enumToInt(multihash_function) >> 4]; - result[1] = hex_charset[@enumToInt(multihash_function) & 15]; - - result[2] = hex_charset[Hash.digest_length >> 4]; - result[3] = hex_charset[Hash.digest_length & 15]; - - for (digest) |byte, i| { - result[4 + i * 2] = hex_charset[byte >> 4]; - result[5 + i * 2] = hex_charset[byte & 15]; - } - return result; -} - fn renameTmpIntoCache( cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, @@ -669,21 +612,3 @@ fn renameTmpIntoCache( break; } } - -const MultihashFunction = enum(u16) { - identity = 0x00, - sha1 = 0x11, - @"sha2-256" = 0x12, - @"sha2-512" = 0x13, - @"sha3-512" = 0x14, - @"sha3-384" = 0x15, - @"sha3-256" = 0x16, - @"sha3-224" = 0x17, - @"sha2-384" = 0x20, - @"sha2-256-trunc254-padded" = 0x1012, - @"sha2-224" = 0x1013, - @"sha2-512-224" = 0x1014, - @"sha2-512-256" = 0x1015, - @"blake2b-256" = 0xb220, - _, -}; diff --git a/src/main.zig b/src/main.zig index 72e7e094e6..f634c259ff 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3915,6 +3915,7 @@ pub const usage_build = ; pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + var color: Color = .auto; var prominent_compile_errors: bool = false; // We want to release all the locks before executing the child process, so we make a nice @@ -4117,6 +4118,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // Here we borrow main package's table and will replace it with a fresh // one after this process completes. main_pkg.fetchAndAddDependencies( + arena, &thread_pool, &http_client, build_directory, @@ -4125,6 +4127,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &dependencies_source, &build_roots_source, "", + color, ) catch |err| switch (err) { error.PackageFetchFailed => process.exit(1), else => |e| return e, @@ -4361,12 +4364,12 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void }; defer gpa.free(source_code); - var tree = std.zig.parse(gpa, source_code) catch |err| { + var tree = Ast.parse(gpa, source_code, .zig) catch |err| { fatal("error parsing stdin: {}", .{err}); }; defer tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, tree.errors, tree, "<stdin>", color); + try printErrsMsgToStdErr(gpa, arena, tree, "<stdin>", color); var has_ast_error = false; if (check_ast_flag) { const Module = @import("Module.zig"); @@ -4566,10 +4569,10 @@ fn fmtPathFile( // Add to set after no longer possible to get error.IsDir. if (try fmt.seen.fetchPut(stat.inode, {})) |_| return; - var tree = try std.zig.parse(fmt.gpa, source_code); + var tree = try Ast.parse(fmt.gpa, source_code, .zig); defer tree.deinit(fmt.gpa); - try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree.errors, tree, file_path, fmt.color); + try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree, file_path, fmt.color); if (tree.errors.len != 0) { fmt.any_error = true; return; @@ -4649,14 +4652,14 @@ fn fmtPathFile( } } -fn printErrsMsgToStdErr( +pub fn printErrsMsgToStdErr( gpa: mem.Allocator, arena: mem.Allocator, - parse_errors: []const Ast.Error, tree: Ast, path: []const u8, color: Color, ) !void { + const parse_errors: []const Ast.Error = tree.errors; var i: usize = 0; while (i < parse_errors.len) : (i += 1) { const parse_error = parse_errors[i]; @@ -5312,11 +5315,11 @@ pub fn cmdAstCheck( file.pkg = try Package.create(gpa, "root", null, file.sub_file_path); defer file.pkg.destroy(gpa); - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree.errors, file.tree, file.sub_file_path, color); + try printErrsMsgToStdErr(gpa, arena, file.tree, file.sub_file_path, color); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5438,11 +5441,11 @@ pub fn cmdChangelist( file.source = source; file.source_loaded = true; - file.tree = try std.zig.parse(gpa, file.source); + file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree.errors, file.tree, old_source_file, .auto); + try printErrsMsgToStdErr(gpa, arena, file.tree, old_source_file, .auto); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5476,10 +5479,10 @@ pub fn cmdChangelist( if (new_amt != new_stat.size) return error.UnexpectedEndOfFile; - var new_tree = try std.zig.parse(gpa, new_source); + var new_tree = try Ast.parse(gpa, new_source, .zig); defer new_tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, new_tree.errors, new_tree, new_source_file, .auto); + try printErrsMsgToStdErr(gpa, arena, new_tree, new_source_file, .auto); if (new_tree.errors.len != 0) { process.exit(1); } |