From 5b90fa05a4e5b155f25319713acfc67ad9516c69 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 13 Feb 2023 13:39:06 -0700 Subject: extract ThreadPool and WaitGroup from compiler to std lib --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index 95cfca1463..dd0faa628c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,6 +9,7 @@ const Allocator = mem.Allocator; const ArrayList = std.ArrayList; const Ast = std.zig.Ast; const warn = std.log.warn; +const ThreadPool = std.Thread.Pool; const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); @@ -22,7 +23,6 @@ const translate_c = @import("translate_c.zig"); const clang = @import("clang.zig"); const Cache = std.Build.Cache; const target_util = @import("target.zig"); -const ThreadPool = @import("ThreadPool.zig"); const crash_report = @import("crash_report.zig"); pub const std_options = struct { -- cgit v1.2.3 From cb094700631ea1ae238ea678c192ce4f85fbecc0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 19 Feb 2023 16:21:36 -0700 Subject: zig build: add a -j option for limiting concurrency --- lib/build_runner.zig | 20 ++++++++++++++++++-- lib/std/Thread/Pool.zig | 11 +++++++++-- src/main.zig | 4 ++-- 3 files changed, 29 insertions(+), 6 deletions(-) (limited to 'src/main.zig') diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 24fd3440a3..e12d46f03d 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -87,6 +87,7 @@ pub fn main() !void { var targets = ArrayList([]const u8).init(allocator); var debug_log_scopes = ArrayList([]const u8).init(allocator); + var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = allocator }; const stderr_stream = io.getStdErr().writer(); const stdout_stream = io.getStdOut().writer(); @@ -231,6 +232,19 @@ pub fn main() !void { }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { builder.reference_trace = null; + } else if (mem.startsWith(u8, arg, "-j")) { + const num = arg["-j".len..]; + const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse jobs count '{s}': {s}", .{ + num, @errorName(err), + }); + process.exit(1); + }; + if (n_jobs < 1) { + std.debug.print("number of jobs must be at least 1\n", .{}); + process.exit(1); + } + thread_pool_options.n_jobs = n_jobs; } else if (mem.eql(u8, arg, "--")) { builder.args = argsRest(args, arg_idx); break; @@ -258,7 +272,7 @@ pub fn main() !void { if (builder.validateUserInputDidItFail()) usageAndErr(builder, true, stderr_stream); - runStepNames(builder, targets.items, main_progress_node) catch |err| { + runStepNames(builder, targets.items, main_progress_node, thread_pool_options) catch |err| { switch (err) { error.UncleanExit => process.exit(1), else => return err, @@ -270,6 +284,7 @@ fn runStepNames( b: *std.Build, step_names: []const []const u8, parent_prog_node: *std.Progress.Node, + thread_pool_options: std.Thread.Pool.Options, ) !void { var step_stack = ArrayList(*Step).init(b.allocator); defer step_stack.deinit(); @@ -297,7 +312,7 @@ fn runStepNames( } var thread_pool: std.Thread.Pool = undefined; - try thread_pool.init(b.allocator); + try thread_pool.init(thread_pool_options); defer thread_pool.deinit(); { @@ -523,6 +538,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\ --verbose Print commands before executing them \\ --color [auto|off|on] Enable or disable colored error messages \\ --prominent-compile-errors Output compile errors formatted for a human to read + \\ -j Limit concurrent jobs (default is to use all CPU cores) \\ \\Project-Specific Options: \\ diff --git a/lib/std/Thread/Pool.zig b/lib/std/Thread/Pool.zig index 930befbac5..ed1a4dc052 100644 --- a/lib/std/Thread/Pool.zig +++ b/lib/std/Thread/Pool.zig @@ -17,7 +17,14 @@ const Runnable = struct { const RunProto = *const fn (*Runnable) void; -pub fn init(pool: *Pool, allocator: std.mem.Allocator) !void { +pub const Options = struct { + allocator: std.mem.Allocator, + n_jobs: ?u32 = null, +}; + +pub fn init(pool: *Pool, options: Options) !void { + const allocator = options.allocator; + pool.* = .{ .allocator = allocator, .threads = &[_]std.Thread{}, @@ -27,7 +34,7 @@ pub fn init(pool: *Pool, allocator: std.mem.Allocator) !void { return; } - const thread_count = std.math.max(1, std.Thread.getCpuCount() catch 1); + const thread_count = options.n_jobs orelse @max(1, std.Thread.getCpuCount() catch 1); pool.threads = try allocator.alloc(std.Thread, thread_count); errdefer allocator.free(pool.threads); diff --git a/src/main.zig b/src/main.zig index dd0faa628c..3a06d0272f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2999,7 +2999,7 @@ fn buildOutputType( defer zig_lib_directory.handle.close(); var thread_pool: ThreadPool = undefined; - try thread_pool.init(gpa); + try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); var libc_installation: ?LibCInstallation = null; @@ -4201,7 +4201,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .basename = exe_basename, }; var thread_pool: ThreadPool = undefined; - try thread_pool.init(gpa); + try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); var cleanup_build_runner_dir: ?fs.Dir = null; -- cgit v1.2.3 From ae8e7c8f5a6065d12087aa547971001a399667b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 14 Jan 2022 17:04:03 -0700 Subject: stage2: hot code swapping PoC * CLI supports --listen to accept commands on a socket * make it able to produce an updated executable while it is running --- lib/std/child_process.zig | 1 + src/Compilation.zig | 7 ++ src/link.zig | 13 ++++ src/main.zig | 187 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+) (limited to 'src/main.zig') diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 3bdef3177a..f8cba85874 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -185,6 +185,7 @@ pub const ChildProcess = struct { } /// Blocks until child process terminates and then cleans up all resources. + /// TODO: set the pid to undefined in this function. pub fn wait(self: *ChildProcess) !Term { const term = if (builtin.os.tag == .windows) try self.waitWindows() diff --git a/src/Compilation.zig b/src/Compilation.zig index 63dd229ec5..83e0f0d5d9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5663,3 +5663,10 @@ pub fn compilerRtStrip(comp: Compilation) bool { return true; } } + +pub fn hotCodeSwap(comp: *Compilation, pid: std.os.pid_t) !void { + comp.bin_file.child_pid = pid; + try comp.makeBinFileWritable(); + try comp.update(); + try comp.makeBinFileExecutable(); +} diff --git a/src/link.zig b/src/link.zig index 1ecbeadd7f..34a16a1a1b 100644 --- a/src/link.zig +++ b/src/link.zig @@ -264,6 +264,8 @@ pub const File = struct { /// of this linking operation. lock: ?Cache.Lock = null, + child_pid: ?std.os.pid_t = null, + /// Attempts incremental linking, if the file already exists. If /// incremental linking fails, falls back to truncating the file and /// rewriting it. A malicious file is detected as incremental link failure @@ -376,6 +378,17 @@ pub const File = struct { if (build_options.only_c) unreachable; if (base.file != null) return; const emit = base.options.emit orelse return; + if (base.child_pid != null) { + // If we try to open the output file in write mode while it is running, + // it will return ETXTBSY. So instead, we copy the file, atomically rename it + // over top of the exe path, and then proceed normally. This changes the inode, + // avoiding the error. + const tmp_sub_path = try std.fmt.allocPrint(base.allocator, "{s}-{x}", .{ + emit.sub_path, std.crypto.random.int(u32), + }); + try emit.directory.handle.copyFile(emit.sub_path, emit.directory.handle, tmp_sub_path, .{}); + try emit.directory.handle.rename(tmp_sub_path, emit.sub_path); + } base.file = try emit.directory.handle.createFile(emit.sub_path, .{ .truncate = false, .read = true, diff --git a/src/main.zig b/src/main.zig index 3a06d0272f..a485fdb9e2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -687,6 +687,7 @@ fn buildOutputType( var function_sections = false; var no_builtin = false; var watch = false; + var listen_addr: ?std.net.Ip4Address = null; var debug_compile_errors = false; var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK"); var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_CC"); @@ -1144,6 +1145,17 @@ fn buildOutputType( } else { try log_scopes.append(gpa, args_iter.nextOrFatal()); } + } else if (mem.eql(u8, arg, "--listen")) { + const next_arg = args_iter.nextOrFatal(); + // example: --listen 127.0.0.1:9000 + var it = std.mem.split(u8, next_arg, ":"); + const host = it.next().?; + const port_text = it.next() orelse "14735"; + const port = std.fmt.parseInt(u16, port_text, 10) catch |err| + fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) }); + listen_addr = std.net.Ip4Address.parse(host, port) catch |err| + fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }); + watch = true; } else if (mem.eql(u8, arg, "--debug-link-snapshot")) { if (!build_options.enable_link_snapshots) { std.log.warn("Zig was compiled without linker snapshots enabled (-Dlink-snapshot). --debug-link-snapshot has no effect.", .{}); @@ -3353,6 +3365,125 @@ fn buildOutputType( var last_cmd: ReplCmd = .help; + if (listen_addr) |ip4_addr| { + var server = std.net.StreamServer.init(.{ + .reuse_address = true, + }); + defer server.deinit(); + + try server.listen(.{ .in = ip4_addr }); + + while (true) { + const conn = try server.accept(); + defer conn.stream.close(); + + var buf: [100]u8 = undefined; + var child_pid: ?i32 = null; + + while (true) { + try comp.makeBinFileExecutable(); + + const amt = try conn.stream.read(&buf); + const line = buf[0..amt]; + const actual_line = mem.trimRight(u8, line, "\r\n "); + + const cmd: ReplCmd = blk: { + if (mem.eql(u8, actual_line, "update")) { + break :blk .update; + } else if (mem.eql(u8, actual_line, "exit")) { + break; + } else if (mem.eql(u8, actual_line, "help")) { + break :blk .help; + } else if (mem.eql(u8, actual_line, "run")) { + break :blk .run; + } else if (mem.eql(u8, actual_line, "update-and-run")) { + break :blk .update_and_run; + } else if (actual_line.len == 0) { + break :blk last_cmd; + } else { + try stderr.print("unknown command: {s}\n", .{actual_line}); + continue; + } + }; + last_cmd = cmd; + switch (cmd) { + .update => { + tracy.frameMark(); + if (output_mode == .Exe) { + try comp.makeBinFileWritable(); + } + updateModule(gpa, comp, hook) catch |err| switch (err) { + error.SemanticAnalyzeFail => continue, + else => |e| return e, + }; + }, + .help => { + try stderr.writeAll(repl_help); + }, + .run => { + tracy.frameMark(); + try runOrTest( + comp, + gpa, + arena, + test_exec_args.items, + self_exe_path.?, + arg_mode, + target_info, + watch, + &comp_destroyed, + all_args, + runtime_args_start, + link_libc, + ); + }, + .update_and_run => { + tracy.frameMark(); + if (child_pid) |pid| { + try conn.stream.writer().print("hot code swap requested for pid {d}", .{pid}); + try comp.hotCodeSwap(pid); + + var errors = try comp.getAllErrorsAlloc(); + defer errors.deinit(comp.gpa); + + if (errors.list.len != 0) { + const ttyconf: std.debug.TTY.Config = switch (comp.color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + for (errors.list) |full_err_msg| { + try full_err_msg.renderToWriter(ttyconf, conn.stream.writer(), "error:", .Red, 0); + } + continue; + } + } else { + if (output_mode == .Exe) { + try comp.makeBinFileWritable(); + } + updateModule(gpa, comp, hook) catch |err| switch (err) { + error.SemanticAnalyzeFail => continue, + else => |e| return e, + }; + try comp.makeBinFileExecutable(); + + child_pid = try runOrTestHotSwap( + comp, + gpa, + arena, + test_exec_args.items, + self_exe_path.?, + arg_mode, + all_args, + runtime_args_start, + ); + } + }, + } + } + } + } + while (watch) { try stderr.print("(zig) ", .{}); try comp.makeBinFileExecutable(); @@ -3631,6 +3762,62 @@ fn runOrTest( } } +fn runOrTestHotSwap( + comp: *Compilation, + gpa: Allocator, + arena: Allocator, + test_exec_args: []const ?[]const u8, + self_exe_path: []const u8, + arg_mode: ArgMode, + all_args: []const []const u8, + runtime_args_start: ?usize, +) !i32 { + const exe_emit = comp.bin_file.options.emit.?; + // A naive `directory.join` here will indeed get the correct path to the binary, + // however, in the case of cwd, we actually want `./foo` so that the path can be executed. + const exe_path = try fs.path.join(arena, &[_][]const u8{ + exe_emit.directory.path orelse ".", exe_emit.sub_path, + }); + + var argv = std.ArrayList([]const u8).init(gpa); + defer argv.deinit(); + + if (test_exec_args.len == 0) { + // when testing pass the zig_exe_path to argv + if (arg_mode == .zig_test) + try argv.appendSlice(&[_][]const u8{ + exe_path, self_exe_path, + }) + // when running just pass the current exe + else + try argv.appendSlice(&[_][]const u8{ + exe_path, + }); + } else { + for (test_exec_args) |arg| { + if (arg) |a| { + try argv.append(a); + } else { + try argv.appendSlice(&[_][]const u8{ + exe_path, self_exe_path, + }); + } + } + } + if (runtime_args_start) |i| { + try argv.appendSlice(all_args[i..]); + } + var child = std.ChildProcess.init(argv.items, arena); + + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + try child.spawn(); + + return child.pid; +} + const AfterUpdateHook = union(enum) { none, print_emit_bin_dir_path, -- cgit v1.2.3 From 572cb24d1a4f70c662ddf17df72d27dec44bc4fc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 23 Feb 2023 16:18:43 -0700 Subject: progress towards semantic error serialization Introduces std.zig.ErrorBundle which is a trivially serializeable set of compilation errors. This is in the standard library so that both the compiler and the build runner can use it. The idea is they will use it to communicate compilation errors over a binary protocol. The binary encoding of ErrorBundle is a bit problematic - I got a little too aggressive with compaction. I need to change it in a follow-up commit to use some indirection in the error message list, otherwise iteration is too unergonomic. In fact it's so problematic right now that the logic getAllErrorsAlloc() actually fails to produce a viable ErrorBundle because it puts SourceLocation data in between the root level ErrorMessage data. This commit has a simplification - redundant logic for rendering AST errors to stderr has been removed in favor of moving the logic for lowering AST errors into AstGen. So even if we get parse errors, the errors will get lowered into ZIR before being reported. I believe this will be useful when working on --autofix. Either way, some redundant brittle logic was happily deleted. In Compilation, updateSubCompilation() is improved to properly perform error reporting when a sub-compilation object fails. It no longer dumps directly to stderr; instead it populates an ErrorBundle object, which gets added to the parent one during getAllErrorsAlloc(). In package fetching code, instead of dumping directly to stderr, it now populates an ErrorBundle object, and gets properly reported at the CLI layer of abstraction. --- lib/std/zig.zig | 1 + lib/std/zig/ErrorBundle.zig | 419 ++++++++++++++++++++ src/AstGen.zig | 96 +++-- src/Compilation.zig | 945 ++++++++++++++++++-------------------------- src/Module.zig | 60 +-- src/Package.zig | 108 +++-- src/Sema.zig | 25 +- src/glibc.zig | 10 +- src/libcxx.zig | 4 +- src/libtsan.zig | 2 +- src/libunwind.zig | 2 +- src/main.zig | 263 +++++------- src/mingw.zig | 12 +- src/musl.zig | 14 +- src/wasi_libc.zig | 14 +- 15 files changed, 1067 insertions(+), 908 deletions(-) create mode 100644 lib/std/zig/ErrorBundle.zig (limited to 'src/main.zig') diff --git a/lib/std/zig.zig b/lib/std/zig.zig index f85cf75e60..ecff8e99bd 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -3,6 +3,7 @@ const tokenizer = @import("zig/tokenizer.zig"); const fmt = @import("zig/fmt.zig"); const assert = std.debug.assert; +pub const ErrorBundle = @import("zig/ErrorBundle.zig"); pub const Token = tokenizer.Token; pub const Tokenizer = tokenizer.Tokenizer; pub const fmtId = fmt.fmtId; diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig new file mode 100644 index 0000000000..dee19818a8 --- /dev/null +++ b/lib/std/zig/ErrorBundle.zig @@ -0,0 +1,419 @@ +//! To support incremental compilation, errors are stored in various places +//! so that they can be created and destroyed appropriately. This structure +//! is used to collect all the errors from the various places into one +//! convenient place for API users to consume. + +string_bytes: std.ArrayListUnmanaged(u8), +/// The first thing in this array is a ErrorMessageListIndex. +extra: std.ArrayListUnmanaged(u32), + +// An index into `extra` pointing at an `ErrorMessage`. +pub const MessageIndex = enum(u32) { + _, +}; + +/// After the header is: +/// * string_bytes +/// * extra (little endian) +pub const Header = struct { + string_bytes_len: u32, + extra_len: u32, +}; + +/// Trailing: ErrorMessage for each len +pub const ErrorMessageList = struct { + len: u32, + start: u32, +}; + +/// Trailing: +/// * ReferenceTrace for each reference_trace_len +pub const SourceLocation = struct { + /// null terminated string index + src_path: u32, + line: u32, + column: u32, + /// byte offset of starting token + span_start: u32, + /// byte offset of main error location + span_main: u32, + /// byte offset of end of last token + span_end: u32, + /// null terminated string index, possibly null. + /// Does not include the trailing newline. + source_line: u32 = 0, + reference_trace_len: u32 = 0, +}; + +/// Trailing: +/// * ErrorMessage for each notes_len. +pub const ErrorMessage = struct { + /// null terminated string index + msg: u32, + /// Usually one, but incremented for redundant messages. + count: u32 = 1, + /// 0 or the index into extra of a SourceLocation + src_loc: u32 = 0, + notes_len: u32 = 0, +}; + +pub const ReferenceTrace = struct { + /// null terminated string index + /// Except for the sentinel ReferenceTrace element, in which case: + /// * 0 means remaining references hidden + /// * >0 means N references hidden + decl_name: u32, + /// Index into extra of a SourceLocation + /// If this is 0, this is the sentinel ReferenceTrace element. + src_loc: u32, +}; + +pub fn init(eb: *ErrorBundle, gpa: Allocator) !void { + eb.* = .{ + .string_bytes = .{}, + .extra = .{}, + }; + + // So that 0 can be used to indicate a null string. + try eb.string_bytes.append(gpa, 0); + + _ = try addExtra(eb, gpa, ErrorMessageList{ + .len = 0, + .start = 0, + }); +} + +pub fn deinit(eb: *ErrorBundle, gpa: Allocator) void { + eb.string_bytes.deinit(gpa); + eb.extra.deinit(gpa); + eb.* = undefined; +} + +pub fn addString(eb: *ErrorBundle, gpa: Allocator, s: []const u8) !u32 { + const index = @intCast(u32, eb.string_bytes.items.len); + try eb.string_bytes.ensureUnusedCapacity(gpa, s.len + 1); + eb.string_bytes.appendSliceAssumeCapacity(s); + eb.string_bytes.appendAssumeCapacity(0); + return index; +} + +pub fn printString(eb: *ErrorBundle, gpa: Allocator, comptime fmt: []const u8, args: anytype) !u32 { + const index = @intCast(u32, eb.string_bytes.items.len); + try eb.string_bytes.writer(gpa).print(fmt, args); + try eb.string_bytes.append(gpa, 0); + return index; +} + +pub fn addErrorMessage(eb: *ErrorBundle, gpa: Allocator, em: ErrorMessage) !void { + if (eb.errorMessageCount() == 0) { + eb.setStartIndex(@intCast(u32, eb.extra.items.len)); + } + _ = try addExtra(eb, gpa, em); +} + +pub fn addSourceLocation(eb: *ErrorBundle, gpa: Allocator, sl: SourceLocation) !u32 { + return addExtra(eb, gpa, sl); +} + +pub fn addReferenceTrace(eb: *ErrorBundle, gpa: Allocator, rt: ReferenceTrace) !void { + _ = try addExtra(eb, gpa, rt); +} + +pub fn addBundle(eb: *ErrorBundle, gpa: Allocator, other: ErrorBundle) !void { + // Skip over the initial ErrorMessageList len field. + const root_fields_len = @typeInfo(ErrorMessageList).Struct.fields.len; + const other_list = other.extraData(ErrorMessageList, 0).data; + const other_extra = other.extra.items[root_fields_len..]; + + try eb.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.items.len); + try eb.extra.ensureUnusedCapacity(gpa, other_extra.len); + + const new_string_base = @intCast(u32, eb.string_bytes.items.len); + const new_data_base = @intCast(u32, eb.extra.items.len - root_fields_len); + + eb.string_bytes.appendSliceAssumeCapacity(other.string_bytes.items); + eb.extra.appendSliceAssumeCapacity(other_extra); + + // Now we must offset the string indexes and extra indexes of the newly + // added extra. + var index = new_data_base + other_list.start; + for (0..other_list.len) |_| { + index = try patchMessage(eb, index, new_string_base, new_data_base); + } +} + +fn patchMessage(eb: *ErrorBundle, msg_idx: usize, new_string_base: u32, new_data_base: u32) !u32 { + var msg = eb.extraData(ErrorMessage, msg_idx); + if (msg.data.msg != 0) msg.data.msg += new_string_base; + if (msg.data.src_loc != 0) msg.data.src_loc += new_data_base; + eb.setExtra(msg_idx, msg.data); + + try patchSrcLoc(eb, msg.data.src_loc, new_string_base, new_data_base); + + var index = @intCast(u32, msg.end); + for (0..msg.data.notes_len) |_| { + index = try patchMessage(eb, index, new_string_base, new_data_base); + } + return index; +} + +fn patchSrcLoc(eb: *ErrorBundle, idx: usize, new_string_base: u32, new_data_base: u32) !void { + if (idx == 0) return; + + var src_loc = eb.extraData(SourceLocation, idx); + if (src_loc.data.src_path != 0) src_loc.data.src_path += new_string_base; + if (src_loc.data.source_line != 0) src_loc.data.source_line += new_string_base; + eb.setExtra(idx, src_loc.data); + + var index = src_loc.end; + for (0..src_loc.data.reference_trace_len) |_| { + var ref_trace = eb.extraData(ReferenceTrace, index); + if (ref_trace.data.decl_name != 0) ref_trace.data.decl_name += new_string_base; + if (ref_trace.data.src_loc != 0) ref_trace.data.src_loc += new_data_base; + eb.setExtra(index, ref_trace.data); + try patchSrcLoc(eb, ref_trace.data.src_loc, new_string_base, new_data_base); + index = ref_trace.end; + } +} + +fn addExtra(eb: *ErrorBundle, gpa: Allocator, extra: anytype) Allocator.Error!u32 { + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + try eb.extra.ensureUnusedCapacity(gpa, fields.len); + return addExtraAssumeCapacity(eb, extra); +} + +fn addExtraAssumeCapacity(eb: *ErrorBundle, extra: anytype) u32 { + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + const result = @intCast(u32, eb.extra.items.len); + eb.extra.items.len += fields.len; + setExtra(eb, result, extra); + return result; +} + +fn setExtra(eb: *ErrorBundle, index: usize, extra: anytype) void { + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + var i = index; + inline for (fields) |field| { + eb.extra.items[i] = switch (field.type) { + u32 => @field(extra, field.name), + else => @compileError("bad field type"), + }; + i += 1; + } +} + +pub fn errorMessageCount(eb: ErrorBundle) u32 { + return eb.extra.items[0]; +} + +pub fn setErrorMessageCount(eb: *ErrorBundle, count: u32) void { + eb.extra.items[0] = count; +} + +pub fn incrementCount(eb: *ErrorBundle, delta: u32) void { + eb.extra.items[0] += delta; +} + +pub fn getStartIndex(eb: ErrorBundle) u32 { + return eb.extra.items[1]; +} + +pub fn setStartIndex(eb: *ErrorBundle, index: u32) void { + eb.extra.items[1] = index; +} + +pub fn getErrorMessage(eb: ErrorBundle, index: MessageIndex) ErrorMessage { + return eb.extraData(ErrorMessage, @enumToInt(index)).data; +} + +pub fn getSourceLocation(eb: ErrorBundle, index: u32) SourceLocation { + assert(index != 0); + return eb.extraData(SourceLocation, index).data; +} + +/// Returns the requested data, as well as the new index which is at the start of the +/// trailers for the object. +fn extraData(eb: ErrorBundle, comptime T: type, index: usize) struct { data: T, end: usize } { + const fields = @typeInfo(T).Struct.fields; + var i: usize = index; + var result: T = undefined; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + u32 => eb.extra.items[i], + else => @compileError("bad field type"), + }; + i += 1; + } + return .{ + .data = result, + .end = i, + }; +} + +/// Given an index into `string_bytes` returns the null-terminated string found there. +pub fn nullTerminatedString(eb: ErrorBundle, index: usize) [:0]const u8 { + const string_bytes = eb.string_bytes.items; + var end: usize = index; + while (string_bytes[end] != 0) { + end += 1; + } + return string_bytes[index..end :0]; +} + +pub fn renderToStdErr(eb: ErrorBundle, ttyconf: std.debug.TTY.Config) void { + std.debug.getStderrMutex().lock(); + defer std.debug.getStderrMutex().unlock(); + const stderr = std.io.getStdErr(); + return renderToWriter(eb, ttyconf, stderr.writer()) catch return; +} + +pub fn renderToWriter( + eb: ErrorBundle, + ttyconf: std.debug.TTY.Config, + writer: anytype, +) anyerror!void { + const list = eb.extraData(ErrorMessageList, 0).data; + var index: usize = list.start; + for (0..list.len) |_| { + const err_msg = eb.extraData(ErrorMessage, index); + index = try renderErrorMessageToWriter(eb, err_msg.data, err_msg.end, ttyconf, writer, "error", .Red, 0); + } +} + +fn renderErrorMessageToWriter( + eb: ErrorBundle, + err_msg: ErrorMessage, + end_index: usize, + ttyconf: std.debug.TTY.Config, + stderr: anytype, + kind: []const u8, + color: std.debug.TTY.Color, + indent: usize, +) anyerror!usize { + var counting_writer = std.io.countingWriter(stderr); + const counting_stderr = counting_writer.writer(); + if (err_msg.src_loc != 0) { + const src = eb.extraData(SourceLocation, err_msg.src_loc); + try counting_stderr.writeByteNTimes(' ', indent); + try ttyconf.setColor(stderr, .Bold); + try counting_stderr.print("{s}:{d}:{d}: ", .{ + eb.nullTerminatedString(src.data.src_path), + src.data.line + 1, + src.data.column + 1, + }); + try ttyconf.setColor(stderr, color); + try counting_stderr.writeAll(kind); + try counting_stderr.writeAll(": "); + // This is the length of the part before the error message: + // e.g. "file.zig:4:5: error: " + const prefix_len = @intCast(usize, counting_stderr.context.bytes_written); + try ttyconf.setColor(stderr, .Reset); + try ttyconf.setColor(stderr, .Bold); + if (err_msg.count == 1) { + try writeMsg(eb, err_msg, stderr, prefix_len); + try stderr.writeByte('\n'); + } else { + try writeMsg(eb, err_msg, stderr, prefix_len); + try ttyconf.setColor(stderr, .Dim); + try stderr.print(" ({d} times)\n", .{err_msg.count}); + } + try ttyconf.setColor(stderr, .Reset); + if (src.data.source_line != 0) { + const line = eb.nullTerminatedString(src.data.source_line); + for (line) |b| switch (b) { + '\t' => try stderr.writeByte(' '), + else => try stderr.writeByte(b), + }; + try stderr.writeByte('\n'); + // TODO basic unicode code point monospace width + const before_caret = src.data.span_main - src.data.span_start; + // -1 since span.main includes the caret + const after_caret = src.data.span_end - src.data.span_main -| 1; + try stderr.writeByteNTimes(' ', src.data.column - before_caret); + try ttyconf.setColor(stderr, .Green); + try stderr.writeByteNTimes('~', before_caret); + try stderr.writeByte('^'); + try stderr.writeByteNTimes('~', after_caret); + try stderr.writeByte('\n'); + try ttyconf.setColor(stderr, .Reset); + } + var index = end_index; + for (0..err_msg.notes_len) |_| { + const note = eb.extraData(ErrorMessage, index); + index = try renderErrorMessageToWriter(eb, note.data, note.end, ttyconf, stderr, "note", .Cyan, indent); + } + if (src.data.reference_trace_len > 0) { + try ttyconf.setColor(stderr, .Reset); + try ttyconf.setColor(stderr, .Dim); + try stderr.print("referenced by:\n", .{}); + var ref_index = src.end; + for (0..src.data.reference_trace_len) |_| { + const ref_trace = eb.extraData(ReferenceTrace, ref_index); + ref_index = ref_trace.end; + if (ref_trace.data.src_loc != 0) { + const ref_src = eb.getSourceLocation(ref_trace.data.src_loc); + try stderr.print(" {s}: {s}:{d}:{d}\n", .{ + eb.nullTerminatedString(ref_trace.data.decl_name), + eb.nullTerminatedString(ref_src.src_path), + ref_src.line + 1, + ref_src.column + 1, + }); + } else if (ref_trace.data.decl_name != 0) { + const count = ref_trace.data.decl_name; + try stderr.print( + " {d} reference(s) hidden; use '-freference-trace={d}' to see all references\n", + .{ count, count + src.data.reference_trace_len - 1 }, + ); + } else { + try stderr.print( + " remaining reference traces hidden; use '-freference-trace' to see all reference traces\n", + .{}, + ); + } + } + try stderr.writeByte('\n'); + try ttyconf.setColor(stderr, .Reset); + } + return index; + } else { + try ttyconf.setColor(stderr, color); + try stderr.writeByteNTimes(' ', indent); + try stderr.writeAll(kind); + try stderr.writeAll(": "); + try ttyconf.setColor(stderr, .Reset); + const msg = eb.nullTerminatedString(err_msg.msg); + if (err_msg.count == 1) { + try stderr.print("{s}\n", .{msg}); + } else { + try stderr.print("{s}", .{msg}); + try ttyconf.setColor(stderr, .Dim); + try stderr.print(" ({d} times)\n", .{err_msg.count}); + } + try ttyconf.setColor(stderr, .Reset); + var index = end_index; + for (0..err_msg.notes_len) |_| { + const note = eb.extraData(ErrorMessage, index); + index = try renderErrorMessageToWriter(eb, note.data, note.end, ttyconf, stderr, "note", .Cyan, indent + 4); + } + return index; + } +} + +/// Splits the error message up into lines to properly indent them +/// to allow for long, good-looking error messages. +/// +/// This is used to split the message in `@compileError("hello\nworld")` for example. +fn writeMsg(eb: ErrorBundle, err_msg: ErrorMessage, stderr: anytype, indent: usize) !void { + var lines = std.mem.split(u8, eb.nullTerminatedString(err_msg.msg), "\n"); + while (lines.next()) |line| { + try stderr.writeAll(line); + if (lines.index == null) break; + try stderr.writeByte('\n'); + try stderr.writeByteNTimes(' ', indent); + } +} + +const std = @import("std"); +const ErrorBundle = @This(); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; diff --git a/src/AstGen.zig b/src/AstGen.zig index 7b2138a535..8ac67a7107 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -133,6 +133,8 @@ pub fn generate(gpa: Allocator, tree: Ast) Allocator.Error!Zir { try astgen.extra.ensureTotalCapacity(gpa, tree.nodes.len + reserved_count); astgen.extra.items.len += reserved_count; + try lowerAstErrors(&astgen); + var top_scope: Scope.Top = .{}; var gz_instructions: std.ArrayListUnmanaged(Zir.Inst.Index) = .{}; @@ -10401,27 +10403,11 @@ fn appendErrorTokNotes( args: anytype, notes: []const u32, ) !void { - @setCold(true); - const string_bytes = &astgen.string_bytes; - const msg = @intCast(u32, string_bytes.items.len); - try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args); - const notes_index: u32 = if (notes.len != 0) blk: { - const notes_start = astgen.extra.items.len; - try astgen.extra.ensureTotalCapacity(astgen.gpa, notes_start + 1 + notes.len); - astgen.extra.appendAssumeCapacity(@intCast(u32, notes.len)); - astgen.extra.appendSliceAssumeCapacity(notes); - break :blk @intCast(u32, notes_start); - } else 0; - try astgen.compile_errors.append(astgen.gpa, .{ - .msg = msg, - .node = 0, - .token = token, - .byte_offset = 0, - .notes = notes_index, - }); + return appendErrorTokNotesOff(astgen, token, 0, format, args, notes); } -/// Same as `fail`, except given an absolute byte offset. +/// Same as `fail`, except given a token plus an offset from its starting byte +/// offset. fn failOff( astgen: *AstGen, token: Ast.TokenIndex, @@ -10429,27 +10415,36 @@ fn failOff( comptime format: []const u8, args: anytype, ) InnerError { - try appendErrorOff(astgen, token, byte_offset, format, args); + try appendErrorTokNotesOff(astgen, token, byte_offset, format, args, &.{}); return error.AnalysisFail; } -fn appendErrorOff( +fn appendErrorTokNotesOff( astgen: *AstGen, token: Ast.TokenIndex, byte_offset: u32, comptime format: []const u8, args: anytype, -) Allocator.Error!void { + notes: []const u32, +) !void { @setCold(true); + const gpa = astgen.gpa; const string_bytes = &astgen.string_bytes; const msg = @intCast(u32, string_bytes.items.len); - try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args); - try astgen.compile_errors.append(astgen.gpa, .{ + try string_bytes.writer(gpa).print(format ++ "\x00", args); + const notes_index: u32 = if (notes.len != 0) blk: { + const notes_start = astgen.extra.items.len; + try astgen.extra.ensureTotalCapacity(gpa, notes_start + 1 + notes.len); + astgen.extra.appendAssumeCapacity(@intCast(u32, notes.len)); + astgen.extra.appendSliceAssumeCapacity(notes); + break :blk @intCast(u32, notes_start); + } else 0; + try astgen.compile_errors.append(gpa, .{ .msg = msg, .node = 0, .token = token, .byte_offset = byte_offset, - .notes = 0, + .notes = notes_index, }); } @@ -10458,6 +10453,16 @@ fn errNoteTok( token: Ast.TokenIndex, comptime format: []const u8, args: anytype, +) Allocator.Error!u32 { + return errNoteTokOff(astgen, token, 0, format, args); +} + +fn errNoteTokOff( + astgen: *AstGen, + token: Ast.TokenIndex, + byte_offset: u32, + comptime format: []const u8, + args: anytype, ) Allocator.Error!u32 { @setCold(true); const string_bytes = &astgen.string_bytes; @@ -10467,7 +10472,7 @@ fn errNoteTok( .msg = msg, .node = 0, .token = token, - .byte_offset = 0, + .byte_offset = byte_offset, .notes = 0, }); } @@ -12634,3 +12639,42 @@ fn emitDbgStmt(gz: *GenZir, line: u32, column: u32) !void { }, } }); } + +fn lowerAstErrors(astgen: *AstGen) !void { + const tree = astgen.tree; + if (tree.errors.len == 0) return; + + const gpa = astgen.gpa; + const parse_err = tree.errors[0]; + + var msg: std.ArrayListUnmanaged(u8) = .{}; + defer msg.deinit(gpa); + + const token_starts = tree.tokens.items(.start); + const token_tags = tree.tokens.items(.tag); + + var notes: std.ArrayListUnmanaged(u32) = .{}; + defer notes.deinit(gpa); + + if (token_tags[parse_err.token + @boolToInt(parse_err.token_is_prev)] == .invalid) { + const tok = parse_err.token + @boolToInt(parse_err.token_is_prev); + const bad_off = @intCast(u32, tree.tokenSlice(parse_err.token + @boolToInt(parse_err.token_is_prev)).len); + const byte_abs = token_starts[parse_err.token + @boolToInt(parse_err.token_is_prev)] + bad_off; + try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{ + std.zig.fmtEscapes(tree.source[byte_abs..][0..1]), + })); + } + + for (tree.errors[1..]) |note| { + if (!note.is_note) break; + + msg.clearRetainingCapacity(); + try tree.renderError(note, msg.writer(gpa)); + try notes.append(gpa, try astgen.errNoteTok(note.token, "{s}", .{msg.items})); + } + + const extra_offset = tree.errorOffset(parse_err); + msg.clearRetainingCapacity(); + try tree.renderError(parse_err, msg.writer(gpa)); + try astgen.appendErrorTokNotesOff(parse_err.token, extra_offset, "{s}", .{msg.items}, notes.items); +} diff --git a/src/Compilation.zig b/src/Compilation.zig index 83e0f0d5d9..f5a92c66ea 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -9,6 +9,7 @@ const log = std.log.scoped(.compilation); const Target = std.Target; const ThreadPool = std.Thread.Pool; const WaitGroup = std.Thread.WaitGroup; +const ErrorBundle = std.zig.ErrorBundle; const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; @@ -334,12 +335,41 @@ pub const MiscTask = enum { libssp, zig_libc, analyze_pkg, + + @"musl crti.o", + @"musl crtn.o", + @"musl crt1.o", + @"musl rcrt1.o", + @"musl Scrt1.o", + @"musl libc.a", + @"musl libc.so", + + @"wasi crt1-reactor.o", + @"wasi crt1-command.o", + @"wasi libc.a", + @"libwasi-emulated-process-clocks.a", + @"libwasi-emulated-getpid.a", + @"libwasi-emulated-mman.a", + @"libwasi-emulated-signal.a", + + @"glibc crti.o", + @"glibc crtn.o", + @"glibc Scrt1.o", + @"glibc libc_nonshared.a", + @"glibc shared object", + + @"mingw-w64 crt2.o", + @"mingw-w64 dllcrt2.o", + @"mingw-w64 mingw32.lib", + @"mingw-w64 msvcrt-os.lib", + @"mingw-w64 mingwex.lib", + @"mingw-w64 uuid.lib", }; pub const MiscError = struct { /// Allocated with gpa. msg: []u8, - children: ?AllErrors = null, + children: ?ErrorBundle = null, pub fn deinit(misc_err: *MiscError, gpa: Allocator) void { gpa.free(misc_err.msg); @@ -365,448 +395,6 @@ pub const LldError = struct { } }; -/// To support incremental compilation, errors are stored in various places -/// so that they can be created and destroyed appropriately. This structure -/// is used to collect all the errors from the various places into one -/// convenient place for API users to consume. It is allocated into 1 arena -/// and freed all at once. -pub const AllErrors = struct { - arena: std.heap.ArenaAllocator.State, - list: []const Message, - - pub const Message = union(enum) { - src: struct { - msg: []const u8, - src_path: []const u8, - line: u32, - column: u32, - span: Module.SrcLoc.Span, - /// Usually one, but incremented for redundant messages. - count: u32 = 1, - /// Does not include the trailing newline. - source_line: ?[]const u8, - notes: []const Message = &.{}, - reference_trace: []Message = &.{}, - - /// Splits the error message up into lines to properly indent them - /// to allow for long, good-looking error messages. - /// - /// This is used to split the message in `@compileError("hello\nworld")` for example. - fn writeMsg(src: @This(), stderr: anytype, indent: usize) !void { - var lines = mem.split(u8, src.msg, "\n"); - while (lines.next()) |line| { - try stderr.writeAll(line); - if (lines.index == null) break; - try stderr.writeByte('\n'); - try stderr.writeByteNTimes(' ', indent); - } - } - }, - plain: struct { - msg: []const u8, - notes: []Message = &.{}, - /// Usually one, but incremented for redundant messages. - count: u32 = 1, - }, - - pub fn incrementCount(msg: *Message) void { - switch (msg.*) { - .src => |*src| { - src.count += 1; - }, - .plain => |*plain| { - plain.count += 1; - }, - } - } - - pub fn renderToStdErr(msg: Message, ttyconf: std.debug.TTY.Config) void { - std.debug.getStderrMutex().lock(); - defer std.debug.getStderrMutex().unlock(); - const stderr = std.io.getStdErr(); - return msg.renderToWriter(ttyconf, stderr.writer(), "error", .Red, 0) catch return; - } - - pub fn renderToWriter( - msg: Message, - ttyconf: std.debug.TTY.Config, - stderr: anytype, - kind: []const u8, - color: std.debug.TTY.Color, - indent: usize, - ) anyerror!void { - var counting_writer = std.io.countingWriter(stderr); - const counting_stderr = counting_writer.writer(); - switch (msg) { - .src => |src| { - try counting_stderr.writeByteNTimes(' ', indent); - try ttyconf.setColor(stderr, .Bold); - try counting_stderr.print("{s}:{d}:{d}: ", .{ - src.src_path, - src.line + 1, - src.column + 1, - }); - try ttyconf.setColor(stderr, color); - try counting_stderr.writeAll(kind); - try counting_stderr.writeAll(": "); - // This is the length of the part before the error message: - // e.g. "file.zig:4:5: error: " - const prefix_len = @intCast(usize, counting_stderr.context.bytes_written); - try ttyconf.setColor(stderr, .Reset); - try ttyconf.setColor(stderr, .Bold); - if (src.count == 1) { - try src.writeMsg(stderr, prefix_len); - try stderr.writeByte('\n'); - } else { - try src.writeMsg(stderr, prefix_len); - try ttyconf.setColor(stderr, .Dim); - try stderr.print(" ({d} times)\n", .{src.count}); - } - try ttyconf.setColor(stderr, .Reset); - if (src.source_line) |line| { - for (line) |b| switch (b) { - '\t' => try stderr.writeByte(' '), - else => try stderr.writeByte(b), - }; - try stderr.writeByte('\n'); - // TODO basic unicode code point monospace width - const before_caret = src.span.main - src.span.start; - // -1 since span.main includes the caret - const after_caret = src.span.end - src.span.main -| 1; - try stderr.writeByteNTimes(' ', src.column - before_caret); - try ttyconf.setColor(stderr, .Green); - try stderr.writeByteNTimes('~', before_caret); - try stderr.writeByte('^'); - try stderr.writeByteNTimes('~', after_caret); - try stderr.writeByte('\n'); - try ttyconf.setColor(stderr, .Reset); - } - for (src.notes) |note| { - try note.renderToWriter(ttyconf, stderr, "note", .Cyan, indent); - } - if (src.reference_trace.len != 0) { - try ttyconf.setColor(stderr, .Reset); - try ttyconf.setColor(stderr, .Dim); - try stderr.print("referenced by:\n", .{}); - for (src.reference_trace) |reference| { - switch (reference) { - .src => |ref_src| try stderr.print(" {s}: {s}:{d}:{d}\n", .{ - ref_src.msg, - ref_src.src_path, - ref_src.line + 1, - ref_src.column + 1, - }), - .plain => |plain| if (plain.count != 0) { - try stderr.print( - " {d} reference(s) hidden; use '-freference-trace={d}' to see all references\n", - .{ plain.count, plain.count + src.reference_trace.len - 1 }, - ); - } else { - try stderr.print( - " remaining reference traces hidden; use '-freference-trace' to see all reference traces\n", - .{}, - ); - }, - } - } - try stderr.writeByte('\n'); - try ttyconf.setColor(stderr, .Reset); - } - }, - .plain => |plain| { - try ttyconf.setColor(stderr, color); - try stderr.writeByteNTimes(' ', indent); - try stderr.writeAll(kind); - try stderr.writeAll(": "); - try ttyconf.setColor(stderr, .Reset); - if (plain.count == 1) { - try stderr.print("{s}\n", .{plain.msg}); - } else { - try stderr.print("{s}", .{plain.msg}); - try ttyconf.setColor(stderr, .Dim); - try stderr.print(" ({d} times)\n", .{plain.count}); - } - try ttyconf.setColor(stderr, .Reset); - for (plain.notes) |note| { - try note.renderToWriter(ttyconf, stderr, "note", .Cyan, indent + 4); - } - }, - } - } - - pub const HashContext = struct { - pub fn hash(ctx: HashContext, key: *Message) u64 { - _ = ctx; - var hasher = std.hash.Wyhash.init(0); - - switch (key.*) { - .src => |src| { - hasher.update(src.msg); - hasher.update(src.src_path); - std.hash.autoHash(&hasher, src.line); - std.hash.autoHash(&hasher, src.column); - std.hash.autoHash(&hasher, src.span.main); - }, - .plain => |plain| { - hasher.update(plain.msg); - }, - } - - return hasher.final(); - } - - pub fn eql(ctx: HashContext, a: *Message, b: *Message) bool { - _ = ctx; - switch (a.*) { - .src => |a_src| switch (b.*) { - .src => |b_src| { - return mem.eql(u8, a_src.msg, b_src.msg) and - mem.eql(u8, a_src.src_path, b_src.src_path) and - a_src.line == b_src.line and - a_src.column == b_src.column and - a_src.span.main == b_src.span.main; - }, - .plain => return false, - }, - .plain => |a_plain| switch (b.*) { - .src => return false, - .plain => |b_plain| { - return mem.eql(u8, a_plain.msg, b_plain.msg); - }, - }, - } - } - }; - }; - - pub fn deinit(self: *AllErrors, gpa: Allocator) void { - self.arena.promote(gpa).deinit(); - } - - pub fn add( - module: *Module, - arena: *std.heap.ArenaAllocator, - errors: *std.ArrayList(Message), - module_err_msg: Module.ErrorMsg, - ) !void { - const allocator = arena.allocator(); - - const notes_buf = try allocator.alloc(Message, module_err_msg.notes.len); - var note_i: usize = 0; - - // De-duplicate error notes. The main use case in mind for this is - // too many "note: called from here" notes when eval branch quota is reached. - var seen_notes = std.HashMap( - *Message, - void, - Message.HashContext, - std.hash_map.default_max_load_percentage, - ).init(allocator); - const err_source = module_err_msg.src_loc.file_scope.getSource(module.gpa) catch |err| { - const file_path = try module_err_msg.src_loc.file_scope.fullPath(allocator); - try errors.append(.{ - .plain = .{ - .msg = try std.fmt.allocPrint(allocator, "unable to load '{s}': {s}", .{ - file_path, @errorName(err), - }), - }, - }); - return; - }; - const err_span = try module_err_msg.src_loc.span(module.gpa); - const err_loc = std.zig.findLineColumn(err_source.bytes, err_span.main); - - for (module_err_msg.notes) |module_note| { - const source = try module_note.src_loc.file_scope.getSource(module.gpa); - const span = try module_note.src_loc.span(module.gpa); - const loc = std.zig.findLineColumn(source.bytes, span.main); - const file_path = try module_note.src_loc.file_scope.fullPath(allocator); - const note = ¬es_buf[note_i]; - note.* = .{ - .src = .{ - .src_path = file_path, - .msg = try allocator.dupe(u8, module_note.msg), - .span = span, - .line = @intCast(u32, loc.line), - .column = @intCast(u32, loc.column), - .source_line = if (err_loc.eql(loc)) null else try allocator.dupe(u8, loc.source_line), - }, - }; - const gop = try seen_notes.getOrPut(note); - if (gop.found_existing) { - gop.key_ptr.*.incrementCount(); - } else { - note_i += 1; - } - } - - const reference_trace = try allocator.alloc(Message, module_err_msg.reference_trace.len); - for (reference_trace, 0..) |*reference, i| { - const module_reference = module_err_msg.reference_trace[i]; - if (module_reference.hidden != 0) { - reference.* = .{ .plain = .{ .msg = undefined, .count = module_reference.hidden } }; - break; - } else if (module_reference.decl == null) { - reference.* = .{ .plain = .{ .msg = undefined, .count = 0 } }; - break; - } - const source = try module_reference.src_loc.file_scope.getSource(module.gpa); - const span = try module_reference.src_loc.span(module.gpa); - const loc = std.zig.findLineColumn(source.bytes, span.main); - const file_path = try module_reference.src_loc.file_scope.fullPath(allocator); - reference.* = .{ - .src = .{ - .src_path = file_path, - .msg = try allocator.dupe(u8, std.mem.sliceTo(module_reference.decl.?, 0)), - .span = span, - .line = @intCast(u32, loc.line), - .column = @intCast(u32, loc.column), - .source_line = null, - }, - }; - } - const file_path = try module_err_msg.src_loc.file_scope.fullPath(allocator); - try errors.append(.{ - .src = .{ - .src_path = file_path, - .msg = try allocator.dupe(u8, module_err_msg.msg), - .span = err_span, - .line = @intCast(u32, err_loc.line), - .column = @intCast(u32, err_loc.column), - .notes = notes_buf[0..note_i], - .reference_trace = reference_trace, - .source_line = if (module_err_msg.src_loc.lazy == .entire_file) null else try allocator.dupe(u8, err_loc.source_line), - }, - }); - } - - pub fn addZir( - arena: Allocator, - errors: *std.ArrayList(Message), - file: *Module.File, - ) !void { - assert(file.zir_loaded); - assert(file.tree_loaded); - assert(file.source_loaded); - const payload_index = file.zir.extra[@enumToInt(Zir.ExtraIndex.compile_errors)]; - assert(payload_index != 0); - - const header = file.zir.extraData(Zir.Inst.CompileErrors, payload_index); - const items_len = header.data.items_len; - var extra_index = header.end; - var item_i: usize = 0; - while (item_i < items_len) : (item_i += 1) { - const item = file.zir.extraData(Zir.Inst.CompileErrors.Item, extra_index); - extra_index = item.end; - const err_span = blk: { - if (item.data.node != 0) { - break :blk Module.SrcLoc.nodeToSpan(&file.tree, item.data.node); - } - const token_starts = file.tree.tokens.items(.start); - const start = token_starts[item.data.token] + item.data.byte_offset; - const end = start + @intCast(u32, file.tree.tokenSlice(item.data.token).len) - item.data.byte_offset; - break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start }; - }; - const err_loc = std.zig.findLineColumn(file.source, err_span.main); - - var notes: []Message = &[0]Message{}; - if (item.data.notes != 0) { - const block = file.zir.extraData(Zir.Inst.Block, item.data.notes); - const body = file.zir.extra[block.end..][0..block.data.body_len]; - notes = try arena.alloc(Message, body.len); - for (notes, 0..) |*note, i| { - const note_item = file.zir.extraData(Zir.Inst.CompileErrors.Item, body[i]); - const msg = file.zir.nullTerminatedString(note_item.data.msg); - const span = blk: { - if (note_item.data.node != 0) { - break :blk Module.SrcLoc.nodeToSpan(&file.tree, note_item.data.node); - } - const token_starts = file.tree.tokens.items(.start); - const start = token_starts[note_item.data.token] + note_item.data.byte_offset; - const end = start + @intCast(u32, file.tree.tokenSlice(note_item.data.token).len) - item.data.byte_offset; - break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start }; - }; - const loc = std.zig.findLineColumn(file.source, span.main); - - note.* = .{ - .src = .{ - .src_path = try file.fullPath(arena), - .msg = try arena.dupe(u8, msg), - .span = span, - .line = @intCast(u32, loc.line), - .column = @intCast(u32, loc.column), - .notes = &.{}, // TODO rework this function to be recursive - .source_line = if (loc.eql(err_loc)) null else try arena.dupe(u8, loc.source_line), - }, - }; - } - } - - const msg = file.zir.nullTerminatedString(item.data.msg); - try errors.append(.{ - .src = .{ - .src_path = try file.fullPath(arena), - .msg = try arena.dupe(u8, msg), - .span = err_span, - .line = @intCast(u32, err_loc.line), - .column = @intCast(u32, err_loc.column), - .notes = notes, - .source_line = try arena.dupe(u8, err_loc.source_line), - }, - }); - } - } - - fn addPlain( - arena: *std.heap.ArenaAllocator, - errors: *std.ArrayList(Message), - msg: []const u8, - ) !void { - _ = arena; - try errors.append(.{ .plain = .{ .msg = msg } }); - } - - fn addPlainWithChildren( - arena: *std.heap.ArenaAllocator, - errors: *std.ArrayList(Message), - msg: []const u8, - optional_children: ?AllErrors, - ) !void { - const allocator = arena.allocator(); - const duped_msg = try allocator.dupe(u8, msg); - if (optional_children) |*children| { - try errors.append(.{ .plain = .{ - .msg = duped_msg, - .notes = try dupeList(children.list, allocator), - } }); - } else { - try errors.append(.{ .plain = .{ .msg = duped_msg } }); - } - } - - fn dupeList(list: []const Message, arena: Allocator) Allocator.Error![]Message { - const duped_list = try arena.alloc(Message, list.len); - for (list, 0..) |item, i| { - duped_list[i] = switch (item) { - .src => |src| .{ .src = .{ - .msg = try arena.dupe(u8, src.msg), - .src_path = try arena.dupe(u8, src.src_path), - .line = src.line, - .column = src.column, - .span = src.span, - .source_line = if (src.source_line) |s| try arena.dupe(u8, s) else null, - .notes = try dupeList(src.notes, arena), - } }, - .plain => |plain| .{ .plain = .{ - .msg = try arena.dupe(u8, plain.msg), - .notes = try dupeList(plain.notes, arena), - } }, - }; - } - return duped_list; - } -}; - pub const Directory = Cache.Directory; pub const EmitLoc = struct { @@ -2891,7 +2479,7 @@ pub fn makeBinFileWritable(self: *Compilation) !void { } /// This function is temporally single-threaded. -pub fn totalErrorCount(self: *Compilation) usize { +pub fn totalErrorCount(self: *Compilation) u32 { var total: usize = self.failed_c_objects.count() + self.misc_failures.count() + @boolToInt(self.alloc_failure_occurred) + self.lld_errors.items.len; @@ -2951,17 +2539,16 @@ pub fn totalErrorCount(self: *Compilation) usize { } } - return total; + return @intCast(u32, total); } /// This function is temporally single-threaded. -pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { - var arena = std.heap.ArenaAllocator.init(self.gpa); - errdefer arena.deinit(); - const arena_allocator = arena.allocator(); +pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { + const gpa = self.gpa; - var errors = std.ArrayList(AllErrors.Message).init(self.gpa); - defer errors.deinit(); + var bundle: ErrorBundle = undefined; + try bundle.init(gpa); + errdefer bundle.deinit(gpa); { var it = self.failed_c_objects.iterator(); @@ -2970,53 +2557,63 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { const err_msg = entry.value_ptr.*; // TODO these fields will need to be adjusted when we have proper // C error reporting bubbling up. - try errors.append(.{ - .src = .{ - .src_path = try arena_allocator.dupe(u8, c_object.src.src_path), - .msg = try std.fmt.allocPrint(arena_allocator, "unable to build C object: {s}", .{ - err_msg.msg, - }), - .span = .{ .start = 0, .end = 1, .main = 0 }, + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.printString(gpa, "unable to build C object: {s}", .{ + err_msg.msg, + }), + .src_loc = try bundle.addSourceLocation(gpa, .{ + .src_path = try bundle.addString(gpa, c_object.src.src_path), + .span_start = 0, + .span_main = 0, + .span_end = 1, .line = err_msg.line, .column = err_msg.column, - .source_line = null, // TODO - }, + .source_line = 0, // TODO + }), }); + bundle.incrementCount(1); } } - for (self.lld_errors.items) |lld_error| { - const notes = try arena_allocator.alloc(AllErrors.Message, lld_error.context_lines.len); - for (lld_error.context_lines, 0..) |context_line, i| { - notes[i] = .{ .plain = .{ - .msg = try arena_allocator.dupe(u8, context_line), - } }; - } - try errors.append(.{ - .plain = .{ - .msg = try arena_allocator.dupe(u8, lld_error.msg), - .notes = notes, - }, + for (self.lld_errors.items) |lld_error| { + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, lld_error.msg), + .notes_len = @intCast(u32, lld_error.context_lines.len), }); + bundle.incrementCount(1); + + for (lld_error.context_lines) |context_line| { + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, context_line), + }); + } } for (self.misc_failures.values()) |*value| { - try AllErrors.addPlainWithChildren(&arena, &errors, value.msg, value.children); + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, value.msg), + .notes_len = if (value.children) |b| b.errorMessageCount() else 0, + }); + if (value.children) |b| try bundle.addBundle(gpa, b); + bundle.incrementCount(1); } if (self.alloc_failure_occurred) { - try AllErrors.addPlain(&arena, &errors, "memory allocation failure"); + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, "memory allocation failure"), + }); + bundle.incrementCount(1); } if (self.bin_file.options.module) |module| { { var it = module.failed_files.iterator(); while (it.next()) |entry| { if (entry.value_ptr.*) |msg| { - try AllErrors.add(module, &arena, &errors, msg.*); + try addModuleErrorMsg(gpa, &bundle, msg.*); } else { // Must be ZIR errors. In order for ZIR errors to exist, the parsing // must have completed successfully. const tree = try entry.key_ptr.*.getTree(module.gpa); assert(tree.errors.len == 0); - try AllErrors.addZir(arena_allocator, &errors, entry.key_ptr.*); + try addZirErrorMessages(gpa, &bundle, entry.key_ptr.*); } } } @@ -3024,7 +2621,7 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { var it = module.failed_embed_files.iterator(); while (it.next()) |entry| { const msg = entry.value_ptr.*; - try AllErrors.add(module, &arena, &errors, msg.*); + try addModuleErrorMsg(gpa, &bundle, msg.*); } } { @@ -3034,23 +2631,21 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { // Skip errors for Decls within files that had a parse failure. // We'll try again once parsing succeeds. if (decl.getFileScope().okToReportErrors()) { - try AllErrors.add(module, &arena, &errors, entry.value_ptr.*.*); + try addModuleErrorMsg(gpa, &bundle, entry.value_ptr.*.*); if (module.cimport_errors.get(entry.key_ptr.*)) |cimport_errors| for (cimport_errors) |c_error| { - if (c_error.path) |some| - try errors.append(.{ - .src = .{ - .src_path = try arena_allocator.dupe(u8, std.mem.span(some)), - .span = .{ .start = c_error.offset, .end = c_error.offset + 1, .main = c_error.offset }, - .msg = try arena_allocator.dupe(u8, std.mem.span(c_error.msg)), - .line = c_error.line, - .column = c_error.column, - .source_line = if (c_error.source_line) |line| try arena_allocator.dupe(u8, std.mem.span(line)) else null, - }, - }) - else - try errors.append(.{ - .plain = .{ .msg = try arena_allocator.dupe(u8, std.mem.span(c_error.msg)) }, - }); + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, std.mem.span(c_error.msg)), + .src_loc = if (c_error.path) |some| try bundle.addSourceLocation(gpa, .{ + .src_path = try bundle.addString(gpa, std.mem.span(some)), + .span_start = c_error.offset, + .span_main = c_error.offset, + .span_end = c_error.offset + 1, + .line = c_error.line, + .column = c_error.column, + .source_line = if (c_error.source_line) |line| try bundle.addString(gpa, std.mem.span(line)) else 0, + }) else 0, + }); + bundle.incrementCount(1); }; } } @@ -3062,45 +2657,40 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { // Skip errors for Decls within files that had a parse failure. // We'll try again once parsing succeeds. if (decl.getFileScope().okToReportErrors()) { - try AllErrors.add(module, &arena, &errors, entry.value_ptr.*.*); + try addModuleErrorMsg(gpa, &bundle, entry.value_ptr.*.*); } } } for (module.failed_exports.values()) |value| { - try AllErrors.add(module, &arena, &errors, value.*); + try addModuleErrorMsg(gpa, &bundle, value.*); } } - if (errors.items.len == 0) { + if (bundle.errorMessageCount() == 0) { if (self.link_error_flags.no_entry_point_found) { - try errors.append(.{ - .plain = .{ - .msg = try std.fmt.allocPrint(arena_allocator, "no entry point found", .{}), - }, + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, "no entry point found"), }); + bundle.incrementCount(1); } } if (self.link_error_flags.missing_libc) { - const notes = try arena_allocator.create([2]AllErrors.Message); - notes.* = .{ - .{ .plain = .{ - .msg = try arena_allocator.dupe(u8, "run 'zig libc -h' to learn about libc installations"), - } }, - .{ .plain = .{ - .msg = try arena_allocator.dupe(u8, "run 'zig targets' to see the targets for which zig can always provide libc"), - } }, - }; - try errors.append(.{ - .plain = .{ - .msg = try std.fmt.allocPrint(arena_allocator, "libc not available", .{}), - .notes = notes, - }, + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, "libc not available"), + .notes_len = 2, + }); + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, "run 'zig libc -h' to learn about libc installations"), }); + try bundle.addErrorMessage(gpa, .{ + .msg = try bundle.addString(gpa, "run 'zig targets' to see the targets for which zig can always provide libc"), + }); + bundle.incrementCount(1); } if (self.bin_file.options.module) |module| { - if (errors.items.len == 0 and module.compile_log_decls.count() != 0) { + if (bundle.errorMessageCount() == 0 and module.compile_log_decls.count() != 0) { const keys = module.compile_log_decls.keys(); const values = module.compile_log_decls.values(); // First one will be the error; subsequent ones will be notes. @@ -3121,16 +2711,259 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors { }; } - try AllErrors.add(module, &arena, &errors, err_msg); + try addModuleErrorMsg(gpa, &bundle, err_msg); } } - assert(errors.items.len == self.totalErrorCount()); + assert(self.totalErrorCount() == bundle.errorMessageCount()); + + return bundle; +} + +pub const ErrorNoteHashContext = struct { + eb: *const ErrorBundle, + + pub fn hash(ctx: ErrorNoteHashContext, key: ErrorBundle.ErrorMessage) u32 { + var hasher = std.hash.Wyhash.init(0); - return AllErrors{ - .list = try arena_allocator.dupe(AllErrors.Message, errors.items), - .arena = arena.state, + hasher.update(ctx.eb.nullTerminatedString(key.msg)); + if (key.src_loc != 0) { + const src = ctx.eb.getSourceLocation(key.src_loc); + hasher.update(ctx.eb.nullTerminatedString(src.src_path)); + std.hash.autoHash(&hasher, src.line); + std.hash.autoHash(&hasher, src.column); + std.hash.autoHash(&hasher, src.span_main); + } + + return @truncate(u32, hasher.final()); + } + + pub fn eql( + ctx: ErrorNoteHashContext, + a: ErrorBundle.ErrorMessage, + b: ErrorBundle.ErrorMessage, + b_index: usize, + ) bool { + _ = b_index; + const msg_a = ctx.eb.nullTerminatedString(a.msg); + const msg_b = ctx.eb.nullTerminatedString(b.msg); + if (!std.mem.eql(u8, msg_a, msg_b)) return false; + + if (a.src_loc == 0 and b.src_loc == 0) return true; + if (a.src_loc == 0 or b.src_loc == 0) return false; + const src_a = ctx.eb.getSourceLocation(a.src_loc); + const src_b = ctx.eb.getSourceLocation(b.src_loc); + + const src_path_a = ctx.eb.nullTerminatedString(src_a.src_path); + const src_path_b = ctx.eb.nullTerminatedString(src_b.src_path); + + return std.mem.eql(u8, src_path_a, src_path_b) and + src_a.line == src_b.line and + src_a.column == src_b.column and + src_a.span_main == src_b.span_main; + } +}; + +pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Module.ErrorMsg) !void { + const err_source = module_err_msg.src_loc.file_scope.getSource(gpa) catch |err| { + const file_path = try module_err_msg.src_loc.file_scope.fullPath(gpa); + defer gpa.free(file_path); + try eb.addErrorMessage(gpa, .{ + .msg = try eb.printString(gpa, "unable to load '{s}': {s}", .{ + file_path, @errorName(err), + }), + }); + eb.incrementCount(1); + return; }; + const err_span = try module_err_msg.src_loc.span(gpa); + const err_loc = std.zig.findLineColumn(err_source.bytes, err_span.main); + const file_path = try module_err_msg.src_loc.file_scope.fullPath(gpa); + defer gpa.free(file_path); + + var ref_traces: std.ArrayListUnmanaged(ErrorBundle.ReferenceTrace) = .{}; + defer ref_traces.deinit(gpa); + + for (module_err_msg.reference_trace) |module_reference| { + if (module_reference.hidden != 0) { + try ref_traces.append(gpa, .{ + .decl_name = module_reference.hidden, + .src_loc = 0, + }); + break; + } else if (module_reference.decl == null) { + try ref_traces.append(gpa, .{ + .decl_name = 0, + .src_loc = 0, + }); + break; + } + const source = try module_reference.src_loc.file_scope.getSource(gpa); + const span = try module_reference.src_loc.span(gpa); + const loc = std.zig.findLineColumn(source.bytes, span.main); + const rt_file_path = try module_reference.src_loc.file_scope.fullPath(gpa); + defer gpa.free(rt_file_path); + try ref_traces.append(gpa, .{ + .decl_name = try eb.addString(gpa, std.mem.sliceTo(module_reference.decl.?, 0)), + .src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, rt_file_path), + .span_start = span.start, + .span_main = span.main, + .span_end = span.end, + .line = @intCast(u32, loc.line), + .column = @intCast(u32, loc.column), + .source_line = 0, + }), + }); + } + + const src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, file_path), + .span_start = err_span.start, + .span_main = err_span.main, + .span_end = err_span.end, + .line = @intCast(u32, err_loc.line), + .column = @intCast(u32, err_loc.column), + .source_line = if (module_err_msg.src_loc.lazy == .entire_file) + 0 + else + try eb.addString(gpa, err_loc.source_line), + .reference_trace_len = @intCast(u32, ref_traces.items.len), + }); + + for (ref_traces.items) |rt| { + try eb.addReferenceTrace(gpa, rt); + } + + // De-duplicate error notes. The main use case in mind for this is + // too many "note: called from here" notes when eval branch quota is reached. + var notes: std.ArrayHashMapUnmanaged(ErrorBundle.ErrorMessage, void, ErrorNoteHashContext, true) = .{}; + defer notes.deinit(gpa); + + for (module_err_msg.notes) |module_note| { + const source = try module_note.src_loc.file_scope.getSource(gpa); + const span = try module_note.src_loc.span(gpa); + const loc = std.zig.findLineColumn(source.bytes, span.main); + const note_file_path = try module_note.src_loc.file_scope.fullPath(gpa); + defer gpa.free(note_file_path); + + const gop = try notes.getOrPutContext(gpa, .{ + .msg = try eb.addString(gpa, module_note.msg), + .src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, note_file_path), + .span_start = span.start, + .span_main = span.main, + .span_end = span.end, + .line = @intCast(u32, loc.line), + .column = @intCast(u32, loc.column), + .source_line = if (err_loc.eql(loc)) 0 else try eb.addString(gpa, loc.source_line), + }), + }, .{ .eb = eb }); + if (gop.found_existing) { + gop.key_ptr.count += 1; + } + } + + try eb.addErrorMessage(gpa, .{ + .msg = try eb.addString(gpa, module_err_msg.msg), + .src_loc = src_loc, + .notes_len = @intCast(u32, notes.entries.len), + }); + eb.incrementCount(1); + + for (notes.keys()) |note| { + try eb.addErrorMessage(gpa, note); + } +} + +pub fn addZirErrorMessages(gpa: Allocator, eb: *ErrorBundle, file: *Module.File) !void { + assert(file.zir_loaded); + assert(file.tree_loaded); + assert(file.source_loaded); + const payload_index = file.zir.extra[@enumToInt(Zir.ExtraIndex.compile_errors)]; + assert(payload_index != 0); + + const header = file.zir.extraData(Zir.Inst.CompileErrors, payload_index); + const items_len = header.data.items_len; + var extra_index = header.end; + for (0..items_len) |_| { + const item = file.zir.extraData(Zir.Inst.CompileErrors.Item, extra_index); + extra_index = item.end; + const err_span = blk: { + if (item.data.node != 0) { + break :blk Module.SrcLoc.nodeToSpan(&file.tree, item.data.node); + } + const token_starts = file.tree.tokens.items(.start); + const start = token_starts[item.data.token] + item.data.byte_offset; + const end = start + @intCast(u32, file.tree.tokenSlice(item.data.token).len) - item.data.byte_offset; + break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start }; + }; + const err_loc = std.zig.findLineColumn(file.source, err_span.main); + + var notes: []ErrorBundle.ErrorMessage = &.{}; + defer gpa.free(notes); + + if (item.data.notes != 0) { + const block = file.zir.extraData(Zir.Inst.Block, item.data.notes); + const body = file.zir.extra[block.end..][0..block.data.body_len]; + notes = try gpa.alloc(ErrorBundle.ErrorMessage, body.len); + for (notes, body) |*note, body_elem| { + const note_item = file.zir.extraData(Zir.Inst.CompileErrors.Item, body_elem); + const msg = file.zir.nullTerminatedString(note_item.data.msg); + const span = blk: { + if (note_item.data.node != 0) { + break :blk Module.SrcLoc.nodeToSpan(&file.tree, note_item.data.node); + } + const token_starts = file.tree.tokens.items(.start); + const start = token_starts[note_item.data.token] + note_item.data.byte_offset; + const end = start + @intCast(u32, file.tree.tokenSlice(note_item.data.token).len) - item.data.byte_offset; + break :blk Module.SrcLoc.Span{ .start = start, .end = end, .main = start }; + }; + const loc = std.zig.findLineColumn(file.source, span.main); + const src_path = try file.fullPath(gpa); + defer gpa.free(src_path); + + note.* = .{ + .msg = try eb.addString(gpa, msg), + .src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, src_path), + .span_start = span.start, + .span_main = span.main, + .span_end = span.end, + .line = @intCast(u32, loc.line), + .column = @intCast(u32, loc.column), + .source_line = if (loc.eql(err_loc)) + 0 + else + try eb.addString(gpa, loc.source_line), + }), + .notes_len = 0, // TODO rework this function to be recursive + }; + } + } + + const msg = file.zir.nullTerminatedString(item.data.msg); + const src_path = try file.fullPath(gpa); + defer gpa.free(src_path); + try eb.addErrorMessage(gpa, .{ + .msg = try eb.addString(gpa, msg), + .src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, src_path), + .span_start = err_span.start, + .span_main = err_span.main, + .span_end = err_span.end, + .line = @intCast(u32, err_loc.line), + .column = @intCast(u32, err_loc.column), + .source_line = try eb.addString(gpa, err_loc.source_line), + }), + .notes_len = @intCast(u32, notes.len), + }); + + for (notes) |note| { + try eb.addErrorMessage(gpa, note); + } + } + eb.incrementCount(items_len); } pub fn getCompileLogOutput(self: *Compilation) []const u8 { @@ -5417,34 +5250,29 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: Allocator) Alloca return buffer.toOwnedSliceSentinel(0); } -pub fn updateSubCompilation(sub_compilation: *Compilation) !void { - try sub_compilation.update(); - - // Look for compilation errors in this sub_compilation - // TODO instead of logging these errors, handle them in the callsites - // of updateSubCompilation and attach them as sub-errors, properly - // surfacing the errors. You can see an example of this already - // done inside buildOutputFromZig. - var errors = try sub_compilation.getAllErrorsAlloc(); - defer errors.deinit(sub_compilation.gpa); - - if (errors.list.len != 0) { - for (errors.list) |full_err_msg| { - switch (full_err_msg) { - .src => |src| { - log.err("{s}:{d}:{d}: {s}", .{ - src.src_path, - src.line + 1, - src.column + 1, - src.msg, - }); - }, - .plain => |plain| { - log.err("{s}", .{plain.msg}); - }, - } - } - return error.BuildingLibCObjectFailed; +pub fn updateSubCompilation( + parent_comp: *Compilation, + sub_comp: *Compilation, + misc_task: MiscTask, +) !void { + try sub_comp.update(); + + // Look for compilation errors in this sub compilation + const gpa = parent_comp.gpa; + var keep_errors = false; + var errors = try sub_comp.getAllErrorsAlloc(); + defer if (!keep_errors) errors.deinit(gpa); + + if (errors.errorMessageCount() > 0) { + try parent_comp.misc_failures.ensureUnusedCapacity(gpa, 1); + parent_comp.misc_failures.putAssumeCapacityNoClobber(misc_task, .{ + .msg = try std.fmt.allocPrint(gpa, "sub-compilation of {s} failed", .{ + @tagName(misc_task), + }), + .children = errors, + }); + keep_errors = true; + return error.SubCompilationFailed; } } @@ -5520,23 +5348,7 @@ fn buildOutputFromZig( }); defer sub_compilation.destroy(); - try sub_compilation.update(); - // Look for compilation errors in this sub_compilation. - var keep_errors = false; - var errors = try sub_compilation.getAllErrorsAlloc(); - defer if (!keep_errors) errors.deinit(sub_compilation.gpa); - - if (errors.list.len != 0) { - try comp.misc_failures.ensureUnusedCapacity(comp.gpa, 1); - comp.misc_failures.putAssumeCapacityNoClobber(misc_task_tag, .{ - .msg = try std.fmt.allocPrint(comp.gpa, "sub-compilation of {s} failed", .{ - @tagName(misc_task_tag), - }), - .children = errors, - }); - keep_errors = true; - return error.SubCompilationFailed; - } + try comp.updateSubCompilation(sub_compilation, misc_task_tag); assert(out.* == null); out.* = Compilation.CRTFile{ @@ -5551,6 +5363,7 @@ pub fn build_crt_file( comp: *Compilation, root_name: []const u8, output_mode: std.builtin.OutputMode, + misc_task_tag: MiscTask, c_source_files: []const Compilation.CSourceFile, ) !void { const tracy_trace = trace(@src()); @@ -5611,7 +5424,7 @@ pub fn build_crt_file( }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, misc_task_tag); try comp.crt_files.ensureUnusedCapacity(comp.gpa, 1); diff --git a/src/Module.zig b/src/Module.zig index 8c52176edd..1520a7d1b2 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -3756,67 +3756,9 @@ pub fn astGenFile(mod: *Module, file: *File) !void { file.source_loaded = true; file.tree = try Ast.parse(gpa, source, .zig); - defer if (!file.tree_loaded) file.tree.deinit(gpa); - - if (file.tree.errors.len != 0) { - const parse_err = file.tree.errors[0]; - - var msg = std.ArrayList(u8).init(gpa); - defer msg.deinit(); - - const token_starts = file.tree.tokens.items(.start); - const token_tags = file.tree.tokens.items(.tag); - - const extra_offset = file.tree.errorOffset(parse_err); - try file.tree.renderError(parse_err, msg.writer()); - const err_msg = try gpa.create(ErrorMsg); - err_msg.* = .{ - .src_loc = .{ - .file_scope = file, - .parent_decl_node = 0, - .lazy = if (extra_offset == 0) .{ - .token_abs = parse_err.token, - } else .{ - .byte_abs = token_starts[parse_err.token] + extra_offset, - }, - }, - .msg = try msg.toOwnedSlice(), - }; - if (token_tags[parse_err.token + @boolToInt(parse_err.token_is_prev)] == .invalid) { - const bad_off = @intCast(u32, file.tree.tokenSlice(parse_err.token + @boolToInt(parse_err.token_is_prev)).len); - const byte_abs = token_starts[parse_err.token + @boolToInt(parse_err.token_is_prev)] + bad_off; - try mod.errNoteNonLazy(.{ - .file_scope = file, - .parent_decl_node = 0, - .lazy = .{ .byte_abs = byte_abs }, - }, err_msg, "invalid byte: '{'}'", .{std.zig.fmtEscapes(source[byte_abs..][0..1])}); - } - - for (file.tree.errors[1..]) |note| { - if (!note.is_note) break; - - try file.tree.renderError(note, msg.writer()); - err_msg.notes = try mod.gpa.realloc(err_msg.notes, err_msg.notes.len + 1); - err_msg.notes[err_msg.notes.len - 1] = .{ - .src_loc = .{ - .file_scope = file, - .parent_decl_node = 0, - .lazy = .{ .token_abs = note.token }, - }, - .msg = try msg.toOwnedSlice(), - }; - } - - { - comp.mutex.lock(); - defer comp.mutex.unlock(); - try mod.failed_files.putNoClobber(gpa, file, err_msg); - } - file.status = .parse_failure; - return error.AnalysisFail; - } file.tree_loaded = true; + // Any potential AST errors are converted to ZIR errors here. file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; file.status = .success_zir; diff --git a/src/Package.zig b/src/Package.zig index 87d52197bd..febcc51788 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -225,7 +225,7 @@ pub fn fetchAndAddDependencies( dependencies_source: *std.ArrayList(u8), build_roots_source: *std.ArrayList(u8), name_prefix: []const u8, - color: main.Color, + error_bundle: *std.zig.ErrorBundle, all_modules: *AllModules, ) !void { const max_bytes = 10 * 1024 * 1024; @@ -250,7 +250,7 @@ pub fn fetchAndAddDependencies( if (ast.errors.len > 0) { const file_path = try directory.join(arena, &.{Manifest.basename}); - try main.printErrsMsgToStdErr(gpa, arena, ast, file_path, color); + try main.putAstErrorsIntoBundle(gpa, ast, file_path, error_bundle); return error.PackageFetchFailed; } @@ -258,23 +258,18 @@ pub fn fetchAndAddDependencies( defer manifest.deinit(gpa); if (manifest.errors.len > 0) { - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; const file_path = try directory.join(arena, &.{Manifest.basename}); for (manifest.errors) |msg| { - Report.renderErrorMessage(ast, file_path, ttyconf, msg, &.{}); + try Report.addErrorMessage(gpa, ast, file_path, error_bundle, 0, msg); } return error.PackageFetchFailed; } const report: Report = .{ + .gpa = gpa, .ast = &ast, .directory = directory, - .color = color, - .arena = arena, + .error_bundle = error_bundle, }; var any_error = false; @@ -307,7 +302,7 @@ pub fn fetchAndAddDependencies( dependencies_source, build_roots_source, sub_prefix, - color, + error_bundle, all_modules, ); @@ -348,10 +343,10 @@ pub fn createFilePkg( } const Report = struct { + gpa: Allocator, ast: *const std.zig.Ast, directory: Compilation.Directory, - color: main.Color, - arena: Allocator, + error_bundle: *std.zig.ErrorBundle, fn fail( report: Report, @@ -359,52 +354,48 @@ const Report = struct { comptime fmt_string: []const u8, fmt_args: anytype, ) error{ PackageFetchFailed, OutOfMemory } { - return failWithNotes(report, &.{}, tok, fmt_string, fmt_args); - } + const gpa = report.gpa; - fn failWithNotes( - report: Report, - notes: []const Compilation.AllErrors.Message, - tok: std.zig.Ast.TokenIndex, - comptime fmt_string: []const u8, - fmt_args: anytype, - ) error{ PackageFetchFailed, OutOfMemory } { - const ttyconf: std.debug.TTY.Config = switch (report.color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; - const file_path = try report.directory.join(report.arena, &.{Manifest.basename}); - renderErrorMessage(report.ast.*, file_path, ttyconf, .{ + const file_path = try report.directory.join(gpa, &.{Manifest.basename}); + defer gpa.free(file_path); + + const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args); + defer gpa.free(msg); + + try addErrorMessage(report.gpa, report.ast.*, file_path, report.error_bundle, 0, .{ .tok = tok, .off = 0, - .msg = try std.fmt.allocPrint(report.arena, fmt_string, fmt_args), - }, notes); + .msg = msg, + }); + return error.PackageFetchFailed; } - fn renderErrorMessage( + fn addErrorMessage( + gpa: Allocator, ast: std.zig.Ast, file_path: []const u8, - ttyconf: std.debug.TTY.Config, + eb: *std.zig.ErrorBundle, + notes_len: u32, msg: Manifest.ErrorMessage, - notes: []const Compilation.AllErrors.Message, - ) void { + ) error{OutOfMemory}!void { const token_starts = ast.tokens.items(.start); const start_loc = ast.tokenLocation(0, msg.tok); - Compilation.AllErrors.Message.renderToStdErr(.{ .src = .{ - .msg = msg.msg, - .src_path = file_path, - .line = @intCast(u32, start_loc.line), - .column = @intCast(u32, start_loc.column), - .span = .{ - .start = token_starts[msg.tok], - .end = @intCast(u32, token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), - .main = token_starts[msg.tok] + msg.off, - }, - .source_line = ast.source[start_loc.line_start..start_loc.line_end], - .notes = notes, - } }, ttyconf); + + try eb.addErrorMessage(gpa, .{ + .msg = try eb.addString(gpa, msg.msg), + .src_loc = try eb.addSourceLocation(gpa, .{ + .src_path = try eb.addString(gpa, file_path), + .span_start = token_starts[msg.tok], + .span_end = @intCast(u32, token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), + .span_main = token_starts[msg.tok] + msg.off, + .line = @intCast(u32, start_loc.line), + .column = @intCast(u32, start_loc.column), + .source_line = try eb.addString(gpa, ast.source[start_loc.line_start..start_loc.line_end]), + }), + .notes_len = notes_len, + }); + eb.incrementCount(1); } }; @@ -504,9 +495,7 @@ fn fetchAndUnpack( // by default, so the same logic applies for buffering the reader as for gzip. try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz); } else { - return report.fail(dep.url_tok, "unknown file extension for path '{s}'", .{ - uri.path, - }); + return report.fail(dep.url_tok, "unknown file extension for path '{s}'", .{uri.path}); } // TODO: delete files not included in the package prior to computing the package hash. @@ -533,10 +522,19 @@ fn fetchAndUnpack( }); } } else { - const notes: [1]Compilation.AllErrors.Message = .{.{ .plain = .{ - .msg = try std.fmt.allocPrint(report.arena, "expected .hash = \"{s}\",", .{&actual_hex}), - } }}; - return report.failWithNotes(¬es, dep.url_tok, "url field is missing corresponding hash field", .{}); + const file_path = try report.directory.join(gpa, &.{Manifest.basename}); + defer gpa.free(file_path); + + const eb = report.error_bundle; + try Report.addErrorMessage(gpa, report.ast.*, file_path, eb, 1, .{ + .tok = dep.url_tok, + .off = 0, + .msg = "url field is missing corresponding hash field", + }); + try eb.addErrorMessage(gpa, .{ + .msg = try eb.printString(gpa, "expected .hash = \"{s}\",", .{&actual_hex}), + }); + return error.PackageFetchFailed; } const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); diff --git a/src/Sema.zig b/src/Sema.zig index 8b6c269246..237936547e 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2211,29 +2211,26 @@ pub fn fail( fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { @setCold(true); + const gpa = sema.gpa; if (crash_report.is_enabled and sema.mod.comp.debug_compile_errors) { if (err_msg.src_loc.lazy == .unneeded) return error.NeededSourceLocation; - var arena = std.heap.ArenaAllocator.init(sema.gpa); - errdefer arena.deinit(); - var errors = std.ArrayList(Compilation.AllErrors.Message).init(sema.gpa); - defer errors.deinit(); - - Compilation.AllErrors.add(sema.mod, &arena, &errors, err_msg.*) catch unreachable; - + var errors: std.zig.ErrorBundle = undefined; + errors.init(gpa) catch unreachable; + Compilation.addModuleErrorMsg(gpa, &errors, err_msg.*) catch unreachable; std.debug.print("compile error during Sema:\n", .{}); - Compilation.AllErrors.Message.renderToStdErr(errors.items[0], .no_color); + errors.renderToStdErr(.no_color); crash_report.compilerPanic("unexpected compile error occurred", null, null); } const mod = sema.mod; ref: { - errdefer err_msg.destroy(mod.gpa); + errdefer err_msg.destroy(gpa); if (err_msg.src_loc.lazy == .unneeded) { return error.NeededSourceLocation; } - try mod.failed_decls.ensureUnusedCapacity(mod.gpa, 1); - try mod.failed_files.ensureUnusedCapacity(mod.gpa, 1); + try mod.failed_decls.ensureUnusedCapacity(gpa, 1); + try mod.failed_files.ensureUnusedCapacity(gpa, 1); const max_references = blk: { if (sema.mod.comp.reference_trace) |num| break :blk num; @@ -2243,11 +2240,11 @@ fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { }; var referenced_by = if (sema.func) |some| some.owner_decl else sema.owner_decl_index; - var reference_stack = std.ArrayList(Module.ErrorMsg.Trace).init(sema.gpa); + var reference_stack = std.ArrayList(Module.ErrorMsg.Trace).init(gpa); defer reference_stack.deinit(); // Avoid infinite loops. - var seen = std.AutoHashMap(Module.Decl.Index, void).init(sema.gpa); + var seen = std.AutoHashMap(Module.Decl.Index, void).init(gpa); defer seen.deinit(); var cur_reference_trace: u32 = 0; @@ -2288,7 +2285,7 @@ fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { if (gop.found_existing) { // If there are multiple errors for the same Decl, prefer the first one added. sema.err = null; - err_msg.destroy(mod.gpa); + err_msg.destroy(gpa); } else { sema.err = err_msg; gop.value_ptr.* = err_msg; diff --git a/src/glibc.zig b/src/glibc.zig index 3021e7c7ba..530f35531a 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -196,7 +196,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-DASSEMBLER", "-Wa,--noexecstack", }); - return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crti", .Obj, .@"glibc crti.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crti.S"), .cache_exempt_flags = args.items, @@ -215,7 +215,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-DASSEMBLER", "-Wa,--noexecstack", }); - return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crtn", .Obj, .@"glibc crtn.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crtn.S"), .cache_exempt_flags = args.items, @@ -265,7 +265,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .cache_exempt_flags = args.items, }; }; - return comp.build_crt_file("Scrt1", .Obj, &[_]Compilation.CSourceFile{ start_o, abi_note_o }); + return comp.build_crt_file("Scrt1", .Obj, .@"glibc Scrt1.o", &[_]Compilation.CSourceFile{ start_o, abi_note_o }); }, .libc_nonshared_a => { const s = path.sep_str; @@ -366,7 +366,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { files_index += 1; } const files = files_buf[0..files_index]; - return comp.build_crt_file("c_nonshared", .Lib, files); + return comp.build_crt_file("c_nonshared", .Lib, .@"glibc libc_nonshared.a", files); }, } } @@ -1105,7 +1105,7 @@ fn buildSharedLib( }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .@"glibc shared object"); } // Return true if glibc has crti/crtn sources for that architecture. diff --git a/src/libcxx.zig b/src/libcxx.zig index 7ca405cf15..c17352c562 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -258,7 +258,7 @@ pub fn buildLibCXX(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .libcxx); assert(comp.libcxx_static_lib == null); comp.libcxx_static_lib = Compilation.CRTFile{ @@ -418,7 +418,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .libcxxabi); assert(comp.libcxxabi_static_lib == null); comp.libcxxabi_static_lib = Compilation.CRTFile{ diff --git a/src/libtsan.zig b/src/libtsan.zig index 16e40c16f8..1399b6b76c 100644 --- a/src/libtsan.zig +++ b/src/libtsan.zig @@ -235,7 +235,7 @@ pub fn buildTsan(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .libtsan); assert(comp.tsan_static_lib == null); comp.tsan_static_lib = Compilation.CRTFile{ diff --git a/src/libunwind.zig b/src/libunwind.zig index a20b5e81f7..667195a369 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -130,7 +130,7 @@ pub fn buildStaticLib(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .libunwind); assert(comp.libunwind_static_lib == null); diff --git a/src/main.zig b/src/main.zig index a485fdb9e2..14b63017e5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -24,6 +24,8 @@ const clang = @import("clang.zig"); const Cache = std.Build.Cache; const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); +const Module = @import("Module.zig"); +const AstGen = @import("AstGen.zig"); pub const std_options = struct { pub const wasiCwd = wasi_cwd; @@ -3446,15 +3448,13 @@ fn buildOutputType( var errors = try comp.getAllErrorsAlloc(); defer errors.deinit(comp.gpa); - if (errors.list.len != 0) { + if (errors.errorMessageCount() > 0) { const ttyconf: std.debug.TTY.Config = switch (comp.color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - for (errors.list) |full_err_msg| { - try full_err_msg.renderToWriter(ttyconf, conn.stream.writer(), "error:", .Red, 0); - } + try errors.renderToWriter(ttyconf, conn.stream.writer()); continue; } } else { @@ -3830,15 +3830,13 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void var errors = try comp.getAllErrorsAlloc(); defer errors.deinit(comp.gpa); - if (errors.list.len != 0) { + if (errors.errorMessageCount() > 0) { const ttyconf: std.debug.TTY.Config = switch (comp.color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - for (errors.list) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); const log_text = comp.getCompileLogOutput(); if (log_text.len != 0) { std.debug.print("\nCompile Log Output:\n{s}", .{log_text}); @@ -4438,9 +4436,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var all_modules: Package.AllModules = .{}; defer all_modules.deinit(gpa); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + // Here we borrow main package's table and will replace it with a fresh // one after this process completes. - build_pkg.fetchAndAddDependencies( + const fetch_result = build_pkg.fetchAndAddDependencies( &main_pkg, arena, &thread_pool, @@ -4451,12 +4453,19 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &dependencies_source, &build_roots_source, "", - color, + &errors, &all_modules, - ) catch |err| switch (err) { - error.PackageFetchFailed => process.exit(1), - else => |e| return e, - }; + ); + if (errors.errorMessageCount() > 0) { + const ttyconf: std.debug.TTY.Config = switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + errors.renderToStdErr(ttyconf); + process.exit(1); + } + try fetch_result; try dependencies_source.appendSlice("};\npub const build_root = struct {\n"); try dependencies_source.appendSlice(build_roots_source.items); @@ -4543,7 +4552,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } fn readSourceFileToEndAlloc( - allocator: mem.Allocator, + allocator: Allocator, input: *const fs.File, size_hint: ?usize, ) ![:0]u8 { @@ -4687,12 +4696,9 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void }; defer tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, tree, "", color); + try printAstErrorsToStderr(gpa, tree, "", color); var has_ast_error = false; if (check_ast_flag) { - const Module = @import("Module.zig"); - const AstGen = @import("AstGen.zig"); - var file: Module.File = .{ .status = .never_loaded, .source_loaded = true, @@ -4715,20 +4721,16 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - var errors = std.ArrayList(Compilation.AllErrors.Message).init(gpa); - defer errors.deinit(); - - try Compilation.AllErrors.addZir(arena_instance.allocator(), &errors, &file); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + try Compilation.addZirErrorMessages(gpa, &errors, &file); const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - for (errors.items) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); has_ast_error = true; } } @@ -4875,12 +4877,13 @@ fn fmtPathFile( if (stat.kind == .Directory) return error.IsDir; + const gpa = fmt.gpa; const source_code = try readSourceFileToEndAlloc( - fmt.gpa, + gpa, &source_file, std.math.cast(usize, stat.size) orelse return error.FileTooBig, ); - defer fmt.gpa.free(source_code); + defer gpa.free(source_code); source_file.close(); file_closed = true; @@ -4888,19 +4891,16 @@ fn fmtPathFile( // Add to set after no longer possible to get error.IsDir. if (try fmt.seen.fetchPut(stat.inode, {})) |_| return; - var tree = try Ast.parse(fmt.gpa, source_code, .zig); - defer tree.deinit(fmt.gpa); + var tree = try Ast.parse(gpa, source_code, .zig); + defer tree.deinit(gpa); - try printErrsMsgToStdErr(fmt.gpa, fmt.arena, tree, file_path, fmt.color); + try printAstErrorsToStderr(gpa, tree, file_path, fmt.color); if (tree.errors.len != 0) { fmt.any_error = true; return; } if (fmt.check_ast) { - const Module = @import("Module.zig"); - const AstGen = @import("AstGen.zig"); - var file: Module.File = .{ .status = .never_loaded, .source_loaded = true, @@ -4919,31 +4919,27 @@ fn fmtPathFile( .root_decl = .none, }; - file.pkg = try Package.create(fmt.gpa, null, file.sub_file_path); - defer file.pkg.destroy(fmt.gpa); + file.pkg = try Package.create(gpa, null, file.sub_file_path); + defer file.pkg.destroy(gpa); if (stat.size > max_src_size) return error.FileTooBig; - file.zir = try AstGen.generate(fmt.gpa, file.tree); + file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; - defer file.zir.deinit(fmt.gpa); + defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var arena_instance = std.heap.ArenaAllocator.init(fmt.gpa); - defer arena_instance.deinit(); - var errors = std.ArrayList(Compilation.AllErrors.Message).init(fmt.gpa); - defer errors.deinit(); - - try Compilation.AllErrors.addZir(arena_instance.allocator(), &errors, &file); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + try Compilation.addZirErrorMessages(gpa, &errors, &file); const ttyconf: std.debug.TTY.Config = switch (fmt.color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - for (errors.items) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); fmt.any_error = true; } } @@ -4971,100 +4967,53 @@ fn fmtPathFile( } } -pub fn printErrsMsgToStdErr( - gpa: mem.Allocator, - arena: mem.Allocator, +fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Color) !void { + var error_bundle: std.zig.ErrorBundle = undefined; + try error_bundle.init(gpa); + defer error_bundle.deinit(gpa); + + try putAstErrorsIntoBundle(gpa, tree, path, &error_bundle); + + const ttyconf: std.debug.TTY.Config = switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + error_bundle.renderToStdErr(ttyconf); +} + +pub fn putAstErrorsIntoBundle( + gpa: Allocator, tree: Ast, path: []const u8, - color: Color, + error_bundle: *std.zig.ErrorBundle, ) !void { - const parse_errors: []const Ast.Error = tree.errors; - var i: usize = 0; - while (i < parse_errors.len) : (i += 1) { - const parse_error = parse_errors[i]; - const lok_token = parse_error.token; - const token_tags = tree.tokens.items(.tag); - const start_loc = tree.tokenLocation(0, lok_token); - const source_line = tree.source[start_loc.line_start..start_loc.line_end]; - - var text_buf = std.ArrayList(u8).init(gpa); - defer text_buf.deinit(); - const writer = text_buf.writer(); - try tree.renderError(parse_error, writer); - const text = try arena.dupe(u8, text_buf.items); - - var notes_buffer: [2]Compilation.AllErrors.Message = undefined; - var notes_len: usize = 0; - - if (token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)] == .invalid) { - const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token + @boolToInt(parse_error.token_is_prev)).len); - const byte_offset = @intCast(u32, start_loc.line_start) + @intCast(u32, start_loc.column) + bad_off; - notes_buffer[notes_len] = .{ - .src = .{ - .src_path = path, - .msg = try std.fmt.allocPrint(arena, "invalid byte: '{'}'", .{ - std.zig.fmtEscapes(tree.source[byte_offset..][0..1]), - }), - .span = .{ .start = byte_offset, .end = byte_offset + 1, .main = byte_offset }, - .line = @intCast(u32, start_loc.line), - .column = @intCast(u32, start_loc.column) + bad_off, - .source_line = source_line, - }, - }; - notes_len += 1; - } - - for (parse_errors[i + 1 ..]) |note| { - if (!note.is_note) break; - - text_buf.items.len = 0; - try tree.renderError(note, writer); - const note_loc = tree.tokenLocation(0, note.token); - const byte_offset = @intCast(u32, note_loc.line_start); - notes_buffer[notes_len] = .{ - .src = .{ - .src_path = path, - .msg = try arena.dupe(u8, text_buf.items), - .span = .{ - .start = byte_offset, - .end = byte_offset + @intCast(u32, tree.tokenSlice(note.token).len), - .main = byte_offset, - }, - .line = @intCast(u32, note_loc.line), - .column = @intCast(u32, note_loc.column), - .source_line = tree.source[note_loc.line_start..note_loc.line_end], - }, - }; - i += 1; - notes_len += 1; - } + var file: Module.File = .{ + .status = .never_loaded, + .source_loaded = true, + .zir_loaded = false, + .sub_file_path = path, + .source = tree.source, + .stat = .{ + .size = 0, + .inode = 0, + .mtime = 0, + }, + .tree = tree, + .tree_loaded = true, + .zir = undefined, + .pkg = undefined, + .root_decl = .none, + }; - const extra_offset = tree.errorOffset(parse_error); - const byte_offset = @intCast(u32, start_loc.line_start) + extra_offset; - const message: Compilation.AllErrors.Message = .{ - .src = .{ - .src_path = path, - .msg = text, - .span = .{ - .start = byte_offset, - .end = byte_offset + @intCast(u32, tree.tokenSlice(lok_token).len), - .main = byte_offset, - }, - .line = @intCast(u32, start_loc.line), - .column = @intCast(u32, start_loc.column) + extra_offset, - .source_line = source_line, - .notes = notes_buffer[0..notes_len], - }, - }; + file.pkg = try Package.create(gpa, null, path); + defer file.pkg.destroy(gpa); - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; + file.zir = try AstGen.generate(gpa, file.tree); + file.zir_loaded = true; + defer file.zir.deinit(gpa); - message.renderToStdErr(ttyconf); - } + try Compilation.addZirErrorMessages(gpa, error_bundle, &file); } pub const info_zen = @@ -5547,8 +5496,6 @@ pub fn cmdAstCheck( arena: Allocator, args: []const []const u8, ) !void { - const Module = @import("Module.zig"); - const AstGen = @import("AstGen.zig"); const Zir = @import("Zir.zig"); var color: Color = .auto; @@ -5638,7 +5585,7 @@ pub fn cmdAstCheck( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree, file.sub_file_path, color); + try printAstErrorsToStderr(gpa, file.tree, file.sub_file_path, color); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5648,16 +5595,16 @@ pub fn cmdAstCheck( defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors = std.ArrayList(Compilation.AllErrors.Message).init(arena); - try Compilation.AllErrors.addZir(arena, &errors, &file); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + try Compilation.addZirErrorMessages(gpa, &errors, &file); const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - for (errors.items) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); process.exit(1); } @@ -5715,8 +5662,6 @@ pub fn cmdChangelist( arena: Allocator, args: []const []const u8, ) !void { - const Module = @import("Module.zig"); - const AstGen = @import("AstGen.zig"); const Zir = @import("Zir.zig"); const old_source_file = args[0]; @@ -5764,7 +5709,7 @@ pub fn cmdChangelist( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, file.tree, old_source_file, .auto); + try printAstErrorsToStderr(gpa, file.tree, old_source_file, .auto); if (file.tree.errors.len != 0) { process.exit(1); } @@ -5774,12 +5719,12 @@ pub fn cmdChangelist( defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors = std.ArrayList(Compilation.AllErrors.Message).init(arena); - try Compilation.AllErrors.addZir(arena, &errors, &file); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + try Compilation.addZirErrorMessages(gpa, &errors, &file); const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); - for (errors.items) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); process.exit(1); } @@ -5801,7 +5746,7 @@ pub fn cmdChangelist( var new_tree = try Ast.parse(gpa, new_source, .zig); defer new_tree.deinit(gpa); - try printErrsMsgToStdErr(gpa, arena, new_tree, new_source_file, .auto); + try printAstErrorsToStderr(gpa, new_tree, new_source_file, .auto); if (new_tree.errors.len != 0) { process.exit(1); } @@ -5813,12 +5758,12 @@ pub fn cmdChangelist( file.zir_loaded = true; if (file.zir.hasCompileErrors()) { - var errors = std.ArrayList(Compilation.AllErrors.Message).init(arena); - try Compilation.AllErrors.addZir(arena, &errors, &file); + var errors: std.zig.ErrorBundle = undefined; + try errors.init(gpa); + defer errors.deinit(gpa); + try Compilation.addZirErrorMessages(gpa, &errors, &file); const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); - for (errors.items) |full_err_msg| { - full_err_msg.renderToStdErr(ttyconf); - } + errors.renderToStdErr(ttyconf); process.exit(1); } diff --git a/src/mingw.zig b/src/mingw.zig index 9e9e180945..d5c42c3ccc 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -41,7 +41,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { //"-D_UNICODE", //"-DWPRFLAG=1", }); - return comp.build_crt_file("crt2", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt2", .Obj, .@"mingw-w64 crt2.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "crt", "crtexe.c", @@ -60,7 +60,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-U__CRTDLL__", "-D__MSVCRT__", }); - return comp.build_crt_file("dllcrt2", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("dllcrt2", .Obj, .@"mingw-w64 dllcrt2.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "crt", "crtdll.c", @@ -100,7 +100,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; } - return comp.build_crt_file("mingw32", .Lib, &c_source_files); + return comp.build_crt_file("mingw32", .Lib, .@"mingw-w64 mingw32.lib", &c_source_files); }, .msvcrt_os_lib => { @@ -148,7 +148,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { }; } } - return comp.build_crt_file("msvcrt-os", .Lib, c_source_files.items); + return comp.build_crt_file("msvcrt-os", .Lib, .@"mingw-w64 msvcrt-os.lib", c_source_files.items); }, .mingwex_lib => { @@ -211,7 +211,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } else { @panic("unsupported arch"); } - return comp.build_crt_file("mingwex", .Lib, c_source_files.items); + return comp.build_crt_file("mingwex", .Lib, .@"mingw-w64 mingwex.lib", c_source_files.items); }, .uuid_lib => { @@ -244,7 +244,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = extra_flags, }; } - return comp.build_crt_file("uuid", .Lib, &c_source_files); + return comp.build_crt_file("uuid", .Lib, .@"mingw-w64 uuid.lib", &c_source_files); }, } } diff --git a/src/musl.zig b/src/musl.zig index 18e618df8f..7dd224604f 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -33,7 +33,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { try args.appendSlice(&[_][]const u8{ "-Qunused-arguments", }); - return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crti", .Obj, .@"musl crti.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crti.s"), .extra_flags = args.items, @@ -46,7 +46,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { try args.appendSlice(&[_][]const u8{ "-Qunused-arguments", }); - return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crtn", .Obj, .@"musl crtn.o", &[1]Compilation.CSourceFile{ .{ .src_path = try start_asm_path(comp, arena, "crtn.s"), .extra_flags = args.items, @@ -60,7 +60,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("crt1", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1", .Obj, .@"musl crt1.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "crt1.c", @@ -77,7 +77,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("rcrt1", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("rcrt1", .Obj, .@"musl rcrt1.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "rcrt1.c", @@ -94,7 +94,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("Scrt1", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("Scrt1", .Obj, .@"musl Scrt1.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "Scrt1.c", @@ -187,7 +187,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; } - return comp.build_crt_file("c", .Lib, c_source_files.items); + return comp.build_crt_file("c", .Lib, .@"musl libc.a", c_source_files.items); }, .libc_so => { const target = comp.getTarget(); @@ -241,7 +241,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { }); defer sub_compilation.destroy(); - try sub_compilation.updateSubCompilation(); + try comp.updateSubCompilation(sub_compilation, .@"musl libc.so"); try comp.crt_files.ensureUnusedCapacity(comp.gpa, 1); diff --git a/src/wasi_libc.zig b/src/wasi_libc.zig index fc8c81d5af..c4a4cbc4a5 100644 --- a/src/wasi_libc.zig +++ b/src/wasi_libc.zig @@ -74,7 +74,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, false); try addLibcBottomHalfIncludes(comp, arena, &args); - return comp.build_crt_file("crt1-reactor", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1-reactor", .Obj, .@"wasi crt1-reactor.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, crt1_reactor_src_file), @@ -87,7 +87,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, false); try addLibcBottomHalfIncludes(comp, arena, &args); - return comp.build_crt_file("crt1-command", .Obj, &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1-command", .Obj, .@"wasi crt1-command.o", &[1]Compilation.CSourceFile{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, crt1_command_src_file), @@ -145,7 +145,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } } - try comp.build_crt_file("c", .Lib, libc_sources.items); + try comp.build_crt_file("c", .Lib, .@"wasi libc.a", libc_sources.items); }, .libwasi_emulated_process_clocks_a => { var args = std.ArrayList([]const u8).init(arena); @@ -161,7 +161,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, emu_clocks_sources.items); + try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, .@"libwasi-emulated-process-clocks.a", emu_clocks_sources.items); }, .libwasi_emulated_getpid_a => { var args = std.ArrayList([]const u8).init(arena); @@ -177,7 +177,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-getpid", .Lib, emu_getpid_sources.items); + try comp.build_crt_file("wasi-emulated-getpid", .Lib, .@"libwasi-emulated-getpid.a", emu_getpid_sources.items); }, .libwasi_emulated_mman_a => { var args = std.ArrayList([]const u8).init(arena); @@ -193,7 +193,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-mman", .Lib, emu_mman_sources.items); + try comp.build_crt_file("wasi-emulated-mman", .Lib, .@"libwasi-emulated-mman.a", emu_mman_sources.items); }, .libwasi_emulated_signal_a => { var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena); @@ -228,7 +228,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } } - try comp.build_crt_file("wasi-emulated-signal", .Lib, emu_signal_sources.items); + try comp.build_crt_file("wasi-emulated-signal", .Lib, .@"libwasi-emulated-signal.a", emu_signal_sources.items); }, } } -- cgit v1.2.3 From 6f717b18f05ba02439603e0987e9c9551fbadedb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 23 Feb 2023 19:21:44 -0700 Subject: std.zig.ErrorBundle: rework binary encoding * Separate into a "WIP" struct and a "finished" struct. * Use a bit of indirection for error notes to simplify ergonomics of this data structure. --- lib/std/zig/ErrorBundle.zig | 410 ++++++++++++++++++++++++-------------------- src/Compilation.zig | 253 +++++++++++++-------------- src/Package.zig | 36 ++-- src/Sema.zig | 9 +- src/main.zig | 86 ++++++---- src/test.zig | 21 +-- 6 files changed, 425 insertions(+), 390 deletions(-) (limited to 'src/main.zig') diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index dee19818a8..1d2eca9703 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -3,24 +3,22 @@ //! is used to collect all the errors from the various places into one //! convenient place for API users to consume. -string_bytes: std.ArrayListUnmanaged(u8), -/// The first thing in this array is a ErrorMessageListIndex. -extra: std.ArrayListUnmanaged(u32), +string_bytes: []const u8, +/// The first thing in this array is an `ErrorMessageList`. +extra: []const u32, // An index into `extra` pointing at an `ErrorMessage`. pub const MessageIndex = enum(u32) { _, }; -/// After the header is: -/// * string_bytes -/// * extra (little endian) -pub const Header = struct { - string_bytes_len: u32, - extra_len: u32, +// An index into `extra` pointing at an `SourceLocation`. +pub const SourceLocationIndex = enum(u32) { + none = 0, + _, }; -/// Trailing: ErrorMessage for each len +/// There will be a MessageIndex for each len at start. pub const ErrorMessageList = struct { len: u32, start: u32, @@ -46,14 +44,13 @@ pub const SourceLocation = struct { }; /// Trailing: -/// * ErrorMessage for each notes_len. +/// * MessageIndex for each notes_len. pub const ErrorMessage = struct { /// null terminated string index msg: u32, /// Usually one, but incremented for redundant messages. count: u32 = 1, - /// 0 or the index into extra of a SourceLocation - src_loc: u32 = 0, + src_loc: SourceLocationIndex = .none, notes_len: u32 = 0, }; @@ -65,170 +62,41 @@ pub const ReferenceTrace = struct { decl_name: u32, /// Index into extra of a SourceLocation /// If this is 0, this is the sentinel ReferenceTrace element. - src_loc: u32, + src_loc: SourceLocationIndex, }; -pub fn init(eb: *ErrorBundle, gpa: Allocator) !void { - eb.* = .{ - .string_bytes = .{}, - .extra = .{}, - }; - - // So that 0 can be used to indicate a null string. - try eb.string_bytes.append(gpa, 0); - - _ = try addExtra(eb, gpa, ErrorMessageList{ - .len = 0, - .start = 0, - }); -} - pub fn deinit(eb: *ErrorBundle, gpa: Allocator) void { - eb.string_bytes.deinit(gpa); - eb.extra.deinit(gpa); + gpa.free(eb.string_bytes); + gpa.free(eb.extra); eb.* = undefined; } -pub fn addString(eb: *ErrorBundle, gpa: Allocator, s: []const u8) !u32 { - const index = @intCast(u32, eb.string_bytes.items.len); - try eb.string_bytes.ensureUnusedCapacity(gpa, s.len + 1); - eb.string_bytes.appendSliceAssumeCapacity(s); - eb.string_bytes.appendAssumeCapacity(0); - return index; -} - -pub fn printString(eb: *ErrorBundle, gpa: Allocator, comptime fmt: []const u8, args: anytype) !u32 { - const index = @intCast(u32, eb.string_bytes.items.len); - try eb.string_bytes.writer(gpa).print(fmt, args); - try eb.string_bytes.append(gpa, 0); - return index; -} - -pub fn addErrorMessage(eb: *ErrorBundle, gpa: Allocator, em: ErrorMessage) !void { - if (eb.errorMessageCount() == 0) { - eb.setStartIndex(@intCast(u32, eb.extra.items.len)); - } - _ = try addExtra(eb, gpa, em); -} - -pub fn addSourceLocation(eb: *ErrorBundle, gpa: Allocator, sl: SourceLocation) !u32 { - return addExtra(eb, gpa, sl); -} - -pub fn addReferenceTrace(eb: *ErrorBundle, gpa: Allocator, rt: ReferenceTrace) !void { - _ = try addExtra(eb, gpa, rt); -} - -pub fn addBundle(eb: *ErrorBundle, gpa: Allocator, other: ErrorBundle) !void { - // Skip over the initial ErrorMessageList len field. - const root_fields_len = @typeInfo(ErrorMessageList).Struct.fields.len; - const other_list = other.extraData(ErrorMessageList, 0).data; - const other_extra = other.extra.items[root_fields_len..]; - - try eb.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.items.len); - try eb.extra.ensureUnusedCapacity(gpa, other_extra.len); - - const new_string_base = @intCast(u32, eb.string_bytes.items.len); - const new_data_base = @intCast(u32, eb.extra.items.len - root_fields_len); - - eb.string_bytes.appendSliceAssumeCapacity(other.string_bytes.items); - eb.extra.appendSliceAssumeCapacity(other_extra); - - // Now we must offset the string indexes and extra indexes of the newly - // added extra. - var index = new_data_base + other_list.start; - for (0..other_list.len) |_| { - index = try patchMessage(eb, index, new_string_base, new_data_base); - } -} - -fn patchMessage(eb: *ErrorBundle, msg_idx: usize, new_string_base: u32, new_data_base: u32) !u32 { - var msg = eb.extraData(ErrorMessage, msg_idx); - if (msg.data.msg != 0) msg.data.msg += new_string_base; - if (msg.data.src_loc != 0) msg.data.src_loc += new_data_base; - eb.setExtra(msg_idx, msg.data); - - try patchSrcLoc(eb, msg.data.src_loc, new_string_base, new_data_base); - - var index = @intCast(u32, msg.end); - for (0..msg.data.notes_len) |_| { - index = try patchMessage(eb, index, new_string_base, new_data_base); - } - return index; -} - -fn patchSrcLoc(eb: *ErrorBundle, idx: usize, new_string_base: u32, new_data_base: u32) !void { - if (idx == 0) return; - - var src_loc = eb.extraData(SourceLocation, idx); - if (src_loc.data.src_path != 0) src_loc.data.src_path += new_string_base; - if (src_loc.data.source_line != 0) src_loc.data.source_line += new_string_base; - eb.setExtra(idx, src_loc.data); - - var index = src_loc.end; - for (0..src_loc.data.reference_trace_len) |_| { - var ref_trace = eb.extraData(ReferenceTrace, index); - if (ref_trace.data.decl_name != 0) ref_trace.data.decl_name += new_string_base; - if (ref_trace.data.src_loc != 0) ref_trace.data.src_loc += new_data_base; - eb.setExtra(index, ref_trace.data); - try patchSrcLoc(eb, ref_trace.data.src_loc, new_string_base, new_data_base); - index = ref_trace.end; - } -} - -fn addExtra(eb: *ErrorBundle, gpa: Allocator, extra: anytype) Allocator.Error!u32 { - const fields = @typeInfo(@TypeOf(extra)).Struct.fields; - try eb.extra.ensureUnusedCapacity(gpa, fields.len); - return addExtraAssumeCapacity(eb, extra); -} - -fn addExtraAssumeCapacity(eb: *ErrorBundle, extra: anytype) u32 { - const fields = @typeInfo(@TypeOf(extra)).Struct.fields; - const result = @intCast(u32, eb.extra.items.len); - eb.extra.items.len += fields.len; - setExtra(eb, result, extra); - return result; -} - -fn setExtra(eb: *ErrorBundle, index: usize, extra: anytype) void { - const fields = @typeInfo(@TypeOf(extra)).Struct.fields; - var i = index; - inline for (fields) |field| { - eb.extra.items[i] = switch (field.type) { - u32 => @field(extra, field.name), - else => @compileError("bad field type"), - }; - i += 1; - } -} - pub fn errorMessageCount(eb: ErrorBundle) u32 { - return eb.extra.items[0]; -} - -pub fn setErrorMessageCount(eb: *ErrorBundle, count: u32) void { - eb.extra.items[0] = count; + return eb.getErrorMessageList().len; } -pub fn incrementCount(eb: *ErrorBundle, delta: u32) void { - eb.extra.items[0] += delta; +pub fn getErrorMessageList(eb: ErrorBundle) ErrorMessageList { + return eb.extraData(ErrorMessageList, 0).data; } -pub fn getStartIndex(eb: ErrorBundle) u32 { - return eb.extra.items[1]; -} - -pub fn setStartIndex(eb: *ErrorBundle, index: u32) void { - eb.extra.items[1] = index; +pub fn getMessages(eb: ErrorBundle) []const MessageIndex { + const list = eb.getErrorMessageList(); + return @ptrCast([]const MessageIndex, eb.extra[list.start..][0..list.len]); } pub fn getErrorMessage(eb: ErrorBundle, index: MessageIndex) ErrorMessage { return eb.extraData(ErrorMessage, @enumToInt(index)).data; } -pub fn getSourceLocation(eb: ErrorBundle, index: u32) SourceLocation { - assert(index != 0); - return eb.extraData(SourceLocation, index).data; +pub fn getSourceLocation(eb: ErrorBundle, index: SourceLocationIndex) SourceLocation { + assert(index != .none); + return eb.extraData(SourceLocation, @enumToInt(index)).data; +} + +pub fn getNotes(eb: ErrorBundle, index: MessageIndex) []const MessageIndex { + const notes_len = eb.getErrorMessage(index).notes_len; + const start = @enumToInt(index) + @typeInfo(ErrorMessage).Struct.fields.len; + return @ptrCast([]const MessageIndex, eb.extra[start..][0..notes_len]); } /// Returns the requested data, as well as the new index which is at the start of the @@ -239,7 +107,9 @@ fn extraData(eb: ErrorBundle, comptime T: type, index: usize) struct { data: T, var result: T = undefined; inline for (fields) |field| { @field(result, field.name) = switch (field.type) { - u32 => eb.extra.items[i], + u32 => eb.extra[i], + MessageIndex => @intToEnum(MessageIndex, eb.extra[i]), + SourceLocationIndex => @intToEnum(SourceLocationIndex, eb.extra[i]), else => @compileError("bad field type"), }; i += 1; @@ -252,7 +122,7 @@ fn extraData(eb: ErrorBundle, comptime T: type, index: usize) struct { data: T, /// Given an index into `string_bytes` returns the null-terminated string found there. pub fn nullTerminatedString(eb: ErrorBundle, index: usize) [:0]const u8 { - const string_bytes = eb.string_bytes.items; + const string_bytes = eb.string_bytes; var end: usize = index; while (string_bytes[end] != 0) { end += 1; @@ -272,28 +142,25 @@ pub fn renderToWriter( ttyconf: std.debug.TTY.Config, writer: anytype, ) anyerror!void { - const list = eb.extraData(ErrorMessageList, 0).data; - var index: usize = list.start; - for (0..list.len) |_| { - const err_msg = eb.extraData(ErrorMessage, index); - index = try renderErrorMessageToWriter(eb, err_msg.data, err_msg.end, ttyconf, writer, "error", .Red, 0); + for (eb.getMessages()) |err_msg| { + try renderErrorMessageToWriter(eb, err_msg, ttyconf, writer, "error", .Red, 0); } } fn renderErrorMessageToWriter( eb: ErrorBundle, - err_msg: ErrorMessage, - end_index: usize, + err_msg_index: MessageIndex, ttyconf: std.debug.TTY.Config, stderr: anytype, kind: []const u8, color: std.debug.TTY.Color, indent: usize, -) anyerror!usize { +) anyerror!void { var counting_writer = std.io.countingWriter(stderr); const counting_stderr = counting_writer.writer(); - if (err_msg.src_loc != 0) { - const src = eb.extraData(SourceLocation, err_msg.src_loc); + const err_msg = eb.getErrorMessage(err_msg_index); + if (err_msg.src_loc != .none) { + const src = eb.extraData(SourceLocation, @enumToInt(err_msg.src_loc)); try counting_stderr.writeByteNTimes(' ', indent); try ttyconf.setColor(stderr, .Bold); try counting_stderr.print("{s}:{d}:{d}: ", .{ @@ -337,10 +204,8 @@ fn renderErrorMessageToWriter( try stderr.writeByte('\n'); try ttyconf.setColor(stderr, .Reset); } - var index = end_index; - for (0..err_msg.notes_len) |_| { - const note = eb.extraData(ErrorMessage, index); - index = try renderErrorMessageToWriter(eb, note.data, note.end, ttyconf, stderr, "note", .Cyan, indent); + for (eb.getNotes(err_msg_index)) |note| { + try renderErrorMessageToWriter(eb, note, ttyconf, stderr, "note", .Cyan, indent); } if (src.data.reference_trace_len > 0) { try ttyconf.setColor(stderr, .Reset); @@ -350,7 +215,7 @@ fn renderErrorMessageToWriter( for (0..src.data.reference_trace_len) |_| { const ref_trace = eb.extraData(ReferenceTrace, ref_index); ref_index = ref_trace.end; - if (ref_trace.data.src_loc != 0) { + if (ref_trace.data.src_loc != .none) { const ref_src = eb.getSourceLocation(ref_trace.data.src_loc); try stderr.print(" {s}: {s}:{d}:{d}\n", .{ eb.nullTerminatedString(ref_trace.data.decl_name), @@ -374,7 +239,6 @@ fn renderErrorMessageToWriter( try stderr.writeByte('\n'); try ttyconf.setColor(stderr, .Reset); } - return index; } else { try ttyconf.setColor(stderr, color); try stderr.writeByteNTimes(' ', indent); @@ -390,12 +254,9 @@ fn renderErrorMessageToWriter( try stderr.print(" ({d} times)\n", .{err_msg.count}); } try ttyconf.setColor(stderr, .Reset); - var index = end_index; - for (0..err_msg.notes_len) |_| { - const note = eb.extraData(ErrorMessage, index); - index = try renderErrorMessageToWriter(eb, note.data, note.end, ttyconf, stderr, "note", .Cyan, indent + 4); + for (eb.getNotes(err_msg_index)) |note| { + try renderErrorMessageToWriter(eb, note, ttyconf, stderr, "note", .Cyan, indent + 4); } - return index; } } @@ -417,3 +278,186 @@ const std = @import("std"); const ErrorBundle = @This(); const Allocator = std.mem.Allocator; const assert = std.debug.assert; + +pub const Wip = struct { + gpa: Allocator, + string_bytes: std.ArrayListUnmanaged(u8), + /// The first thing in this array is a ErrorMessageList. + extra: std.ArrayListUnmanaged(u32), + root_list: std.ArrayListUnmanaged(MessageIndex), + + pub fn init(wip: *Wip, gpa: Allocator) !void { + wip.* = .{ + .gpa = gpa, + .string_bytes = .{}, + .extra = .{}, + .root_list = .{}, + }; + + // So that 0 can be used to indicate a null string. + try wip.string_bytes.append(gpa, 0); + + assert(0 == try addExtra(wip, ErrorMessageList{ + .len = 0, + .start = 0, + })); + } + + pub fn deinit(wip: *Wip) void { + const gpa = wip.gpa; + wip.root_list.deinit(gpa); + wip.string_bytes.deinit(gpa); + wip.extra.deinit(gpa); + wip.* = undefined; + } + + pub fn toOwnedBundle(wip: *Wip) !ErrorBundle { + const gpa = wip.gpa; + wip.setExtra(0, ErrorMessageList{ + .len = @intCast(u32, wip.root_list.items.len), + .start = @intCast(u32, wip.extra.items.len), + }); + try wip.extra.appendSlice(gpa, @ptrCast([]const u32, wip.root_list.items)); + wip.root_list.clearAndFree(gpa); + return .{ + .string_bytes = try wip.string_bytes.toOwnedSlice(gpa), + .extra = try wip.extra.toOwnedSlice(gpa), + }; + } + + pub fn tmpBundle(wip: Wip) ErrorBundle { + return .{ + .string_bytes = wip.string_bytes.items, + .extra = wip.extra.items, + }; + } + + pub fn addString(wip: *Wip, s: []const u8) !u32 { + const gpa = wip.gpa; + const index = @intCast(u32, wip.string_bytes.items.len); + try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1); + wip.string_bytes.appendSliceAssumeCapacity(s); + wip.string_bytes.appendAssumeCapacity(0); + return index; + } + + pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 { + const gpa = wip.gpa; + const index = @intCast(u32, wip.string_bytes.items.len); + try wip.string_bytes.writer(gpa).print(fmt, args); + try wip.string_bytes.append(gpa, 0); + return index; + } + + pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) !void { + try wip.root_list.ensureUnusedCapacity(wip.gpa, 1); + wip.root_list.appendAssumeCapacity(try addErrorMessage(wip, em)); + } + + pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex { + return @intToEnum(MessageIndex, try addExtra(wip, em)); + } + + pub fn addErrorMessageAssumeCapacity(wip: *Wip, em: ErrorMessage) MessageIndex { + return @intToEnum(MessageIndex, addExtraAssumeCapacity(wip, em)); + } + + pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex { + return @intToEnum(SourceLocationIndex, try addExtra(wip, sl)); + } + + pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void { + _ = try addExtra(wip, rt); + } + + pub fn addBundle(wip: *Wip, other: ErrorBundle) !void { + const gpa = wip.gpa; + + try wip.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.len); + try wip.extra.ensureUnusedCapacity(gpa, other.extra.len); + + const other_list = other.getMessages(); + + // The ensureUnusedCapacity call above guarantees this. + const notes_start = wip.reserveNotes(@intCast(u32, other_list.len)) catch unreachable; + for (notes_start.., other_list) |note, message| { + wip.extra.items[note] = @enumToInt(wip.addOtherMessage(other, message) catch unreachable); + } + } + + pub fn reserveNotes(wip: *Wip, notes_len: u32) !u32 { + try wip.extra.ensureUnusedCapacity(wip.gpa, notes_len + + notes_len * @typeInfo(ErrorBundle.ErrorMessage).Struct.fields.len); + wip.extra.items.len += notes_len; + return @intCast(u32, wip.extra.items.len - notes_len); + } + + fn addOtherMessage(wip: *Wip, other: ErrorBundle, msg_index: MessageIndex) !MessageIndex { + const other_msg = other.getErrorMessage(msg_index); + const src_loc = try wip.addOtherSourceLocation(other, other_msg.src_loc); + const msg = try wip.addErrorMessage(.{ + .msg = try wip.addString(other.nullTerminatedString(other_msg.msg)), + .count = other_msg.count, + .src_loc = src_loc, + .notes_len = other_msg.notes_len, + }); + const notes_start = try wip.reserveNotes(other_msg.notes_len); + for (notes_start.., other.getNotes(msg_index)) |note, other_note| { + wip.extra.items[note] = @enumToInt(try wip.addOtherMessage(other, other_note)); + } + return msg; + } + + fn addOtherSourceLocation( + wip: *Wip, + other: ErrorBundle, + index: SourceLocationIndex, + ) !SourceLocationIndex { + if (index == .none) return .none; + const other_sl = other.getSourceLocation(index); + + const src_loc = try wip.addSourceLocation(.{ + .src_path = try wip.addString(other.nullTerminatedString(other_sl.src_path)), + .line = other_sl.line, + .column = other_sl.column, + .span_start = other_sl.span_start, + .span_main = other_sl.span_main, + .span_end = other_sl.span_end, + .source_line = try wip.addString(other.nullTerminatedString(other_sl.source_line)), + .reference_trace_len = other_sl.reference_trace_len, + }); + + // TODO: also add the reference trace + + return src_loc; + } + + fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 { + const gpa = wip.gpa; + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + try wip.extra.ensureUnusedCapacity(gpa, fields.len); + return addExtraAssumeCapacity(wip, extra); + } + + fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 { + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + const result = @intCast(u32, wip.extra.items.len); + wip.extra.items.len += fields.len; + setExtra(wip, result, extra); + return result; + } + + fn setExtra(wip: *Wip, index: usize, extra: anytype) void { + const fields = @typeInfo(@TypeOf(extra)).Struct.fields; + var i = index; + inline for (fields) |field| { + wip.extra.items[i] = switch (field.type) { + u32 => @field(extra, field.name), + MessageIndex => @enumToInt(@field(extra, field.name)), + SourceLocationIndex => @enumToInt(@field(extra, field.name)), + else => @compileError("bad field type"), + }; + i += 1; + } + } +}; diff --git a/src/Compilation.zig b/src/Compilation.zig index f5a92c66ea..e19dc59445 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2546,9 +2546,9 @@ pub fn totalErrorCount(self: *Compilation) u32 { pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { const gpa = self.gpa; - var bundle: ErrorBundle = undefined; + var bundle: ErrorBundle.Wip = undefined; try bundle.init(gpa); - errdefer bundle.deinit(gpa); + defer bundle.deinit(); { var it = self.failed_c_objects.iterator(); @@ -2557,12 +2557,10 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { const err_msg = entry.value_ptr.*; // TODO these fields will need to be adjusted when we have proper // C error reporting bubbling up. - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.printString(gpa, "unable to build C object: {s}", .{ - err_msg.msg, - }), - .src_loc = try bundle.addSourceLocation(gpa, .{ - .src_path = try bundle.addString(gpa, c_object.src.src_path), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.printString("unable to build C object: {s}", .{err_msg.msg}), + .src_loc = try bundle.addSourceLocation(.{ + .src_path = try bundle.addString(c_object.src.src_path), .span_start = 0, .span_main = 0, .span_end = 1, @@ -2571,49 +2569,46 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { .source_line = 0, // TODO }), }); - bundle.incrementCount(1); } } for (self.lld_errors.items) |lld_error| { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, lld_error.msg), - .notes_len = @intCast(u32, lld_error.context_lines.len), - }); - bundle.incrementCount(1); + const notes_len = @intCast(u32, lld_error.context_lines.len); - for (lld_error.context_lines) |context_line| { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, context_line), - }); + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString(lld_error.msg), + .notes_len = notes_len, + }); + const notes_start = try bundle.reserveNotes(notes_len); + for (notes_start.., lld_error.context_lines) |note, context_line| { + bundle.extra.items[note] = @enumToInt(bundle.addErrorMessageAssumeCapacity(.{ + .msg = try bundle.addString(context_line), + })); } } for (self.misc_failures.values()) |*value| { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, value.msg), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString(value.msg), .notes_len = if (value.children) |b| b.errorMessageCount() else 0, }); - if (value.children) |b| try bundle.addBundle(gpa, b); - bundle.incrementCount(1); + if (value.children) |b| try bundle.addBundle(b); } if (self.alloc_failure_occurred) { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, "memory allocation failure"), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString("memory allocation failure"), }); - bundle.incrementCount(1); } if (self.bin_file.options.module) |module| { { var it = module.failed_files.iterator(); while (it.next()) |entry| { if (entry.value_ptr.*) |msg| { - try addModuleErrorMsg(gpa, &bundle, msg.*); + try addModuleErrorMsg(&bundle, msg.*); } else { - // Must be ZIR errors. In order for ZIR errors to exist, the parsing - // must have completed successfully. - const tree = try entry.key_ptr.*.getTree(module.gpa); - assert(tree.errors.len == 0); - try addZirErrorMessages(gpa, &bundle, entry.key_ptr.*); + // Must be ZIR errors. Note that this may include AST errors. + // addZirErrorMessages asserts that the tree is loaded. + _ = try entry.key_ptr.*.getTree(gpa); + try addZirErrorMessages(&bundle, entry.key_ptr.*); } } } @@ -2621,7 +2616,7 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { var it = module.failed_embed_files.iterator(); while (it.next()) |entry| { const msg = entry.value_ptr.*; - try addModuleErrorMsg(gpa, &bundle, msg.*); + try addModuleErrorMsg(&bundle, msg.*); } } { @@ -2631,21 +2626,20 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { // Skip errors for Decls within files that had a parse failure. // We'll try again once parsing succeeds. if (decl.getFileScope().okToReportErrors()) { - try addModuleErrorMsg(gpa, &bundle, entry.value_ptr.*.*); + try addModuleErrorMsg(&bundle, entry.value_ptr.*.*); if (module.cimport_errors.get(entry.key_ptr.*)) |cimport_errors| for (cimport_errors) |c_error| { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, std.mem.span(c_error.msg)), - .src_loc = if (c_error.path) |some| try bundle.addSourceLocation(gpa, .{ - .src_path = try bundle.addString(gpa, std.mem.span(some)), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString(std.mem.span(c_error.msg)), + .src_loc = if (c_error.path) |some| try bundle.addSourceLocation(.{ + .src_path = try bundle.addString(std.mem.span(some)), .span_start = c_error.offset, .span_main = c_error.offset, .span_end = c_error.offset + 1, .line = c_error.line, .column = c_error.column, - .source_line = if (c_error.source_line) |line| try bundle.addString(gpa, std.mem.span(line)) else 0, - }) else 0, + .source_line = if (c_error.source_line) |line| try bundle.addString(std.mem.span(line)) else 0, + }) else .none, }); - bundle.incrementCount(1); }; } } @@ -2657,40 +2651,39 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { // Skip errors for Decls within files that had a parse failure. // We'll try again once parsing succeeds. if (decl.getFileScope().okToReportErrors()) { - try addModuleErrorMsg(gpa, &bundle, entry.value_ptr.*.*); + try addModuleErrorMsg(&bundle, entry.value_ptr.*.*); } } } for (module.failed_exports.values()) |value| { - try addModuleErrorMsg(gpa, &bundle, value.*); + try addModuleErrorMsg(&bundle, value.*); } } - if (bundle.errorMessageCount() == 0) { + if (bundle.root_list.items.len == 0) { if (self.link_error_flags.no_entry_point_found) { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, "no entry point found"), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString("no entry point found"), }); - bundle.incrementCount(1); } } if (self.link_error_flags.missing_libc) { - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, "libc not available"), + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString("libc not available"), .notes_len = 2, }); - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, "run 'zig libc -h' to learn about libc installations"), - }); - try bundle.addErrorMessage(gpa, .{ - .msg = try bundle.addString(gpa, "run 'zig targets' to see the targets for which zig can always provide libc"), - }); - bundle.incrementCount(1); + const notes_start = try bundle.reserveNotes(2); + bundle.extra.items[notes_start + 0] = @enumToInt(try bundle.addErrorMessage(.{ + .msg = try bundle.addString("run 'zig libc -h' to learn about libc installations"), + })); + bundle.extra.items[notes_start + 1] = @enumToInt(try bundle.addErrorMessage(.{ + .msg = try bundle.addString("run 'zig targets' to see the targets for which zig can always provide libc"), + })); } if (self.bin_file.options.module) |module| { - if (bundle.errorMessageCount() == 0 and module.compile_log_decls.count() != 0) { + if (bundle.root_list.items.len == 0 and module.compile_log_decls.count() != 0) { const keys = module.compile_log_decls.keys(); const values = module.compile_log_decls.values(); // First one will be the error; subsequent ones will be notes. @@ -2699,9 +2692,9 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { const err_msg = Module.ErrorMsg{ .src_loc = src_loc, .msg = "found compile log statement", - .notes = try self.gpa.alloc(Module.ErrorMsg, module.compile_log_decls.count() - 1), + .notes = try gpa.alloc(Module.ErrorMsg, module.compile_log_decls.count() - 1), }; - defer self.gpa.free(err_msg.notes); + defer gpa.free(err_msg.notes); for (keys[1..], 0..) |key, i| { const note_decl = module.declPtr(key); @@ -2711,25 +2704,26 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { }; } - try addModuleErrorMsg(gpa, &bundle, err_msg); + try addModuleErrorMsg(&bundle, err_msg); } } - assert(self.totalErrorCount() == bundle.errorMessageCount()); + assert(self.totalErrorCount() == bundle.root_list.items.len); - return bundle; + return bundle.toOwnedBundle(); } pub const ErrorNoteHashContext = struct { - eb: *const ErrorBundle, + eb: *const ErrorBundle.Wip, pub fn hash(ctx: ErrorNoteHashContext, key: ErrorBundle.ErrorMessage) u32 { var hasher = std.hash.Wyhash.init(0); + const eb = ctx.eb.tmpBundle(); - hasher.update(ctx.eb.nullTerminatedString(key.msg)); - if (key.src_loc != 0) { - const src = ctx.eb.getSourceLocation(key.src_loc); - hasher.update(ctx.eb.nullTerminatedString(src.src_path)); + hasher.update(eb.nullTerminatedString(key.msg)); + if (key.src_loc != .none) { + const src = eb.getSourceLocation(key.src_loc); + hasher.update(eb.nullTerminatedString(src.src_path)); std.hash.autoHash(&hasher, src.line); std.hash.autoHash(&hasher, src.column); std.hash.autoHash(&hasher, src.span_main); @@ -2745,17 +2739,18 @@ pub const ErrorNoteHashContext = struct { b_index: usize, ) bool { _ = b_index; - const msg_a = ctx.eb.nullTerminatedString(a.msg); - const msg_b = ctx.eb.nullTerminatedString(b.msg); + const eb = ctx.eb.tmpBundle(); + const msg_a = eb.nullTerminatedString(a.msg); + const msg_b = eb.nullTerminatedString(b.msg); if (!std.mem.eql(u8, msg_a, msg_b)) return false; - if (a.src_loc == 0 and b.src_loc == 0) return true; - if (a.src_loc == 0 or b.src_loc == 0) return false; - const src_a = ctx.eb.getSourceLocation(a.src_loc); - const src_b = ctx.eb.getSourceLocation(b.src_loc); + if (a.src_loc == .none and b.src_loc == .none) return true; + if (a.src_loc == .none or b.src_loc == .none) return false; + const src_a = eb.getSourceLocation(a.src_loc); + const src_b = eb.getSourceLocation(b.src_loc); - const src_path_a = ctx.eb.nullTerminatedString(src_a.src_path); - const src_path_b = ctx.eb.nullTerminatedString(src_b.src_path); + const src_path_a = eb.nullTerminatedString(src_a.src_path); + const src_path_b = eb.nullTerminatedString(src_b.src_path); return std.mem.eql(u8, src_path_a, src_path_b) and src_a.line == src_b.line and @@ -2764,16 +2759,16 @@ pub const ErrorNoteHashContext = struct { } }; -pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Module.ErrorMsg) !void { +pub fn addModuleErrorMsg(eb: *ErrorBundle.Wip, module_err_msg: Module.ErrorMsg) !void { + const gpa = eb.gpa; const err_source = module_err_msg.src_loc.file_scope.getSource(gpa) catch |err| { const file_path = try module_err_msg.src_loc.file_scope.fullPath(gpa); defer gpa.free(file_path); - try eb.addErrorMessage(gpa, .{ - .msg = try eb.printString(gpa, "unable to load '{s}': {s}", .{ + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load '{s}': {s}", .{ file_path, @errorName(err), }), }); - eb.incrementCount(1); return; }; const err_span = try module_err_msg.src_loc.span(gpa); @@ -2788,13 +2783,13 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul if (module_reference.hidden != 0) { try ref_traces.append(gpa, .{ .decl_name = module_reference.hidden, - .src_loc = 0, + .src_loc = .none, }); break; } else if (module_reference.decl == null) { try ref_traces.append(gpa, .{ .decl_name = 0, - .src_loc = 0, + .src_loc = .none, }); break; } @@ -2804,9 +2799,9 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul const rt_file_path = try module_reference.src_loc.file_scope.fullPath(gpa); defer gpa.free(rt_file_path); try ref_traces.append(gpa, .{ - .decl_name = try eb.addString(gpa, std.mem.sliceTo(module_reference.decl.?, 0)), - .src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, rt_file_path), + .decl_name = try eb.addString(std.mem.sliceTo(module_reference.decl.?, 0)), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(rt_file_path), .span_start = span.start, .span_main = span.main, .span_end = span.end, @@ -2817,8 +2812,8 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul }); } - const src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, file_path), + const src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(file_path), .span_start = err_span.start, .span_main = err_span.main, .span_end = err_span.end, @@ -2827,12 +2822,12 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul .source_line = if (module_err_msg.src_loc.lazy == .entire_file) 0 else - try eb.addString(gpa, err_loc.source_line), + try eb.addString(err_loc.source_line), .reference_trace_len = @intCast(u32, ref_traces.items.len), }); for (ref_traces.items) |rt| { - try eb.addReferenceTrace(gpa, rt); + try eb.addReferenceTrace(rt); } // De-duplicate error notes. The main use case in mind for this is @@ -2848,15 +2843,15 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul defer gpa.free(note_file_path); const gop = try notes.getOrPutContext(gpa, .{ - .msg = try eb.addString(gpa, module_note.msg), - .src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, note_file_path), + .msg = try eb.addString(module_note.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(note_file_path), .span_start = span.start, .span_main = span.main, .span_end = span.end, .line = @intCast(u32, loc.line), .column = @intCast(u32, loc.column), - .source_line = if (err_loc.eql(loc)) 0 else try eb.addString(gpa, loc.source_line), + .source_line = if (err_loc.eql(loc)) 0 else try eb.addString(loc.source_line), }), }, .{ .eb = eb }); if (gop.found_existing) { @@ -2864,24 +2859,28 @@ pub fn addModuleErrorMsg(gpa: Allocator, eb: *ErrorBundle, module_err_msg: Modul } } - try eb.addErrorMessage(gpa, .{ - .msg = try eb.addString(gpa, module_err_msg.msg), + const notes_len = @intCast(u32, notes.entries.len); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(module_err_msg.msg), .src_loc = src_loc, - .notes_len = @intCast(u32, notes.entries.len), + .notes_len = notes_len, }); - eb.incrementCount(1); - for (notes.keys()) |note| { - try eb.addErrorMessage(gpa, note); + const notes_start = try eb.reserveNotes(notes_len); + + for (notes_start.., notes.keys()) |i, note| { + eb.extra.items[i] = @enumToInt(try eb.addErrorMessage(note)); } } -pub fn addZirErrorMessages(gpa: Allocator, eb: *ErrorBundle, file: *Module.File) !void { +pub fn addZirErrorMessages(eb: *ErrorBundle.Wip, file: *Module.File) !void { assert(file.zir_loaded); assert(file.tree_loaded); assert(file.source_loaded); const payload_index = file.zir.extra[@enumToInt(Zir.ExtraIndex.compile_errors)]; assert(payload_index != 0); + const gpa = eb.gpa; const header = file.zir.extraData(Zir.Inst.CompileErrors, payload_index); const items_len = header.data.items_len; @@ -2900,14 +2899,30 @@ pub fn addZirErrorMessages(gpa: Allocator, eb: *ErrorBundle, file: *Module.File) }; const err_loc = std.zig.findLineColumn(file.source, err_span.main); - var notes: []ErrorBundle.ErrorMessage = &.{}; - defer gpa.free(notes); + { + const msg = file.zir.nullTerminatedString(item.data.msg); + const src_path = try file.fullPath(gpa); + defer gpa.free(src_path); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(src_path), + .span_start = err_span.start, + .span_main = err_span.main, + .span_end = err_span.end, + .line = @intCast(u32, err_loc.line), + .column = @intCast(u32, err_loc.column), + .source_line = try eb.addString(err_loc.source_line), + }), + .notes_len = item.data.notes, + }); + } if (item.data.notes != 0) { + const notes_start = try eb.reserveNotes(item.data.notes); const block = file.zir.extraData(Zir.Inst.Block, item.data.notes); const body = file.zir.extra[block.end..][0..block.data.body_len]; - notes = try gpa.alloc(ErrorBundle.ErrorMessage, body.len); - for (notes, body) |*note, body_elem| { + for (notes_start.., body) |note_i, body_elem| { const note_item = file.zir.extraData(Zir.Inst.CompileErrors.Item, body_elem); const msg = file.zir.nullTerminatedString(note_item.data.msg); const span = blk: { @@ -2923,10 +2938,10 @@ pub fn addZirErrorMessages(gpa: Allocator, eb: *ErrorBundle, file: *Module.File) const src_path = try file.fullPath(gpa); defer gpa.free(src_path); - note.* = .{ - .msg = try eb.addString(gpa, msg), - .src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, src_path), + eb.extra.items[note_i] = @enumToInt(try eb.addErrorMessage(.{ + .msg = try eb.addString(msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(src_path), .span_start = span.start, .span_main = span.main, .span_end = span.end, @@ -2935,35 +2950,13 @@ pub fn addZirErrorMessages(gpa: Allocator, eb: *ErrorBundle, file: *Module.File) .source_line = if (loc.eql(err_loc)) 0 else - try eb.addString(gpa, loc.source_line), + try eb.addString(loc.source_line), }), .notes_len = 0, // TODO rework this function to be recursive - }; + })); } } - - const msg = file.zir.nullTerminatedString(item.data.msg); - const src_path = try file.fullPath(gpa); - defer gpa.free(src_path); - try eb.addErrorMessage(gpa, .{ - .msg = try eb.addString(gpa, msg), - .src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, src_path), - .span_start = err_span.start, - .span_main = err_span.main, - .span_end = err_span.end, - .line = @intCast(u32, err_loc.line), - .column = @intCast(u32, err_loc.column), - .source_line = try eb.addString(gpa, err_loc.source_line), - }), - .notes_len = @intCast(u32, notes.len), - }); - - for (notes) |note| { - try eb.addErrorMessage(gpa, note); - } } - eb.incrementCount(items_len); } pub fn getCompileLogOutput(self: *Compilation) []const u8 { diff --git a/src/Package.zig b/src/Package.zig index febcc51788..d26daf5a0c 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -225,7 +225,7 @@ pub fn fetchAndAddDependencies( dependencies_source: *std.ArrayList(u8), build_roots_source: *std.ArrayList(u8), name_prefix: []const u8, - error_bundle: *std.zig.ErrorBundle, + error_bundle: *std.zig.ErrorBundle.Wip, all_modules: *AllModules, ) !void { const max_bytes = 10 * 1024 * 1024; @@ -260,13 +260,12 @@ pub fn fetchAndAddDependencies( if (manifest.errors.len > 0) { const file_path = try directory.join(arena, &.{Manifest.basename}); for (manifest.errors) |msg| { - try Report.addErrorMessage(gpa, ast, file_path, error_bundle, 0, msg); + try Report.addErrorMessage(ast, file_path, error_bundle, 0, msg); } return error.PackageFetchFailed; } const report: Report = .{ - .gpa = gpa, .ast = &ast, .directory = directory, .error_bundle = error_bundle, @@ -343,10 +342,9 @@ pub fn createFilePkg( } const Report = struct { - gpa: Allocator, ast: *const std.zig.Ast, directory: Compilation.Directory, - error_bundle: *std.zig.ErrorBundle, + error_bundle: *std.zig.ErrorBundle.Wip, fn fail( report: Report, @@ -354,7 +352,7 @@ const Report = struct { comptime fmt_string: []const u8, fmt_args: anytype, ) error{ PackageFetchFailed, OutOfMemory } { - const gpa = report.gpa; + const gpa = report.error_bundle.gpa; const file_path = try report.directory.join(gpa, &.{Manifest.basename}); defer gpa.free(file_path); @@ -362,7 +360,7 @@ const Report = struct { const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args); defer gpa.free(msg); - try addErrorMessage(report.gpa, report.ast.*, file_path, report.error_bundle, 0, .{ + try addErrorMessage(report.ast.*, file_path, report.error_bundle, 0, .{ .tok = tok, .off = 0, .msg = msg, @@ -372,30 +370,28 @@ const Report = struct { } fn addErrorMessage( - gpa: Allocator, ast: std.zig.Ast, file_path: []const u8, - eb: *std.zig.ErrorBundle, + eb: *std.zig.ErrorBundle.Wip, notes_len: u32, msg: Manifest.ErrorMessage, ) error{OutOfMemory}!void { const token_starts = ast.tokens.items(.start); const start_loc = ast.tokenLocation(0, msg.tok); - try eb.addErrorMessage(gpa, .{ - .msg = try eb.addString(gpa, msg.msg), - .src_loc = try eb.addSourceLocation(gpa, .{ - .src_path = try eb.addString(gpa, file_path), + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = try eb.addString(file_path), .span_start = token_starts[msg.tok], .span_end = @intCast(u32, token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), .span_main = token_starts[msg.tok] + msg.off, .line = @intCast(u32, start_loc.line), .column = @intCast(u32, start_loc.column), - .source_line = try eb.addString(gpa, ast.source[start_loc.line_start..start_loc.line_end]), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), }), .notes_len = notes_len, }); - eb.incrementCount(1); } }; @@ -526,14 +522,16 @@ fn fetchAndUnpack( defer gpa.free(file_path); const eb = report.error_bundle; - try Report.addErrorMessage(gpa, report.ast.*, file_path, eb, 1, .{ + const notes_len = 1; + try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{ .tok = dep.url_tok, .off = 0, .msg = "url field is missing corresponding hash field", }); - try eb.addErrorMessage(gpa, .{ - .msg = try eb.printString(gpa, "expected .hash = \"{s}\",", .{&actual_hex}), - }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @enumToInt(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), + })); return error.PackageFetchFailed; } diff --git a/src/Sema.zig b/src/Sema.zig index 237936547e..03ebbbdbac 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2215,11 +2215,12 @@ fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { if (crash_report.is_enabled and sema.mod.comp.debug_compile_errors) { if (err_msg.src_loc.lazy == .unneeded) return error.NeededSourceLocation; - var errors: std.zig.ErrorBundle = undefined; - errors.init(gpa) catch unreachable; - Compilation.addModuleErrorMsg(gpa, &errors, err_msg.*) catch unreachable; + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + wip_errors.init(gpa) catch unreachable; + Compilation.addModuleErrorMsg(&wip_errors, err_msg.*) catch unreachable; std.debug.print("compile error during Sema:\n", .{}); - errors.renderToStdErr(.no_color); + var error_bundle = wip_errors.toOwnedBundle() catch unreachable; + error_bundle.renderToStdErr(.no_color); crash_report.compilerPanic("unexpected compile error occurred", null, null); } diff --git a/src/main.zig b/src/main.zig index 14b63017e5..0a16aa5a46 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4436,9 +4436,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var all_modules: Package.AllModules = .{}; defer all_modules.deinit(gpa); - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); // Here we borrow main package's table and will replace it with a fresh // one after this process completes. @@ -4453,15 +4453,17 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &dependencies_source, &build_roots_source, "", - &errors, + &wip_errors, &all_modules, ); - if (errors.errorMessageCount() > 0) { + if (wip_errors.root_list.items.len > 0) { const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; + var errors = try wip_errors.toOwnedBundle(); + defer errors.deinit(gpa); errors.renderToStdErr(ttyconf); process.exit(1); } @@ -4721,16 +4723,18 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, &errors, &file); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + try Compilation.addZirErrorMessages(&wip_errors, &file); const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - errors.renderToStdErr(ttyconf); + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(ttyconf); has_ast_error = true; } } @@ -4930,16 +4934,18 @@ fn fmtPathFile( defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, &errors, &file); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + try Compilation.addZirErrorMessages(&wip_errors, &file); const ttyconf: std.debug.TTY.Config = switch (fmt.color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - errors.renderToStdErr(ttyconf); + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(ttyconf); fmt.any_error = true; } } @@ -4968,17 +4974,19 @@ fn fmtPathFile( } fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Color) !void { - var error_bundle: std.zig.ErrorBundle = undefined; - try error_bundle.init(gpa); - defer error_bundle.deinit(gpa); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); - try putAstErrorsIntoBundle(gpa, tree, path, &error_bundle); + try putAstErrorsIntoBundle(gpa, tree, path, &wip_errors); const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(ttyconf); } @@ -4986,7 +4994,7 @@ pub fn putAstErrorsIntoBundle( gpa: Allocator, tree: Ast, path: []const u8, - error_bundle: *std.zig.ErrorBundle, + wip_errors: *std.zig.ErrorBundle.Wip, ) !void { var file: Module.File = .{ .status = .never_loaded, @@ -5013,7 +5021,7 @@ pub fn putAstErrorsIntoBundle( file.zir_loaded = true; defer file.zir.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, error_bundle, &file); + try Compilation.addZirErrorMessages(wip_errors, &file); } pub const info_zen = @@ -5595,16 +5603,18 @@ pub fn cmdAstCheck( defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, &errors, &file); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + try Compilation.addZirErrorMessages(&wip_errors, &file); const ttyconf: std.debug.TTY.Config = switch (color) { .auto => std.debug.detectTTYConfig(std.io.getStdErr()), .on => .escape_codes, .off => .no_color, }; - errors.renderToStdErr(ttyconf); + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(ttyconf); process.exit(1); } @@ -5719,12 +5729,14 @@ pub fn cmdChangelist( defer file.zir.deinit(gpa); if (file.zir.hasCompileErrors()) { - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, &errors, &file); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + try Compilation.addZirErrorMessages(&wip_errors, &file); const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); - errors.renderToStdErr(ttyconf); + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(ttyconf); process.exit(1); } @@ -5758,12 +5770,14 @@ pub fn cmdChangelist( file.zir_loaded = true; if (file.zir.hasCompileErrors()) { - var errors: std.zig.ErrorBundle = undefined; - try errors.init(gpa); - defer errors.deinit(gpa); - try Compilation.addZirErrorMessages(gpa, &errors, &file); + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + try Compilation.addZirErrorMessages(&wip_errors, &file); const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); - errors.renderToStdErr(ttyconf); + var error_bundle = try wip_errors.toOwnedBundle(); + defer error_bundle.deinit(gpa); + error_bundle.renderToStdErr(ttyconf); process.exit(1); } diff --git a/src/test.zig b/src/test.zig index ce87742606..663c4f1aff 100644 --- a/src/test.zig +++ b/src/test.zig @@ -1242,7 +1242,7 @@ pub const TestContext = struct { defer self.gpa.free(zig_lib_directory.path.?); var aux_thread_pool: ThreadPool = undefined; - try aux_thread_pool.init(self.gpa); + try aux_thread_pool.init(.{ .allocator = self.gpa }); defer aux_thread_pool.deinit(); // Use the same global cache dir for all the tests, such that we for example don't have to @@ -1614,23 +1614,8 @@ pub const TestContext = struct { if (update.case != .Error) { var all_errors = try comp.getAllErrorsAlloc(); defer all_errors.deinit(allocator); - if (all_errors.list.len != 0) { - print( - "\nCase '{s}': unexpected errors at update_index={d}:\n{s}\n", - .{ case.name, update_index, hr }, - ); - for (all_errors.list) |err_msg| { - switch (err_msg) { - .src => |src| { - print("{s}:{d}:{d}: error: {s}\n{s}\n", .{ - src.src_path, src.line + 1, src.column + 1, src.msg, hr, - }); - }, - .plain => |plain| { - print("error: {s}\n{s}\n", .{ plain.msg, hr }); - }, - } - } + if (all_errors.errorMessageCount() > 0) { + all_errors.renderToStdErr(std.debug.detectTTYConfig(std.io.getStdErr())); // TODO print generated C code return error.UnexpectedCompileErrors; } -- cgit v1.2.3 From 986a30e373f6b2f0da2de64570013c83cacc17b6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Feb 2023 16:52:35 -0700 Subject: integrate the build runner and the compiler server The compiler now provides a server protocol for an interactive session with another process. The build runner uses this protocol to communicate compilation errors semantically from zig compiler subprocesses to the build runner. The protocol is exposed via stdin/stdout, or on a network socket, depending on whether the CLI flag `--listen=-` or e.g. `--listen=127.0.0.1:1337` is used. Additionally: * add the zig version string to the build runner cache prefix * remove --prominent-compile-errors CLI flag because it no longer does anything. Compilation errors are now unconditionally displayed at the bottom of the build summary output when using the terminal-based build runner. * Remove the color field from std.Build. The build steps are no longer supposed to interact with stderr directly. Instead they communicate semantically back to the build runner, which has its own logic about TTY configuration. * Use the cleanExit() pattern in the build runner. * Build steps can now use error.MakeFailed when they have already properly reported an error, or they can fail with any other error code in which case the build runner will create a simple message based on this error code. --- lib/build_runner.zig | 113 +++++++++---- lib/std/Build.zig | 147 ++++++++++++---- lib/std/Build/CompileStep.zig | 6 +- lib/std/Build/RunStep.zig | 8 +- lib/std/Build/Step.zig | 29 ++-- lib/std/zig.zig | 2 + lib/std/zig/Client.zig | 32 ++++ lib/std/zig/Server.zig | 28 ++++ src/main.zig | 380 +++++++++++++++++++++++++++--------------- 9 files changed, 524 insertions(+), 221 deletions(-) create mode 100644 lib/std/zig/Client.zig create mode 100644 lib/std/zig/Server.zig (limited to 'src/main.zig') diff --git a/lib/build_runner.zig b/lib/build_runner.zig index e12d46f03d..b95c36a196 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -1,6 +1,7 @@ const root = @import("@build"); const std = @import("std"); const builtin = @import("builtin"); +const assert = std.debug.assert; const io = std.io; const fmt = std.fmt; const mem = std.mem; @@ -71,8 +72,7 @@ pub fn main() !void { cache.addPrefix(build_root_directory); cache.addPrefix(local_cache_directory); cache.addPrefix(global_cache_directory); - - //cache.hash.addBytes(builtin.zig_version); + cache.hash.addBytes(builtin.zig_version_string); const builder = try std.Build.create( allocator, @@ -95,10 +95,8 @@ pub fn main() !void { var install_prefix: ?[]const u8 = null; var dir_list = std.Build.DirList{}; - // before arg parsing, check for the NO_COLOR environment variable - // if it exists, default the color setting to .off - // explicit --color arguments will still override this setting. - builder.color = if (process.hasEnvVarConstant("NO_COLOR")) .off else .auto; + const Color = enum { auto, off, on }; + var color: Color = .auto; while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-D")) { @@ -166,7 +164,7 @@ pub fn main() !void { std.debug.print("expected [auto|on|off] after --color", .{}); usageAndErr(builder, false, stderr_stream); }; - builder.color = std.meta.stringToEnum(@TypeOf(builder.color), next_arg) orelse { + color = std.meta.stringToEnum(Color, next_arg) orelse { std.debug.print("expected [auto|on|off] after --color, found '{s}'", .{next_arg}); usageAndErr(builder, false, stderr_stream); }; @@ -200,8 +198,6 @@ pub fn main() !void { builder.verbose_cc = true; } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { builder.verbose_llvm_cpu_features = true; - } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { - builder.prominent_compile_errors = true; } else if (mem.eql(u8, arg, "-fwine")) { builder.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { @@ -257,6 +253,12 @@ pub fn main() !void { } } + const ttyconf: std.debug.TTY.Config = switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; + var progress: std.Progress = .{}; const main_progress_node = progress.start("", 0); defer main_progress_node.end(); @@ -272,11 +274,15 @@ pub fn main() !void { if (builder.validateUserInputDidItFail()) usageAndErr(builder, true, stderr_stream); - runStepNames(builder, targets.items, main_progress_node, thread_pool_options) catch |err| { - switch (err) { - error.UncleanExit => process.exit(1), - else => return err, - } + runStepNames( + builder, + targets.items, + main_progress_node, + thread_pool_options, + ttyconf, + ) catch |err| switch (err) { + error.UncleanExit => process.exit(1), + else => return err, }; } @@ -285,6 +291,7 @@ fn runStepNames( step_names: []const []const u8, parent_prog_node: *std.Progress.Node, thread_pool_options: std.Thread.Pool.Options, + ttyconf: std.debug.TTY.Config, ) !void { var step_stack = ArrayList(*Step).init(b.allocator); defer step_stack.deinit(); @@ -332,12 +339,14 @@ fn runStepNames( wait_group.start(); thread_pool.spawn(workerMakeOneStep, .{ - &wait_group, &thread_pool, b, step, &step_prog, + &wait_group, &thread_pool, b, step, &step_prog, ttyconf, }) catch @panic("OOM"); } } - var any_failed = false; + var success_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; for (step_stack.items) |s| { switch (s.state) { @@ -349,20 +358,42 @@ fn runStepNames( // A -> B -> C (failure) // B will be marked as dependency_failure, while A may never be queued, and thus // remain in the initial state of precheck_done. - .dependency_failure, .precheck_done => continue, - .success => continue, - .failure => { - any_failed = true; - std.debug.print("{s}: {s}\n", .{ - s.name, @errorName(s.result.err_code), - }); - }, + .dependency_failure, .precheck_done => pending_count += 1, + .success => success_count += 1, + .failure => failure_count += 1, } } - if (any_failed) { - process.exit(1); - } + const stderr = std.io.getStdErr(); + + const total_count = success_count + failure_count + pending_count; + stderr.writer().print("build summary: {d}/{d} steps succeeded; {d} failed\n", .{ + success_count, total_count, failure_count, + }) catch {}; + if (failure_count == 0) return cleanExit(); + + for (step_stack.items) |s| switch (s.state) { + .failure => { + // TODO print the dep prefix too + ttyconf.setColor(stderr, .Bold) catch break; + stderr.writeAll(s.name) catch break; + ttyconf.setColor(stderr, .Reset) catch break; + + if (s.result_error_bundle.errorMessageCount() > 0) { + stderr.writer().print(": {d} compilation errors:\n", .{ + s.result_error_bundle.errorMessageCount(), + }) catch break; + s.result_error_bundle.renderToStdErr(ttyconf); + } else { + stderr.writer().print(": {d} error messages (printed above)\n", .{ + s.result_error_msgs.items.len, + }) catch break; + } + }, + else => continue, + }; + + process.exit(1); } fn checkForDependencyLoop( @@ -407,6 +438,7 @@ fn workerMakeOneStep( b: *std.Build, s: *Step, prog_node: *std.Progress.Node, + ttyconf: std.debug.TTY.Config, ) void { defer wg.finish(); @@ -446,17 +478,26 @@ fn workerMakeOneStep( const make_result = s.make(); // No matter the result, we want to display error/warning messages. - if (s.result.error_msgs.items.len > 0) { + if (s.result_error_msgs.items.len > 0) { sub_prog_node.context.lock_stderr(); defer sub_prog_node.context.unlock_stderr(); - for (s.result.error_msgs.items) |msg| { - std.io.getStdErr().writeAll(msg) catch break; + const stderr = std.io.getStdErr(); + + for (s.result_error_msgs.items) |msg| { + // TODO print the dep prefix too + ttyconf.setColor(stderr, .Bold) catch break; + stderr.writeAll(s.name) catch break; + stderr.writeAll(": ") catch break; + ttyconf.setColor(stderr, .Red) catch break; + stderr.writeAll("error: ") catch break; + ttyconf.setColor(stderr, .Reset) catch break; + stderr.writeAll(msg) catch break; } } make_result catch |err| { - s.result.err_code = err; + assert(err == error.MakeFailed); @atomicStore(Step.State, &s.state, .failure, .SeqCst); return; }; @@ -467,7 +508,7 @@ fn workerMakeOneStep( for (s.dependants.items) |dep| { wg.start(); thread_pool.spawn(workerMakeOneStep, .{ - wg, thread_pool, b, dep, prog_node, + wg, thread_pool, b, dep, prog_node, ttyconf, }) catch @panic("OOM"); } } @@ -601,3 +642,11 @@ fn argsRest(args: [][]const u8, idx: usize) ?[][]const u8 { if (idx >= args.len) return null; return args[idx..]; } + +fn cleanExit() void { + if (builtin.mode == .Debug) { + return; + } else { + process.exit(0); + } +} diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 92ace4e60b..fc8ec26723 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -59,9 +59,6 @@ verbose_air: bool, verbose_llvm_ir: bool, verbose_cimport: bool, verbose_llvm_cpu_features: bool, -/// The purpose of executing the command is for a human to read compile errors from the terminal -prominent_compile_errors: bool, -color: enum { auto, on, off } = .auto, reference_trace: ?u32 = null, invalid_user_input: bool, zig_exe: []const u8, @@ -211,7 +208,6 @@ pub fn create( .verbose_llvm_ir = false, .verbose_cimport = false, .verbose_llvm_cpu_features = false, - .prominent_compile_errors = false, .invalid_user_input = false, .allocator = allocator, .user_input_options = UserInputOptionsMap.init(allocator), @@ -295,8 +291,6 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc .verbose_llvm_ir = parent.verbose_llvm_ir, .verbose_cimport = parent.verbose_cimport, .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, - .prominent_compile_errors = parent.prominent_compile_errors, - .color = parent.color, .reference_trace = parent.reference_trace, .invalid_user_input = false, .zig_exe = parent.zig_exe, @@ -1409,54 +1403,149 @@ pub fn execAllowFail( } } -pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]u8 { +/// This function is used exclusively for spawning and communicating with the zig compiler. +/// TODO: move to build_runner.zig +pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { assert(argv.len != 0); if (b.verbose) { - printCmd(b.allocator, null, argv); + const text = try allocPrintCmd(b.allocator, null, argv); + try s.result_error_msgs.append(b.allocator, text); } if (!process.can_spawn) { - try s.result.error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{ + try s.result_error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{ try allocPrintCmd(b.allocator, null, argv), })); - return error.CannotSpawnProcesses; + return error.MakeFailed; } - const result = std.ChildProcess.exec(.{ - .allocator = b.allocator, - .argv = argv, - .env_map = b.env_map, - .max_output_bytes = 10 * 1024 * 1024, - }) catch |err| { - try s.result.error_msgs.append(b.allocator, b.fmt("unable to spawn the following command: {s}\n{s}", .{ - @errorName(err), try allocPrintCmd(b.allocator, null, argv), - })); - return error.ExecFailed; - }; + var child = std.ChildProcess.init(argv, b.allocator); + child.env_map = b.env_map; + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + try child.spawn(); - if (result.stderr.len != 0) { - try s.result.error_msgs.append(b.allocator, result.stderr); + var poller = std.io.poll(b.allocator, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try sendMessage(child.stdin.?, .update); + try sendMessage(child.stdin.?, .exit); + + const Header = std.zig.Server.Message.Header; + var result: ?[]const u8 = null; + + while (try poller.poll()) { + const stdout = poller.fifo(.stdout); + const buf = stdout.readableSlice(0); + assert(stdout.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + const header_and_msg_len = header.bytes_len + @sizeOf(Header); + if (buf.len >= header_and_msg_len) { + const body = buf[@sizeOf(Header)..]; + switch (header.tag) { + .zig_version => { + if (!mem.eql(u8, builtin.zig_version_string, body)) { + try s.result_error_msgs.append( + b.allocator, + b.fmt("zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ + builtin.zig_version_string, body, + }), + ); + return error.MakeFailed; + } + }, + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @ptrCast(*align(1) const EbHdr, body); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + // TODO: use @ptrCast when the compiler supports it + const unaligned_extra = mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try b.allocator.alloc(u32, unaligned_extra.len); + // TODO: use @memcpy when it supports slices + for (extra_array, unaligned_extra) |*dst, src| dst.* = src; + s.result_error_bundle = .{ + .string_bytes = try b.allocator.dupe(u8, string_bytes), + .extra = extra_array, + }; + }, + .progress => { + @panic("TODO handle progress message"); + }, + .emit_bin_path => { + @panic("TODO handle emit_bin_path message"); + }, + _ => { + // Unrecognized message. + }, + } + stdout.discard(header_and_msg_len); + } + } + } + + const stderr = poller.fifo(.stderr); + if (stderr.readableLength() > 0) { + try s.result_error_msgs.append(b.allocator, try stderr.toOwnedSlice()); } - switch (result.term) { + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + const term = try child.wait(); + switch (term) { .Exited => |code| { if (code != 0) { - try s.result.error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{ + try s.result_error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{ code, try allocPrintCmd(b.allocator, null, argv), })); - return error.ExitCodeFailure; + return error.MakeFailed; } - return result.stdout; }, .Signal, .Stopped, .Unknown => |code| { _ = code; - try s.result.error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{ + try s.result_error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{ try allocPrintCmd(b.allocator, null, argv), })); - return error.ProcessTerminated; + return error.MakeFailed; }, } + + if (s.result_error_bundle.errorMessageCount() > 0) { + try s.result_error_msgs.append( + b.allocator, + b.fmt("the following command failed with {d} compilation errors:\n{s}", .{ + s.result_error_bundle.errorMessageCount(), + try allocPrintCmd(b.allocator, null, argv), + }), + ); + return error.MakeFailed; + } + + return result orelse { + try s.result_error_msgs.append(b.allocator, b.fmt("the following command failed to communicate the compilation result:\n{s}", .{ + try allocPrintCmd(b.allocator, null, argv), + })); + return error.MakeFailed; + }; +} + +fn sendMessage(file: fs.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + try file.writeAll(std.mem.asBytes(&header)); } /// This is a helper function to be called from build.zig scripts, *not* from diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index ce0ede9510..c5c2b2a440 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -1177,11 +1177,6 @@ fn make(step: *Step) !void { }; try zig_args.append(cmd); - if (builder.color != .auto) { - try zig_args.append("--color"); - try zig_args.append(@tagName(builder.color)); - } - if (builder.reference_trace) |some| { try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-freference-trace={d}", .{some})); } @@ -1834,6 +1829,7 @@ fn make(step: *Step) !void { } try zig_args.append("--enable-cache"); + try zig_args.append("--listen=-"); // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 815916f380..904ef0935f 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -419,12 +419,8 @@ pub fn runCommand( }; if (!termMatches(expected_term, term)) { - if (builder.prominent_compile_errors) { - std.debug.print("Run step {} (expected {})\n", .{ fmtTerm(term), fmtTerm(expected_term) }); - } else { - std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) }); - printCmd(cwd, argv); - } + std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) }); + printCmd(cwd, argv); return error.UnexpectedExit; } diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 42698f2190..4edece8038 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -6,15 +6,13 @@ dependencies: std.ArrayList(*Step), /// then populated during dependency loop checking in the build runner. dependants: std.ArrayListUnmanaged(*Step), state: State, -/// Populated only if state is success. -result: struct { - err_code: anyerror, - error_msgs: std.ArrayListUnmanaged([]const u8), -}, /// The return addresss associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: [n_debug_stack_frames]usize, +result_error_msgs: std.ArrayListUnmanaged([]const u8), +result_error_bundle: std.zig.ErrorBundle, + const n_debug_stack_frames = 4; pub const State = enum { @@ -94,16 +92,25 @@ pub fn init(allocator: Allocator, options: Options) Step { .dependencies = std.ArrayList(*Step).init(allocator), .dependants = .{}, .state = .precheck_unstarted, - .result = .{ - .err_code = undefined, - .error_msgs = .{}, - }, .debug_stack_trace = addresses, + .result_error_msgs = .{}, + .result_error_bundle = std.zig.ErrorBundle.empty, }; } -pub fn make(self: *Step) !void { - try self.makeFn(self); +/// If the Step's `make` function reports `error.MakeFailed`, it indicates they +/// have already reported the error. Otherwise, we add a simple error report +/// here. +pub fn make(s: *Step) error{MakeFailed}!void { + return s.makeFn(s) catch |err| { + if (err != error.MakeFailed) { + const gpa = s.dependencies.allocator; + s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{ + s.name, @errorName(err), + }) catch @panic("OOM")) catch @panic("OOM"); + } + return error.MakeFailed; + }; } pub fn dependOn(self: *Step, other: *Step) void { diff --git a/lib/std/zig.zig b/lib/std/zig.zig index ecff8e99bd..98edeabd10 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -4,6 +4,8 @@ const fmt = @import("zig/fmt.zig"); const assert = std.debug.assert; pub const ErrorBundle = @import("zig/ErrorBundle.zig"); +pub const Server = @import("zig/Server.zig"); +pub const Client = @import("zig/Client.zig"); pub const Token = tokenizer.Token; pub const Tokenizer = tokenizer.Tokenizer; pub const fmtId = fmt.fmtId; diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig new file mode 100644 index 0000000000..a68c189e57 --- /dev/null +++ b/lib/std/zig/Client.zig @@ -0,0 +1,32 @@ +pub const Message = struct { + pub const Header = extern struct { + tag: Tag, + /// Size of the body only; does not include this Header. + bytes_len: u32, + }; + + pub const Tag = enum(u32) { + /// Tells the compiler to shut down cleanly. + /// No body. + exit, + /// Tells the compiler to detect changes in source files and update the + /// affected output compilation artifacts. + /// If one of the compilation artifacts is an executable that is + /// running as a child process, the compiler will wait for it to exit + /// before performing the update. + /// No body. + update, + /// Tells the compiler to execute the executable as a child process. + /// No body. + run, + /// Tells the compiler to detect changes in source files and update the + /// affected output compilation artifacts. + /// If one of the compilation artifacts is an executable that is + /// running as a child process, the compiler will perform a hot code + /// swap. + /// No body. + hot_update, + + _, + }; +}; diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig new file mode 100644 index 0000000000..76f2303f6b --- /dev/null +++ b/lib/std/zig/Server.zig @@ -0,0 +1,28 @@ +pub const Message = struct { + pub const Header = extern struct { + tag: Tag, + /// Size of the body only; does not include this Header. + bytes_len: u32, + }; + + pub const Tag = enum(u32) { + /// Body is a UTF-8 string. + zig_version, + /// Body is an ErrorBundle. + error_bundle, + /// Body is a UTF-8 string. + progress, + /// Body is a UTF-8 string. + emit_bin_path, + _, + }; + + /// Trailing: + /// * extra: [extra_len]u32, + /// * string_bytes: [string_bytes_len]u8, + /// See `std.zig.ErrorBundle`. + pub const ErrorBundle = extern struct { + extra_len: u32, + string_bytes_len: u32, + }; +}; diff --git a/src/main.zig b/src/main.zig index 0a16aa5a46..d574681bcd 100644 --- a/src/main.zig +++ b/src/main.zig @@ -668,6 +668,12 @@ const ArgMode = union(enum) { run, }; +const Listen = union(enum) { + none, + ip4: std.net.Ip4Address, + stdio, +}; + fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -689,7 +695,7 @@ fn buildOutputType( var function_sections = false; var no_builtin = false; var watch = false; - var listen_addr: ?std.net.Ip4Address = null; + var listen: Listen = .none; var debug_compile_errors = false; var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK"); var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_CC"); @@ -1149,14 +1155,22 @@ fn buildOutputType( } } else if (mem.eql(u8, arg, "--listen")) { const next_arg = args_iter.nextOrFatal(); - // example: --listen 127.0.0.1:9000 - var it = std.mem.split(u8, next_arg, ":"); - const host = it.next().?; - const port_text = it.next() orelse "14735"; - const port = std.fmt.parseInt(u16, port_text, 10) catch |err| - fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) }); - listen_addr = std.net.Ip4Address.parse(host, port) catch |err| - fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }); + if (mem.eql(u8, next_arg, "-")) { + listen = .stdio; + watch = true; + } else { + // example: --listen 127.0.0.1:9000 + var it = std.mem.split(u8, next_arg, ":"); + const host = it.next().?; + const port_text = it.next() orelse "14735"; + const port = std.fmt.parseInt(u16, port_text, 10) catch |err| + fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) }); + listen = .{ .ip4 = std.net.Ip4Address.parse(host, port) catch |err| + fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }) }; + watch = true; + } + } else if (mem.eql(u8, arg, "--listen=-")) { + listen = .stdio; watch = true; } else if (mem.eql(u8, arg, "--debug-link-snapshot")) { if (!build_options.enable_link_snapshots) { @@ -3277,6 +3291,47 @@ fn buildOutputType( return cmdTranslateC(comp, arena, have_enable_cache); } + switch (listen) { + .none => {}, + .stdio => { + try serve( + comp, + std.io.getStdIn(), + std.io.getStdOut(), + test_exec_args.items, + self_exe_path, + arg_mode, + all_args, + runtime_args_start, + ); + return cleanExit(); + }, + .ip4 => |ip4_addr| { + var server = std.net.StreamServer.init(.{ + .reuse_address = true, + }); + defer server.deinit(); + + try server.listen(.{ .in = ip4_addr }); + + while (true) { + const conn = try server.accept(); + defer conn.stream.close(); + + try serve( + comp, + .{ .handle = conn.stream.handle }, + .{ .handle = conn.stream.handle }, + test_exec_args.items, + self_exe_path, + arg_mode, + all_args, + runtime_args_start, + ); + } + }, + } + const hook: AfterUpdateHook = blk: { if (!have_enable_cache) break :blk .none; @@ -3354,6 +3409,12 @@ fn buildOutputType( ); } + // TODO move this REPL implementation to the standard library / build + // system and have it be a CLI abstraction layer on top of the real, actual + // binary protocol of the compiler. Make it actually interface through the + // server protocol. This way the REPL does not have any special powers that + // an IDE couldn't also have. + const stdin = std.io.getStdIn().reader(); const stderr = std.io.getStdErr().writer(); var repl_buf: [1024]u8 = undefined; @@ -3367,123 +3428,6 @@ fn buildOutputType( var last_cmd: ReplCmd = .help; - if (listen_addr) |ip4_addr| { - var server = std.net.StreamServer.init(.{ - .reuse_address = true, - }); - defer server.deinit(); - - try server.listen(.{ .in = ip4_addr }); - - while (true) { - const conn = try server.accept(); - defer conn.stream.close(); - - var buf: [100]u8 = undefined; - var child_pid: ?i32 = null; - - while (true) { - try comp.makeBinFileExecutable(); - - const amt = try conn.stream.read(&buf); - const line = buf[0..amt]; - const actual_line = mem.trimRight(u8, line, "\r\n "); - - const cmd: ReplCmd = blk: { - if (mem.eql(u8, actual_line, "update")) { - break :blk .update; - } else if (mem.eql(u8, actual_line, "exit")) { - break; - } else if (mem.eql(u8, actual_line, "help")) { - break :blk .help; - } else if (mem.eql(u8, actual_line, "run")) { - break :blk .run; - } else if (mem.eql(u8, actual_line, "update-and-run")) { - break :blk .update_and_run; - } else if (actual_line.len == 0) { - break :blk last_cmd; - } else { - try stderr.print("unknown command: {s}\n", .{actual_line}); - continue; - } - }; - last_cmd = cmd; - switch (cmd) { - .update => { - tracy.frameMark(); - if (output_mode == .Exe) { - try comp.makeBinFileWritable(); - } - updateModule(gpa, comp, hook) catch |err| switch (err) { - error.SemanticAnalyzeFail => continue, - else => |e| return e, - }; - }, - .help => { - try stderr.writeAll(repl_help); - }, - .run => { - tracy.frameMark(); - try runOrTest( - comp, - gpa, - arena, - test_exec_args.items, - self_exe_path.?, - arg_mode, - target_info, - watch, - &comp_destroyed, - all_args, - runtime_args_start, - link_libc, - ); - }, - .update_and_run => { - tracy.frameMark(); - if (child_pid) |pid| { - try conn.stream.writer().print("hot code swap requested for pid {d}", .{pid}); - try comp.hotCodeSwap(pid); - - var errors = try comp.getAllErrorsAlloc(); - defer errors.deinit(comp.gpa); - - if (errors.errorMessageCount() > 0) { - const ttyconf: std.debug.TTY.Config = switch (comp.color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; - try errors.renderToWriter(ttyconf, conn.stream.writer()); - continue; - } - } else { - if (output_mode == .Exe) { - try comp.makeBinFileWritable(); - } - updateModule(gpa, comp, hook) catch |err| switch (err) { - error.SemanticAnalyzeFail => continue, - else => |e| return e, - }; - try comp.makeBinFileExecutable(); - - child_pid = try runOrTestHotSwap( - comp, - gpa, - arena, - test_exec_args.items, - self_exe_path.?, - arg_mode, - all_args, - runtime_args_start, - ); - } - }, - } - } - } - } - while (watch) { try stderr.print("(zig) ", .{}); try comp.makeBinFileExecutable(); @@ -3576,6 +3520,173 @@ fn buildOutputType( return cleanExit(); } +fn serve( + comp: *Compilation, + in: fs.File, + out: fs.File, + test_exec_args: []const ?[]const u8, + self_exe_path: ?[]const u8, + arg_mode: ArgMode, + all_args: []const []const u8, + runtime_args_start: ?usize, +) !void { + const gpa = comp.gpa; + + try serveMessage(out, .{ + .tag = .zig_version, + .bytes_len = build_options.version.len, + }, &.{ + build_options.version, + }); + + var child_pid: ?i32 = null; + var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa); + defer receive_fifo.deinit(); + + while (true) { + const hdr = try receiveMessage(in, &receive_fifo); + + switch (hdr.tag) { + .exit => { + return cleanExit(); + }, + .update => { + tracy.frameMark(); + if (comp.bin_file.options.output_mode == .Exe) { + try comp.makeBinFileWritable(); + } + try comp.update(); + try comp.makeBinFileExecutable(); + try serveUpdateResults(out, comp); + }, + .run => { + if (child_pid != null) { + @panic("TODO block until the child exits"); + } + @panic("TODO call runOrTest"); + //try runOrTest( + // comp, + // gpa, + // arena, + // test_exec_args, + // self_exe_path.?, + // arg_mode, + // target_info, + // true, + // &comp_destroyed, + // all_args, + // runtime_args_start, + // link_libc, + //); + }, + .hot_update => { + tracy.frameMark(); + if (child_pid) |pid| { + try comp.hotCodeSwap(pid); + try serveUpdateResults(out, comp); + } else { + if (comp.bin_file.options.output_mode == .Exe) { + try comp.makeBinFileWritable(); + } + try comp.update(); + try comp.makeBinFileExecutable(); + try serveUpdateResults(out, comp); + + child_pid = try runOrTestHotSwap( + comp, + gpa, + test_exec_args, + self_exe_path.?, + arg_mode, + all_args, + runtime_args_start, + ); + } + }, + _ => { + @panic("TODO unrecognized message from client"); + }, + } + } +} + +fn serveMessage( + out: fs.File, + header: std.zig.Server.Message.Header, + bufs: []const []const u8, +) !void { + var iovecs: [10]std.os.iovec_const = undefined; + iovecs[0] = .{ + .iov_base = @ptrCast([*]const u8, &header), + .iov_len = @sizeOf(std.zig.Server.Message.Header), + }; + for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { + iovec.* = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + try out.writevAll(iovecs[0 .. bufs.len + 1]); +} + +fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void { + const eb_hdr: std.zig.Server.Message.ErrorBundle = .{ + .extra_len = @intCast(u32, error_bundle.extra.len), + .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), + }; + const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) + + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; + try serveMessage(out, .{ + .tag = .error_bundle, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&eb_hdr), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(error_bundle.extra), + error_bundle.string_bytes, + }); +} + +fn serveUpdateResults(out: fs.File, comp: *Compilation) !void { + const gpa = comp.gpa; + var error_bundle = try comp.getAllErrorsAlloc(); + defer error_bundle.deinit(gpa); + if (error_bundle.errorMessageCount() > 0) { + try serveErrorBundle(out, error_bundle); + } else if (comp.bin_file.options.emit) |emit| { + const full_path = try emit.directory.join(gpa, &.{emit.sub_path}); + defer gpa.free(full_path); + + try serveMessage(out, .{ + .tag = .emit_bin_path, + .bytes_len = @intCast(u32, full_path.len), + }, &.{ + full_path, + }); + } +} + +fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header { + const Header = std.zig.Client.Message.Header; + + while (true) { + const buf = fifo.readableSlice(0); + assert(fifo.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + if (header.bytes_len != 0) + return error.InvalidClientMessage; + const result = header.*; + fifo.discard(@sizeOf(Header)); + return result; + } + + const write_buffer = try fifo.writableWithSize(256); + const amt = try in.read(write_buffer); + fifo.update(amt); + } +} + const ModuleDepIterator = struct { split: mem.SplitIterator(u8), @@ -3765,7 +3876,6 @@ fn runOrTest( fn runOrTestHotSwap( comp: *Compilation, gpa: Allocator, - arena: Allocator, test_exec_args: []const ?[]const u8, self_exe_path: []const u8, arg_mode: ArgMode, @@ -3775,9 +3885,10 @@ fn runOrTestHotSwap( const exe_emit = comp.bin_file.options.emit.?; // A naive `directory.join` here will indeed get the correct path to the binary, // however, in the case of cwd, we actually want `./foo` so that the path can be executed. - const exe_path = try fs.path.join(arena, &[_][]const u8{ + const exe_path = try fs.path.join(gpa, &[_][]const u8{ exe_emit.directory.path orelse ".", exe_emit.sub_path, }); + defer gpa.free(exe_path); var argv = std.ArrayList([]const u8).init(gpa); defer argv.deinit(); @@ -3807,7 +3918,7 @@ fn runOrTestHotSwap( if (runtime_args_start) |i| { try argv.appendSlice(all_args[i..]); } - var child = std.ChildProcess.init(argv.items, arena); + var child = std.ChildProcess.init(argv.items, gpa); child.stdin_behavior = .Inherit; child.stdout_behavior = .Inherit; @@ -4206,7 +4317,6 @@ pub const usage_build = pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var color: Color = .auto; - var prominent_compile_errors: bool = false; // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. @@ -4267,8 +4377,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi i += 1; override_global_cache_dir = args[i]; continue; - } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { - prominent_compile_errors = true; } else if (mem.eql(u8, arg, "-freference-trace")) { try child_argv.append(arg); reference_trace = 256; @@ -4535,12 +4643,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .Exited => |code| { if (code == 0) return cleanExit(); - if (prominent_compile_errors) { - fatal("the build command failed with exit code {d}", .{code}); - } else { - const cmd = try std.mem.join(arena, " ", child_argv); - fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); - } + const cmd = try std.mem.join(arena, " ", child_argv); + fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); }, else => { const cmd = try std.mem.join(arena, " ", child_argv); -- cgit v1.2.3 From 8acbfafefb635390f6b4da102966b5c4a2f3332e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Feb 2023 21:14:32 -0700 Subject: compiler: update function accepts a std.Progress.Node This makes progress be exposed to the top-level caller of update(). I tossed in a bonus change: when the `zig build` subcommand sees exit code 2, it omits the "following command failed" line, and the build runner uses exit code 2 when there are compile errors. This tidies up the output on build failure by a little bit. --- src/Compilation.zig | 85 +++++++++++++++++++++++++++-------------------------- src/glibc.zig | 19 +++++++----- src/libcxx.zig | 8 ++--- src/libtsan.zig | 4 +-- src/libunwind.zig | 4 +-- src/main.zig | 48 +++++++++++++++++++++++++++--- src/mingw.zig | 14 ++++----- src/musl.zig | 16 +++++----- src/wasi_libc.zig | 16 +++++----- 9 files changed, 130 insertions(+), 84 deletions(-) (limited to 'src/main.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index 3a75b44861..478f931718 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1847,8 +1847,15 @@ fn cleanupTmpArtifactDirectory( } } +pub fn hotCodeSwap(comp: *Compilation, prog_node: *std.Progress.Node, pid: std.os.pid_t) !void { + comp.bin_file.child_pid = pid; + try comp.makeBinFileWritable(); + try comp.update(prog_node); + try comp.makeBinFileExecutable(); +} + /// Detect changes to source files, perform semantic analysis, and update the output files. -pub fn update(comp: *Compilation) !void { +pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void { const tracy_trace = trace(@src()); defer tracy_trace.end(); @@ -1995,21 +2002,6 @@ pub fn update(comp: *Compilation) !void { } } - // If the terminal is dumb, we dont want to show the user all the output. - var progress: std.Progress = .{ .dont_print_on_dumb = true }; - const main_progress_node = progress.start("", 0); - defer main_progress_node.end(); - switch (comp.color) { - .off => { - progress.terminal = null; - }, - .on => { - progress.terminal = std.io.getStdErr(); - progress.supports_ansi_escape_codes = true; - }, - .auto => {}, - } - try comp.performAllTheWork(main_progress_node); if (comp.bin_file.options.module) |module| { @@ -3057,11 +3049,11 @@ pub fn performAllTheWork( // backend, preventing anonymous Decls from being prematurely destroyed. while (true) { if (comp.work_queue.readItem()) |work_item| { - try processOneJob(comp, work_item); + try processOneJob(comp, work_item, main_progress_node); continue; } if (comp.anon_work_queue.readItem()) |work_item| { - try processOneJob(comp, work_item); + try processOneJob(comp, work_item, main_progress_node); continue; } break; @@ -3069,16 +3061,16 @@ pub fn performAllTheWork( if (comp.job_queued_compiler_rt_lib) { comp.job_queued_compiler_rt_lib = false; - buildCompilerRtOneShot(comp, .Lib, &comp.compiler_rt_lib); + buildCompilerRtOneShot(comp, .Lib, &comp.compiler_rt_lib, main_progress_node); } if (comp.job_queued_compiler_rt_obj) { comp.job_queued_compiler_rt_obj = false; - buildCompilerRtOneShot(comp, .Obj, &comp.compiler_rt_obj); + buildCompilerRtOneShot(comp, .Obj, &comp.compiler_rt_obj, main_progress_node); } } -fn processOneJob(comp: *Compilation, job: Job) !void { +fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !void { switch (job) { .codegen_decl => |decl_index| { const module = comp.bin_file.options.module.?; @@ -3230,7 +3222,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("glibc_crt_file"); defer named_frame.end(); - glibc.buildCRTFile(comp, crt_file) catch |err| { + glibc.buildCRTFile(comp, crt_file, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure(.glibc_crt_file, "unable to build glibc CRT file: {s}", .{ @errorName(err), @@ -3241,7 +3233,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("glibc_shared_objects"); defer named_frame.end(); - glibc.buildSharedObjects(comp) catch |err| { + glibc.buildSharedObjects(comp, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .glibc_shared_objects, @@ -3254,7 +3246,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("musl_crt_file"); defer named_frame.end(); - musl.buildCRTFile(comp, crt_file) catch |err| { + musl.buildCRTFile(comp, crt_file, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .musl_crt_file, @@ -3267,7 +3259,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("mingw_crt_file"); defer named_frame.end(); - mingw.buildCRTFile(comp, crt_file) catch |err| { + mingw.buildCRTFile(comp, crt_file, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .mingw_crt_file, @@ -3294,7 +3286,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("libunwind"); defer named_frame.end(); - libunwind.buildStaticLib(comp) catch |err| { + libunwind.buildStaticLib(comp, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .libunwind, @@ -3307,7 +3299,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("libcxx"); defer named_frame.end(); - libcxx.buildLibCXX(comp) catch |err| { + libcxx.buildLibCXX(comp, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .libcxx, @@ -3320,7 +3312,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("libcxxabi"); defer named_frame.end(); - libcxx.buildLibCXXABI(comp) catch |err| { + libcxx.buildLibCXXABI(comp, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .libcxxabi, @@ -3333,7 +3325,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("libtsan"); defer named_frame.end(); - libtsan.buildTsan(comp) catch |err| { + libtsan.buildTsan(comp, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .libtsan, @@ -3346,7 +3338,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { const named_frame = tracy.namedFrame("wasi_libc_crt_file"); defer named_frame.end(); - wasi_libc.buildCRTFile(comp, crt_file) catch |err| { + wasi_libc.buildCRTFile(comp, crt_file, prog_node) catch |err| { // TODO Surface more error details. comp.lockAndSetMiscFailure( .wasi_libc_crt_file, @@ -3364,6 +3356,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { .Lib, &comp.libssp_static_lib, .libssp, + prog_node, ) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.SubCompilationFailed => return, // error reported already @@ -3383,6 +3376,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void { .Lib, &comp.libc_static_lib, .zig_libc, + prog_node, ) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.SubCompilationFailed => return, // error reported already @@ -3723,8 +3717,15 @@ fn buildCompilerRtOneShot( comp: *Compilation, output_mode: std.builtin.OutputMode, out: *?CRTFile, + prog_node: *std.Progress.Node, ) void { - comp.buildOutputFromZig("compiler_rt.zig", output_mode, out, .compiler_rt) catch |err| switch (err) { + comp.buildOutputFromZig( + "compiler_rt.zig", + output_mode, + out, + .compiler_rt, + prog_node, + ) catch |err| switch (err) { error.SubCompilationFailed => return, // error reported already else => comp.lockAndSetMiscFailure( .compiler_rt, @@ -5248,8 +5249,15 @@ pub fn updateSubCompilation( parent_comp: *Compilation, sub_comp: *Compilation, misc_task: MiscTask, + prog_node: *std.Progress.Node, ) !void { - try sub_comp.update(); + { + var sub_node = prog_node.start(@tagName(misc_task), 0); + sub_node.activate(); + defer sub_node.end(); + + try sub_comp.update(prog_node); + } // Look for compilation errors in this sub compilation const gpa = parent_comp.gpa; @@ -5276,6 +5284,7 @@ fn buildOutputFromZig( output_mode: std.builtin.OutputMode, out: *?CRTFile, misc_task_tag: MiscTask, + prog_node: *std.Progress.Node, ) !void { const tracy_trace = trace(@src()); defer tracy_trace.end(); @@ -5342,7 +5351,7 @@ fn buildOutputFromZig( }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, misc_task_tag); + try comp.updateSubCompilation(sub_compilation, misc_task_tag, prog_node); assert(out.* == null); out.* = Compilation.CRTFile{ @@ -5358,6 +5367,7 @@ pub fn build_crt_file( root_name: []const u8, output_mode: std.builtin.OutputMode, misc_task_tag: MiscTask, + prog_node: *std.Progress.Node, c_source_files: []const Compilation.CSourceFile, ) !void { const tracy_trace = trace(@src()); @@ -5418,7 +5428,7 @@ pub fn build_crt_file( }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, misc_task_tag); + try comp.updateSubCompilation(sub_compilation, misc_task_tag, prog_node); try comp.crt_files.ensureUnusedCapacity(comp.gpa, 1); @@ -5470,10 +5480,3 @@ pub fn compilerRtStrip(comp: Compilation) bool { return true; } } - -pub fn hotCodeSwap(comp: *Compilation, pid: std.os.pid_t) !void { - comp.bin_file.child_pid = pid; - try comp.makeBinFileWritable(); - try comp.update(); - try comp.makeBinFileExecutable(); -} diff --git a/src/glibc.zig b/src/glibc.zig index 530f35531a..b37398bffd 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -161,7 +161,7 @@ pub const CRTFile = enum { libc_nonshared_a, }; -pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -196,7 +196,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-DASSEMBLER", "-Wa,--noexecstack", }); - return comp.build_crt_file("crti", .Obj, .@"glibc crti.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crti", .Obj, .@"glibc crti.o", prog_node, &.{ .{ .src_path = try start_asm_path(comp, arena, "crti.S"), .cache_exempt_flags = args.items, @@ -215,7 +215,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-DASSEMBLER", "-Wa,--noexecstack", }); - return comp.build_crt_file("crtn", .Obj, .@"glibc crtn.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crtn", .Obj, .@"glibc crtn.o", prog_node, &.{ .{ .src_path = try start_asm_path(comp, arena, "crtn.S"), .cache_exempt_flags = args.items, @@ -265,7 +265,9 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .cache_exempt_flags = args.items, }; }; - return comp.build_crt_file("Scrt1", .Obj, .@"glibc Scrt1.o", &[_]Compilation.CSourceFile{ start_o, abi_note_o }); + return comp.build_crt_file("Scrt1", .Obj, .@"glibc Scrt1.o", prog_node, &.{ + start_o, abi_note_o, + }); }, .libc_nonshared_a => { const s = path.sep_str; @@ -366,7 +368,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { files_index += 1; } const files = files_buf[0..files_index]; - return comp.build_crt_file("c_nonshared", .Lib, .@"glibc libc_nonshared.a", files); + return comp.build_crt_file("c_nonshared", .Lib, .@"glibc libc_nonshared.a", prog_node, files); }, } } @@ -639,7 +641,7 @@ pub const BuiltSharedObjects = struct { const all_map_basename = "all.map"; -pub fn buildSharedObjects(comp: *Compilation) !void { +pub fn buildSharedObjects(comp: *Compilation, prog_node: *std.Progress.Node) !void { const tracy = trace(@src()); defer tracy.end(); @@ -1023,7 +1025,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const asm_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.s", .{lib.name}) catch unreachable; try o_directory.handle.writeFile(asm_file_basename, stubs_asm.items); - try buildSharedLib(comp, arena, comp.global_cache_directory, o_directory, asm_file_basename, lib); + try buildSharedLib(comp, arena, comp.global_cache_directory, o_directory, asm_file_basename, lib, prog_node); } man.writeManifest() catch |err| { @@ -1046,6 +1048,7 @@ fn buildSharedLib( bin_directory: Compilation.Directory, asm_file_basename: []const u8, lib: Lib, + prog_node: *std.Progress.Node, ) !void { const tracy = trace(@src()); defer tracy.end(); @@ -1105,7 +1108,7 @@ fn buildSharedLib( }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .@"glibc shared object"); + try comp.updateSubCompilation(sub_compilation, .@"glibc shared object", prog_node); } // Return true if glibc has crti/crtn sources for that architecture. diff --git a/src/libcxx.zig b/src/libcxx.zig index c17352c562..9c5dc9426f 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -96,7 +96,7 @@ const libcxx_files = [_][]const u8{ "src/verbose_abort.cpp", }; -pub fn buildLibCXX(comp: *Compilation) !void { +pub fn buildLibCXX(comp: *Compilation, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -258,7 +258,7 @@ pub fn buildLibCXX(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .libcxx); + try comp.updateSubCompilation(sub_compilation, .libcxx, prog_node); assert(comp.libcxx_static_lib == null); comp.libcxx_static_lib = Compilation.CRTFile{ @@ -269,7 +269,7 @@ pub fn buildLibCXX(comp: *Compilation) !void { }; } -pub fn buildLibCXXABI(comp: *Compilation) !void { +pub fn buildLibCXXABI(comp: *Compilation, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -418,7 +418,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .libcxxabi); + try comp.updateSubCompilation(sub_compilation, .libcxxabi, prog_node); assert(comp.libcxxabi_static_lib == null); comp.libcxxabi_static_lib = Compilation.CRTFile{ diff --git a/src/libtsan.zig b/src/libtsan.zig index 1399b6b76c..54bf00e4b6 100644 --- a/src/libtsan.zig +++ b/src/libtsan.zig @@ -5,7 +5,7 @@ const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; -pub fn buildTsan(comp: *Compilation) !void { +pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -235,7 +235,7 @@ pub fn buildTsan(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .libtsan); + try comp.updateSubCompilation(sub_compilation, .libtsan, prog_node); assert(comp.tsan_static_lib == null); comp.tsan_static_lib = Compilation.CRTFile{ diff --git a/src/libunwind.zig b/src/libunwind.zig index 667195a369..aefbfb457d 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -7,7 +7,7 @@ const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; -pub fn buildStaticLib(comp: *Compilation) !void { +pub fn buildStaticLib(comp: *Compilation, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -130,7 +130,7 @@ pub fn buildStaticLib(comp: *Compilation) !void { }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .libunwind); + try comp.updateSubCompilation(sub_compilation, .libunwind, prog_node); assert(comp.libunwind_static_lib == null); diff --git a/src/main.zig b/src/main.zig index d574681bcd..495cc40ed5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3543,6 +3543,23 @@ fn serve( var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa); defer receive_fifo.deinit(); + var progress: std.Progress = .{ + .terminal = null, + .root = .{ + .context = undefined, + .parent = null, + .name = "", + .unprotected_estimated_total_items = 0, + .unprotected_completed_items = 0, + }, + .columns_written = 0, + .prev_refresh_timestamp = 0, + .timer = null, + .done = false, + }; + const main_progress_node = &progress.root; + main_progress_node.context = &progress; + while (true) { const hdr = try receiveMessage(in, &receive_fifo); @@ -3551,11 +3568,12 @@ fn serve( return cleanExit(); }, .update => { + assert(main_progress_node.recently_updated_child == null); tracy.frameMark(); if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } - try comp.update(); + try comp.update(main_progress_node); try comp.makeBinFileExecutable(); try serveUpdateResults(out, comp); }, @@ -3581,14 +3599,15 @@ fn serve( }, .hot_update => { tracy.frameMark(); + assert(main_progress_node.recently_updated_child == null); if (child_pid) |pid| { - try comp.hotCodeSwap(pid); + try comp.hotCodeSwap(main_progress_node, pid); try serveUpdateResults(out, comp); } else { if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } - try comp.update(); + try comp.update(main_progress_node); try comp.makeBinFileExecutable(); try serveUpdateResults(out, comp); @@ -3936,7 +3955,24 @@ const AfterUpdateHook = union(enum) { }; fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void { - try comp.update(); + { + // If the terminal is dumb, we dont want to show the user all the output. + var progress: std.Progress = .{ .dont_print_on_dumb = true }; + const main_progress_node = progress.start("", 0); + defer main_progress_node.end(); + switch (comp.color) { + .off => { + progress.terminal = null; + }, + .on => { + progress.terminal = std.io.getStdErr(); + progress.supports_ansi_escape_codes = true; + }, + .auto => {}, + } + + try comp.update(main_progress_node); + } var errors = try comp.getAllErrorsAlloc(); defer errors.deinit(comp.gpa); @@ -4642,6 +4678,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi switch (term) { .Exited => |code| { if (code == 0) return cleanExit(); + // Indicates that the build runner has reported compile errors + // and this parent process does not need to report any further + // diagnostics. + if (code == 2) process.exit(2); const cmd = try std.mem.join(arena, " ", child_argv); fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); diff --git a/src/mingw.zig b/src/mingw.zig index d5c42c3ccc..a85645e80b 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -19,7 +19,7 @@ pub const CRTFile = enum { uuid_lib, }; -pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -41,7 +41,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { //"-D_UNICODE", //"-DWPRFLAG=1", }); - return comp.build_crt_file("crt2", .Obj, .@"mingw-w64 crt2.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt2", .Obj, .@"mingw-w64 crt2.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "crt", "crtexe.c", @@ -60,7 +60,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-U__CRTDLL__", "-D__MSVCRT__", }); - return comp.build_crt_file("dllcrt2", .Obj, .@"mingw-w64 dllcrt2.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("dllcrt2", .Obj, .@"mingw-w64 dllcrt2.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "crt", "crtdll.c", @@ -100,7 +100,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; } - return comp.build_crt_file("mingw32", .Lib, .@"mingw-w64 mingw32.lib", &c_source_files); + return comp.build_crt_file("mingw32", .Lib, .@"mingw-w64 mingw32.lib", prog_node, &c_source_files); }, .msvcrt_os_lib => { @@ -148,7 +148,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { }; } } - return comp.build_crt_file("msvcrt-os", .Lib, .@"mingw-w64 msvcrt-os.lib", c_source_files.items); + return comp.build_crt_file("msvcrt-os", .Lib, .@"mingw-w64 msvcrt-os.lib", prog_node, c_source_files.items); }, .mingwex_lib => { @@ -211,7 +211,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } else { @panic("unsupported arch"); } - return comp.build_crt_file("mingwex", .Lib, .@"mingw-w64 mingwex.lib", c_source_files.items); + return comp.build_crt_file("mingwex", .Lib, .@"mingw-w64 mingwex.lib", prog_node, c_source_files.items); }, .uuid_lib => { @@ -244,7 +244,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = extra_flags, }; } - return comp.build_crt_file("uuid", .Lib, .@"mingw-w64 uuid.lib", &c_source_files); + return comp.build_crt_file("uuid", .Lib, .@"mingw-w64 uuid.lib", prog_node, &c_source_files); }, } } diff --git a/src/musl.zig b/src/musl.zig index 7dd224604f..4a3f1e6dde 100644 --- a/src/musl.zig +++ b/src/musl.zig @@ -17,7 +17,7 @@ pub const CRTFile = enum { libc_so, }; -pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -33,7 +33,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { try args.appendSlice(&[_][]const u8{ "-Qunused-arguments", }); - return comp.build_crt_file("crti", .Obj, .@"musl crti.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crti", .Obj, .@"musl crti.o", prog_node, &.{ .{ .src_path = try start_asm_path(comp, arena, "crti.s"), .extra_flags = args.items, @@ -46,7 +46,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { try args.appendSlice(&[_][]const u8{ "-Qunused-arguments", }); - return comp.build_crt_file("crtn", .Obj, .@"musl crtn.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crtn", .Obj, .@"musl crtn.o", prog_node, &.{ .{ .src_path = try start_asm_path(comp, arena, "crtn.s"), .extra_flags = args.items, @@ -60,7 +60,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("crt1", .Obj, .@"musl crt1.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1", .Obj, .@"musl crt1.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "crt1.c", @@ -77,7 +77,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("rcrt1", .Obj, .@"musl rcrt1.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("rcrt1", .Obj, .@"musl rcrt1.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "rcrt1.c", @@ -94,7 +94,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { "-fno-stack-protector", "-DCRT", }); - return comp.build_crt_file("Scrt1", .Obj, .@"musl Scrt1.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("Scrt1", .Obj, .@"musl Scrt1.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "crt", "Scrt1.c", @@ -187,7 +187,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }; } - return comp.build_crt_file("c", .Lib, .@"musl libc.a", c_source_files.items); + return comp.build_crt_file("c", .Lib, .@"musl libc.a", prog_node, c_source_files.items); }, .libc_so => { const target = comp.getTarget(); @@ -241,7 +241,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { }); defer sub_compilation.destroy(); - try comp.updateSubCompilation(sub_compilation, .@"musl libc.so"); + try comp.updateSubCompilation(sub_compilation, .@"musl libc.so", prog_node); try comp.crt_files.ensureUnusedCapacity(comp.gpa, 1); diff --git a/src/wasi_libc.zig b/src/wasi_libc.zig index c4a4cbc4a5..38a4f17190 100644 --- a/src/wasi_libc.zig +++ b/src/wasi_libc.zig @@ -59,7 +59,7 @@ pub fn execModelCrtFileFullName(wasi_exec_model: std.builtin.WasiExecModel) []co }; } -pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { +pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile, prog_node: *std.Progress.Node) !void { if (!build_options.have_llvm) { return error.ZigCompilerNotBuiltWithLLVMExtensions; } @@ -74,7 +74,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, false); try addLibcBottomHalfIncludes(comp, arena, &args); - return comp.build_crt_file("crt1-reactor", .Obj, .@"wasi crt1-reactor.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1-reactor", .Obj, .@"wasi crt1-reactor.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, crt1_reactor_src_file), @@ -87,7 +87,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { var args = std.ArrayList([]const u8).init(arena); try addCCArgs(comp, arena, &args, false); try addLibcBottomHalfIncludes(comp, arena, &args); - return comp.build_crt_file("crt1-command", .Obj, .@"wasi crt1-command.o", &[1]Compilation.CSourceFile{ + return comp.build_crt_file("crt1-command", .Obj, .@"wasi crt1-command.o", prog_node, &.{ .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", try sanitize(arena, crt1_command_src_file), @@ -145,7 +145,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } } - try comp.build_crt_file("c", .Lib, .@"wasi libc.a", libc_sources.items); + try comp.build_crt_file("c", .Lib, .@"wasi libc.a", prog_node, libc_sources.items); }, .libwasi_emulated_process_clocks_a => { var args = std.ArrayList([]const u8).init(arena); @@ -161,7 +161,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, .@"libwasi-emulated-process-clocks.a", emu_clocks_sources.items); + try comp.build_crt_file("wasi-emulated-process-clocks", .Lib, .@"libwasi-emulated-process-clocks.a", prog_node, emu_clocks_sources.items); }, .libwasi_emulated_getpid_a => { var args = std.ArrayList([]const u8).init(arena); @@ -177,7 +177,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-getpid", .Lib, .@"libwasi-emulated-getpid.a", emu_getpid_sources.items); + try comp.build_crt_file("wasi-emulated-getpid", .Lib, .@"libwasi-emulated-getpid.a", prog_node, emu_getpid_sources.items); }, .libwasi_emulated_mman_a => { var args = std.ArrayList([]const u8).init(arena); @@ -193,7 +193,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { .extra_flags = args.items, }); } - try comp.build_crt_file("wasi-emulated-mman", .Lib, .@"libwasi-emulated-mman.a", emu_mman_sources.items); + try comp.build_crt_file("wasi-emulated-mman", .Lib, .@"libwasi-emulated-mman.a", prog_node, emu_mman_sources.items); }, .libwasi_emulated_signal_a => { var emu_signal_sources = std.ArrayList(Compilation.CSourceFile).init(arena); @@ -228,7 +228,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void { } } - try comp.build_crt_file("wasi-emulated-signal", .Lib, .@"libwasi-emulated-signal.a", emu_signal_sources.items); + try comp.build_crt_file("wasi-emulated-signal", .Lib, .@"libwasi-emulated-signal.a", prog_node, emu_signal_sources.items); }, } } -- cgit v1.2.3 From 0e078790feaf49964d7a0da3042117ebd10de13b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Feb 2023 23:58:13 -0700 Subject: multiplex compiler progress messages into the build runner --- lib/build_runner.zig | 2 +- lib/std/Build.zig | 16 ++++++-- lib/std/Build/CheckFileStep.zig | 3 +- lib/std/Build/CheckObjectStep.zig | 3 +- lib/std/Build/CompileStep.zig | 6 +-- lib/std/Build/ConfigHeaderStep.zig | 3 +- lib/std/Build/EmulatableRunStep.zig | 3 +- lib/std/Build/FmtStep.zig | 3 +- lib/std/Build/InstallArtifactStep.zig | 3 +- lib/std/Build/InstallDirStep.zig | 3 +- lib/std/Build/InstallFileStep.zig | 3 +- lib/std/Build/LogStep.zig | 3 +- lib/std/Build/ObjCopyStep.zig | 3 +- lib/std/Build/OptionsStep.zig | 3 +- lib/std/Build/RemoveDirStep.zig | 3 +- lib/std/Build/RunStep.zig | 3 +- lib/std/Build/Step.zig | 13 ++++--- lib/std/Build/TranslateCStep.zig | 4 +- lib/std/Build/WriteFileStep.zig | 3 +- src/main.zig | 73 ++++++++++++++++++++++++++++++++++- test/tests.zig | 6 ++- 21 files changed, 131 insertions(+), 31 deletions(-) (limited to 'src/main.zig') diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 9a8f138979..ea4703bfbb 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -571,7 +571,7 @@ fn workerMakeOneStep( // For example, CompileStep does some sus things with modifying the saved // *Build object in install header steps that might be able to be removed // by passing the *Build object through the make() functions. - const make_result = s.make(); + const make_result = s.make(&sub_prog_node); // No matter the result, we want to display error/warning messages. if (s.result_error_msgs.items.len > 0) { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a9d7c31733..ca7ddb591c 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -718,7 +718,8 @@ pub fn getUninstallStep(self: *Build) *Step { return &self.uninstall_tls.step; } -fn makeUninstall(uninstall_step: *Step) anyerror!void { +fn makeUninstall(uninstall_step: *Step, prog_node: *std.Progress.Node) anyerror!void { + _ = prog_node; const uninstall_tls = @fieldParentPtr(TopLevelStep, "step", uninstall_step); const self = @fieldParentPtr(Build, "uninstall_tls", uninstall_tls); @@ -1404,7 +1405,7 @@ pub fn execAllowFail( /// This function is used exclusively for spawning and communicating with the zig compiler. /// TODO: move to build_runner.zig -pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { +pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *std.Progress.Node) ![]const u8 { assert(argv.len != 0); if (b.verbose) { @@ -1439,6 +1440,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { const Header = std.zig.Server.Message.Header; var result: ?[]const u8 = null; + var node_name: std.ArrayListUnmanaged(u8) = .{}; + defer node_name.deinit(b.allocator); + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |*n| n.end(); + while (try poller.poll()) { const stdout = poller.fifo(.stdout); const buf = stdout.readableSlice(0); @@ -1478,7 +1484,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { }; }, .progress => { - @panic("TODO handle progress message"); + if (sub_prog_node) |*n| n.end(); + node_name.clearRetainingCapacity(); + try node_name.appendSlice(b.allocator, body); + sub_prog_node = prog_node.start(node_name.items, 0); + sub_prog_node.?.activate(); }, .emit_bin_path => { result = try b.allocator.dupe(u8, body); diff --git a/lib/std/Build/CheckFileStep.zig b/lib/std/Build/CheckFileStep.zig index 5b2e5b4e5b..9fee947386 100644 --- a/lib/std/Build/CheckFileStep.zig +++ b/lib/std/Build/CheckFileStep.zig @@ -33,7 +33,8 @@ pub fn create( return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(CheckFileStep, "step", step); const src_path = self.source.getPath(self.builder); diff --git a/lib/std/Build/CheckObjectStep.zig b/lib/std/Build/CheckObjectStep.zig index 1775a5d39a..eccbbd1696 100644 --- a/lib/std/Build/CheckObjectStep.zig +++ b/lib/std/Build/CheckObjectStep.zig @@ -300,7 +300,8 @@ pub fn checkComputeCompare( self.checks.append(new_check) catch @panic("OOM"); } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(CheckObjectStep, "step", step); const gpa = self.builder.allocator; diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 491c8ac8e4..711092c762 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -1160,7 +1160,7 @@ fn constructDepString( } } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { const self = @fieldParentPtr(CompileStep, "step", step); const builder = self.builder; @@ -1718,7 +1718,7 @@ fn make(step: *Step) !void { } if (other.installed_headers.items.len > 0) { for (other.installed_headers.items) |install_step| { - try install_step.make(); + try install_step.make(prog_node); } try zig_args.append("-I"); try zig_args.append(builder.pathJoin(&.{ @@ -1894,7 +1894,7 @@ fn make(step: *Step) !void { try zig_args.append(resolved_args_file); } - const output_bin_path = try builder.execFromStep(zig_args.items, &self.step); + const output_bin_path = try builder.execFromStep(zig_args.items, &self.step, prog_node); const build_output_dir = fs.path.dirname(output_bin_path).?; if (self.output_dir) |output_dir| { diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index fb8d92e4e2..c74c03718e 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -152,7 +152,8 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v } } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(ConfigHeaderStep, "step", step); const gpa = self.builder.allocator; diff --git a/lib/std/Build/EmulatableRunStep.zig b/lib/std/Build/EmulatableRunStep.zig index 6556f4b8c0..44387c36f6 100644 --- a/lib/std/Build/EmulatableRunStep.zig +++ b/lib/std/Build/EmulatableRunStep.zig @@ -71,7 +71,8 @@ pub fn create(builder: *std.Build, name: []const u8, artifact: *CompileStep) *Em return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(EmulatableRunStep, "step", step); const host_info = self.builder.host; diff --git a/lib/std/Build/FmtStep.zig b/lib/std/Build/FmtStep.zig index 04b8ccb3e4..5efada7507 100644 --- a/lib/std/Build/FmtStep.zig +++ b/lib/std/Build/FmtStep.zig @@ -29,7 +29,8 @@ pub fn create(builder: *std.Build, paths: []const []const u8) *FmtStep { return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(FmtStep, "step", step); return self.builder.spawnChild(self.argv); diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/InstallArtifactStep.zig index b652ade38f..d8907eb59f 100644 --- a/lib/std/Build/InstallArtifactStep.zig +++ b/lib/std/Build/InstallArtifactStep.zig @@ -64,7 +64,8 @@ pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallArtifactStep, "step", step); const builder = self.builder; diff --git a/lib/std/Build/InstallDirStep.zig b/lib/std/Build/InstallDirStep.zig index 54a37af1d4..bf89d9e7c7 100644 --- a/lib/std/Build/InstallDirStep.zig +++ b/lib/std/Build/InstallDirStep.zig @@ -56,7 +56,8 @@ pub fn init( }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallDirStep, "step", step); const dest_prefix = self.builder.getInstallPath(self.options.install_dir, self.options.install_subdir); const src_builder = self.override_source_builder orelse self.builder; diff --git a/lib/std/Build/InstallFileStep.zig b/lib/std/Build/InstallFileStep.zig index e6c57a8050..f77b22c112 100644 --- a/lib/std/Build/InstallFileStep.zig +++ b/lib/std/Build/InstallFileStep.zig @@ -35,7 +35,8 @@ pub fn init( }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallFileStep, "step", step); const src_builder = self.override_source_builder orelse self.builder; const full_src_path = self.source.getPath2(src_builder, step); diff --git a/lib/std/Build/LogStep.zig b/lib/std/Build/LogStep.zig index 008ed6e8bb..25bba747bf 100644 --- a/lib/std/Build/LogStep.zig +++ b/lib/std/Build/LogStep.zig @@ -21,7 +21,8 @@ pub fn init(builder: *std.Build, data: []const u8) LogStep { }; } -fn make(step: *Step) anyerror!void { +fn make(step: *Step, prog_node: *std.Progress.Node) anyerror!void { + _ = prog_node; const self = @fieldParentPtr(LogStep, "step", step); log.info("{s}", .{self.data}); } diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/ObjCopyStep.zig index 549bb1ed00..839d95903c 100644 --- a/lib/std/Build/ObjCopyStep.zig +++ b/lib/std/Build/ObjCopyStep.zig @@ -66,7 +66,8 @@ pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource { return .{ .generated = &self.output_file }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(ObjCopyStep, "step", step); const b = self.builder; diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index 8e86578e30..2cb3bb13be 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -219,7 +219,8 @@ pub fn getSource(self: *OptionsStep) FileSource { return .{ .generated = &self.generated_file }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(OptionsStep, "step", step); for (self.artifact_args.items) |item| { diff --git a/lib/std/Build/RemoveDirStep.zig b/lib/std/Build/RemoveDirStep.zig index acde60a745..4fc8e6d338 100644 --- a/lib/std/Build/RemoveDirStep.zig +++ b/lib/std/Build/RemoveDirStep.zig @@ -22,7 +22,8 @@ pub fn init(builder: *std.Build, dir_path: []const u8) RemoveDirStep { }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 904ef0935f..7ec60b6d22 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -206,7 +206,8 @@ fn needOutputCheck(self: RunStep) bool { return false; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RunStep, "step", step); const need_output_check = self.needOutputCheck(); diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 4edece8038..53861683ac 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -1,6 +1,6 @@ id: Id, name: []const u8, -makeFn: *const fn (self: *Step) anyerror!void, +makeFn: MakeFn, dependencies: std.ArrayList(*Step), /// This field is empty during execution of the user's build script, and /// then populated during dependency loop checking in the build runner. @@ -13,6 +13,8 @@ debug_stack_trace: [n_debug_stack_frames]usize, result_error_msgs: std.ArrayListUnmanaged([]const u8), result_error_bundle: std.zig.ErrorBundle, +pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void; + const n_debug_stack_frames = 4; pub const State = enum { @@ -72,7 +74,7 @@ pub const Id = enum { pub const Options = struct { id: Id, name: []const u8, - makeFn: *const fn (self: *Step) anyerror!void = makeNoOp, + makeFn: MakeFn = makeNoOp, first_ret_addr: ?usize = null, }; @@ -101,8 +103,8 @@ pub fn init(allocator: Allocator, options: Options) Step { /// If the Step's `make` function reports `error.MakeFailed`, it indicates they /// have already reported the error. Otherwise, we add a simple error report /// here. -pub fn make(s: *Step) error{MakeFailed}!void { - return s.makeFn(s) catch |err| { +pub fn make(s: *Step, prog_node: *std.Progress.Node) error{MakeFailed}!void { + return s.makeFn(s, prog_node) catch |err| { if (err != error.MakeFailed) { const gpa = s.dependencies.allocator; s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{ @@ -129,8 +131,9 @@ pub fn getStackTrace(s: *Step) std.builtin.StackTrace { }; } -fn makeNoOp(self: *Step) anyerror!void { +fn makeNoOp(self: *Step, prog_node: *std.Progress.Node) anyerror!void { _ = self; + _ = prog_node; } pub fn cast(step: *Step, comptime T: type) ?*T { diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/TranslateCStep.zig index 8c3a254d6a..d19e598d93 100644 --- a/lib/std/Build/TranslateCStep.zig +++ b/lib/std/Build/TranslateCStep.zig @@ -88,7 +88,7 @@ pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void { self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM"); } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { const self = @fieldParentPtr(TranslateCStep, "step", step); var argv_list = std.ArrayList([]const u8).init(self.builder.allocator); @@ -120,7 +120,7 @@ fn make(step: *Step) !void { try argv_list.append(self.source.getPath(self.builder)); - const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step); + const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step, prog_node); const output_path = mem.trimRight(u8, output_path_nl, "\r\n"); self.out_basename = fs.path.basename(output_path); diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 4f5e768237..62acd6e8ee 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -99,7 +99,8 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo return null; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const wf = @fieldParentPtr(WriteFileStep, "step", step); // Writing to source files is kind of an extra capability of this diff --git a/src/main.zig b/src/main.zig index 495cc40ed5..1d5cf57388 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3573,7 +3573,21 @@ fn serve( if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } - try comp.update(main_progress_node); + + { + var reset: std.Thread.ResetEvent = .{}; + + var progress_thread = try std.Thread.spawn(.{}, progressThread, .{ + &progress, out, &reset, + }); + defer { + reset.set(); + progress_thread.join(); + } + + try comp.update(main_progress_node); + } + try comp.makeBinFileExecutable(); try serveUpdateResults(out, comp); }, @@ -3629,6 +3643,63 @@ fn serve( } } +fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void { + while (true) { + if (reset.timedWait(500 * std.time.ns_per_ms)) |_| { + // The Compilation update has completed. + return; + } else |err| switch (err) { + error.Timeout => {}, + } + + var buf: std.BoundedArray(u8, 160) = .{}; + + { + progress.update_mutex.lock(); + defer progress.update_mutex.unlock(); + + var need_ellipse = false; + var maybe_node: ?*std.Progress.Node = &progress.root; + while (maybe_node) |node| { + if (need_ellipse) { + buf.appendSlice("... ") catch {}; + } + need_ellipse = false; + const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic); + const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic); + const current_item = completed_items + 1; + if (node.name.len != 0 or eti > 0) { + if (node.name.len != 0) { + buf.appendSlice(node.name) catch {}; + need_ellipse = true; + } + if (eti > 0) { + if (need_ellipse) buf.appendSlice(" ") catch {}; + buf.writer().print("[{d}/{d}] ", .{ current_item, eti }) catch {}; + need_ellipse = false; + } else if (completed_items != 0) { + if (need_ellipse) buf.appendSlice(" ") catch {}; + buf.writer().print("[{d}] ", .{current_item}) catch {}; + need_ellipse = false; + } + } + maybe_node = @atomicLoad(?*std.Progress.Node, &node.recently_updated_child, .Acquire); + } + } + + const progress_string = buf.slice(); + + serveMessage(out, .{ + .tag = .progress, + .bytes_len = @intCast(u32, progress_string.len), + }, &.{ + progress_string, + }) catch |err| { + fatal("unable to write to client: {s}", .{@errorName(err)}); + }; + } +} + fn serveMessage( out: fs.File, header: std.zig.Server.Message.Header, diff --git a/test/tests.zig b/test/tests.zig index fe2efbc06e..fceaa173d1 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -875,7 +875,8 @@ pub const StackTracesContext = struct { return ptr; } - fn make(step: *Step) !void { + fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RunAndCompareStep, "step", step); const b = self.context.b; @@ -1218,7 +1219,8 @@ pub const GenHContext = struct { return ptr; } - fn make(step: *Step) !void { + fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(GenHCmpOutputStep, "step", step); const b = self.context.b; -- cgit v1.2.3 From 8b2d872020bf8139404d0655e5ab70792cc67873 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 1 Mar 2023 00:34:30 -0700 Subject: fix std.Build.TranslateCStep --- lib/std/Build.zig | 2 +- lib/std/Build/TranslateCStep.zig | 4 ++-- src/main.zig | 51 +++++++++++++++++++++++----------------- 3 files changed, 32 insertions(+), 25 deletions(-) (limited to 'src/main.zig') diff --git a/lib/std/Build.zig b/lib/std/Build.zig index ca7ddb591c..05decec36f 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1453,7 +1453,7 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *s const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); const header_and_msg_len = header.bytes_len + @sizeOf(Header); if (buf.len >= header_and_msg_len) { - const body = buf[@sizeOf(Header)..]; + const body = buf[@sizeOf(Header)..][0..header.bytes_len]; switch (header.tag) { .zig_version => { if (!mem.eql(u8, builtin.zig_version_string, body)) { diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/TranslateCStep.zig index d19e598d93..fef644f03c 100644 --- a/lib/std/Build/TranslateCStep.zig +++ b/lib/std/Build/TranslateCStep.zig @@ -97,6 +97,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try argv_list.append("-lc"); try argv_list.append("--enable-cache"); + try argv_list.append("--listen=-"); if (!self.target.isNative()) { try argv_list.append("-target"); @@ -120,8 +121,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try argv_list.append(self.source.getPath(self.builder)); - const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step, prog_node); - const output_path = mem.trimRight(u8, output_path_nl, "\r\n"); + const output_path = try self.builder.execFromStep(argv_list.items, &self.step, prog_node); self.out_basename = fs.path.basename(output_path); const output_dir = fs.path.dirname(output_path).?; diff --git a/src/main.zig b/src/main.zig index 1d5cf57388..630f454272 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3287,10 +3287,6 @@ fn buildOutputType( if (show_builtin) { return std.io.getStdOut().writeAll(try comp.generateBuiltinZigSource(arena)); } - if (arg_mode == .translate_c) { - return cmdTranslateC(comp, arena, have_enable_cache); - } - switch (listen) { .none => {}, .stdio => { @@ -3332,6 +3328,10 @@ fn buildOutputType( }, } + if (arg_mode == .translate_c) { + return cmdTranslateC(comp, arena, null); + } + const hook: AfterUpdateHook = blk: { if (!have_enable_cache) break :blk .none; @@ -3532,12 +3532,7 @@ fn serve( ) !void { const gpa = comp.gpa; - try serveMessage(out, .{ - .tag = .zig_version, - .bytes_len = build_options.version.len, - }, &.{ - build_options.version, - }); + try serveStringMessage(out, .zig_version, build_options.version); var child_pid: ?i32 = null; var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa); @@ -3570,6 +3565,17 @@ fn serve( .update => { assert(main_progress_node.recently_updated_child == null); tracy.frameMark(); + + if (arg_mode == .translate_c) { + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + var output_path: []const u8 = undefined; + try cmdTranslateC(comp, arena, &output_path); + try serveStringMessage(out, .emit_bin_path, output_path); + continue; + } + if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } @@ -3746,16 +3752,17 @@ fn serveUpdateResults(out: fs.File, comp: *Compilation) !void { } else if (comp.bin_file.options.emit) |emit| { const full_path = try emit.directory.join(gpa, &.{emit.sub_path}); defer gpa.free(full_path); - - try serveMessage(out, .{ - .tag = .emit_bin_path, - .bytes_len = @intCast(u32, full_path.len), - }, &.{ - full_path, - }); + try serveStringMessage(out, .emit_bin_path, full_path); } } +fn serveStringMessage(out: fs.File, tag: std.zig.Server.Message.Tag, s: []const u8) !void { + try serveMessage(out, .{ + .tag = tag, + .bytes_len = @intCast(u32, s.len), + }, &.{s}); +} + fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header { const Header = std.zig.Client.Message.Header; @@ -4100,7 +4107,7 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void } } -fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void { +fn cmdTranslateC(comp: *Compilation, arena: Allocator, output_path: ?*[]const u8) !void { if (!build_options.have_llvm) fatal("cannot translate-c: compiler built without LLVM extensions", .{}); @@ -4111,7 +4118,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void var man: Cache.Manifest = comp.obtainCObjectCacheManifest(); man.want_shared_lock = false; - defer if (enable_cache) man.deinit(); + defer if (output_path != null) man.deinit(); man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| { @@ -4169,6 +4176,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void error.OutOfMemory => return error.OutOfMemory, error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}), error.SemanticAnalyzeFail => { + // TODO convert these to zig errors for (clang_errors) |clang_err| { std.debug.print("{s}:{d}:{d}: {s}\n", .{ if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)", @@ -4213,12 +4221,11 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void break :digest digest; }; - if (enable_cache) { + if (output_path) |out_path| { const full_zig_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename, }); - try io.getStdOut().writer().print("{s}\n", .{full_zig_path}); - return cleanExit(); + out_path.* = full_zig_path; } else { const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename }); const zig_file = comp.local_cache_directory.handle.openFile(out_zig_path, .{}) catch |err| { -- cgit v1.2.3 From b5baa41077dce5ef61667e76d80835c0792a2fc4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 1 Mar 2023 13:31:02 -0700 Subject: fix zig fmt crash --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index 630f454272..fcdb52c342 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5120,8 +5120,8 @@ fn fmtPathFile( var tree = try Ast.parse(gpa, source_code, .zig); defer tree.deinit(gpa); - try printAstErrorsToStderr(gpa, tree, file_path, fmt.color); if (tree.errors.len != 0) { + try printAstErrorsToStderr(gpa, tree, file_path, fmt.color); fmt.any_error = true; return; } -- cgit v1.2.3 From 58edefc6d1716c0731ee2fe672ec8d073651aafb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 1 Mar 2023 22:56:37 -0700 Subject: zig build: many enhancements related to parallel building Rework std.Build.Step to have an `owner: *Build` field. This simplified the implementation of installation steps, as well as provided some much-needed common API for the new parallelized build system. --verbose is now defined very concretely: it prints to stderr just before spawning a child process. Child process execution is updated to conform to the new parallel-friendly make() function semantics. DRY up the failWithCacheError handling code. It now integrates properly with the step graph instead of incorrectly dumping to stderr and calling process exit. In the main CLI, fix `zig fmt` crash when there are no errors and stdin is used. Deleted steps: * EmulatableRunStep - this entire thing can be removed in favor of a flag added to std.Build.RunStep called `skip_foreign_checks`. * LogStep - this doesn't really fit with a multi-threaded build runner and is effectively superseded by the new build summary output. build runner: * add -fsummary and -fno-summary to override the default behavior, which is to print a summary if any of the build steps fail. * print the dep prefix when emitting error messages for steps. std.Build.FmtStep: * This step now supports exclude paths as well as a check flag. * The check flag decides between two modes, modify mode, and check mode. These can be used to update source files in place, or to fail the build, respectively. Zig's own build.zig: * The `test-fmt` step will do all the `zig fmt` checking that we expect to be done. Since the `test` step depends on this one, we can simply remove the explicit call to `zig fmt` in the CI. * The new `fmt` step will actually perform `zig fmt` and update source files in place. std.Build.RunStep: * expose max_stdio_size is a field (previously an unchangeable hard-coded value). * rework the API. Instead of configuring each stream independently, there is a `stdio` field where you can choose between `infer_from_args`, `inherit`, or `check`. These determine whether the RunStep is considered to have side-effects or not. The previous field, `condition` is gone. * when stdio mode is set to `check` there is a slice of any number of checks to make, which include things like exit code, stderr matching, or stdout matching. * remove the ill-defined `print` field. * when adding an output arg, it takes the opportunity to give itself a better name. * The flag `skip_foreign_checks` is added. If this is true, a RunStep which is configured to check the output of the executed binary will not fail the build if the binary cannot be executed due to being for a foreign binary to the host system which is running the build graph. Command-line arguments such as -fqemu and -fwasmtime may affect whether a binary is detected as foreign, as well as system configuration such as Rosetta (macOS) and binfmt_misc (Linux). - This makes EmulatableRunStep no longer needed. * Fix the child process handling to properly integrate with the new bulid API and to avoid deadlocks in stdout/stderr streams by polling if necessary. std.Build.RemoveDirStep now uses the open build_root directory handle instead of an absolute path. --- build.zig | 24 +- lib/build_runner.zig | 71 +++-- lib/std/Build.zig | 226 +-------------- lib/std/Build/CheckFileStep.zig | 18 +- lib/std/Build/CheckObjectStep.zig | 37 ++- lib/std/Build/CompileStep.zig | 456 ++++++++++++++++-------------- lib/std/Build/ConfigHeaderStep.zig | 27 +- lib/std/Build/EmulatableRunStep.zig | 218 --------------- lib/std/Build/FmtStep.zig | 80 ++++-- lib/std/Build/InstallArtifactStep.zig | 49 ++-- lib/std/Build/InstallDirStep.zig | 34 ++- lib/std/Build/InstallFileStep.zig | 29 +- lib/std/Build/LogStep.zig | 28 -- lib/std/Build/ObjCopyStep.zig | 33 +-- lib/std/Build/OptionsStep.zig | 34 +-- lib/std/Build/RemoveDirStep.zig | 29 +- lib/std/Build/RunStep.zig | 511 ++++++++++++++++++++-------------- lib/std/Build/Step.zig | 241 +++++++++++++++- lib/std/Build/TranslateCStep.zig | 40 +-- lib/std/Build/WriteFileStep.zig | 63 ++--- src/main.zig | 12 +- test/src/compare_output.zig | 4 +- test/tests.zig | 16 +- 23 files changed, 1113 insertions(+), 1167 deletions(-) delete mode 100644 lib/std/Build/EmulatableRunStep.zig delete mode 100644 lib/std/Build/LogStep.zig (limited to 'src/main.zig') diff --git a/build.zig b/build.zig index 189f50407d..5f7e214d35 100644 --- a/build.zig +++ b/build.zig @@ -61,8 +61,6 @@ pub fn build(b: *std.Build) !void { test_cases.stack_size = stack_size; test_cases.single_threaded = single_threaded; - const fmt_build_zig = b.addFmt(&[_][]const u8{"build.zig"}); - const skip_debug = b.option(bool, "skip-debug", "Main test suite skips debug builds") orelse false; const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false; const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release; @@ -386,10 +384,24 @@ pub fn build(b: *std.Build) !void { } const optimization_modes = chosen_opt_modes_buf[0..chosen_mode_index]; - // run stage1 `zig fmt` on this build.zig file just to make sure it works - test_step.dependOn(&fmt_build_zig.step); - const fmt_step = b.step("test-fmt", "Run zig fmt against build.zig to make sure it works"); - fmt_step.dependOn(&fmt_build_zig.step); + const fmt_include_paths = &.{ "doc", "lib", "src", "test", "tools", "build.zig" }; + const fmt_exclude_paths = &.{ "test/cases" }; + const check_fmt = b.addFmt(.{ + .paths = fmt_include_paths, + .exclude_paths = fmt_exclude_paths, + .check = true, + }); + const do_fmt = b.addFmt(.{ + .paths = fmt_include_paths, + .exclude_paths = fmt_exclude_paths, + }); + + const test_fmt_step = b.step("test-fmt", "Check whether source files have conforming formatting"); + test_fmt_step.dependOn(&check_fmt.step); + + const do_fmt_step = b.step("fmt", "Modify source files in place to have conforming formatting"); + do_fmt_step.dependOn(&do_fmt.step); + test_step.dependOn(tests.addPkgTests( b, diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 2c5bcdd617..40f45d9ac8 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -93,6 +93,7 @@ pub fn main() !void { var install_prefix: ?[]const u8 = null; var dir_list = std.Build.DirList{}; + var enable_summary: ?bool = null; const Color = enum { auto, off, on }; var color: Color = .auto; @@ -217,6 +218,10 @@ pub fn main() !void { builder.enable_darling = true; } else if (mem.eql(u8, arg, "-fno-darling")) { builder.enable_darling = false; + } else if (mem.eql(u8, arg, "-fsummary")) { + enable_summary = true; + } else if (mem.eql(u8, arg, "-fno-summary")) { + enable_summary = false; } else if (mem.eql(u8, arg, "-freference-trace")) { builder.reference_trace = 256; } else if (mem.startsWith(u8, arg, "-freference-trace=")) { @@ -252,8 +257,9 @@ pub fn main() !void { } } + const stderr = std.io.getStdErr(); const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .auto => std.debug.detectTTYConfig(stderr), .on => .escape_codes, .off => .no_color, }; @@ -279,6 +285,8 @@ pub fn main() !void { main_progress_node, thread_pool_options, ttyconf, + stderr, + enable_summary, ) catch |err| switch (err) { error.UncleanExit => process.exit(1), else => return err, @@ -292,6 +300,8 @@ fn runStepNames( parent_prog_node: *std.Progress.Node, thread_pool_options: std.Thread.Pool.Options, ttyconf: std.debug.TTY.Config, + stderr: std.fs.File, + enable_summary: ?bool, ) !void { const gpa = b.allocator; var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{}; @@ -382,28 +392,35 @@ fn runStepNames( // A proper command line application defaults to silently succeeding. // The user may request verbose mode if they have a different preference. - if (failure_count == 0 and !b.verbose) return cleanExit(); - - const stderr = std.io.getStdErr(); - - const total_count = success_count + failure_count + pending_count; - ttyconf.setColor(stderr, .Cyan) catch {}; - stderr.writeAll("Build Summary: ") catch {}; - ttyconf.setColor(stderr, .Reset) catch {}; - stderr.writer().print("{d}/{d} steps succeeded; {d} failed; {d} total compile errors\n", .{ - success_count, total_count, failure_count, total_compile_errors, - }) catch {}; + if (failure_count == 0 and enable_summary != true) return cleanExit(); + + if (enable_summary != false) { + const total_count = success_count + failure_count + pending_count; + ttyconf.setColor(stderr, .Cyan) catch {}; + stderr.writeAll("Build Summary:") catch {}; + ttyconf.setColor(stderr, .Reset) catch {}; + stderr.writer().print(" {d}/{d} steps succeeded; {d} failed", .{ + success_count, total_count, failure_count, + }) catch {}; + + if (enable_summary == null) { + ttyconf.setColor(stderr, .Dim) catch {}; + stderr.writeAll(" (disable with -fno-summary)") catch {}; + ttyconf.setColor(stderr, .Reset) catch {}; + } + stderr.writeAll("\n") catch {}; - // Print a fancy tree with build results. - var print_node: PrintNode = .{ .parent = null }; - if (step_names.len == 0) { - print_node.last = true; - printTreeStep(b, b.default_step, stderr, ttyconf, &print_node, &step_stack) catch {}; - } else { - for (step_names, 0..) |step_name, i| { - const tls = b.top_level_steps.get(step_name).?; - print_node.last = i + 1 == b.top_level_steps.count(); - printTreeStep(b, &tls.step, stderr, ttyconf, &print_node, &step_stack) catch {}; + // Print a fancy tree with build results. + var print_node: PrintNode = .{ .parent = null }; + if (step_names.len == 0) { + print_node.last = true; + printTreeStep(b, b.default_step, stderr, ttyconf, &print_node, &step_stack) catch {}; + } else { + for (step_names, 0..) |step_name, i| { + const tls = b.top_level_steps.get(step_name).?; + print_node.last = i + 1 == b.top_level_steps.count(); + printTreeStep(b, &tls.step, stderr, ttyconf, &print_node, &step_stack) catch {}; + } } } @@ -453,9 +470,9 @@ fn printTreeStep( step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), ) !void { const first = step_stack.swapRemove(s); - if (!first) try ttyconf.setColor(stderr, .Dim); try printPrefix(parent_node, stderr); + if (!first) try ttyconf.setColor(stderr, .Dim); if (parent_node.parent != null) { if (parent_node.last) { try stderr.writeAll("└─ "); @@ -464,7 +481,7 @@ fn printTreeStep( } } - // TODO print the dep prefix too? + // dep_prefix omitted here because it is redundant with the tree. try stderr.writeAll(s.name); if (first) { @@ -608,8 +625,10 @@ fn workerMakeOneStep( const stderr = std.io.getStdErr(); for (s.result_error_msgs.items) |msg| { - // TODO print the dep prefix too + // Sometimes it feels like you just can't catch a break. Finally, + // with Zig, you can. ttyconf.setColor(stderr, .Bold) catch break; + stderr.writeAll(s.owner.dep_prefix) catch break; stderr.writeAll(s.name) catch break; stderr.writeAll(": ") catch break; ttyconf.setColor(stderr, .Red) catch break; @@ -735,6 +754,8 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\Advanced Options: \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error \\ -fno-reference-trace Disable reference trace + \\ -fsummary Print the build summary, even on success + \\ -fno-summary Omit the build summary, even on failure \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 3e765be4ba..1d87ea961f 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -32,14 +32,12 @@ pub const Step = @import("Build/Step.zig"); pub const CheckFileStep = @import("Build/CheckFileStep.zig"); pub const CheckObjectStep = @import("Build/CheckObjectStep.zig"); pub const ConfigHeaderStep = @import("Build/ConfigHeaderStep.zig"); -pub const EmulatableRunStep = @import("Build/EmulatableRunStep.zig"); pub const FmtStep = @import("Build/FmtStep.zig"); pub const InstallArtifactStep = @import("Build/InstallArtifactStep.zig"); pub const InstallDirStep = @import("Build/InstallDirStep.zig"); pub const InstallFileStep = @import("Build/InstallFileStep.zig"); pub const ObjCopyStep = @import("Build/ObjCopyStep.zig"); pub const CompileStep = @import("Build/CompileStep.zig"); -pub const LogStep = @import("Build/LogStep.zig"); pub const OptionsStep = @import("Build/OptionsStep.zig"); pub const RemoveDirStep = @import("Build/RemoveDirStep.zig"); pub const RunStep = @import("Build/RunStep.zig"); @@ -195,7 +193,7 @@ pub fn create( env_map.* = try process.getEnvMap(allocator); const self = try allocator.create(Build); - self.* = Build{ + self.* = .{ .zig_exe = zig_exe, .build_root = build_root, .cache_root = cache_root, @@ -224,16 +222,18 @@ pub fn create( .dest_dir = env_map.get("DESTDIR"), .installed_files = ArrayList(InstalledFile).init(allocator), .install_tls = .{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .top_level, .name = "install", + .owner = self, }), .description = "Copy build artifacts to prefix path", }, .uninstall_tls = .{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .top_level, .name = "uninstall", + .owner = self, .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", @@ -267,16 +267,18 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc child.* = .{ .allocator = allocator, .install_tls = .{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .top_level, .name = "install", + .owner = child, }), .description = "Copy build artifacts to prefix path", }, .uninstall_tls = .{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .top_level, .name = "uninstall", + .owner = child, .makeFn = makeUninstall, }), .description = "Remove build artifacts from prefix path", @@ -689,21 +691,14 @@ pub fn addWriteFiles(self: *Build) *WriteFileStep { return write_file_step; } -pub fn addLog(self: *Build, comptime format: []const u8, args: anytype) *LogStep { - const data = self.fmt(format, args); - const log_step = self.allocator.create(LogStep) catch @panic("OOM"); - log_step.* = LogStep.init(self, data); - return log_step; -} - pub fn addRemoveDirTree(self: *Build, dir_path: []const u8) *RemoveDirStep { const remove_dir_step = self.allocator.create(RemoveDirStep) catch @panic("OOM"); remove_dir_step.* = RemoveDirStep.init(self, dir_path); return remove_dir_step; } -pub fn addFmt(self: *Build, paths: []const []const u8) *FmtStep { - return FmtStep.create(self, paths); +pub fn addFmt(b: *Build, options: FmtStep.Options) *FmtStep { + return FmtStep.create(b, options); } pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCStep { @@ -870,10 +865,11 @@ pub fn option(self: *Build, comptime T: type, name_raw: []const u8, description_ pub fn step(self: *Build, name: []const u8, description: []const u8) *Step { const step_info = self.allocator.create(TopLevelStep) catch @panic("OOM"); - step_info.* = TopLevelStep{ - .step = Step.init(self.allocator, .{ + step_info.* = .{ + .step = Step.init(.{ .id = .top_level, .name = name, + .owner = self, }), .description = self.dupe(description), }; @@ -1145,10 +1141,6 @@ pub fn validateUserInputDidItFail(self: *Build) bool { return self.invalid_user_input; } -pub fn spawnChild(self: *Build, argv: []const []const u8) !void { - return self.spawnChildEnvMap(null, self.env_map, argv); -} - fn allocPrintCmd(ally: Allocator, opt_cwd: ?[]const u8, argv: []const []const u8) ![]u8 { var buf = ArrayList(u8).init(ally); if (opt_cwd) |cwd| try buf.writer().print("cd {s} && ", .{cwd}); @@ -1163,40 +1155,6 @@ fn printCmd(ally: Allocator, cwd: ?[]const u8, argv: []const []const u8) void { std.debug.print("{s}\n", .{text}); } -pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap, argv: []const []const u8) !void { - if (self.verbose) { - printCmd(self.allocator, cwd, argv); - } - - if (!process.can_spawn) - return error.ExecNotSupported; - - var child = std.ChildProcess.init(argv, self.allocator); - child.cwd = cwd; - child.env_map = env_map; - - const term = child.spawnAndWait() catch |err| { - log.err("Unable to spawn {s}: {s}", .{ argv[0], @errorName(err) }); - return err; - }; - - switch (term) { - .Exited => |code| { - if (code != 0) { - log.err("The following command exited with error code {}:", .{code}); - printCmd(self.allocator, cwd, argv); - return error.UncleanExit; - } - }, - else => { - log.err("The following command terminated unexpectedly:", .{}); - printCmd(self.allocator, cwd, argv); - - return error.UncleanExit; - }, - } -} - pub fn installArtifact(self: *Build, artifact: *CompileStep) void { self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step); } @@ -1403,160 +1361,6 @@ pub fn execAllowFail( } } -/// This function is used exclusively for spawning and communicating with the zig compiler. -/// TODO: move to build_runner.zig -pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *std.Progress.Node) ![]const u8 { - assert(argv.len != 0); - - if (b.verbose) { - const text = try allocPrintCmd(b.allocator, null, argv); - try s.result_error_msgs.append(b.allocator, text); - } - - if (!process.can_spawn) { - try s.result_error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{ - try allocPrintCmd(b.allocator, null, argv), - })); - return error.MakeFailed; - } - - var child = std.ChildProcess.init(argv, b.allocator); - child.env_map = b.env_map; - child.stdin_behavior = .Pipe; - child.stdout_behavior = .Pipe; - child.stderr_behavior = .Pipe; - - try child.spawn(); - - var poller = std.io.poll(b.allocator, enum { stdout, stderr }, .{ - .stdout = child.stdout.?, - .stderr = child.stderr.?, - }); - defer poller.deinit(); - - try sendMessage(child.stdin.?, .update); - try sendMessage(child.stdin.?, .exit); - - const Header = std.zig.Server.Message.Header; - var result: ?[]const u8 = null; - - var node_name: std.ArrayListUnmanaged(u8) = .{}; - defer node_name.deinit(b.allocator); - var sub_prog_node: ?std.Progress.Node = null; - defer if (sub_prog_node) |*n| n.end(); - - while (try poller.poll()) { - const stdout = poller.fifo(.stdout); - const buf = stdout.readableSlice(0); - assert(stdout.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); - const header_and_msg_len = header.bytes_len + @sizeOf(Header); - if (buf.len >= header_and_msg_len) { - const body = buf[@sizeOf(Header)..][0..header.bytes_len]; - switch (header.tag) { - .zig_version => { - if (!mem.eql(u8, builtin.zig_version_string, body)) { - try s.result_error_msgs.append( - b.allocator, - b.fmt("zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{ - builtin.zig_version_string, body, - }), - ); - return error.MakeFailed; - } - }, - .error_bundle => { - const EbHdr = std.zig.Server.Message.ErrorBundle; - const eb_hdr = @ptrCast(*align(1) const EbHdr, body); - const extra_bytes = - body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; - const string_bytes = - body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; - // TODO: use @ptrCast when the compiler supports it - const unaligned_extra = mem.bytesAsSlice(u32, extra_bytes); - const extra_array = try b.allocator.alloc(u32, unaligned_extra.len); - // TODO: use @memcpy when it supports slices - for (extra_array, unaligned_extra) |*dst, src| dst.* = src; - s.result_error_bundle = .{ - .string_bytes = try b.allocator.dupe(u8, string_bytes), - .extra = extra_array, - }; - }, - .progress => { - if (sub_prog_node) |*n| n.end(); - node_name.clearRetainingCapacity(); - try node_name.appendSlice(b.allocator, body); - sub_prog_node = prog_node.start(node_name.items, 0); - sub_prog_node.?.activate(); - }, - .emit_bin_path => { - result = try b.allocator.dupe(u8, body); - }, - _ => { - // Unrecognized message. - }, - } - stdout.discard(header_and_msg_len); - } - } - } - - const stderr = poller.fifo(.stderr); - if (stderr.readableLength() > 0) { - try s.result_error_msgs.append(b.allocator, try stderr.toOwnedSlice()); - } - - // Send EOF to stdin. - child.stdin.?.close(); - child.stdin = null; - - const term = try child.wait(); - switch (term) { - .Exited => |code| { - if (code != 0) { - try s.result_error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{ - code, try allocPrintCmd(b.allocator, null, argv), - })); - return error.MakeFailed; - } - }, - .Signal, .Stopped, .Unknown => |code| { - _ = code; - try s.result_error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{ - try allocPrintCmd(b.allocator, null, argv), - })); - return error.MakeFailed; - }, - } - - if (s.result_error_bundle.errorMessageCount() > 0) { - try s.result_error_msgs.append( - b.allocator, - b.fmt("the following command failed with {d} compilation errors:\n{s}", .{ - s.result_error_bundle.errorMessageCount(), - try allocPrintCmd(b.allocator, null, argv), - }), - ); - return error.MakeFailed; - } - - return result orelse { - try s.result_error_msgs.append(b.allocator, b.fmt("the following command failed to communicate the compilation result:\n{s}", .{ - try allocPrintCmd(b.allocator, null, argv), - })); - return error.MakeFailed; - }; -} - -fn sendMessage(file: fs.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - try file.writeAll(std.mem.asBytes(&header)); -} - /// This is a helper function to be called from build.zig scripts, *not* from /// inside step make() functions. If any errors occur, it fails the build with /// a helpful message. @@ -1910,14 +1714,12 @@ pub fn serializeCpu(allocator: Allocator, cpu: std.Target.Cpu) ![]const u8 { test { _ = CheckFileStep; _ = CheckObjectStep; - _ = EmulatableRunStep; _ = FmtStep; _ = InstallArtifactStep; _ = InstallDirStep; _ = InstallFileStep; _ = ObjCopyStep; _ = CompileStep; - _ = LogStep; _ = OptionsStep; _ = RemoveDirStep; _ = RunStep; diff --git a/lib/std/Build/CheckFileStep.zig b/lib/std/Build/CheckFileStep.zig index 9fee947386..f70a29840e 100644 --- a/lib/std/Build/CheckFileStep.zig +++ b/lib/std/Build/CheckFileStep.zig @@ -8,26 +8,25 @@ const CheckFileStep = @This(); pub const base_id = .check_file; step: Step, -builder: *std.Build, expected_matches: []const []const u8, source: std.Build.FileSource, max_bytes: usize = 20 * 1024 * 1024, pub fn create( - builder: *std.Build, + owner: *std.Build, source: std.Build.FileSource, expected_matches: []const []const u8, ) *CheckFileStep { - const self = builder.allocator.create(CheckFileStep) catch @panic("OOM"); + const self = owner.allocator.create(CheckFileStep) catch @panic("OOM"); self.* = CheckFileStep{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = .check_file, .name = "CheckFile", + .owner = owner, .makeFn = make, }), - .source = source.dupe(builder), - .expected_matches = builder.dupeStrings(expected_matches), + .source = source.dupe(owner), + .expected_matches = owner.dupeStrings(expected_matches), }; self.source.addStepDependencies(&self.step); return self; @@ -35,10 +34,11 @@ pub fn create( fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const b = step.owner; const self = @fieldParentPtr(CheckFileStep, "step", step); - const src_path = self.source.getPath(self.builder); - const contents = try fs.cwd().readFileAlloc(self.builder.allocator, src_path, self.max_bytes); + const src_path = self.source.getPath(b); + const contents = try fs.cwd().readFileAlloc(b.allocator, src_path, self.max_bytes); for (self.expected_matches) |expected_match| { if (mem.indexOf(u8, contents, expected_match) == null) { diff --git a/lib/std/Build/CheckObjectStep.zig b/lib/std/Build/CheckObjectStep.zig index eccbbd1696..57d280da0e 100644 --- a/lib/std/Build/CheckObjectStep.zig +++ b/lib/std/Build/CheckObjectStep.zig @@ -10,29 +10,31 @@ const CheckObjectStep = @This(); const Allocator = mem.Allocator; const Step = std.Build.Step; -const EmulatableRunStep = std.Build.EmulatableRunStep; pub const base_id = .check_object; step: Step, -builder: *std.Build, source: std.Build.FileSource, max_bytes: usize = 20 * 1024 * 1024, checks: std.ArrayList(Check), dump_symtab: bool = false, obj_format: std.Target.ObjectFormat, -pub fn create(builder: *std.Build, source: std.Build.FileSource, obj_format: std.Target.ObjectFormat) *CheckObjectStep { - const gpa = builder.allocator; +pub fn create( + owner: *std.Build, + source: std.Build.FileSource, + obj_format: std.Target.ObjectFormat, +) *CheckObjectStep { + const gpa = owner.allocator; const self = gpa.create(CheckObjectStep) catch @panic("OOM"); self.* = .{ - .builder = builder, - .step = Step.init(gpa, .{ + .step = Step.init(.{ .id = .check_file, .name = "CheckObject", + .owner = owner, .makeFn = make, }), - .source = source.dupe(builder), + .source = source.dupe(owner), .checks = std.ArrayList(Check).init(gpa), .obj_format = obj_format, }; @@ -42,14 +44,18 @@ pub fn create(builder: *std.Build, source: std.Build.FileSource, obj_format: std /// Runs and (optionally) compares the output of a binary. /// Asserts `self` was generated from an executable step. -pub fn runAndCompare(self: *CheckObjectStep) *EmulatableRunStep { +/// TODO this doesn't actually compare, and there's no apparent reason for it +/// to depend on the check object step. I don't see why this function should exist, +/// the caller could just add the run step directly. +pub fn runAndCompare(self: *CheckObjectStep) *std.Build.RunStep { const dependencies_len = self.step.dependencies.items.len; assert(dependencies_len > 0); const exe_step = self.step.dependencies.items[dependencies_len - 1]; const exe = exe_step.cast(std.Build.CompileStep).?; - const emulatable_step = EmulatableRunStep.create(self.builder, "EmulatableRun", exe); - emulatable_step.step.dependOn(&self.step); - return emulatable_step; + const run = self.step.owner.addRunArtifact(exe); + run.skip_foreign_checks = true; + run.step.dependOn(&self.step); + return run; } /// There two types of actions currently suported: @@ -253,7 +259,7 @@ const Check = struct { /// Creates a new sequence of actions with `phrase` as the first anchor searched phrase. pub fn checkStart(self: *CheckObjectStep, phrase: []const u8) void { - var new_check = Check.create(self.builder); + var new_check = Check.create(self.step.owner); new_check.match(phrase); self.checks.append(new_check) catch @panic("OOM"); } @@ -295,17 +301,18 @@ pub fn checkComputeCompare( program: []const u8, expected: ComputeCompareExpected, ) void { - var new_check = Check.create(self.builder); + var new_check = Check.create(self.step.owner); new_check.computeCmp(program, expected); self.checks.append(new_check) catch @panic("OOM"); } fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const b = step.owner; + const gpa = b.allocator; const self = @fieldParentPtr(CheckObjectStep, "step", step); - const gpa = self.builder.allocator; - const src_path = self.source.getPath(self.builder); + const src_path = self.source.getPath(b); const contents = try fs.cwd().readFileAllocOptions( gpa, src_path, diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 711092c762..f1a8f71334 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -22,7 +22,6 @@ const InstallDir = std.Build.InstallDir; const InstallArtifactStep = std.Build.InstallArtifactStep; const GeneratedFile = std.Build.GeneratedFile; const ObjCopyStep = std.Build.ObjCopyStep; -const EmulatableRunStep = std.Build.EmulatableRunStep; const CheckObjectStep = std.Build.CheckObjectStep; const RunStep = std.Build.RunStep; const OptionsStep = std.Build.OptionsStep; @@ -32,7 +31,6 @@ const CompileStep = @This(); pub const base_id: Step.Id = .compile; step: Step, -builder: *std.Build, name: []const u8, target: CrossTarget, target_info: NativeTargetInfo, @@ -305,24 +303,23 @@ pub const EmitOption = union(enum) { } }; -pub fn create(builder: *std.Build, options: Options) *CompileStep { - const name = builder.dupe(options.name); - const root_src: ?FileSource = if (options.root_source_file) |rsrc| rsrc.dupe(builder) else null; +pub fn create(owner: *std.Build, options: Options) *CompileStep { + const name = owner.dupe(options.name); + const root_src: ?FileSource = if (options.root_source_file) |rsrc| rsrc.dupe(owner) else null; if (mem.indexOf(u8, name, "/") != null or mem.indexOf(u8, name, "\\") != null) { panic("invalid name: '{s}'. It looks like a file path, but it is supposed to be the library or application name.", .{name}); } - const step_name = builder.fmt("compile {s} {s} {s}", .{ + const step_name = owner.fmt("compile {s} {s} {s}", .{ name, @tagName(options.optimize), - options.target.zigTriple(builder.allocator) catch @panic("OOM"), + options.target.zigTriple(owner.allocator) catch @panic("OOM"), }); - const self = builder.allocator.create(CompileStep) catch @panic("OOM"); + const self = owner.allocator.create(CompileStep) catch @panic("OOM"); self.* = CompileStep{ .strip = null, .unwind_tables = null, - .builder = builder, .verbose_link = false, .verbose_cc = false, .optimize = options.optimize, @@ -331,27 +328,28 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep { .kind = options.kind, .root_src = root_src, .name = name, - .frameworks = StringHashMap(FrameworkLinkInfo).init(builder.allocator), - .step = Step.init(builder.allocator, .{ + .frameworks = StringHashMap(FrameworkLinkInfo).init(owner.allocator), + .step = Step.init(.{ .id = base_id, .name = step_name, + .owner = owner, .makeFn = make, }), .version = options.version, .out_filename = undefined, - .out_h_filename = builder.fmt("{s}.h", .{name}), + .out_h_filename = owner.fmt("{s}.h", .{name}), .out_lib_filename = undefined, - .out_pdb_filename = builder.fmt("{s}.pdb", .{name}), + .out_pdb_filename = owner.fmt("{s}.pdb", .{name}), .major_only_filename = null, .name_only_filename = null, - .modules = std.StringArrayHashMap(*Module).init(builder.allocator), - .include_dirs = ArrayList(IncludeDir).init(builder.allocator), - .link_objects = ArrayList(LinkObject).init(builder.allocator), - .c_macros = ArrayList([]const u8).init(builder.allocator), - .lib_paths = ArrayList([]const u8).init(builder.allocator), - .rpaths = ArrayList([]const u8).init(builder.allocator), - .framework_dirs = ArrayList([]const u8).init(builder.allocator), - .installed_headers = ArrayList(*Step).init(builder.allocator), + .modules = std.StringArrayHashMap(*Module).init(owner.allocator), + .include_dirs = ArrayList(IncludeDir).init(owner.allocator), + .link_objects = ArrayList(LinkObject).init(owner.allocator), + .c_macros = ArrayList([]const u8).init(owner.allocator), + .lib_paths = ArrayList([]const u8).init(owner.allocator), + .rpaths = ArrayList([]const u8).init(owner.allocator), + .framework_dirs = ArrayList([]const u8).init(owner.allocator), + .installed_headers = ArrayList(*Step).init(owner.allocator), .object_src = undefined, .c_std = std.Build.CStd.C99, .zig_lib_dir = null, @@ -382,9 +380,10 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep { } fn computeOutFileNames(self: *CompileStep) void { + const b = self.step.owner; const target = self.target_info.target; - self.out_filename = std.zig.binNameAlloc(self.builder.allocator, .{ + self.out_filename = std.zig.binNameAlloc(b.allocator, .{ .root_name = self.name, .target = target, .output_mode = switch (self.kind) { @@ -404,30 +403,30 @@ fn computeOutFileNames(self: *CompileStep) void { self.out_lib_filename = self.out_filename; } else if (self.version) |version| { if (target.isDarwin()) { - self.major_only_filename = self.builder.fmt("lib{s}.{d}.dylib", .{ + self.major_only_filename = b.fmt("lib{s}.{d}.dylib", .{ self.name, version.major, }); - self.name_only_filename = self.builder.fmt("lib{s}.dylib", .{self.name}); + self.name_only_filename = b.fmt("lib{s}.dylib", .{self.name}); self.out_lib_filename = self.out_filename; } else if (target.os.tag == .windows) { - self.out_lib_filename = self.builder.fmt("{s}.lib", .{self.name}); + self.out_lib_filename = b.fmt("{s}.lib", .{self.name}); } else { - self.major_only_filename = self.builder.fmt("lib{s}.so.{d}", .{ self.name, version.major }); - self.name_only_filename = self.builder.fmt("lib{s}.so", .{self.name}); + self.major_only_filename = b.fmt("lib{s}.so.{d}", .{ self.name, version.major }); + self.name_only_filename = b.fmt("lib{s}.so", .{self.name}); self.out_lib_filename = self.out_filename; } } else { if (target.isDarwin()) { self.out_lib_filename = self.out_filename; } else if (target.os.tag == .windows) { - self.out_lib_filename = self.builder.fmt("{s}.lib", .{self.name}); + self.out_lib_filename = b.fmt("{s}.lib", .{self.name}); } else { self.out_lib_filename = self.out_filename; } } if (self.output_dir != null) { - self.output_lib_path_source.path = self.builder.pathJoin( + self.output_lib_path_source.path = b.pathJoin( &.{ self.output_dir.?, self.out_lib_filename }, ); } @@ -435,17 +434,20 @@ fn computeOutFileNames(self: *CompileStep) void { } pub fn setOutputDir(self: *CompileStep, dir: []const u8) void { - self.output_dir = self.builder.dupePath(dir); + const b = self.step.owner; + self.output_dir = b.dupePath(dir); } pub fn install(self: *CompileStep) void { - self.builder.installArtifact(self); + const b = self.step.owner; + b.installArtifact(self); } -pub fn installHeader(a: *CompileStep, src_path: []const u8, dest_rel_path: []const u8) void { - const install_file = a.builder.addInstallHeaderFile(src_path, dest_rel_path); - a.builder.getInstallStep().dependOn(&install_file.step); - a.installed_headers.append(&install_file.step) catch @panic("OOM"); +pub fn installHeader(cs: *CompileStep, src_path: []const u8, dest_rel_path: []const u8) void { + const b = cs.step.owner; + const install_file = b.addInstallHeaderFile(src_path, dest_rel_path); + b.getInstallStep().dependOn(&install_file.step); + cs.installed_headers.append(&install_file.step) catch @panic("OOM"); } pub const InstallConfigHeaderOptions = struct { @@ -459,13 +461,14 @@ pub fn installConfigHeader( options: InstallConfigHeaderOptions, ) void { const dest_rel_path = options.dest_rel_path orelse config_header.include_path; - const install_file = cs.builder.addInstallFileWithDir( + const b = cs.step.owner; + const install_file = b.addInstallFileWithDir( .{ .generated = &config_header.output_file }, options.install_dir, dest_rel_path, ); install_file.step.dependOn(&config_header.step); - cs.builder.getInstallStep().dependOn(&install_file.step); + b.getInstallStep().dependOn(&install_file.step); cs.installed_headers.append(&install_file.step) catch @panic("OOM"); } @@ -482,91 +485,84 @@ pub fn installHeadersDirectory( } pub fn installHeadersDirectoryOptions( - a: *CompileStep, + cs: *CompileStep, options: std.Build.InstallDirStep.Options, ) void { - const install_dir = a.builder.addInstallDirectory(options); - a.builder.getInstallStep().dependOn(&install_dir.step); - a.installed_headers.append(&install_dir.step) catch @panic("OOM"); + const b = cs.step.owner; + const install_dir = b.addInstallDirectory(options); + b.getInstallStep().dependOn(&install_dir.step); + cs.installed_headers.append(&install_dir.step) catch @panic("OOM"); } -pub fn installLibraryHeaders(a: *CompileStep, l: *CompileStep) void { +pub fn installLibraryHeaders(cs: *CompileStep, l: *CompileStep) void { assert(l.kind == .lib); - const install_step = a.builder.getInstallStep(); + const b = cs.step.owner; + const install_step = b.getInstallStep(); // Copy each element from installed_headers, modifying the builder // to be the new parent's builder. for (l.installed_headers.items) |step| { const step_copy = switch (step.id) { inline .install_file, .install_dir => |id| blk: { const T = id.Type(); - const ptr = a.builder.allocator.create(T) catch @panic("OOM"); + const ptr = b.allocator.create(T) catch @panic("OOM"); ptr.* = step.cast(T).?.*; - ptr.override_source_builder = ptr.builder; - ptr.builder = a.builder; + ptr.dest_builder = b; break :blk &ptr.step; }, else => unreachable, }; - a.installed_headers.append(step_copy) catch @panic("OOM"); + cs.installed_headers.append(step_copy) catch @panic("OOM"); install_step.dependOn(step_copy); } - a.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM"); + cs.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM"); } pub fn addObjCopy(cs: *CompileStep, options: ObjCopyStep.Options) *ObjCopyStep { + const b = cs.step.owner; var copy = options; if (copy.basename == null) { if (options.format) |f| { - copy.basename = cs.builder.fmt("{s}.{s}", .{ cs.name, @tagName(f) }); + copy.basename = b.fmt("{s}.{s}", .{ cs.name, @tagName(f) }); } else { copy.basename = cs.name; } } - return cs.builder.addObjCopy(cs.getOutputSource(), copy); + return b.addObjCopy(cs.getOutputSource(), copy); } /// Deprecated: use `std.Build.addRunArtifact` /// This function will run in the context of the package that created the executable, /// which is undesirable when running an executable provided by a dependency package. -pub fn run(exe: *CompileStep) *RunStep { - return exe.builder.addRunArtifact(exe); -} - -/// Creates an `EmulatableRunStep` with an executable built with `addExecutable`. -/// Allows running foreign binaries through emulation platforms such as Qemu or Rosetta. -/// When a binary cannot be ran through emulation or the option is disabled, a warning -/// will be printed and the binary will *NOT* be ran. -pub fn runEmulatable(exe: *CompileStep) *EmulatableRunStep { - assert(exe.kind == .exe or exe.kind == .test_exe); - - const run_step = EmulatableRunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}), exe); - if (exe.vcpkg_bin_path) |path| { - RunStep.addPathDirInternal(&run_step.step, exe.builder, path); - } - return run_step; +pub fn run(cs: *CompileStep) *RunStep { + return cs.step.owner.addRunArtifact(cs); } pub fn checkObject(self: *CompileStep, obj_format: std.Target.ObjectFormat) *CheckObjectStep { - return CheckObjectStep.create(self.builder, self.getOutputSource(), obj_format); + const b = self.step.owner; + return CheckObjectStep.create(b, self.getOutputSource(), obj_format); } pub fn setLinkerScriptPath(self: *CompileStep, source: FileSource) void { - self.linker_script = source.dupe(self.builder); + const b = self.step.owner; + self.linker_script = source.dupe(b); source.addStepDependencies(&self.step); } pub fn linkFramework(self: *CompileStep, framework_name: []const u8) void { - self.frameworks.put(self.builder.dupe(framework_name), .{}) catch @panic("OOM"); + const b = self.step.owner; + self.frameworks.put(b.dupe(framework_name), .{}) catch @panic("OOM"); } pub fn linkFrameworkNeeded(self: *CompileStep, framework_name: []const u8) void { - self.frameworks.put(self.builder.dupe(framework_name), .{ + const b = self.step.owner; + self.frameworks.put(b.dupe(framework_name), .{ .needed = true, }) catch @panic("OOM"); } pub fn linkFrameworkWeak(self: *CompileStep, framework_name: []const u8) void { - self.frameworks.put(self.builder.dupe(framework_name), .{ + const b = self.step.owner; + self.frameworks.put(b.dupe(framework_name), .{ .weak = true, }) catch @panic("OOM"); } @@ -619,21 +615,24 @@ pub fn linkLibCpp(self: *CompileStep) void { /// If the value is omitted, it is set to 1. /// `name` and `value` need not live longer than the function call. pub fn defineCMacro(self: *CompileStep, name: []const u8, value: ?[]const u8) void { - const macro = std.Build.constructCMacro(self.builder.allocator, name, value); + const b = self.step.owner; + const macro = std.Build.constructCMacro(b.allocator, name, value); self.c_macros.append(macro) catch @panic("OOM"); } /// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1. pub fn defineCMacroRaw(self: *CompileStep, name_and_value: []const u8) void { - self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM"); + const b = self.step.owner; + self.c_macros.append(b.dupe(name_and_value)) catch @panic("OOM"); } /// This one has no integration with anything, it just puts -lname on the command line. /// Prefer to use `linkSystemLibrary` instead. pub fn linkSystemLibraryName(self: *CompileStep, name: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(name), + .name = b.dupe(name), .needed = false, .weak = false, .use_pkg_config = .no, @@ -644,9 +643,10 @@ pub fn linkSystemLibraryName(self: *CompileStep, name: []const u8) void { /// This one has no integration with anything, it just puts -needed-lname on the command line. /// Prefer to use `linkSystemLibraryNeeded` instead. pub fn linkSystemLibraryNeededName(self: *CompileStep, name: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(name), + .name = b.dupe(name), .needed = true, .weak = false, .use_pkg_config = .no, @@ -657,9 +657,10 @@ pub fn linkSystemLibraryNeededName(self: *CompileStep, name: []const u8) void { /// Darwin-only. This one has no integration with anything, it just puts -weak-lname on the /// command line. Prefer to use `linkSystemLibraryWeak` instead. pub fn linkSystemLibraryWeakName(self: *CompileStep, name: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(name), + .name = b.dupe(name), .needed = false, .weak = true, .use_pkg_config = .no, @@ -670,9 +671,10 @@ pub fn linkSystemLibraryWeakName(self: *CompileStep, name: []const u8) void { /// This links against a system library, exclusively using pkg-config to find the library. /// Prefer to use `linkSystemLibrary` instead. pub fn linkSystemLibraryPkgConfigOnly(self: *CompileStep, lib_name: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(lib_name), + .name = b.dupe(lib_name), .needed = false, .weak = false, .use_pkg_config = .force, @@ -683,9 +685,10 @@ pub fn linkSystemLibraryPkgConfigOnly(self: *CompileStep, lib_name: []const u8) /// This links against a system library, exclusively using pkg-config to find the library. /// Prefer to use `linkSystemLibraryNeeded` instead. pub fn linkSystemLibraryNeededPkgConfigOnly(self: *CompileStep, lib_name: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(lib_name), + .name = b.dupe(lib_name), .needed = true, .weak = false, .use_pkg_config = .force, @@ -696,13 +699,14 @@ pub fn linkSystemLibraryNeededPkgConfigOnly(self: *CompileStep, lib_name: []cons /// Run pkg-config for the given library name and parse the output, returning the arguments /// that should be passed to zig to link the given library. pub fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u8 { + const b = self.step.owner; const pkg_name = match: { // First we have to map the library name to pkg config name. Unfortunately, // there are several examples where this is not straightforward: // -lSDL2 -> pkg-config sdl2 // -lgdk-3 -> pkg-config gdk-3.0 // -latk-1.0 -> pkg-config atk - const pkgs = try getPkgConfigList(self.builder); + const pkgs = try getPkgConfigList(b); // Exact match means instant winner. for (pkgs) |pkg| { @@ -742,7 +746,7 @@ pub fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u }; var code: u8 = undefined; - const stdout = if (self.builder.execAllowFail(&[_][]const u8{ + const stdout = if (b.execAllowFail(&[_][]const u8{ "pkg-config", pkg_name, "--cflags", @@ -755,7 +759,7 @@ pub fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u else => return err, }; - var zig_args = ArrayList([]const u8).init(self.builder.allocator); + var zig_args = ArrayList([]const u8).init(b.allocator); defer zig_args.deinit(); var it = mem.tokenize(u8, stdout, " \r\n\t"); @@ -780,7 +784,7 @@ pub fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u try zig_args.appendSlice(&[_][]const u8{ "-D", macro }); } else if (mem.startsWith(u8, tok, "-D")) { try zig_args.append(tok); - } else if (self.builder.verbose) { + } else if (b.verbose) { log.warn("Ignoring pkg-config flag '{s}'", .{tok}); } } @@ -804,6 +808,7 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct { needed: bool = false, weak: bool = false, }) void { + const b = self.step.owner; if (isLibCLibrary(name)) { self.linkLibC(); return; @@ -815,7 +820,7 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct { self.link_objects.append(.{ .system_lib = .{ - .name = self.builder.dupe(name), + .name = b.dupe(name), .needed = opts.needed, .weak = opts.weak, .use_pkg_config = .yes, @@ -824,26 +829,30 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct { } pub fn setNamePrefix(self: *CompileStep, text: []const u8) void { + const b = self.step.owner; assert(self.kind == .@"test" or self.kind == .test_exe); - self.name_prefix = self.builder.dupe(text); + self.name_prefix = b.dupe(text); } pub fn setFilter(self: *CompileStep, text: ?[]const u8) void { + const b = self.step.owner; assert(self.kind == .@"test" or self.kind == .test_exe); - self.filter = if (text) |t| self.builder.dupe(t) else null; + self.filter = if (text) |t| b.dupe(t) else null; } pub fn setTestRunner(self: *CompileStep, path: ?[]const u8) void { + const b = self.step.owner; assert(self.kind == .@"test" or self.kind == .test_exe); - self.test_runner = if (path) |p| self.builder.dupePath(p) else null; + self.test_runner = if (path) |p| b.dupePath(p) else null; } /// Handy when you have many C/C++ source files and want them all to have the same flags. pub fn addCSourceFiles(self: *CompileStep, files: []const []const u8, flags: []const []const u8) void { - const c_source_files = self.builder.allocator.create(CSourceFiles) catch @panic("OOM"); + const b = self.step.owner; + const c_source_files = b.allocator.create(CSourceFiles) catch @panic("OOM"); - const files_copy = self.builder.dupeStrings(files); - const flags_copy = self.builder.dupeStrings(flags); + const files_copy = b.dupeStrings(files); + const flags_copy = b.dupeStrings(flags); c_source_files.* = .{ .files = files_copy, @@ -860,8 +869,9 @@ pub fn addCSourceFile(self: *CompileStep, file: []const u8, flags: []const []con } pub fn addCSourceFileSource(self: *CompileStep, source: CSourceFile) void { - const c_source_file = self.builder.allocator.create(CSourceFile) catch @panic("OOM"); - c_source_file.* = source.dupe(self.builder); + const b = self.step.owner; + const c_source_file = b.allocator.create(CSourceFile) catch @panic("OOM"); + c_source_file.* = source.dupe(b); self.link_objects.append(.{ .c_source_file = c_source_file }) catch @panic("OOM"); source.source.addStepDependencies(&self.step); } @@ -875,15 +885,18 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void { } pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void { - self.zig_lib_dir = self.builder.dupePath(dir_path); + const b = self.step.owner; + self.zig_lib_dir = b.dupePath(dir_path); } pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void { - self.main_pkg_path = self.builder.dupePath(dir_path); + const b = self.step.owner; + self.main_pkg_path = b.dupePath(dir_path); } pub fn setLibCFile(self: *CompileStep, libc_file: ?FileSource) void { - self.libc_file = if (libc_file) |f| f.dupe(self.builder) else null; + const b = self.step.owner; + self.libc_file = if (libc_file) |f| f.dupe(b) else null; } /// Returns the generated executable, library or object file. @@ -914,13 +927,15 @@ pub fn getOutputPdbSource(self: *CompileStep) FileSource { } pub fn addAssemblyFile(self: *CompileStep, path: []const u8) void { + const b = self.step.owner; self.link_objects.append(.{ - .assembly_file = .{ .path = self.builder.dupe(path) }, + .assembly_file = .{ .path = b.dupe(path) }, }) catch @panic("OOM"); } pub fn addAssemblyFileSource(self: *CompileStep, source: FileSource) void { - const source_duped = source.dupe(self.builder); + const b = self.step.owner; + const source_duped = source.dupe(b); self.link_objects.append(.{ .assembly_file = source_duped }) catch @panic("OOM"); source_duped.addStepDependencies(&self.step); } @@ -930,7 +945,8 @@ pub fn addObjectFile(self: *CompileStep, source_file: []const u8) void { } pub fn addObjectFileSource(self: *CompileStep, source: FileSource) void { - self.link_objects.append(.{ .static_path = source.dupe(self.builder) }) catch @panic("OOM"); + const b = self.step.owner; + self.link_objects.append(.{ .static_path = source.dupe(b) }) catch @panic("OOM"); source.addStepDependencies(&self.step); } @@ -945,11 +961,13 @@ pub const addLibPath = @compileError("deprecated, use addLibraryPath"); pub const addFrameworkDir = @compileError("deprecated, use addFrameworkPath"); pub fn addSystemIncludePath(self: *CompileStep, path: []const u8) void { - self.include_dirs.append(IncludeDir{ .raw_path_system = self.builder.dupe(path) }) catch @panic("OOM"); + const b = self.step.owner; + self.include_dirs.append(IncludeDir{ .raw_path_system = b.dupe(path) }) catch @panic("OOM"); } pub fn addIncludePath(self: *CompileStep, path: []const u8) void { - self.include_dirs.append(IncludeDir{ .raw_path = self.builder.dupe(path) }) catch @panic("OOM"); + const b = self.step.owner; + self.include_dirs.append(IncludeDir{ .raw_path = b.dupe(path) }) catch @panic("OOM"); } pub fn addConfigHeader(self: *CompileStep, config_header: *ConfigHeaderStep) void { @@ -958,23 +976,27 @@ pub fn addConfigHeader(self: *CompileStep, config_header: *ConfigHeaderStep) voi } pub fn addLibraryPath(self: *CompileStep, path: []const u8) void { - self.lib_paths.append(self.builder.dupe(path)) catch @panic("OOM"); + const b = self.step.owner; + self.lib_paths.append(b.dupe(path)) catch @panic("OOM"); } pub fn addRPath(self: *CompileStep, path: []const u8) void { - self.rpaths.append(self.builder.dupe(path)) catch @panic("OOM"); + const b = self.step.owner; + self.rpaths.append(b.dupe(path)) catch @panic("OOM"); } pub fn addFrameworkPath(self: *CompileStep, dir_path: []const u8) void { - self.framework_dirs.append(self.builder.dupe(dir_path)) catch @panic("OOM"); + const b = self.step.owner; + self.framework_dirs.append(b.dupe(dir_path)) catch @panic("OOM"); } /// Adds a module to be used with `@import` and exposing it in the current /// package's module table using `name`. pub fn addModule(cs: *CompileStep, name: []const u8, module: *Module) void { - cs.modules.put(cs.builder.dupe(name), module) catch @panic("OOM"); + const b = cs.step.owner; + cs.modules.put(b.dupe(name), module) catch @panic("OOM"); - var done = std.AutoHashMap(*Module, void).init(cs.builder.allocator); + var done = std.AutoHashMap(*Module, void).init(b.allocator); defer done.deinit(); cs.addRecursiveBuildDeps(module, &done) catch @panic("OOM"); } @@ -982,7 +1004,8 @@ pub fn addModule(cs: *CompileStep, name: []const u8, module: *Module) void { /// Adds a module to be used with `@import` without exposing it in the current /// package's module table. pub fn addAnonymousModule(cs: *CompileStep, name: []const u8, options: std.Build.CreateModuleOptions) void { - const module = cs.builder.createModule(options); + const b = cs.step.owner; + const module = b.createModule(options); return addModule(cs, name, module); } @@ -1002,12 +1025,13 @@ fn addRecursiveBuildDeps(cs: *CompileStep, module: *Module, done: *std.AutoHashM /// If Vcpkg was found on the system, it will be added to include and lib /// paths for the specified target. pub fn addVcpkgPaths(self: *CompileStep, linkage: CompileStep.Linkage) !void { + const b = self.step.owner; // Ideally in the Unattempted case we would call the function recursively // after findVcpkgRoot and have only one switch statement, but the compiler // cannot resolve the error set. - switch (self.builder.vcpkg_root) { + switch (b.vcpkg_root) { .unattempted => { - self.builder.vcpkg_root = if (try findVcpkgRoot(self.builder.allocator)) |root| + b.vcpkg_root = if (try findVcpkgRoot(b.allocator)) |root| VcpkgRoot{ .found = root } else .not_found; @@ -1016,31 +1040,32 @@ pub fn addVcpkgPaths(self: *CompileStep, linkage: CompileStep.Linkage) !void { .found => {}, } - switch (self.builder.vcpkg_root) { + switch (b.vcpkg_root) { .unattempted => unreachable, .not_found => return error.VcpkgNotFound, .found => |root| { - const allocator = self.builder.allocator; + const allocator = b.allocator; const triplet = try self.target.vcpkgTriplet(allocator, if (linkage == .static) .Static else .Dynamic); - defer self.builder.allocator.free(triplet); + defer b.allocator.free(triplet); - const include_path = self.builder.pathJoin(&.{ root, "installed", triplet, "include" }); + const include_path = b.pathJoin(&.{ root, "installed", triplet, "include" }); errdefer allocator.free(include_path); try self.include_dirs.append(IncludeDir{ .raw_path = include_path }); - const lib_path = self.builder.pathJoin(&.{ root, "installed", triplet, "lib" }); + const lib_path = b.pathJoin(&.{ root, "installed", triplet, "lib" }); try self.lib_paths.append(lib_path); - self.vcpkg_bin_path = self.builder.pathJoin(&.{ root, "installed", triplet, "bin" }); + self.vcpkg_bin_path = b.pathJoin(&.{ root, "installed", triplet, "bin" }); }, } } pub fn setExecCmd(self: *CompileStep, args: []const ?[]const u8) void { + const b = self.step.owner; assert(self.kind == .@"test"); - const duped_args = self.builder.allocator.alloc(?[]u8, args.len) catch @panic("OOM"); + const duped_args = b.allocator.alloc(?[]u8, args.len) catch @panic("OOM"); for (args, 0..) |arg, i| { - duped_args[i] = if (arg) |a| self.builder.dupe(a) else null; + duped_args[i] = if (arg) |a| b.dupe(a) else null; } self.exec_cmd_args = duped_args; } @@ -1055,16 +1080,17 @@ fn appendModuleArgs( cs: *CompileStep, zig_args: *ArrayList([]const u8), ) error{OutOfMemory}!void { + const b = cs.step.owner; // First, traverse the whole dependency graph and give every module a unique name, ideally one // named after what it's called somewhere in the graph. It will help here to have both a mapping // from module to name and a set of all the currently-used names. - var mod_names = std.AutoHashMap(*Module, []const u8).init(cs.builder.allocator); - var names = std.StringHashMap(void).init(cs.builder.allocator); + var mod_names = std.AutoHashMap(*Module, []const u8).init(b.allocator); + var names = std.StringHashMap(void).init(b.allocator); var to_name = std.ArrayList(struct { name: []const u8, mod: *Module, - }).init(cs.builder.allocator); + }).init(b.allocator); { var it = cs.modules.iterator(); while (it.next()) |kv| { @@ -1085,7 +1111,7 @@ fn appendModuleArgs( if (mod_names.contains(dep.mod)) continue; // We'll use this buffer to store the name we decide on - var buf = try cs.builder.allocator.alloc(u8, dep.name.len + 32); + var buf = try b.allocator.alloc(u8, dep.name.len + 32); // First, try just the exposed dependency name std.mem.copy(u8, buf, dep.name); var name = buf[0..dep.name.len]; @@ -1122,15 +1148,15 @@ fn appendModuleArgs( const mod = kv.key_ptr.*; const name = kv.value_ptr.*; - const deps_str = try constructDepString(cs.builder.allocator, mod_names, mod.dependencies); + const deps_str = try constructDepString(b.allocator, mod_names, mod.dependencies); const src = mod.builder.pathFromRoot(mod.source_file.getPath(mod.builder)); try zig_args.append("--mod"); - try zig_args.append(try std.fmt.allocPrint(cs.builder.allocator, "{s}:{s}:{s}", .{ name, deps_str, src })); + try zig_args.append(try std.fmt.allocPrint(b.allocator, "{s}:{s}:{s}", .{ name, deps_str, src })); } } // Lastly, output the root dependencies - const deps_str = try constructDepString(cs.builder.allocator, mod_names, cs.modules); + const deps_str = try constructDepString(b.allocator, mod_names, cs.modules); if (deps_str.len > 0) { try zig_args.append("--deps"); try zig_args.append(deps_str); @@ -1161,18 +1187,18 @@ fn constructDepString( } fn make(step: *Step, prog_node: *std.Progress.Node) !void { + const b = step.owner; const self = @fieldParentPtr(CompileStep, "step", step); - const builder = self.builder; if (self.root_src == null and self.link_objects.items.len == 0) { log.err("{s}: linker needs 1 or more objects to link", .{self.step.name}); return error.NeedAnObject; } - var zig_args = ArrayList([]const u8).init(builder.allocator); + var zig_args = ArrayList([]const u8).init(b.allocator); defer zig_args.deinit(); - try zig_args.append(builder.zig_exe); + try zig_args.append(b.zig_exe); const cmd = switch (self.kind) { .lib => "build-lib", @@ -1183,15 +1209,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }; try zig_args.append(cmd); - if (builder.reference_trace) |some| { - try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-freference-trace={d}", .{some})); + if (b.reference_trace) |some| { + try zig_args.append(try std.fmt.allocPrint(b.allocator, "-freference-trace={d}", .{some})); } try addFlag(&zig_args, "LLVM", self.use_llvm); try addFlag(&zig_args, "LLD", self.use_lld); if (self.target.ofmt) |ofmt| { - try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-ofmt={s}", .{@tagName(ofmt)})); + try zig_args.append(try std.fmt.allocPrint(b.allocator, "-ofmt={s}", .{@tagName(ofmt)})); } if (self.entry_symbol_name) |entry| { @@ -1201,18 +1227,18 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.stack_size) |stack_size| { try zig_args.append("--stack"); - try zig_args.append(try std.fmt.allocPrint(builder.allocator, "{}", .{stack_size})); + try zig_args.append(try std.fmt.allocPrint(b.allocator, "{}", .{stack_size})); } - if (self.root_src) |root_src| try zig_args.append(root_src.getPath(builder)); + if (self.root_src) |root_src| try zig_args.append(root_src.getPath(b)); // We will add link objects from transitive dependencies, but we want to keep // all link objects in the same order provided. // This array is used to keep self.link_objects immutable. var transitive_deps: TransitiveDeps = .{ - .link_objects = ArrayList(LinkObject).init(builder.allocator), - .seen_system_libs = StringHashMap(void).init(builder.allocator), - .seen_steps = std.AutoHashMap(*const Step, void).init(builder.allocator), + .link_objects = ArrayList(LinkObject).init(b.allocator), + .seen_system_libs = StringHashMap(void).init(b.allocator), + .seen_steps = std.AutoHashMap(*const Step, void).init(b.allocator), .is_linking_libcpp = self.is_linking_libcpp, .is_linking_libc = self.is_linking_libc, .frameworks = &self.frameworks, @@ -1225,14 +1251,14 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { for (transitive_deps.link_objects.items) |link_object| { switch (link_object) { - .static_path => |static_path| try zig_args.append(static_path.getPath(builder)), + .static_path => |static_path| try zig_args.append(static_path.getPath(b)), .other_step => |other| switch (other.kind) { .exe => @panic("Cannot link with an executable build artifact"), .test_exe => @panic("Cannot link with an executable build artifact"), .@"test" => @panic("Cannot link with a test"), .obj => { - try zig_args.append(other.getOutputSource().getPath(builder)); + try zig_args.append(other.getOutputSource().getPath(b)); }, .lib => l: { if (self.isStaticLibrary() and other.isStaticLibrary()) { @@ -1240,7 +1266,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { break :l; } - const full_path_lib = other.getOutputLibSource().getPath(builder); + const full_path_lib = other.getOutputLibSource().getPath(b); try zig_args.append(full_path_lib); if (other.linkage == Linkage.dynamic and !self.target.isWindows()) { @@ -1262,7 +1288,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { break :prefix "-l"; }; switch (system_lib.use_pkg_config) { - .no => try zig_args.append(builder.fmt("{s}{s}", .{ prefix, system_lib.name })), + .no => try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name })), .yes, .force => { if (self.runPkgConfig(system_lib.name)) |args| { try zig_args.appendSlice(args); @@ -1276,7 +1302,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .yes => { // pkg-config failed, so fall back to linking the library // by name directly. - try zig_args.append(builder.fmt("{s}{s}", .{ + try zig_args.append(b.fmt("{s}{s}", .{ prefix, system_lib.name, })); @@ -1299,7 +1325,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append("--"); prev_has_extra_flags = false; } - try zig_args.append(asm_file.getPath(builder)); + try zig_args.append(asm_file.getPath(b)); }, .c_source_file => |c_source_file| { @@ -1316,7 +1342,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } try zig_args.append("--"); } - try zig_args.append(c_source_file.source.getPath(builder)); + try zig_args.append(c_source_file.source.getPath(b)); }, .c_source_files => |c_source_files| { @@ -1334,7 +1360,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append("--"); } for (c_source_files.files) |file| { - try zig_args.append(builder.pathFromRoot(file)); + try zig_args.append(b.pathFromRoot(file)); } }, } @@ -1350,7 +1376,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.image_base) |image_base| { try zig_args.append("--image-base"); - try zig_args.append(builder.fmt("0x{x}", .{image_base})); + try zig_args.append(b.fmt("0x{x}", .{image_base})); } if (self.filter) |filter| { @@ -1369,32 +1395,32 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.test_runner) |test_runner| { try zig_args.append("--test-runner"); - try zig_args.append(builder.pathFromRoot(test_runner)); + try zig_args.append(b.pathFromRoot(test_runner)); } - for (builder.debug_log_scopes) |log_scope| { + for (b.debug_log_scopes) |log_scope| { try zig_args.append("--debug-log"); try zig_args.append(log_scope); } - if (builder.debug_compile_errors) { + if (b.debug_compile_errors) { try zig_args.append("--debug-compile-errors"); } - if (builder.verbose_cimport) try zig_args.append("--verbose-cimport"); - if (builder.verbose_air) try zig_args.append("--verbose-air"); - if (builder.verbose_llvm_ir) try zig_args.append("--verbose-llvm-ir"); - if (builder.verbose_link or self.verbose_link) try zig_args.append("--verbose-link"); - if (builder.verbose_cc or self.verbose_cc) try zig_args.append("--verbose-cc"); - if (builder.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); + if (b.verbose_cimport) try zig_args.append("--verbose-cimport"); + if (b.verbose_air) try zig_args.append("--verbose-air"); + if (b.verbose_llvm_ir) try zig_args.append("--verbose-llvm-ir"); + if (b.verbose_link or self.verbose_link) try zig_args.append("--verbose-link"); + if (b.verbose_cc or self.verbose_cc) try zig_args.append("--verbose-cc"); + if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); - if (self.emit_analysis.getArg(builder, "emit-analysis")) |arg| try zig_args.append(arg); - if (self.emit_asm.getArg(builder, "emit-asm")) |arg| try zig_args.append(arg); - if (self.emit_bin.getArg(builder, "emit-bin")) |arg| try zig_args.append(arg); - if (self.emit_docs.getArg(builder, "emit-docs")) |arg| try zig_args.append(arg); - if (self.emit_implib.getArg(builder, "emit-implib")) |arg| try zig_args.append(arg); - if (self.emit_llvm_bc.getArg(builder, "emit-llvm-bc")) |arg| try zig_args.append(arg); - if (self.emit_llvm_ir.getArg(builder, "emit-llvm-ir")) |arg| try zig_args.append(arg); + if (self.emit_analysis.getArg(b, "emit-analysis")) |arg| try zig_args.append(arg); + if (self.emit_asm.getArg(b, "emit-asm")) |arg| try zig_args.append(arg); + if (self.emit_bin.getArg(b, "emit-bin")) |arg| try zig_args.append(arg); + if (self.emit_docs.getArg(b, "emit-docs")) |arg| try zig_args.append(arg); + if (self.emit_implib.getArg(b, "emit-implib")) |arg| try zig_args.append(arg); + if (self.emit_llvm_bc.getArg(b, "emit-llvm-bc")) |arg| try zig_args.append(arg); + if (self.emit_llvm_ir.getArg(b, "emit-llvm-ir")) |arg| try zig_args.append(arg); if (self.emit_h) try zig_args.append("-femit-h"); @@ -1435,31 +1461,31 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } if (self.link_z_common_page_size) |size| { try zig_args.append("-z"); - try zig_args.append(builder.fmt("common-page-size={d}", .{size})); + try zig_args.append(b.fmt("common-page-size={d}", .{size})); } if (self.link_z_max_page_size) |size| { try zig_args.append("-z"); - try zig_args.append(builder.fmt("max-page-size={d}", .{size})); + try zig_args.append(b.fmt("max-page-size={d}", .{size})); } if (self.libc_file) |libc_file| { try zig_args.append("--libc"); - try zig_args.append(libc_file.getPath(builder)); - } else if (builder.libc_file) |libc_file| { + try zig_args.append(libc_file.getPath(b)); + } else if (b.libc_file) |libc_file| { try zig_args.append("--libc"); try zig_args.append(libc_file); } switch (self.optimize) { .Debug => {}, // Skip since it's the default. - else => try zig_args.append(builder.fmt("-O{s}", .{@tagName(self.optimize)})), + else => try zig_args.append(b.fmt("-O{s}", .{@tagName(self.optimize)})), } try zig_args.append("--cache-dir"); - try zig_args.append(builder.cache_root.path orelse "."); + try zig_args.append(b.cache_root.path orelse "."); try zig_args.append("--global-cache-dir"); - try zig_args.append(builder.global_cache_root.path orelse "."); + try zig_args.append(b.global_cache_root.path orelse "."); try zig_args.append("--name"); try zig_args.append(self.name); @@ -1471,11 +1497,11 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.kind == .lib and self.linkage != null and self.linkage.? == .dynamic) { if (self.version) |version| { try zig_args.append("--version"); - try zig_args.append(builder.fmt("{}", .{version})); + try zig_args.append(b.fmt("{}", .{version})); } if (self.target.isDarwin()) { - const install_name = self.install_name orelse builder.fmt("@rpath/{s}{s}{s}", .{ + const install_name = self.install_name orelse b.fmt("@rpath/{s}{s}{s}", .{ self.target.libPrefix(), self.name, self.target.dynamicLibSuffix(), @@ -1489,7 +1515,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.appendSlice(&[_][]const u8{ "--entitlements", entitlements }); } if (self.pagezero_size) |pagezero_size| { - const size = try std.fmt.allocPrint(builder.allocator, "{x}", .{pagezero_size}); + const size = try std.fmt.allocPrint(b.allocator, "{x}", .{pagezero_size}); try zig_args.appendSlice(&[_][]const u8{ "-pagezero_size", size }); } if (self.search_strategy) |strat| switch (strat) { @@ -1497,7 +1523,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .dylibs_first => try zig_args.append("-search_dylibs_first"), }; if (self.headerpad_size) |headerpad_size| { - const size = try std.fmt.allocPrint(builder.allocator, "{x}", .{headerpad_size}); + const size = try std.fmt.allocPrint(b.allocator, "{x}", .{headerpad_size}); try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); } if (self.headerpad_max_install_names) { @@ -1545,16 +1571,16 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append("--export-table"); } if (self.initial_memory) |initial_memory| { - try zig_args.append(builder.fmt("--initial-memory={d}", .{initial_memory})); + try zig_args.append(b.fmt("--initial-memory={d}", .{initial_memory})); } if (self.max_memory) |max_memory| { - try zig_args.append(builder.fmt("--max-memory={d}", .{max_memory})); + try zig_args.append(b.fmt("--max-memory={d}", .{max_memory})); } if (self.shared_memory) { try zig_args.append("--shared-memory"); } if (self.global_base) |global_base| { - try zig_args.append(builder.fmt("--global-base={d}", .{global_base})); + try zig_args.append(b.fmt("--global-base={d}", .{global_base})); } if (self.code_model != .default) { @@ -1562,16 +1588,16 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append(@tagName(self.code_model)); } if (self.wasi_exec_model) |model| { - try zig_args.append(builder.fmt("-mexec-model={s}", .{@tagName(model)})); + try zig_args.append(b.fmt("-mexec-model={s}", .{@tagName(model)})); } for (self.export_symbol_names) |symbol_name| { - try zig_args.append(builder.fmt("--export={s}", .{symbol_name})); + try zig_args.append(b.fmt("--export={s}", .{symbol_name})); } if (!self.target.isNative()) { try zig_args.appendSlice(&.{ - "-target", try self.target.zigTriple(builder.allocator), - "-mcpu", try std.Build.serializeCpu(builder.allocator, self.target.getCpu()), + "-target", try self.target.zigTriple(b.allocator), + "-mcpu", try std.Build.serializeCpu(b.allocator, self.target.getCpu()), }); if (self.target.dynamic_linker.get()) |dynamic_linker| { @@ -1582,12 +1608,12 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.linker_script) |linker_script| { try zig_args.append("--script"); - try zig_args.append(linker_script.getPath(builder)); + try zig_args.append(linker_script.getPath(b)); } if (self.version_script) |version_script| { try zig_args.append("--version-script"); - try zig_args.append(builder.pathFromRoot(version_script)); + try zig_args.append(b.pathFromRoot(version_script)); } if (self.kind == .@"test") { @@ -1603,23 +1629,23 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } else { const need_cross_glibc = self.target.isGnuLibC() and transitive_deps.is_linking_libc; - switch (builder.host.getExternalExecutor(self.target_info, .{ - .qemu_fixes_dl = need_cross_glibc and builder.glibc_runtimes_dir != null, + switch (b.host.getExternalExecutor(self.target_info, .{ + .qemu_fixes_dl = need_cross_glibc and b.glibc_runtimes_dir != null, .link_libc = transitive_deps.is_linking_libc, })) { .native => {}, .bad_dl, .bad_os_or_cpu => { try zig_args.append("--test-no-exec"); }, - .rosetta => if (builder.enable_rosetta) { + .rosetta => if (b.enable_rosetta) { try zig_args.append("--test-cmd-bin"); } else { try zig_args.append("--test-no-exec"); }, .qemu => |bin_name| ok: { - if (builder.enable_qemu) qemu: { + if (b.enable_qemu) qemu: { const glibc_dir_arg = if (need_cross_glibc) - builder.glibc_runtimes_dir orelse break :qemu + b.glibc_runtimes_dir orelse break :qemu else null; try zig_args.append("--test-cmd"); @@ -1636,7 +1662,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { "i686" else @tagName(cpu_arch); - const full_dir = try std.fmt.allocPrint(builder.allocator, fmt_str, .{ + const full_dir = try std.fmt.allocPrint(b.allocator, fmt_str, .{ dir, cpu_arch_name, @tagName(os_tag), @tagName(abi), }); @@ -1650,14 +1676,14 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } try zig_args.append("--test-no-exec"); }, - .wine => |bin_name| if (builder.enable_wine) { + .wine => |bin_name| if (b.enable_wine) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd-bin"); } else { try zig_args.append("--test-no-exec"); }, - .wasmtime => |bin_name| if (builder.enable_wasmtime) { + .wasmtime => |bin_name| if (b.enable_wasmtime) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd"); @@ -1666,7 +1692,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } else { try zig_args.append("--test-no-exec"); }, - .darling => |bin_name| if (builder.enable_darling) { + .darling => |bin_name| if (b.enable_darling) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd-bin"); @@ -1685,18 +1711,18 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { switch (include_dir) { .raw_path => |include_path| { try zig_args.append("-I"); - try zig_args.append(builder.pathFromRoot(include_path)); + try zig_args.append(b.pathFromRoot(include_path)); }, .raw_path_system => |include_path| { - if (builder.sysroot != null) { + if (b.sysroot != null) { try zig_args.append("-iwithsysroot"); } else { try zig_args.append("-isystem"); } - const resolved_include_path = builder.pathFromRoot(include_path); + const resolved_include_path = b.pathFromRoot(include_path); - const common_include_path = if (builtin.os.tag == .windows and builder.sysroot != null and fs.path.isAbsolute(resolved_include_path)) blk: { + const common_include_path = if (builtin.os.tag == .windows and b.sysroot != null and fs.path.isAbsolute(resolved_include_path)) blk: { // We need to check for disk designator and strip it out from dir path so // that zig/clang can concat resolved_include_path with sysroot. const disk_designator = fs.path.diskDesignatorWindows(resolved_include_path); @@ -1712,7 +1738,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }, .other_step => |other| { if (other.emit_h) { - const h_path = other.getOutputHSource().getPath(builder); + const h_path = other.getOutputHSource().getPath(b); try zig_args.append("-isystem"); try zig_args.append(fs.path.dirname(h_path).?); } @@ -1721,8 +1747,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try install_step.make(prog_node); } try zig_args.append("-I"); - try zig_args.append(builder.pathJoin(&.{ - other.builder.install_prefix, "include", + try zig_args.append(b.pathJoin(&.{ + other.step.owner.install_prefix, "include", })); } }, @@ -1751,7 +1777,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.target.isDarwin()) { for (self.framework_dirs.items) |dir| { - if (builder.sysroot != null) { + if (b.sysroot != null) { try zig_args.append("-iframeworkwithsysroot"); } else { try zig_args.append("-iframework"); @@ -1784,17 +1810,17 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } } - if (builder.sysroot) |sysroot| { + if (b.sysroot) |sysroot| { try zig_args.appendSlice(&[_][]const u8{ "--sysroot", sysroot }); } - for (builder.search_prefixes.items) |search_prefix| { + for (b.search_prefixes.items) |search_prefix| { try zig_args.append("-L"); - try zig_args.append(builder.pathJoin(&.{ + try zig_args.append(b.pathJoin(&.{ search_prefix, "lib", })); try zig_args.append("-I"); - try zig_args.append(builder.pathJoin(&.{ + try zig_args.append(b.pathJoin(&.{ search_prefix, "include", })); } @@ -1805,15 +1831,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (self.zig_lib_dir) |dir| { try zig_args.append("--zig-lib-dir"); - try zig_args.append(builder.pathFromRoot(dir)); - } else if (builder.zig_lib_dir) |dir| { + try zig_args.append(b.pathFromRoot(dir)); + } else if (b.zig_lib_dir) |dir| { try zig_args.append("--zig-lib-dir"); try zig_args.append(dir); } if (self.main_pkg_path) |dir| { try zig_args.append("--main-pkg-path"); - try zig_args.append(builder.pathFromRoot(dir)); + try zig_args.append(b.pathFromRoot(dir)); } try addFlag(&zig_args, "PIC", self.force_pic); @@ -1846,15 +1872,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { args_length += arg.len + 1; // +1 to account for null terminator } if (args_length >= 30 * 1024) { - try builder.cache_root.handle.makePath("args"); + try b.cache_root.handle.makePath("args"); const args_to_escape = zig_args.items[2..]; - var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len); + var escaped_args = try ArrayList([]const u8).initCapacity(b.allocator, args_to_escape.len); arg_blk: for (args_to_escape) |arg| { for (arg, 0..) |c, arg_idx| { if (c == '\\' or c == '"') { // Slow path for arguments that need to be escaped. We'll need to allocate and copy - var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1); + var escaped = try ArrayList(u8).initCapacity(b.allocator, arg.len + 1); const writer = escaped.writer(); try writer.writeAll(arg[0..arg_idx]); for (arg[arg_idx..]) |to_escape| { @@ -1870,8 +1896,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { // Write the args to zig-cache/args/ to avoid conflicts with // other zig build commands running in parallel. - const partially_quoted = try std.mem.join(builder.allocator, "\" \"", escaped_args.items); - const args = try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); + const partially_quoted = try std.mem.join(b.allocator, "\" \"", escaped_args.items); + const args = try std.mem.concat(b.allocator, u8, &[_][]const u8{ "\"", partially_quoted, "\"" }); var args_hash: [Sha256.digest_length]u8 = undefined; Sha256.hash(args, &args_hash, .{}); @@ -1883,18 +1909,18 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { ); const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; - try builder.cache_root.handle.writeFile(args_file, args); + try b.cache_root.handle.writeFile(args_file, args); - const resolved_args_file = try mem.concat(builder.allocator, u8, &.{ + const resolved_args_file = try mem.concat(b.allocator, u8, &.{ "@", - try builder.cache_root.join(builder.allocator, &.{args_file}), + try b.cache_root.join(b.allocator, &.{args_file}), }); zig_args.shrinkRetainingCapacity(2); try zig_args.append(resolved_args_file); } - const output_bin_path = try builder.execFromStep(zig_args.items, &self.step, prog_node); + const output_bin_path = try step.evalZigProcess(zig_args.items, prog_node); const build_output_dir = fs.path.dirname(output_bin_path).?; if (self.output_dir) |output_dir| { @@ -1928,25 +1954,25 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { // Update generated files if (self.output_dir != null) { - self.output_path_source.path = builder.pathJoin( + self.output_path_source.path = b.pathJoin( &.{ self.output_dir.?, self.out_filename }, ); if (self.emit_h) { - self.output_h_path_source.path = builder.pathJoin( + self.output_h_path_source.path = b.pathJoin( &.{ self.output_dir.?, self.out_h_filename }, ); } if (self.target.isWindows() or self.target.isUefi()) { - self.output_pdb_path_source.path = builder.pathJoin( + self.output_pdb_path_source.path = b.pathJoin( &.{ self.output_dir.?, self.out_pdb_filename }, ); } } if (self.kind == .lib and self.linkage != null and self.linkage.? == .dynamic and self.version != null and self.target.wantSharedLibSymLinks()) { - try doAtomicSymLinks(builder.allocator, self.getOutputSource().getPath(builder), self.major_only_filename.?, self.name_only_filename.?); + try doAtomicSymLinks(b.allocator, self.getOutputSource().getPath(b), self.major_only_filename.?, self.name_only_filename.?); } } diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index 994b71da32..8b4c05bab7 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -34,7 +34,6 @@ pub const Value = union(enum) { }; step: Step, -builder: *std.Build, values: std.StringArrayHashMap(Value), output_file: std.Build.GeneratedFile, @@ -49,8 +48,8 @@ pub const Options = struct { first_ret_addr: ?usize = null, }; -pub fn create(builder: *std.Build, options: Options) *ConfigHeaderStep { - const self = builder.allocator.create(ConfigHeaderStep) catch @panic("OOM"); +pub fn create(owner: *std.Build, options: Options) *ConfigHeaderStep { + const self = owner.allocator.create(ConfigHeaderStep) catch @panic("OOM"); var include_path: []const u8 = "config.h"; @@ -69,29 +68,28 @@ pub fn create(builder: *std.Build, options: Options) *ConfigHeaderStep { } const name = if (options.style.getFileSource()) |s| - builder.fmt("configure {s} header {s} to {s}", .{ + owner.fmt("configure {s} header {s} to {s}", .{ @tagName(options.style), s.getDisplayName(), include_path, }) else - builder.fmt("configure {s} header to {s}", .{@tagName(options.style), include_path}); + owner.fmt("configure {s} header to {s}", .{ @tagName(options.style), include_path }); self.* = .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = base_id, .name = name, + .owner = owner, .makeFn = make, .first_ret_addr = options.first_ret_addr orelse @returnAddress(), }), .style = options.style, - .values = std.StringArrayHashMap(Value).init(builder.allocator), + .values = std.StringArrayHashMap(Value).init(owner.allocator), .max_bytes = options.max_bytes, .include_path = include_path, .output_file = .{ .step = &self.step }, }; - return self; } @@ -161,8 +159,9 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const b = step.owner; const self = @fieldParentPtr(ConfigHeaderStep, "step", step); - const gpa = self.builder.allocator; + const gpa = b.allocator; // The cache is used here not really as a way to speed things up - because writing // the data to a file would probably be very fast - but as a way to find a canonical @@ -191,13 +190,13 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { switch (self.style) { .autoconf => |file_source| { try output.appendSlice(c_generated_line); - const src_path = file_source.getPath(self.builder); + const src_path = file_source.getPath(b); const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes); try render_autoconf(contents, &output, self.values, src_path); }, .cmake => |file_source| { try output.appendSlice(c_generated_line); - const src_path = file_source.getPath(self.builder); + const src_path = file_source.getPath(b); const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes); try render_cmake(contents, &output, self.values, src_path); }, @@ -222,7 +221,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .{std.fmt.fmtSliceHexLower(&digest)}, ) catch unreachable; - const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename }); + const output_dir = try b.cache_root.join(gpa, &.{ "o", &hash_basename }); // If output_path has directory parts, deal with them. Example: // output_dir is zig-cache/o/HASH @@ -242,7 +241,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try dir.writeFile(std.fs.path.basename(self.include_path), output.items); - self.output_file.path = try std.fs.path.join(self.builder.allocator, &.{ + self.output_file.path = try std.fs.path.join(b.allocator, &.{ output_dir, self.include_path, }); } diff --git a/lib/std/Build/EmulatableRunStep.zig b/lib/std/Build/EmulatableRunStep.zig deleted file mode 100644 index 44387c36f6..0000000000 --- a/lib/std/Build/EmulatableRunStep.zig +++ /dev/null @@ -1,218 +0,0 @@ -//! Unlike `RunStep` this step will provide emulation, when enabled, to run foreign binaries. -//! When a binary is foreign, but emulation for the target is disabled, the specified binary -//! will not be run and therefore also not validated against its output. -//! This step can be useful when wishing to run a built binary on multiple platforms, -//! without having to verify if it's possible to be ran against. - -const std = @import("../std.zig"); -const Step = std.Build.Step; -const CompileStep = std.Build.CompileStep; -const RunStep = std.Build.RunStep; - -const fs = std.fs; -const process = std.process; -const EnvMap = process.EnvMap; - -const EmulatableRunStep = @This(); - -pub const base_id = .emulatable_run; - -const max_stdout_size = 1 * 1024 * 1024; // 1 MiB - -step: Step, -builder: *std.Build, - -/// The artifact (executable) to be run by this step -exe: *CompileStep, - -/// Set this to `null` to ignore the exit code for the purpose of determining a successful execution -expected_term: ?std.ChildProcess.Term = .{ .Exited = 0 }, - -/// Override this field to modify the environment -env_map: ?*EnvMap, - -/// Set this to modify the current working directory -cwd: ?[]const u8, - -stdout_action: RunStep.StdIoAction = .inherit, -stderr_action: RunStep.StdIoAction = .inherit, - -/// When set to true, hides the warning of skipping a foreign binary which cannot be run on the host -/// or through emulation. -hide_foreign_binaries_warning: bool, - -/// Creates a step that will execute the given artifact. This step will allow running the -/// binary through emulation when any of the emulation options such as `enable_rosetta` are set to true. -/// When set to false, and the binary is foreign, running the executable is skipped. -/// Asserts given artifact is an executable. -pub fn create(builder: *std.Build, name: []const u8, artifact: *CompileStep) *EmulatableRunStep { - std.debug.assert(artifact.kind == .exe or artifact.kind == .test_exe); - const self = builder.allocator.create(EmulatableRunStep) catch @panic("OOM"); - - const option_name = "hide-foreign-warnings"; - const hide_warnings = if (builder.available_options_map.get(option_name) == null) warn: { - break :warn builder.option(bool, option_name, "Hide the warning when a foreign binary which is incompatible is skipped") orelse false; - } else false; - - self.* = .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ - .id = .emulatable_run, - .name = name, - .makeFn = make, - }), - .exe = artifact, - .env_map = null, - .cwd = null, - .hide_foreign_binaries_warning = hide_warnings, - }; - self.step.dependOn(&artifact.step); - - return self; -} - -fn make(step: *Step, prog_node: *std.Progress.Node) !void { - _ = prog_node; - const self = @fieldParentPtr(EmulatableRunStep, "step", step); - const host_info = self.builder.host; - - var argv_list = std.ArrayList([]const u8).init(self.builder.allocator); - defer argv_list.deinit(); - - const need_cross_glibc = self.exe.target.isGnuLibC() and self.exe.is_linking_libc; - switch (host_info.getExternalExecutor(self.exe.target_info, .{ - .qemu_fixes_dl = need_cross_glibc and self.builder.glibc_runtimes_dir != null, - .link_libc = self.exe.is_linking_libc, - })) { - .native => {}, - .rosetta => if (!self.builder.enable_rosetta) return warnAboutForeignBinaries(self), - .wine => |bin_name| if (self.builder.enable_wine) { - try argv_list.append(bin_name); - } else return, - .qemu => |bin_name| if (self.builder.enable_qemu) { - const glibc_dir_arg = if (need_cross_glibc) - self.builder.glibc_runtimes_dir orelse return - else - null; - try argv_list.append(bin_name); - if (glibc_dir_arg) |dir| { - // TODO look into making this a call to `linuxTriple`. This - // needs the directory to be called "i686" rather than - // "x86" which is why we do it manually here. - const fmt_str = "{s}" ++ fs.path.sep_str ++ "{s}-{s}-{s}"; - const cpu_arch = self.exe.target.getCpuArch(); - const os_tag = self.exe.target.getOsTag(); - const abi = self.exe.target.getAbi(); - const cpu_arch_name: []const u8 = if (cpu_arch == .x86) - "i686" - else - @tagName(cpu_arch); - const full_dir = try std.fmt.allocPrint(self.builder.allocator, fmt_str, .{ - dir, cpu_arch_name, @tagName(os_tag), @tagName(abi), - }); - - try argv_list.append("-L"); - try argv_list.append(full_dir); - } - } else return warnAboutForeignBinaries(self), - .darling => |bin_name| if (self.builder.enable_darling) { - try argv_list.append(bin_name); - } else return warnAboutForeignBinaries(self), - .wasmtime => |bin_name| if (self.builder.enable_wasmtime) { - try argv_list.append(bin_name); - try argv_list.append("--dir=."); - } else return warnAboutForeignBinaries(self), - else => return warnAboutForeignBinaries(self), - } - - if (self.exe.target.isWindows()) { - // On Windows we don't have rpaths so we have to add .dll search paths to PATH - RunStep.addPathForDynLibsInternal(&self.step, self.builder, self.exe); - } - - const executable_path = self.exe.installed_path orelse self.exe.getOutputSource().getPath(self.builder); - try argv_list.append(executable_path); - - try RunStep.runCommand( - argv_list.items, - self.builder, - self.expected_term, - self.stdout_action, - self.stderr_action, - .Inherit, - self.env_map, - self.cwd, - false, - ); -} - -pub fn expectStdErrEqual(self: *EmulatableRunStep, bytes: []const u8) void { - self.stderr_action = .{ .expect_exact = self.builder.dupe(bytes) }; -} - -pub fn expectStdOutEqual(self: *EmulatableRunStep, bytes: []const u8) void { - self.stdout_action = .{ .expect_exact = self.builder.dupe(bytes) }; -} - -fn warnAboutForeignBinaries(step: *EmulatableRunStep) void { - if (step.hide_foreign_binaries_warning) return; - const builder = step.builder; - const artifact = step.exe; - - const host_name = builder.host.target.zigTriple(builder.allocator) catch @panic("unhandled error"); - const foreign_name = artifact.target.zigTriple(builder.allocator) catch @panic("unhandled error"); - const target_info = std.zig.system.NativeTargetInfo.detect(artifact.target) catch @panic("unhandled error"); - const need_cross_glibc = artifact.target.isGnuLibC() and artifact.is_linking_libc; - switch (builder.host.getExternalExecutor(target_info, .{ - .qemu_fixes_dl = need_cross_glibc and builder.glibc_runtimes_dir != null, - .link_libc = artifact.is_linking_libc, - })) { - .native => unreachable, - .bad_dl => |foreign_dl| { - const host_dl = builder.host.dynamic_linker.get() orelse "(none)"; - std.debug.print("the host system does not appear to be capable of executing binaries from the target because the host dynamic linker is '{s}', while the target dynamic linker is '{s}'. Consider setting the dynamic linker as '{s}'.\n", .{ - host_dl, foreign_dl, host_dl, - }); - }, - .bad_os_or_cpu => { - std.debug.print("the host system ({s}) does not appear to be capable of executing binaries from the target ({s}).\n", .{ - host_name, foreign_name, - }); - }, - .darling => if (!builder.enable_darling) { - std.debug.print( - "the host system ({s}) does not appear to be capable of executing binaries " ++ - "from the target ({s}). Consider enabling darling.\n", - .{ host_name, foreign_name }, - ); - }, - .rosetta => if (!builder.enable_rosetta) { - std.debug.print( - "the host system ({s}) does not appear to be capable of executing binaries " ++ - "from the target ({s}). Consider enabling rosetta.\n", - .{ host_name, foreign_name }, - ); - }, - .wine => if (!builder.enable_wine) { - std.debug.print( - "the host system ({s}) does not appear to be capable of executing binaries " ++ - "from the target ({s}). Consider enabling wine.\n", - .{ host_name, foreign_name }, - ); - }, - .qemu => if (!builder.enable_qemu) { - std.debug.print( - "the host system ({s}) does not appear to be capable of executing binaries " ++ - "from the target ({s}). Consider enabling qemu.\n", - .{ host_name, foreign_name }, - ); - }, - .wasmtime => { - std.debug.print( - "the host system ({s}) does not appear to be capable of executing binaries " ++ - "from the target ({s}). Consider enabling wasmtime.\n", - .{ host_name, foreign_name }, - ); - }, - } -} diff --git a/lib/std/Build/FmtStep.zig b/lib/std/Build/FmtStep.zig index 5efada7507..2a82342336 100644 --- a/lib/std/Build/FmtStep.zig +++ b/lib/std/Build/FmtStep.zig @@ -1,37 +1,73 @@ -const std = @import("../std.zig"); -const Step = std.Build.Step; -const FmtStep = @This(); +//! This step has two modes: +//! * Modify mode: directly modify source files, formatting them in place. +//! * Check mode: fail the step if a non-conforming file is found. + +step: Step, +paths: []const []const u8, +exclude_paths: []const []const u8, +check: bool, pub const base_id = .fmt; -step: Step, -builder: *std.Build, -argv: [][]const u8, - -pub fn create(builder: *std.Build, paths: []const []const u8) *FmtStep { - const self = builder.allocator.create(FmtStep) catch @panic("OOM"); - const name = "zig fmt"; - self.* = FmtStep{ - .step = Step.init(builder.allocator, .{ - .id = .fmt, +pub const Options = struct { + paths: []const []const u8 = &.{}, + exclude_paths: []const []const u8 = &.{}, + /// If true, fails the build step when any non-conforming files are encountered. + check: bool = false, +}; + +pub fn create(owner: *std.Build, options: Options) *FmtStep { + const self = owner.allocator.create(FmtStep) catch @panic("OOM"); + const name = if (options.check) "zig fmt --check" else "zig fmt"; + self.* = .{ + .step = Step.init(.{ + .id = base_id, .name = name, + .owner = owner, .makeFn = make, }), - .builder = builder, - .argv = builder.allocator.alloc([]u8, paths.len + 2) catch @panic("OOM"), + .paths = options.paths, + .exclude_paths = options.exclude_paths, + .check = options.check, }; - - self.argv[0] = builder.zig_exe; - self.argv[1] = "fmt"; - for (paths, 0..) |path, i| { - self.argv[2 + i] = builder.pathFromRoot(path); - } return self; } fn make(step: *Step, prog_node: *std.Progress.Node) !void { + // zig fmt is fast enough that no progress is needed. _ = prog_node; + + // TODO: if check=false, this means we are modifying source files in place, which + // is an operation that could race against other operations also modifying source files + // in place. In this case, this step should obtain a write lock while making those + // modifications. + + const b = step.owner; + const arena = b.allocator; const self = @fieldParentPtr(FmtStep, "step", step); - return self.builder.spawnChild(self.argv); + var argv: std.ArrayListUnmanaged([]const u8) = .{}; + try argv.ensureUnusedCapacity(arena, 2 + 1 + self.paths.len + 2 * self.exclude_paths.len); + + argv.appendAssumeCapacity(b.zig_exe); + argv.appendAssumeCapacity("fmt"); + + if (self.check) { + argv.appendAssumeCapacity("--check"); + } + + for (self.paths) |p| { + argv.appendAssumeCapacity(b.pathFromRoot(p)); + } + + for (self.exclude_paths) |p| { + argv.appendAssumeCapacity("--exclude"); + argv.appendAssumeCapacity(b.pathFromRoot(p)); + } + + return step.evalChildProcess(argv.items); } + +const std = @import("../std.zig"); +const Step = std.Build.Step; +const FmtStep = @This(); diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/InstallArtifactStep.zig index d8907eb59f..7e35d0a5ee 100644 --- a/lib/std/Build/InstallArtifactStep.zig +++ b/lib/std/Build/InstallArtifactStep.zig @@ -7,23 +7,24 @@ const InstallArtifactStep = @This(); pub const base_id = .install_artifact; step: Step, -builder: *std.Build, +dest_builder: *std.Build, artifact: *CompileStep, dest_dir: InstallDir, pdb_dir: ?InstallDir, h_dir: ?InstallDir, -pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep { +pub fn create(owner: *std.Build, artifact: *CompileStep) *InstallArtifactStep { if (artifact.install_step) |s| return s; - const self = builder.allocator.create(InstallArtifactStep) catch @panic("OOM"); + const self = owner.allocator.create(InstallArtifactStep) catch @panic("OOM"); self.* = InstallArtifactStep{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = base_id, - .name = builder.fmt("install {s}", .{artifact.name}), + .name = owner.fmt("install {s}", .{artifact.name}), + .owner = owner, .makeFn = make, }), + .dest_builder = owner, .artifact = artifact, .dest_dir = artifact.override_dest_dir orelse switch (artifact.kind) { .obj => @panic("Cannot install a .obj build artifact."), @@ -43,48 +44,52 @@ pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep self.step.dependOn(&artifact.step); artifact.install_step = self; - builder.pushInstalledFile(self.dest_dir, artifact.out_filename); + owner.pushInstalledFile(self.dest_dir, artifact.out_filename); if (self.artifact.isDynamicLibrary()) { if (artifact.major_only_filename) |name| { - builder.pushInstalledFile(.lib, name); + owner.pushInstalledFile(.lib, name); } if (artifact.name_only_filename) |name| { - builder.pushInstalledFile(.lib, name); + owner.pushInstalledFile(.lib, name); } if (self.artifact.target.isWindows()) { - builder.pushInstalledFile(.lib, artifact.out_lib_filename); + owner.pushInstalledFile(.lib, artifact.out_lib_filename); } } if (self.pdb_dir) |pdb_dir| { - builder.pushInstalledFile(pdb_dir, artifact.out_pdb_filename); + owner.pushInstalledFile(pdb_dir, artifact.out_pdb_filename); } if (self.h_dir) |h_dir| { - builder.pushInstalledFile(h_dir, artifact.out_h_filename); + owner.pushInstalledFile(h_dir, artifact.out_h_filename); } return self; } fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const src_builder = step.owner; const self = @fieldParentPtr(InstallArtifactStep, "step", step); - const builder = self.builder; + const dest_builder = self.dest_builder; - const full_dest_path = builder.getInstallPath(self.dest_dir, self.artifact.out_filename); - try builder.updateFile(self.artifact.getOutputSource().getPath(builder), full_dest_path); + const full_dest_path = dest_builder.getInstallPath(self.dest_dir, self.artifact.out_filename); + try src_builder.updateFile( + self.artifact.getOutputSource().getPath(src_builder), + full_dest_path, + ); if (self.artifact.isDynamicLibrary() and self.artifact.version != null and self.artifact.target.wantSharedLibSymLinks()) { - try CompileStep.doAtomicSymLinks(builder.allocator, full_dest_path, self.artifact.major_only_filename.?, self.artifact.name_only_filename.?); + try CompileStep.doAtomicSymLinks(src_builder.allocator, full_dest_path, self.artifact.major_only_filename.?, self.artifact.name_only_filename.?); } if (self.artifact.isDynamicLibrary() and self.artifact.target.isWindows() and self.artifact.emit_implib != .no_emit) { - const full_implib_path = builder.getInstallPath(self.dest_dir, self.artifact.out_lib_filename); - try builder.updateFile(self.artifact.getOutputLibSource().getPath(builder), full_implib_path); + const full_implib_path = dest_builder.getInstallPath(self.dest_dir, self.artifact.out_lib_filename); + try src_builder.updateFile(self.artifact.getOutputLibSource().getPath(src_builder), full_implib_path); } if (self.pdb_dir) |pdb_dir| { - const full_pdb_path = builder.getInstallPath(pdb_dir, self.artifact.out_pdb_filename); - try builder.updateFile(self.artifact.getOutputPdbSource().getPath(builder), full_pdb_path); + const full_pdb_path = dest_builder.getInstallPath(pdb_dir, self.artifact.out_pdb_filename); + try src_builder.updateFile(self.artifact.getOutputPdbSource().getPath(src_builder), full_pdb_path); } if (self.h_dir) |h_dir| { - const full_h_path = builder.getInstallPath(h_dir, self.artifact.out_h_filename); - try builder.updateFile(self.artifact.getOutputHSource().getPath(builder), full_h_path); + const full_h_path = dest_builder.getInstallPath(h_dir, self.artifact.out_h_filename); + try src_builder.updateFile(self.artifact.getOutputHSource().getPath(src_builder), full_h_path); } self.artifact.installed_path = full_dest_path; } diff --git a/lib/std/Build/InstallDirStep.zig b/lib/std/Build/InstallDirStep.zig index bf89d9e7c7..11553d9bc7 100644 --- a/lib/std/Build/InstallDirStep.zig +++ b/lib/std/Build/InstallDirStep.zig @@ -7,11 +7,10 @@ const InstallDirStep = @This(); const log = std.log; step: Step, -builder: *std.Build, options: Options, /// This is used by the build system when a file being installed comes from one /// package but is being installed by another. -override_source_builder: ?*std.Build = null, +dest_builder: *std.Build, pub const base_id = .install_dir; @@ -40,27 +39,26 @@ pub const Options = struct { } }; -pub fn init( - builder: *std.Build, - options: Options, -) InstallDirStep { - builder.pushInstalledFile(options.install_dir, options.install_subdir); +pub fn init(owner: *std.Build, options: Options) InstallDirStep { + owner.pushInstalledFile(options.install_dir, options.install_subdir); return .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = .install_dir, - .name = builder.fmt("install {s}/", .{options.source_dir}), + .name = owner.fmt("install {s}/", .{options.source_dir}), + .owner = owner, .makeFn = make, }), - .options = options.dupe(builder), + .options = options.dupe(owner), + .dest_builder = owner, }; } fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; const self = @fieldParentPtr(InstallDirStep, "step", step); - const dest_prefix = self.builder.getInstallPath(self.options.install_dir, self.options.install_subdir); - const src_builder = self.override_source_builder orelse self.builder; + const dest_builder = self.dest_builder; + const dest_prefix = dest_builder.getInstallPath(self.options.install_dir, self.options.install_subdir); + const src_builder = self.step.owner; const full_src_dir = src_builder.pathFromRoot(self.options.source_dir); var src_dir = std.fs.cwd().openIterableDir(full_src_dir, .{}) catch |err| { log.err("InstallDirStep: unable to open source directory '{s}': {s}", .{ @@ -69,7 +67,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { return error.StepFailed; }; defer src_dir.close(); - var it = try src_dir.walk(self.builder.allocator); + var it = try src_dir.walk(dest_builder.allocator); next_entry: while (try it.next()) |entry| { for (self.options.exclude_extensions) |ext| { if (mem.endsWith(u8, entry.path, ext)) { @@ -77,20 +75,20 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } } - const full_path = self.builder.pathJoin(&.{ full_src_dir, entry.path }); - const dest_path = self.builder.pathJoin(&.{ dest_prefix, entry.path }); + const full_path = dest_builder.pathJoin(&.{ full_src_dir, entry.path }); + const dest_path = dest_builder.pathJoin(&.{ dest_prefix, entry.path }); switch (entry.kind) { .Directory => try fs.cwd().makePath(dest_path), .File => { for (self.options.blank_extensions) |ext| { if (mem.endsWith(u8, entry.path, ext)) { - try self.builder.truncateFile(dest_path); + try dest_builder.truncateFile(dest_path); continue :next_entry; } } - try self.builder.updateFile(full_path, dest_path); + try dest_builder.updateFile(full_path, dest_path); }, else => continue, } diff --git a/lib/std/Build/InstallFileStep.zig b/lib/std/Build/InstallFileStep.zig index f77b22c112..ed7576f42c 100644 --- a/lib/std/Build/InstallFileStep.zig +++ b/lib/std/Build/InstallFileStep.zig @@ -7,39 +7,40 @@ const InstallFileStep = @This(); pub const base_id = .install_file; step: Step, -builder: *std.Build, source: FileSource, dir: InstallDir, dest_rel_path: []const u8, /// This is used by the build system when a file being installed comes from one /// package but is being installed by another. -override_source_builder: ?*std.Build = null, +dest_builder: *std.Build, pub fn init( - builder: *std.Build, + owner: *std.Build, source: FileSource, dir: InstallDir, dest_rel_path: []const u8, ) InstallFileStep { - builder.pushInstalledFile(dir, dest_rel_path); + owner.pushInstalledFile(dir, dest_rel_path); return InstallFileStep{ - .builder = builder, - .step = Step.init(builder.allocator, .{ - .id = .install_file, - .name = builder.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), + .step = Step.init(.{ + .id = base_id, + .name = owner.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), + .owner = owner, .makeFn = make, }), - .source = source.dupe(builder), - .dir = dir.dupe(builder), - .dest_rel_path = builder.dupePath(dest_rel_path), + .source = source.dupe(owner), + .dir = dir.dupe(owner), + .dest_rel_path = owner.dupePath(dest_rel_path), + .dest_builder = owner, }; } fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const src_builder = step.owner; const self = @fieldParentPtr(InstallFileStep, "step", step); - const src_builder = self.override_source_builder orelse self.builder; + const dest_builder = self.dest_builder; const full_src_path = self.source.getPath2(src_builder, step); - const full_dest_path = self.builder.getInstallPath(self.dir, self.dest_rel_path); - try self.builder.updateFile(full_src_path, full_dest_path); + const full_dest_path = dest_builder.getInstallPath(self.dir, self.dest_rel_path); + try dest_builder.updateFile(full_src_path, full_dest_path); } diff --git a/lib/std/Build/LogStep.zig b/lib/std/Build/LogStep.zig deleted file mode 100644 index 25bba747bf..0000000000 --- a/lib/std/Build/LogStep.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("../std.zig"); -const log = std.log; -const Step = std.Build.Step; -const LogStep = @This(); - -pub const base_id = .log; - -step: Step, -builder: *std.Build, -data: []const u8, - -pub fn init(builder: *std.Build, data: []const u8) LogStep { - return LogStep{ - .builder = builder, - .step = Step.init(builder.allocator, .{ - .id = .log, - .name = builder.fmt("log {s}", .{data}), - .makeFn = make, - }), - .data = builder.dupe(data), - }; -} - -fn make(step: *Step, prog_node: *std.Progress.Node) anyerror!void { - _ = prog_node; - const self = @fieldParentPtr(LogStep, "step", step); - log.info("{s}", .{self.data}); -} diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/ObjCopyStep.zig index 839d95903c..7199431ee6 100644 --- a/lib/std/Build/ObjCopyStep.zig +++ b/lib/std/Build/ObjCopyStep.zig @@ -21,7 +21,6 @@ pub const RawFormat = enum { }; step: Step, -builder: *std.Build, file_source: std.Build.FileSource, basename: []const u8, output_file: std.Build.GeneratedFile, @@ -38,18 +37,18 @@ pub const Options = struct { }; pub fn create( - builder: *std.Build, + owner: *std.Build, file_source: std.Build.FileSource, options: Options, ) *ObjCopyStep { - const self = builder.allocator.create(ObjCopyStep) catch @panic("OOM"); + const self = owner.allocator.create(ObjCopyStep) catch @panic("OOM"); self.* = ObjCopyStep{ - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = base_id, - .name = builder.fmt("objcopy {s}", .{file_source.getDisplayName()}), + .name = owner.fmt("objcopy {s}", .{file_source.getDisplayName()}), + .owner = owner, .makeFn = make, }), - .builder = builder, .file_source = file_source, .basename = options.basename orelse file_source.getDisplayName(), .output_file = std.Build.GeneratedFile{ .step = &self.step }, @@ -67,9 +66,8 @@ pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource { } fn make(step: *Step, prog_node: *std.Progress.Node) !void { - _ = prog_node; + const b = step.owner; const self = @fieldParentPtr(ObjCopyStep, "step", step); - const b = self.builder; var man = b.cache.obtain(); defer man.deinit(); @@ -84,7 +82,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { man.hash.addOptional(self.pad_to); man.hash.addOptional(self.format); - if (man.hit() catch |err| failWithCacheError(man, err)) { + if (try step.cacheHit(&man)) { // Cache hit, skip subprocess execution. const digest = man.final(); self.output_file.path = try b.cache_root.join(b.allocator, &.{ @@ -116,23 +114,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }; try argv.appendSlice(&.{ full_src_path, full_dest_path }); - _ = try self.builder.execFromStep(argv.items, &self.step); + _ = try step.spawnZigProcess(argv.items, prog_node); self.output_file.path = full_dest_path; try man.writeManifest(); } - -/// TODO consolidate this with the same function in RunStep? -/// Also properly deal with concurrency (see open PR) -fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn { - const i = man.failed_file_index orelse failWithSimpleError(err); - const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err); - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path }); - std.process.exit(1); -} - -fn failWithSimpleError(err: anyerror) noreturn { - std.debug.print("{s}\n", .{@errorName(err)}); - std.process.exit(1); -} diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index 2cb3bb13be..859d0b68c9 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -12,25 +12,24 @@ pub const base_id = .options; step: Step, generated_file: GeneratedFile, -builder: *std.Build, contents: std.ArrayList(u8), artifact_args: std.ArrayList(OptionArtifactArg), file_source_args: std.ArrayList(OptionFileSourceArg), -pub fn create(builder: *std.Build) *OptionsStep { - const self = builder.allocator.create(OptionsStep) catch @panic("OOM"); +pub fn create(owner: *std.Build) *OptionsStep { + const self = owner.allocator.create(OptionsStep) catch @panic("OOM"); self.* = .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = base_id, .name = "options", + .owner = owner, .makeFn = make, }), .generated_file = undefined, - .contents = std.ArrayList(u8).init(builder.allocator), - .artifact_args = std.ArrayList(OptionArtifactArg).init(builder.allocator), - .file_source_args = std.ArrayList(OptionFileSourceArg).init(builder.allocator), + .contents = std.ArrayList(u8).init(owner.allocator), + .artifact_args = std.ArrayList(OptionArtifactArg).init(owner.allocator), + .file_source_args = std.ArrayList(OptionFileSourceArg).init(owner.allocator), }; self.generated_file = .{ .step = &self.step }; @@ -196,7 +195,7 @@ pub fn addOptionFileSource( ) void { self.file_source_args.append(.{ .name = name, - .source = source.dupe(self.builder), + .source = source.dupe(self.step.owner), }) catch @panic("OOM"); source.addStepDependencies(&self.step); } @@ -204,12 +203,12 @@ pub fn addOptionFileSource( /// The value is the path in the cache dir. /// Adds a dependency automatically. pub fn addOptionArtifact(self: *OptionsStep, name: []const u8, artifact: *CompileStep) void { - self.artifact_args.append(.{ .name = self.builder.dupe(name), .artifact = artifact }) catch @panic("OOM"); + self.artifact_args.append(.{ .name = self.step.owner.dupe(name), .artifact = artifact }) catch @panic("OOM"); self.step.dependOn(&artifact.step); } pub fn createModule(self: *OptionsStep) *std.Build.Module { - return self.builder.createModule(.{ + return self.step.owner.createModule(.{ .source_file = self.getSource(), .dependencies = &.{}, }); @@ -220,14 +219,17 @@ pub fn getSource(self: *OptionsStep) FileSource { } fn make(step: *Step, prog_node: *std.Progress.Node) !void { + // This step completes so quickly that no progress is necessary. _ = prog_node; + + const b = step.owner; const self = @fieldParentPtr(OptionsStep, "step", step); for (self.artifact_args.items) |item| { self.addOption( []const u8, item.name, - self.builder.pathFromRoot(item.artifact.getOutputSource().getPath(self.builder)), + b.pathFromRoot(item.artifact.getOutputSource().getPath(b)), ); } @@ -235,20 +237,18 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { self.addOption( []const u8, item.name, - item.source.getPath(self.builder), + item.source.getPath(b), ); } - var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{}); + var options_dir = try b.cache_root.handle.makeOpenPath("options", .{}); defer options_dir.close(); const basename = self.hashContentsToFileName(); try options_dir.writeFile(&basename, self.contents.items); - self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{ - "options", &basename, - }); + self.generated_file.path = try b.cache_root.join(b.allocator, &.{ "options", &basename }); } fn hashContentsToFileName(self: *OptionsStep) [64]u8 { diff --git a/lib/std/Build/RemoveDirStep.zig b/lib/std/Build/RemoveDirStep.zig index 4fc8e6d338..9f291c7523 100644 --- a/lib/std/Build/RemoveDirStep.zig +++ b/lib/std/Build/RemoveDirStep.zig @@ -7,28 +7,37 @@ const RemoveDirStep = @This(); pub const base_id = .remove_dir; step: Step, -builder: *std.Build, dir_path: []const u8, -pub fn init(builder: *std.Build, dir_path: []const u8) RemoveDirStep { +pub fn init(owner: *std.Build, dir_path: []const u8) RemoveDirStep { return RemoveDirStep{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = .remove_dir, - .name = builder.fmt("RemoveDir {s}", .{dir_path}), + .name = owner.fmt("RemoveDir {s}", .{dir_path}), + .owner = owner, .makeFn = make, }), - .dir_path = builder.dupePath(dir_path), + .dir_path = owner.dupePath(dir_path), }; } fn make(step: *Step, prog_node: *std.Progress.Node) !void { + // TODO update progress node while walking file system. + // Should the standard library support this use case?? _ = prog_node; + + const b = step.owner; const self = @fieldParentPtr(RemoveDirStep, "step", step); - const full_path = self.builder.pathFromRoot(self.dir_path); - fs.cwd().deleteTree(full_path) catch |err| { - log.err("Unable to remove {s}: {s}", .{ full_path, @errorName(err) }); - return err; + b.build_root.handle.deleteTree(self.dir_path) catch |err| { + if (b.build_root.path) |base| { + return step.fail("unable to recursively delete path '{s}/{s}': {s}", .{ + base, self.dir_path, @errorName(err), + }); + } else { + return step.fail("unable to recursively delete path '{s}': {s}", .{ + self.dir_path, @errorName(err), + }); + } }; } diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 84fd7f975d..087483fea8 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -11,14 +11,11 @@ const EnvMap = process.EnvMap; const Allocator = mem.Allocator; const ExecError = std.Build.ExecError; -const max_stdout_size = 1 * 1024 * 1024; // 1 MiB - const RunStep = @This(); pub const base_id: Step.Id = .run; step: Step, -builder: *std.Build, /// See also addArg and addArgs to modifying this directly argv: ArrayList(Arg), @@ -29,35 +26,68 @@ cwd: ?[]const u8, /// Override this field to modify the environment, or use setEnvironmentVariable env_map: ?*EnvMap, -stdout_action: StdIoAction = .inherit, -stderr_action: StdIoAction = .inherit, - -stdin_behavior: std.ChildProcess.StdIo = .Inherit, - -/// Set this to `null` to ignore the exit code for the purpose of determining a successful execution -expected_term: ?std.ChildProcess.Term = .{ .Exited = 0 }, - -/// Print the command before running it -print: bool, -/// Controls whether execution is skipped if the output file is up-to-date. -/// The default is to always run if there is no output file, and to skip -/// running if all output files are up-to-date. -condition: enum { output_outdated, always } = .output_outdated, +/// Configures whether the RunStep is considered to have side-effects, and also +/// whether the RunStep will inherit stdio streams, forwarding them to the +/// parent process, in which case will require a global lock to prevent other +/// steps from interfering with stdio while the subprocess associated with this +/// RunStep is running. +/// If the RunStep is determined to not have side-effects, then execution will +/// be skipped if all output files are up-to-date and input files are +/// unchanged. +stdio: StdIo = .infer_from_args, /// Additional file paths relative to build.zig that, when modified, indicate /// that the RunStep should be re-executed. +/// If the RunStep is determined to have side-effects, this field is ignored +/// and the RunStep is always executed when it appears in the build graph. extra_file_dependencies: []const []const u8 = &.{}, /// After adding an output argument, this step will by default rename itself /// for a better display name in the build summary. /// This can be disabled by setting this to false. -rename_step_with_output_arg: bool, - -pub const StdIoAction = union(enum) { +rename_step_with_output_arg: bool = true, + +/// If this is true, a RunStep which is configured to check the output of the +/// executed binary will not fail the build if the binary cannot be executed +/// due to being for a foreign binary to the host system which is running the +/// build graph. +/// Command-line arguments such as -fqemu and -fwasmtime may affect whether a +/// binary is detected as foreign, as well as system configuration such as +/// Rosetta (macOS) and binfmt_misc (Linux). +skip_foreign_checks: bool = false, + +/// If stderr or stdout exceeds this amount, the child process is killed and +/// the step fails. +max_stdio_size: usize = 10 * 1024 * 1024, + +pub const StdIo = union(enum) { + /// Whether the RunStep has side-effects will be determined by whether or not one + /// of the args is an output file (added with `addOutputFileArg`). + /// If the RunStep is determined to have side-effects, this is the same as `inherit`. + /// The step will fail if the subprocess crashes or returns a non-zero exit code. + infer_from_args, + /// Causes the RunStep to be considered to have side-effects, and therefore + /// always execute when it appears in the build graph. + /// It also means that this step will obtain a global lock to prevent other + /// steps from running in the meantime. + /// The step will fail if the subprocess crashes or returns a non-zero exit code. inherit, - ignore, - expect_exact: []const u8, - expect_matches: []const []const u8, + /// Causes the RunStep to be considered to *not* have side-effects. The + /// process will be re-executed if any of the input dependencies are + /// modified. The exit code and standard I/O streams will be checked for + /// certain conditions, and the step will succeed or fail based on these + /// conditions. + /// Note that an explicit check for exit code 0 needs to be added to this + /// list if such a check is desireable. + check: []const Check, + + pub const Check = union(enum) { + expect_stderr_exact: []const u8, + expect_stderr_match: []const u8, + expect_stdout_exact: []const u8, + expect_stdout_match: []const u8, + expect_term: std.ChildProcess.Term, + }; }; pub const Arg = union(enum) { @@ -72,20 +102,20 @@ pub const Arg = union(enum) { }; }; -pub fn create(builder: *std.Build, name: []const u8) *RunStep { - const self = builder.allocator.create(RunStep) catch @panic("OOM"); +pub fn create(owner: *std.Build, name: []const u8) *RunStep { + const self = owner.allocator.create(RunStep) catch @panic("OOM"); self.* = .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = base_id, .name = name, + .owner = owner, .makeFn = make, }), - .argv = ArrayList(Arg).init(builder.allocator), + .argv = ArrayList(Arg).init(owner.allocator), .cwd = null, .env_map = null, - .print = builder.verbose, .rename_step_with_output_arg = true, + .max_stdio_size = 10 * 1024 * 1024, }; return self; } @@ -99,16 +129,17 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void { /// run, and returns a FileSource which can be used as inputs to other APIs /// throughout the build system. pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource { - const generated_file = rs.builder.allocator.create(std.Build.GeneratedFile) catch @panic("OOM"); + const b = rs.step.owner; + const generated_file = b.allocator.create(std.Build.GeneratedFile) catch @panic("OOM"); generated_file.* = .{ .step = &rs.step }; rs.argv.append(.{ .output = .{ .generated_file = generated_file, - .basename = rs.builder.dupe(basename), + .basename = b.dupe(basename), } }) catch @panic("OOM"); if (rs.rename_step_with_output_arg) { rs.rename_step_with_output_arg = false; - rs.step.name = rs.builder.fmt("{s} ({s})", .{ rs.step.name, basename }); + rs.step.name = b.fmt("{s} ({s})", .{ rs.step.name, basename }); } return .{ .generated = generated_file }; @@ -116,13 +147,13 @@ pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void { self.argv.append(Arg{ - .file_source = file_source.dupe(self.builder), + .file_source = file_source.dupe(self.step.owner), }) catch @panic("OOM"); file_source.addStepDependencies(&self.step); } pub fn addArg(self: *RunStep, arg: []const u8) void { - self.argv.append(Arg{ .bytes = self.builder.dupe(arg) }) catch @panic("OOM"); + self.argv.append(Arg{ .bytes = self.step.owner.dupe(arg) }) catch @panic("OOM"); } pub fn addArgs(self: *RunStep, args: []const []const u8) void { @@ -132,13 +163,14 @@ pub fn addArgs(self: *RunStep, args: []const []const u8) void { } pub fn clearEnvironment(self: *RunStep) void { - const new_env_map = self.builder.allocator.create(EnvMap) catch @panic("OOM"); - new_env_map.* = EnvMap.init(self.builder.allocator); + const b = self.step.owner; + const new_env_map = b.allocator.create(EnvMap) catch @panic("OOM"); + new_env_map.* = EnvMap.init(b.allocator); self.env_map = new_env_map; } pub fn addPathDir(self: *RunStep, search_path: []const u8) void { - addPathDirInternal(&self.step, self.builder, search_path); + addPathDirInternal(&self.step, self.step.owner, search_path); } /// For internal use only, users of `RunStep` should use `addPathDir` directly. @@ -157,13 +189,12 @@ pub fn addPathDirInternal(step: *Step, builder: *std.Build, search_path: []const } pub fn getEnvMap(self: *RunStep) *EnvMap { - return getEnvMapInternal(&self.step, self.builder.allocator); + return getEnvMapInternal(&self.step, self.step.owner.allocator); } fn getEnvMapInternal(step: *Step, allocator: Allocator) *EnvMap { const maybe_env_map = switch (step.id) { .run => step.cast(RunStep).?.env_map, - .emulatable_run => step.cast(std.Build.EmulatableRunStep).?.env_map, else => unreachable, }; return maybe_env_map orelse { @@ -171,7 +202,6 @@ fn getEnvMapInternal(step: *Step, allocator: Allocator) *EnvMap { env_map.* = process.getEnvMap(allocator) catch @panic("unhandled error"); switch (step.id) { .run => step.cast(RunStep).?.env_map = env_map, - .emulatable_run => step.cast(RunStep).?.env_map = env_map, else => unreachable, } return env_map; @@ -179,41 +209,85 @@ fn getEnvMapInternal(step: *Step, allocator: Allocator) *EnvMap { } pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void { + const b = self.step.owner; const env_map = self.getEnvMap(); - env_map.put( - self.builder.dupe(key), - self.builder.dupe(value), - ) catch @panic("unhandled error"); + env_map.put(b.dupe(key), b.dupe(value)) catch @panic("unhandled error"); } pub fn expectStdErrEqual(self: *RunStep, bytes: []const u8) void { - self.stderr_action = .{ .expect_exact = self.builder.dupe(bytes) }; + const new_check: StdIo.Check = .{ .expect_stderr_exact = self.step.owner.dupe(bytes) }; + self.addCheck(new_check); } pub fn expectStdOutEqual(self: *RunStep, bytes: []const u8) void { - self.stdout_action = .{ .expect_exact = self.builder.dupe(bytes) }; + const new_check: StdIo.Check = .{ .expect_stdout_exact = self.step.owner.dupe(bytes) }; + self.addCheck(new_check); } -fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo { - return switch (action) { - .ignore => .Ignore, - .inherit => .Inherit, - .expect_exact, .expect_matches => .Pipe, - }; +pub fn expectExitCode(self: *RunStep, code: u8) void { + const new_check: StdIo.Check = .{ .expect_term = .{ .Exited = code } }; + self.addCheck(new_check); } -fn needOutputCheck(self: RunStep) bool { - switch (self.condition) { - .always => return false, - .output_outdated => {}, +pub fn addCheck(self: *RunStep, new_check: StdIo.Check) void { + const arena = self.step.owner.allocator; + switch (self.stdio) { + .infer_from_args => { + const list = arena.create([1]StdIo.Check) catch @panic("OOM"); + list.* = .{new_check}; + self.stdio = .{ .check = list }; + }, + .check => |checks| { + const new_list = arena.alloc(StdIo.Check, checks.len + 1) catch @panic("OOM"); + std.mem.copy(StdIo.Check, new_list, checks); + new_list[checks.len] = new_check; + }, + else => @panic("illegal call to addCheck: conflicting helper method calls. Suggest to directly set stdio field of RunStep instead"), } - if (self.extra_file_dependencies.len > 0) return true; +} + +/// Returns whether the RunStep has side effects *other than* updating the output arguments. +fn hasSideEffects(self: RunStep) bool { + return switch (self.stdio) { + .infer_from_args => !self.hasAnyOutputArgs(), + .inherit => true, + .check => false, + }; +} +fn hasAnyOutputArgs(self: RunStep) bool { for (self.argv.items) |arg| switch (arg) { .output => return true, else => continue, }; + return false; +} +fn checksContainStdout(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stderr_exact, + .expect_stderr_match, + .expect_term, + => continue, + + .expect_stdout_exact, + .expect_stdout_match, + => return true, + }; + return false; +} + +fn checksContainStderr(checks: []const StdIo.Check) bool { + for (checks) |check| switch (check) { + .expect_stdout_exact, + .expect_stdout_match, + .expect_term, + => continue, + + .expect_stderr_exact, + .expect_stderr_match, + => return true, + }; return false; } @@ -223,16 +297,17 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { // processes could use to supply progress updates. _ = prog_node; + const b = step.owner; const self = @fieldParentPtr(RunStep, "step", step); - const need_output_check = self.needOutputCheck(); + const has_side_effects = self.hasSideEffects(); - var argv_list = ArrayList([]const u8).init(self.builder.allocator); + var argv_list = ArrayList([]const u8).init(b.allocator); var output_placeholders = ArrayList(struct { index: usize, output: Arg.Output, - }).init(self.builder.allocator); + }).init(b.allocator); - var man = self.builder.cache.obtain(); + var man = b.cache.obtain(); defer man.deinit(); for (self.argv.items) |arg| { @@ -242,7 +317,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { man.hash.addBytes(bytes); }, .file_source => |file| { - const file_path = file.getPath(self.builder); + const file_path = file.getPath(b); try argv_list.append(file_path); _ = try man.addFile(file_path, null); }, @@ -252,7 +327,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { self.addPathForDynLibs(artifact); } const file_path = artifact.installed_path orelse - artifact.getOutputSource().getPath(self.builder); + artifact.getOutputSource().getPath(b); try argv_list.append(file_path); @@ -272,17 +347,17 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } } - if (need_output_check) { + if (!has_side_effects) { for (self.extra_file_dependencies) |file_path| { - _ = try man.addFile(self.builder.pathFromRoot(file_path), null); + _ = try man.addFile(b.pathFromRoot(file_path), null); } - if (man.hit() catch |err| failWithCacheError(man, err)) { + if (try step.cacheHit(&man)) { // cache hit, skip running command const digest = man.final(); for (output_placeholders.items) |placeholder| { - placeholder.output.generated_file.path = try self.builder.cache_root.join( - self.builder.allocator, + placeholder.output.generated_file.path = try b.cache_root.join( + b.allocator, &.{ "o", &digest, placeholder.output.basename }, ); } @@ -292,8 +367,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { const digest = man.final(); for (output_placeholders.items) |placeholder| { - const output_path = try self.builder.cache_root.join( - self.builder.allocator, + const output_path = try b.cache_root.join( + b.allocator, &.{ "o", &digest, placeholder.output.basename }, ); const output_dir = fs.path.dirname(output_path).?; @@ -308,18 +383,16 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } try runCommand( + step, + self.cwd, argv_list.items, - self.builder, - self.expected_term, - self.stdout_action, - self.stderr_action, - self.stdin_behavior, self.env_map, - self.cwd, - self.print, + self.stdio, + has_side_effects, + self.max_stdio_size, ); - if (need_output_check) { + if (!has_side_effects) { try man.writeManifest(); } } @@ -369,165 +442,171 @@ fn termMatches(expected: ?std.ChildProcess.Term, actual: std.ChildProcess.Term) }; } -pub fn runCommand( +fn runCommand( + step: *Step, + opt_cwd: ?[]const u8, argv: []const []const u8, - builder: *std.Build, - expected_term: ?std.ChildProcess.Term, - stdout_action: StdIoAction, - stderr_action: StdIoAction, - stdin_behavior: std.ChildProcess.StdIo, env_map: ?*EnvMap, - maybe_cwd: ?[]const u8, - print: bool, + stdio: StdIo, + has_side_effects: bool, + max_stdio_size: usize, ) !void { - const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path; - - if (!std.process.can_spawn) { - const cmd = try std.mem.join(builder.allocator, " ", argv); - std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ - @tagName(builtin.os.tag), cmd, - }); - builder.allocator.free(cmd); - return ExecError.ExecNotSupported; - } + const b = step.owner; + const arena = b.allocator; + const cwd = if (opt_cwd) |cwd| b.pathFromRoot(cwd) else b.build_root.path; - var child = std.ChildProcess.init(argv, builder.allocator); - child.cwd = cwd; - child.env_map = env_map orelse builder.env_map; + try step.handleChildProcUnsupported(opt_cwd, argv); + try Step.handleVerbose(step.owner, opt_cwd, argv); - child.stdin_behavior = stdin_behavior; - child.stdout_behavior = stdIoActionToBehavior(stdout_action); - child.stderr_behavior = stdIoActionToBehavior(stderr_action); - - if (print) - printCmd(cwd, argv); + var child = std.ChildProcess.init(argv, arena); + child.cwd = cwd; + child.env_map = env_map orelse b.env_map; - child.spawn() catch |err| { - std.debug.print("Unable to spawn {s}: {s}\n", .{ argv[0], @errorName(err) }); - return err; + child.stdin_behavior = switch (stdio) { + .infer_from_args => if (has_side_effects) .Inherit else .Ignore, + .inherit => .Inherit, + .check => .Close, + }; + child.stdout_behavior = switch (stdio) { + .infer_from_args => if (has_side_effects) .Inherit else .Ignore, + .inherit => .Inherit, + .check => |checks| if (checksContainStdout(checks)) .Pipe else .Ignore, + }; + child.stderr_behavior = switch (stdio) { + .infer_from_args => if (has_side_effects) .Inherit else .Pipe, + .inherit => .Inherit, + .check => .Pipe, }; - // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). - - var stdout: ?[]const u8 = null; - defer if (stdout) |s| builder.allocator.free(s); + child.spawn() catch |err| return step.fail("unable to spawn {s}: {s}", .{ + argv[0], @errorName(err), + }); + + var stdout_bytes: ?[]const u8 = null; + var stderr_bytes: ?[]const u8 = null; + + if (child.stdout) |stdout| { + if (child.stderr) |stderr| { + var poller = std.io.poll(arena, enum { stdout, stderr }, .{ + .stdout = stdout, + .stderr = stderr, + }); + defer poller.deinit(); + + while (try poller.poll()) { + if (poller.fifo(.stdout).count > max_stdio_size) + return error.StdoutStreamTooLong; + if (poller.fifo(.stderr).count > max_stdio_size) + return error.StderrStreamTooLong; + } - switch (stdout_action) { - .expect_exact, .expect_matches => { - stdout = try child.stdout.?.reader().readAllAlloc(builder.allocator, max_stdout_size); - }, - .inherit, .ignore => {}, + stdout_bytes = try poller.fifo(.stdout).toOwnedSlice(); + stderr_bytes = try poller.fifo(.stderr).toOwnedSlice(); + } else { + stdout_bytes = try stdout.reader().readAllAlloc(arena, max_stdio_size); + } + } else if (child.stderr) |stderr| { + stderr_bytes = try stderr.reader().readAllAlloc(arena, max_stdio_size); } - var stderr: ?[]const u8 = null; - defer if (stderr) |s| builder.allocator.free(s); - - switch (stderr_action) { - .expect_exact, .expect_matches => { - stderr = try child.stderr.?.reader().readAllAlloc(builder.allocator, max_stdout_size); - }, - .inherit, .ignore => {}, - } + if (stderr_bytes) |stderr| if (stderr.len > 0) { + const stderr_is_diagnostic = switch (stdio) { + .check => |checks| !checksContainStderr(checks), + else => true, + }; + if (stderr_is_diagnostic) { + try step.result_error_msgs.append(arena, stderr); + } + }; const term = child.wait() catch |err| { - std.debug.print("Unable to spawn {s}: {s}\n", .{ argv[0], @errorName(err) }); - return err; + return step.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); }; - if (!termMatches(expected_term, term)) { - std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) }); - printCmd(cwd, argv); - return error.UnexpectedExit; - } - - switch (stderr_action) { - .inherit, .ignore => {}, - .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stderr.?)) { - std.debug.print( - \\ - \\========= Expected this stderr: ========= - \\{s} - \\========= But found: ==================== - \\{s} - \\ - , .{ expected_bytes, stderr.? }); - printCmd(cwd, argv); - return error.TestFailed; - } - }, - .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stderr.?, match) == null) { - std.debug.print( - \\ - \\========= Expected to find in stderr: ========= - \\{s} - \\========= But stderr does not contain it: ===== - \\{s} - \\ - , .{ match, stderr.? }); - printCmd(cwd, argv); - return error.TestFailed; - } - }, - } - - switch (stdout_action) { - .inherit, .ignore => {}, - .expect_exact => |expected_bytes| { - if (!mem.eql(u8, expected_bytes, stdout.?)) { - std.debug.print( - \\ - \\========= Expected this stdout: ========= - \\{s} - \\========= But found: ==================== - \\{s} - \\ - , .{ expected_bytes, stdout.? }); - printCmd(cwd, argv); - return error.TestFailed; - } + switch (stdio) { + .check => |checks| for (checks) |check| switch (check) { + .expect_stderr_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, stderr_bytes.?)) { + return step.fail( + \\========= expected this stderr: ========= + \\{s} + \\========= but found: ==================== + \\{s} + \\========= from the following command: === + \\{s} + , .{ + expected_bytes, + stderr_bytes.?, + try Step.allocPrintCmd(arena, opt_cwd, argv), + }); + } + }, + .expect_stderr_match => |match| { + if (mem.indexOf(u8, stderr_bytes.?, match) == null) { + return step.fail( + \\========= expected to find in stderr: ========= + \\{s} + \\========= but stderr does not contain it: ===== + \\{s} + \\========= from the following command: ========= + \\{s} + , .{ + match, + stderr_bytes.?, + try Step.allocPrintCmd(arena, opt_cwd, argv), + }); + } + }, + .expect_stdout_exact => |expected_bytes| { + if (!mem.eql(u8, expected_bytes, stdout_bytes.?)) { + return step.fail( + \\========= expected this stdout: ========= + \\{s} + \\========= but found: ==================== + \\{s} + \\========= from the following command: === + \\{s} + , .{ + expected_bytes, + stdout_bytes.?, + try Step.allocPrintCmd(arena, opt_cwd, argv), + }); + } + }, + .expect_stdout_match => |match| { + if (mem.indexOf(u8, stdout_bytes.?, match) == null) { + return step.fail( + \\========= expected to find in stdout: ========= + \\{s} + \\========= but stdout does not contain it: ===== + \\{s} + \\========= from the following command: ========= + \\{s} + , .{ + match, + stdout_bytes.?, + try Step.allocPrintCmd(arena, opt_cwd, argv), + }); + } + }, + .expect_term => |expected_term| { + if (!termMatches(expected_term, term)) { + return step.fail("the following command {} (expected {}):\n{s}", .{ + fmtTerm(term), + fmtTerm(expected_term), + try Step.allocPrintCmd(arena, opt_cwd, argv), + }); + } + }, }, - .expect_matches => |matches| for (matches) |match| { - if (mem.indexOf(u8, stdout.?, match) == null) { - std.debug.print( - \\ - \\========= Expected to find in stdout: ========= - \\{s} - \\========= But stdout does not contain it: ===== - \\{s} - \\ - , .{ match, stdout.? }); - printCmd(cwd, argv); - return error.TestFailed; - } + else => { + try step.handleChildProcessTerm(term, opt_cwd, argv); }, } } -fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn { - const i = man.failed_file_index orelse failWithSimpleError(err); - const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err); - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path }); - std.process.exit(1); -} - -fn failWithSimpleError(err: anyerror) noreturn { - std.debug.print("{s}\n", .{@errorName(err)}); - std.process.exit(1); -} - -fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void { - if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd}); - for (argv) |arg| { - std.debug.print("{s} ", .{arg}); - } - std.debug.print("\n", .{}); -} - fn addPathForDynLibs(self: *RunStep, artifact: *CompileStep) void { - addPathForDynLibsInternal(&self.step, self.builder, artifact); + addPathForDynLibsInternal(&self.step, self.step.owner, artifact); } /// This should only be used for internal usage, this is called automatically diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 29cf38b55b..f5ac9e1821 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -1,5 +1,6 @@ id: Id, name: []const u8, +owner: *Build, makeFn: MakeFn, dependencies: std.ArrayList(*Step), /// This field is empty during execution of the user's build script, and @@ -39,7 +40,6 @@ pub const Id = enum { translate_c, write_file, run, - emulatable_run, check_file, check_object, config_header, @@ -60,7 +60,6 @@ pub const Id = enum { .translate_c => Build.TranslateCStep, .write_file => Build.WriteFileStep, .run => Build.RunStep, - .emulatable_run => Build.EmulatableRunStep, .check_file => Build.CheckFileStep, .check_object => Build.CheckObjectStep, .config_header => Build.ConfigHeaderStep, @@ -74,11 +73,14 @@ pub const Id = enum { pub const Options = struct { id: Id, name: []const u8, + owner: *Build, makeFn: MakeFn = makeNoOp, first_ret_addr: ?usize = null, }; -pub fn init(allocator: Allocator, options: Options) Step { +pub fn init(options: Options) Step { + const arena = options.owner.allocator; + var addresses = [1]usize{0} ** n_debug_stack_frames; const first_ret_addr = options.first_ret_addr orelse @returnAddress(); var stack_trace = std.builtin.StackTrace{ @@ -89,9 +91,10 @@ pub fn init(allocator: Allocator, options: Options) Step { return .{ .id = options.id, - .name = allocator.dupe(u8, options.name) catch @panic("OOM"), + .name = arena.dupe(u8, options.name) catch @panic("OOM"), + .owner = options.owner, .makeFn = options.makeFn, - .dependencies = std.ArrayList(*Step).init(allocator), + .dependencies = std.ArrayList(*Step).init(arena), .dependants = .{}, .state = .precheck_unstarted, .debug_stack_trace = addresses, @@ -168,3 +171,231 @@ const std = @import("../std.zig"); const Build = std.Build; const Allocator = std.mem.Allocator; const assert = std.debug.assert; +const builtin = @import("builtin"); + +pub fn evalChildProcess(s: *Step, argv: []const []const u8) !void { + const arena = s.owner.allocator; + + try handleChildProcUnsupported(s, null, argv); + try handleVerbose(s.owner, null, argv); + + const result = std.ChildProcess.exec(.{ + .allocator = arena, + .argv = argv, + }) catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err) }); + + if (result.stderr.len > 0) { + try s.result_error_msgs.append(arena, result.stderr); + } + + try handleChildProcessTerm(s, result.term, null, argv); +} + +pub fn fail(step: *Step, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, MakeFailed } { + const arena = step.owner.allocator; + const msg = try std.fmt.allocPrint(arena, fmt, args); + try step.result_error_msgs.append(arena, msg); + return error.MakeFailed; +} + +/// Assumes that argv contains `--listen=-` and that the process being spawned +/// is the zig compiler - the same version that compiled the build runner. +pub fn evalZigProcess( + s: *Step, + argv: []const []const u8, + prog_node: *std.Progress.Node, +) ![]const u8 { + assert(argv.len != 0); + const b = s.owner; + const arena = b.allocator; + const gpa = arena; + + try handleChildProcUnsupported(s, null, argv); + try handleVerbose(s.owner, null, argv); + + var child = std.ChildProcess.init(argv, arena); + child.env_map = b.env_map; + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{ + argv[0], @errorName(err), + }); + + var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try sendMessage(child.stdin.?, .update); + try sendMessage(child.stdin.?, .exit); + + const Header = std.zig.Server.Message.Header; + var result: ?[]const u8 = null; + + var node_name: std.ArrayListUnmanaged(u8) = .{}; + defer node_name.deinit(gpa); + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |*n| n.end(); + + while (try poller.poll()) { + const stdout = poller.fifo(.stdout); + const buf = stdout.readableSlice(0); + assert(stdout.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + const header_and_msg_len = header.bytes_len + @sizeOf(Header); + if (buf.len >= header_and_msg_len) { + const body = buf[@sizeOf(Header)..][0..header.bytes_len]; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return s.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @ptrCast(*align(1) const EbHdr, body); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + // TODO: use @ptrCast when the compiler supports it + const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try arena.alloc(u32, unaligned_extra.len); + // TODO: use @memcpy when it supports slices + for (extra_array, unaligned_extra) |*dst, src| dst.* = src; + s.result_error_bundle = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .extra = extra_array, + }; + }, + .progress => { + if (sub_prog_node) |*n| n.end(); + node_name.clearRetainingCapacity(); + try node_name.appendSlice(gpa, body); + sub_prog_node = prog_node.start(node_name.items, 0); + sub_prog_node.?.activate(); + }, + .emit_bin_path => { + result = try arena.dupe(u8, body); + }, + _ => { + // Unrecognized message. + }, + } + stdout.discard(header_and_msg_len); + } + } + } + + const stderr = poller.fifo(.stderr); + if (stderr.readableLength() > 0) { + try s.result_error_msgs.append(arena, try stderr.toOwnedSlice()); + } + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + const term = child.wait() catch |err| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); + }; + try handleChildProcessTerm(s, term, null, argv); + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ + s.result_error_bundle.errorMessageCount(), + try allocPrintCmd(arena, null, argv), + }); + } + + return result orelse return s.fail( + "the following command failed to communicate the compilation result:\n{s}", + .{try allocPrintCmd(arena, null, argv)}, + ); +} + +fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + try file.writeAll(std.mem.asBytes(&header)); +} + +pub fn handleVerbose( + b: *Build, + opt_cwd: ?[]const u8, + argv: []const []const u8, +) error{OutOfMemory}!void { + if (b.verbose) { + // Intention of verbose is to print all sub-process command lines to + // stderr before spawning them. + const text = try allocPrintCmd(b.allocator, opt_cwd, argv); + std.debug.print("{s}\n", .{text}); + } +} + +pub inline fn handleChildProcUnsupported( + s: *Step, + opt_cwd: ?[]const u8, + argv: []const []const u8, +) error{ OutOfMemory, MakeFailed }!void { + if (!std.process.can_spawn) { + return s.fail( + "unable to execute the following command: host cannot spawn child processes\n{s}", + .{try allocPrintCmd(s.owner.allocator, opt_cwd, argv)}, + ); + } +} + +pub fn handleChildProcessTerm( + s: *Step, + term: std.ChildProcess.Term, + opt_cwd: ?[]const u8, + argv: []const []const u8, +) error{ MakeFailed, OutOfMemory }!void { + const arena = s.owner.allocator; + switch (term) { + .Exited => |code| { + if (code != 0) { + return s.fail( + "the following command exited with error code {d}:\n{s}", + .{ code, try allocPrintCmd(arena, opt_cwd, argv) }, + ); + } + }, + .Signal, .Stopped, .Unknown => { + return s.fail( + "the following command terminated unexpectedly:\n{s}", + .{try allocPrintCmd(arena, opt_cwd, argv)}, + ); + }, + } +} + +pub fn allocPrintCmd(arena: Allocator, opt_cwd: ?[]const u8, argv: []const []const u8) ![]u8 { + var buf: std.ArrayListUnmanaged(u8) = .{}; + if (opt_cwd) |cwd| try buf.writer(arena).print("cd {s} && ", .{cwd}); + for (argv) |arg| { + try buf.writer(arena).print("{s} ", .{arg}); + } + return buf.toOwnedSlice(arena); +} + +pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool { + return man.hit() catch |err| return failWithCacheError(s, man, err); +} + +fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror { + const i = man.failed_file_index orelse return err; + const pp = man.files.items[i].prefixed_path orelse return err; + const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; + return s.fail("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path }); +} diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/TranslateCStep.zig index fef644f03c..dbb93d8c61 100644 --- a/lib/std/Build/TranslateCStep.zig +++ b/lib/std/Build/TranslateCStep.zig @@ -11,7 +11,6 @@ const TranslateCStep = @This(); pub const base_id = .translate_c; step: Step, -builder: *std.Build, source: std.Build.FileSource, include_dirs: std.ArrayList([]const u8), c_macros: std.ArrayList([]const u8), @@ -26,19 +25,19 @@ pub const Options = struct { optimize: std.builtin.OptimizeMode, }; -pub fn create(builder: *std.Build, options: Options) *TranslateCStep { - const self = builder.allocator.create(TranslateCStep) catch @panic("OOM"); - const source = options.source_file.dupe(builder); +pub fn create(owner: *std.Build, options: Options) *TranslateCStep { + const self = owner.allocator.create(TranslateCStep) catch @panic("OOM"); + const source = options.source_file.dupe(owner); self.* = TranslateCStep{ - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = .translate_c, .name = "translate-c", + .owner = owner, .makeFn = make, }), - .builder = builder, .source = source, - .include_dirs = std.ArrayList([]const u8).init(builder.allocator), - .c_macros = std.ArrayList([]const u8).init(builder.allocator), + .include_dirs = std.ArrayList([]const u8).init(owner.allocator), + .c_macros = std.ArrayList([]const u8).init(owner.allocator), .out_basename = undefined, .target = options.target, .optimize = options.optimize, @@ -58,7 +57,7 @@ pub const AddExecutableOptions = struct { /// Creates a step to build an executable from the translated source. pub fn addExecutable(self: *TranslateCStep, options: AddExecutableOptions) *CompileStep { - return self.builder.addExecutable(.{ + return self.step.owner.addExecutable(.{ .root_source_file = .{ .generated = &self.output_file }, .name = options.name orelse "translated_c", .version = options.version, @@ -69,30 +68,31 @@ pub fn addExecutable(self: *TranslateCStep, options: AddExecutableOptions) *Comp } pub fn addIncludeDir(self: *TranslateCStep, include_dir: []const u8) void { - self.include_dirs.append(self.builder.dupePath(include_dir)) catch @panic("OOM"); + self.include_dirs.append(self.step.owner.dupePath(include_dir)) catch @panic("OOM"); } pub fn addCheckFile(self: *TranslateCStep, expected_matches: []const []const u8) *CheckFileStep { - return CheckFileStep.create(self.builder, .{ .generated = &self.output_file }, self.builder.dupeStrings(expected_matches)); + return CheckFileStep.create(self.step.owner, .{ .generated = &self.output_file }, self.step.owner.dupeStrings(expected_matches)); } /// If the value is omitted, it is set to 1. /// `name` and `value` need not live longer than the function call. pub fn defineCMacro(self: *TranslateCStep, name: []const u8, value: ?[]const u8) void { - const macro = std.Build.constructCMacro(self.builder.allocator, name, value); + const macro = std.Build.constructCMacro(self.step.owner.allocator, name, value); self.c_macros.append(macro) catch @panic("OOM"); } /// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1. pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void { - self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM"); + self.c_macros.append(self.step.owner.dupe(name_and_value)) catch @panic("OOM"); } fn make(step: *Step, prog_node: *std.Progress.Node) !void { + const b = step.owner; const self = @fieldParentPtr(TranslateCStep, "step", step); - var argv_list = std.ArrayList([]const u8).init(self.builder.allocator); - try argv_list.append(self.builder.zig_exe); + var argv_list = std.ArrayList([]const u8).init(b.allocator); + try argv_list.append(b.zig_exe); try argv_list.append("translate-c"); try argv_list.append("-lc"); @@ -101,12 +101,12 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (!self.target.isNative()) { try argv_list.append("-target"); - try argv_list.append(try self.target.zigTriple(self.builder.allocator)); + try argv_list.append(try self.target.zigTriple(b.allocator)); } switch (self.optimize) { .Debug => {}, // Skip since it's the default. - else => try argv_list.append(self.builder.fmt("-O{s}", .{@tagName(self.optimize)})), + else => try argv_list.append(b.fmt("-O{s}", .{@tagName(self.optimize)})), } for (self.include_dirs.items) |include_dir| { @@ -119,15 +119,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try argv_list.append(c_macro); } - try argv_list.append(self.source.getPath(self.builder)); + try argv_list.append(self.source.getPath(b)); - const output_path = try self.builder.execFromStep(argv_list.items, &self.step, prog_node); + const output_path = try step.evalZigProcess(argv_list.items, prog_node); self.out_basename = fs.path.basename(output_path); const output_dir = fs.path.dirname(output_path).?; self.output_file.path = try fs.path.join( - self.builder.allocator, + b.allocator, &[_][]const u8{ output_dir, self.out_basename }, ); } diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 62acd6e8ee..d554d6efc1 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -10,7 +10,6 @@ //! control. step: Step, -builder: *std.Build, /// The elements here are pointers because we need stable pointers for the /// GeneratedFile field. files: std.ArrayListUnmanaged(*File), @@ -34,12 +33,12 @@ pub const Contents = union(enum) { copy: std.Build.FileSource, }; -pub fn init(builder: *std.Build) WriteFileStep { +pub fn init(owner: *std.Build) WriteFileStep { return .{ - .builder = builder, - .step = Step.init(builder.allocator, .{ + .step = Step.init(.{ .id = .write_file, .name = "writefile", + .owner = owner, .makeFn = make, }), .files = .{}, @@ -48,12 +47,13 @@ pub fn init(builder: *std.Build) WriteFileStep { } pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void { - const gpa = wf.builder.allocator; + const b = wf.step.owner; + const gpa = b.allocator; const file = gpa.create(File) catch @panic("OOM"); file.* = .{ .generated_file = .{ .step = &wf.step }, - .sub_path = wf.builder.dupePath(sub_path), - .contents = .{ .bytes = wf.builder.dupe(bytes) }, + .sub_path = b.dupePath(sub_path), + .contents = .{ .bytes = b.dupe(bytes) }, }; wf.files.append(gpa, file) catch @panic("OOM"); } @@ -66,11 +66,12 @@ pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void { /// required sub-path exists. /// This is the option expected to be used most commonly with `addCopyFile`. pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void { - const gpa = wf.builder.allocator; + const b = wf.step.owner; + const gpa = b.allocator; const file = gpa.create(File) catch @panic("OOM"); file.* = .{ .generated_file = .{ .step = &wf.step }, - .sub_path = wf.builder.dupePath(sub_path), + .sub_path = b.dupePath(sub_path), .contents = .{ .copy = source }, }; wf.files.append(gpa, file) catch @panic("OOM"); @@ -83,7 +84,8 @@ pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: [ /// those changes to version control. /// A file added this way is not available with `getFileSource`. pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void { - wf.output_source_files.append(wf.builder.allocator, .{ + const b = wf.step.owner; + wf.output_source_files.append(b.allocator, .{ .contents = .{ .copy = source }, .sub_path = sub_path, }) catch @panic("OOM"); @@ -101,6 +103,7 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo fn make(step: *Step, prog_node: *std.Progress.Node) !void { _ = prog_node; + const b = step.owner; const wf = @fieldParentPtr(WriteFileStep, "step", step); // Writing to source files is kind of an extra capability of this @@ -110,11 +113,11 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { for (wf.output_source_files.items) |output_source_file| { const basename = fs.path.basename(output_source_file.sub_path); if (fs.path.dirname(output_source_file.sub_path)) |dirname| { - var dir = try wf.builder.build_root.handle.makeOpenPath(dirname, .{}); + var dir = try b.build_root.handle.makeOpenPath(dirname, .{}); defer dir.close(); try writeFile(wf, dir, output_source_file.contents, basename); } else { - try writeFile(wf, wf.builder.build_root.handle, output_source_file.contents, basename); + try writeFile(wf, b.build_root.handle, output_source_file.contents, basename); } } @@ -125,7 +128,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { // If, for example, a hard-coded path was used as the location to put WriteFileStep // files, then two WriteFileSteps executing in parallel might clobber each other. - var man = wf.builder.cache.obtain(); + var man = b.cache.obtain(); defer man.deinit(); // Random bytes to make WriteFileStep unique. Refresh this with @@ -140,17 +143,17 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { man.hash.addBytes(bytes); }, .copy => |file_source| { - _ = try man.addFile(file_source.getPath(wf.builder), null); + _ = try man.addFile(file_source.getPath(b), null); }, } } - if (man.hit() catch |err| failWithCacheError(man, err)) { + if (try step.cacheHit(&man)) { // Cache hit, skip writing file data. const digest = man.final(); for (wf.files.items) |file| { - file.generated_file.path = try wf.builder.cache_root.join( - wf.builder.allocator, + file.generated_file.path = try b.cache_root.join( + b.allocator, &.{ "o", &digest, file.sub_path }, ); } @@ -160,7 +163,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { const digest = man.final(); const cache_path = "o" ++ fs.path.sep_str ++ digest; - var cache_dir = wf.builder.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| { + var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| { std.debug.print("unable to make path {s}: {s}\n", .{ cache_path, @errorName(err) }); return err; }; @@ -169,15 +172,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { for (wf.files.items) |file| { const basename = fs.path.basename(file.sub_path); if (fs.path.dirname(file.sub_path)) |dirname| { - var dir = try wf.builder.cache_root.handle.makeOpenPath(dirname, .{}); + var dir = try b.cache_root.handle.makeOpenPath(dirname, .{}); defer dir.close(); try writeFile(wf, dir, file.contents, basename); } else { try writeFile(wf, cache_dir, file.contents, basename); } - file.generated_file.path = try wf.builder.cache_root.join( - wf.builder.allocator, + file.generated_file.path = try b.cache_root.join( + b.allocator, &.{ cache_path, file.sub_path }, ); } @@ -186,32 +189,18 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } fn writeFile(wf: *WriteFileStep, dir: fs.Dir, contents: Contents, basename: []const u8) !void { + const b = wf.step.owner; // TODO after landing concurrency PR, improve error reporting here switch (contents) { .bytes => |bytes| return dir.writeFile(basename, bytes), .copy => |file_source| { - const source_path = file_source.getPath(wf.builder); + const source_path = file_source.getPath(b); const prev_status = try fs.Dir.updateFile(fs.cwd(), source_path, dir, basename, .{}); _ = prev_status; // TODO logging (affected by open PR regarding concurrency) }, } } -/// TODO consolidate this with the same function in RunStep? -/// Also properly deal with concurrency (see open PR) -fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn { - const i = man.failed_file_index orelse failWithSimpleError(err); - const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err); - const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; - std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path }); - std.process.exit(1); -} - -fn failWithSimpleError(err: anyerror) noreturn { - std.debug.print("{s}\n", .{@errorName(err)}); - std.process.exit(1); -} - const std = @import("../std.zig"); const Step = std.Build.Step; const fs = std.fs; diff --git a/src/main.zig b/src/main.zig index fcdb52c342..6bf7177a9e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4419,6 +4419,8 @@ pub const usage_build = \\Options: \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error \\ -fno-reference-trace Disable reference trace + \\ -fsummary Print the build summary, even on success + \\ -fno-summary Omit the build summary, even on failure \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory @@ -4920,8 +4922,6 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void }; defer tree.deinit(gpa); - try printAstErrorsToStderr(gpa, tree, "", color); - var has_ast_error = false; if (check_ast_flag) { var file: Module.File = .{ .status = .never_loaded, @@ -4957,11 +4957,11 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(ttyconf); - has_ast_error = true; + process.exit(2); } - } - if (tree.errors.len != 0 or has_ast_error) { - process.exit(1); + } else if (tree.errors.len != 0) { + try printAstErrorsToStderr(gpa, tree, "", color); + process.exit(2); } const formatted = try tree.render(gpa); defer gpa.free(formatted); diff --git a/test/src/compare_output.zig b/test/src/compare_output.zig index 3bda3bdacd..20bd62d8e2 100644 --- a/test/src/compare_output.zig +++ b/test/src/compare_output.zig @@ -166,9 +166,7 @@ pub const CompareOutputContext = struct { const run = exe.run(); run.addArgs(case.cli_args); - run.stderr_action = .ignore; - run.stdout_action = .ignore; - run.expected_term = .{ .Exited = 126 }; + run.expectExitCode(126); self.step.dependOn(&run.step); }, diff --git a/test/tests.zig b/test/tests.zig index fceaa173d1..494ae0ea41 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -858,10 +858,11 @@ pub const StackTracesContext = struct { const allocator = context.b.allocator; const ptr = allocator.create(RunAndCompareStep) catch unreachable; ptr.* = RunAndCompareStep{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .custom, .name = "StackTraceCompareOutputStep", .makeFn = make, + .owner = context.b, }), .context = context, .exe = exe, @@ -1121,10 +1122,7 @@ pub const StandaloneContext = struct { defer zig_args.resize(zig_args_base_len) catch unreachable; const run_cmd = b.addSystemCommand(zig_args.items); - const log_step = b.addLog("PASS {s} ({s})", .{ annotated_case_name, @tagName(optimize_mode) }); - log_step.step.dependOn(&run_cmd.step); - - self.step.dependOn(&log_step.step); + self.step.dependOn(&run_cmd.step); } } @@ -1150,10 +1148,7 @@ pub const StandaloneContext = struct { exe.linkSystemLibrary("c"); } - const log_step = b.addLog("PASS {s}", .{annotated_case_name}); - log_step.step.dependOn(&exe.step); - - self.step.dependOn(&log_step.step); + self.step.dependOn(&exe.step); } } }; @@ -1203,9 +1198,10 @@ pub const GenHContext = struct { const allocator = context.b.allocator; const ptr = allocator.create(GenHCmpOutputStep) catch unreachable; ptr.* = GenHCmpOutputStep{ - .step = Step.init(allocator, .{ + .step = Step.init(.{ .id = .custom, .name = "ParseCCmpOutput", + .owner = context.b, .makeFn = make, }), .context = context, -- cgit v1.2.3 From 9bf63b09963ca6ea1179dfaa9142498556bfac9d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 22:37:07 -0700 Subject: stage2: avoid linux-only APIs on other operating systems --- src/Compilation.zig | 2 +- src/link.zig | 26 ++++++++++++++++-------- src/link/Elf.zig | 58 +++++++++++++++++++++++++++++++---------------------- src/main.zig | 6 +++--- src/test.zig | 3 +-- 5 files changed, 57 insertions(+), 38 deletions(-) (limited to 'src/main.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index 478f931718..28e7c43702 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1847,7 +1847,7 @@ fn cleanupTmpArtifactDirectory( } } -pub fn hotCodeSwap(comp: *Compilation, prog_node: *std.Progress.Node, pid: std.os.pid_t) !void { +pub fn hotCodeSwap(comp: *Compilation, prog_node: *std.Progress.Node, pid: std.ChildProcess.Id) !void { comp.bin_file.child_pid = pid; try comp.makeBinFileWritable(); try comp.update(prog_node); diff --git a/src/link.zig b/src/link.zig index 96931dd79e..e68f9c97d0 100644 --- a/src/link.zig +++ b/src/link.zig @@ -264,7 +264,7 @@ pub const File = struct { /// of this linking operation. lock: ?Cache.Lock = null, - child_pid: ?std.os.pid_t = null, + child_pid: ?std.ChildProcess.Id = null, /// Attempts incremental linking, if the file already exists. If /// incremental linking fails, falls back to truncating the file and @@ -388,10 +388,14 @@ pub const File = struct { }); try emit.directory.handle.copyFile(emit.sub_path, emit.directory.handle, tmp_sub_path, .{}); try emit.directory.handle.rename(tmp_sub_path, emit.sub_path); - - switch (std.os.errno(std.os.linux.ptrace(std.os.linux.PTRACE.ATTACH, pid, 0, 0, 0))) { - .SUCCESS => {}, - else => |errno| log.warn("ptrace failure: {s}", .{@tagName(errno)}), + switch (builtin.os.tag) { + .linux => { + switch (std.os.errno(std.os.linux.ptrace(std.os.linux.PTRACE.ATTACH, pid, 0, 0, 0))) { + .SUCCESS => {}, + else => |errno| log.warn("ptrace failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, } } base.file = try emit.directory.handle.createFile(emit.sub_path, .{ @@ -444,9 +448,14 @@ pub const File = struct { base.file = null; if (base.child_pid) |pid| { - switch (std.os.errno(std.os.linux.ptrace(std.os.linux.PTRACE.DETACH, pid, 0, 0, 0))) { - .SUCCESS => {}, - else => |errno| log.warn("ptrace failure: {s}", .{@tagName(errno)}), + switch (builtin.os.tag) { + .linux => { + switch (std.os.errno(std.os.linux.ptrace(std.os.linux.PTRACE.DETACH, pid, 0, 0, 0))) { + .SUCCESS => {}, + else => |errno| log.warn("ptrace failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, } } }, @@ -487,6 +496,7 @@ pub const File = struct { NetNameDeleted, DeviceBusy, InvalidArgument, + HotSwapUnavailableOnHostOperatingSystem, }; /// Called from within the CodeGen to lower a local variable instantion as an unnamed diff --git a/src/link/Elf.zig b/src/link/Elf.zig index b849dcc9d2..5943277908 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2453,18 +2453,23 @@ fn updateDeclCode(self: *Elf, decl_index: Module.Decl.Index, code: []const u8, s const file_offset = self.sections.items(.shdr)[shdr_index].sh_offset + section_offset; if (self.base.child_pid) |pid| { - var code_vec: [1]std.os.iovec_const = .{.{ - .iov_base = code.ptr, - .iov_len = code.len, - }}; - var remote_vec: [1]std.os.iovec_const = .{.{ - .iov_base = @intToPtr([*]u8, local_sym.st_value), - .iov_len = code.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); - switch (std.os.errno(rc)) { - .SUCCESS => assert(rc == code.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + switch (builtin.os.tag) { + .linux => { + var code_vec: [1]std.os.iovec_const = .{.{ + .iov_base = code.ptr, + .iov_len = code.len, + }}; + var remote_vec: [1]std.os.iovec_const = .{.{ + .iov_base = @intToPtr([*]u8, local_sym.st_value), + .iov_len = code.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); + switch (std.os.errno(rc)) { + .SUCCESS => assert(rc == code.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, } } @@ -2856,18 +2861,23 @@ fn writeOffsetTableEntry(self: *Elf, index: usize) !void { try self.base.file.?.pwriteAll(&buf, off); if (self.base.child_pid) |pid| { - var local_vec: [1]std.os.iovec_const = .{.{ - .iov_base = &buf, - .iov_len = buf.len, - }}; - var remote_vec: [1]std.os.iovec_const = .{.{ - .iov_base = @intToPtr([*]u8, vaddr), - .iov_len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.errno(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.os.iovec_const = .{.{ + .iov_base = &buf, + .iov_len = buf.len, + }}; + var remote_vec: [1]std.os.iovec_const = .{.{ + .iov_base = @intToPtr([*]u8, vaddr), + .iov_len = buf.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.errno(rc)) { + .SUCCESS => assert(rc == buf.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, } } }, diff --git a/src/main.zig b/src/main.zig index 6bf7177a9e..70051d2cc7 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3534,7 +3534,7 @@ fn serve( try serveStringMessage(out, .zig_version, build_options.version); - var child_pid: ?i32 = null; + var child_pid: ?std.ChildProcess.Id = null; var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa); defer receive_fifo.deinit(); @@ -3978,7 +3978,7 @@ fn runOrTestHotSwap( arg_mode: ArgMode, all_args: []const []const u8, runtime_args_start: ?usize, -) !i32 { +) !std.ChildProcess.Id { const exe_emit = comp.bin_file.options.emit.?; // A naive `directory.join` here will indeed get the correct path to the binary, // however, in the case of cwd, we actually want `./foo` so that the path can be executed. @@ -4023,7 +4023,7 @@ fn runOrTestHotSwap( try child.spawn(); - return child.pid; + return child.id; } const AfterUpdateHook = union(enum) { diff --git a/src/test.zig b/src/test.zig index 663c4f1aff..5b73e516c6 100644 --- a/src/test.zig +++ b/src/test.zig @@ -1606,9 +1606,8 @@ pub const TestContext = struct { var module_node = update_node.start("parse/analysis/codegen", 0); module_node.activate(); - module_node.context.refresh(); try comp.makeBinFileWritable(); - try comp.update(); + try comp.update(&module_node); module_node.end(); if (update.case != .Error) { -- cgit v1.2.3 From d6f7766da298accfa84120dbc7f770a6ab5b25e0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 23:46:25 -0700 Subject: stage2: avoid networking when generating zig2.c Avoid dragging networking into zig2.c because it adds dependencies on some linker symbols that are annoying to satisfy while bootstrapping. --- src/main.zig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index 70051d2cc7..93082a964d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -668,9 +668,13 @@ const ArgMode = union(enum) { run, }; +/// Avoid dragging networking into zig2.c because it adds dependencies on some +/// linker symbols that are annoying to satisfy while bootstrapping. +const Ip4Address = if (build_options.omit_pkg_fetching_code) void else std.net.Ip4Address; + const Listen = union(enum) { none, - ip4: std.net.Ip4Address, + ip4: Ip4Address, stdio, }; @@ -1159,6 +1163,7 @@ fn buildOutputType( listen = .stdio; watch = true; } else { + if (build_options.omit_pkg_fetching_code) unreachable; // example: --listen 127.0.0.1:9000 var it = std.mem.split(u8, next_arg, ":"); const host = it.next().?; @@ -3303,6 +3308,8 @@ fn buildOutputType( return cleanExit(); }, .ip4 => |ip4_addr| { + if (build_options.omit_pkg_fetching_code) unreachable; + var server = std.net.StreamServer.init(.{ .reuse_address = true, }); -- cgit v1.2.3 From 677a0e294116b0fcc1d69eea99c2fa62eb9873fe Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 23:56:56 -0700 Subject: stage2: avoid bloat when using -Donly-c --- build.zig | 2 +- src/main.zig | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/main.zig') diff --git a/build.zig b/build.zig index d32f666dac..40210500d8 100644 --- a/build.zig +++ b/build.zig @@ -192,7 +192,7 @@ pub fn build(b: *std.Build) !void { exe_options.addOption(bool, "llvm_has_arc", llvm_has_arc); exe_options.addOption(bool, "force_gpa", force_gpa); exe_options.addOption(bool, "only_c", only_c); - exe_options.addOption(bool, "omit_pkg_fetching_code", false); + exe_options.addOption(bool, "omit_pkg_fetching_code", only_c); if (link_libc) { exe.linkLibC(); diff --git a/src/main.zig b/src/main.zig index 93082a964d..f81911ffd0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3295,6 +3295,7 @@ fn buildOutputType( switch (listen) { .none => {}, .stdio => { + if (build_options.only_c) unreachable; try serve( comp, std.io.getStdIn(), -- cgit v1.2.3 From 2996eb558756c697e2ccfe9691356e536c88916b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 3 Mar 2023 16:39:08 -0700 Subject: std.Build.RunStep: add maxrss, duration, and cached status --- lib/std/Build/ObjCopyStep.zig | 2 +- lib/std/Build/Step.zig | 10 +++++++++- lib/std/zig/Server.zig | 13 ++++++++++++- src/Compilation.zig | 3 +++ src/main.zig | 41 +++++++++++++++++++++++++++++++++-------- 5 files changed, 58 insertions(+), 11 deletions(-) (limited to 'src/main.zig') diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/ObjCopyStep.zig index 13046b3efe..5f675ed383 100644 --- a/lib/std/Build/ObjCopyStep.zig +++ b/lib/std/Build/ObjCopyStep.zig @@ -113,7 +113,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }; try argv.appendSlice(&.{ full_src_path, full_dest_path }); - _ = try step.spawnZigProcess(argv.items, prog_node); + _ = try step.evalZigProcess(argv.items, prog_node); self.output_file.path = full_dest_path; try man.writeManifest(); diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index be14373afe..37bd66678b 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -234,10 +234,12 @@ pub fn evalZigProcess( child.stdin_behavior = .Pipe; child.stdout_behavior = .Pipe; child.stderr_behavior = .Pipe; + child.request_resource_usage_statistics = true; child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err), }); + var timer = try std.time.Timer.start(); var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ .stdout = child.stdout.?, @@ -301,7 +303,10 @@ pub fn evalZigProcess( sub_prog_node.?.activate(); }, .emit_bin_path => { - result = try arena.dupe(u8, body); + const EbpHdr = std.zig.Server.Message.EmitBinPath; + const ebp_hdr = @ptrCast(*align(1) const EbpHdr, body); + s.result_cached = ebp_hdr.flags.cache_hit; + result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); }, _ => { // Unrecognized message. @@ -323,6 +328,9 @@ pub fn evalZigProcess( const term = child.wait() catch |err| { return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); }; + s.result_duration_ns = timer.read(); + s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + try handleChildProcessTerm(s, term, null, argv); if (s.result_error_bundle.errorMessageCount() > 0) { diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 76f2303f6b..d34b2193e9 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -12,7 +12,7 @@ pub const Message = struct { error_bundle, /// Body is a UTF-8 string. progress, - /// Body is a UTF-8 string. + /// Body is a EmitBinPath. emit_bin_path, _, }; @@ -25,4 +25,15 @@ pub const Message = struct { extra_len: u32, string_bytes_len: u32, }; + + /// Trailing: + /// * the file system path the emitted binary can be found + pub const EmitBinPath = extern struct { + flags: Flags, + + pub const Flags = packed struct(u8) { + cache_hit: bool, + reserved: u7 = 0, + }; + }; }; diff --git a/src/Compilation.zig b/src/Compilation.zig index 28e7c43702..b542c86511 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -100,6 +100,7 @@ job_queued_compiler_rt_lib: bool = false, job_queued_compiler_rt_obj: bool = false, alloc_failure_occurred: bool = false, formatted_panics: bool = false, +last_update_was_cache_hit: bool = false, c_source_files: []const CSourceFile, clang_argv: []const []const u8, @@ -1860,6 +1861,7 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void defer tracy_trace.end(); comp.clearMiscFailures(); + comp.last_update_was_cache_hit = false; var man: Cache.Manifest = undefined; defer if (comp.whole_cache_manifest != null) man.deinit(); @@ -1887,6 +1889,7 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void return err; }; if (is_hit) { + comp.last_update_was_cache_hit = true; log.debug("CacheMode.whole cache hit for {s}", .{comp.bin_file.options.root_name}); const digest = man.final(); diff --git a/src/main.zig b/src/main.zig index f81911ffd0..699122d26f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3578,9 +3578,11 @@ fn serve( var arena_instance = std.heap.ArenaAllocator.init(gpa); defer arena_instance.deinit(); const arena = arena_instance.allocator(); - var output_path: []const u8 = undefined; - try cmdTranslateC(comp, arena, &output_path); - try serveStringMessage(out, .emit_bin_path, output_path); + var output: TranslateCOutput = undefined; + try cmdTranslateC(comp, arena, &output); + try serveEmitBinPath(out, output.path, .{ + .flags = .{ .cache_hit = output.cache_hit }, + }); continue; } @@ -3760,10 +3762,26 @@ fn serveUpdateResults(out: fs.File, comp: *Compilation) !void { } else if (comp.bin_file.options.emit) |emit| { const full_path = try emit.directory.join(gpa, &.{emit.sub_path}); defer gpa.free(full_path); - try serveStringMessage(out, .emit_bin_path, full_path); + try serveEmitBinPath(out, full_path, .{ + .flags = .{ .cache_hit = comp.last_update_was_cache_hit }, + }); } } +fn serveEmitBinPath( + out: fs.File, + fs_path: []const u8, + header: std.zig.Server.Message.EmitBinPath, +) !void { + try serveMessage(out, .{ + .tag = .emit_bin_path, + .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)), + }, &.{ + std.mem.asBytes(&header), + fs_path, + }); +} + fn serveStringMessage(out: fs.File, tag: std.zig.Server.Message.Tag, s: []const u8) !void { try serveMessage(out, .{ .tag = tag, @@ -4115,7 +4133,12 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void } } -fn cmdTranslateC(comp: *Compilation, arena: Allocator, output_path: ?*[]const u8) !void { +const TranslateCOutput = struct { + path: []const u8, + cache_hit: bool, +}; + +fn cmdTranslateC(comp: *Compilation, arena: Allocator, fancy_output: ?*TranslateCOutput) !void { if (!build_options.have_llvm) fatal("cannot translate-c: compiler built without LLVM extensions", .{}); @@ -4126,14 +4149,16 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, output_path: ?*[]const u8 var man: Cache.Manifest = comp.obtainCObjectCacheManifest(); man.want_shared_lock = false; - defer if (output_path != null) man.deinit(); + defer man.deinit(); man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| { fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) }); }; + if (fancy_output) |p| p.cache_hit = true; const digest = if (try man.hit()) man.final() else digest: { + if (fancy_output) |p| p.cache_hit = false; var argv = std.ArrayList([]const u8).init(arena); try argv.append(""); // argv[0] is program name, actual args start at [1] @@ -4229,11 +4254,11 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, output_path: ?*[]const u8 break :digest digest; }; - if (output_path) |out_path| { + if (fancy_output) |p| { const full_zig_path = try comp.local_cache_directory.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename, }); - out_path.* = full_zig_path; + p.path = full_zig_path; } else { const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename }); const zig_file = comp.local_cache_directory.handle.openFile(out_zig_path, .{}) catch |err| { -- cgit v1.2.3 From 7106a91b097c5ac0feb13f328bec5e6788f8c8ef Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Mar 2023 14:18:25 -0700 Subject: CLI: fix ast-check printing ZIR errors twice --- src/main.zig | 67 ++++++++++++++---------------------------------------------- 1 file changed, 15 insertions(+), 52 deletions(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index 699122d26f..669f1afd0c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4082,12 +4082,7 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void defer errors.deinit(comp.gpa); if (errors.errorMessageCount() > 0) { - const ttyconf: std.debug.TTY.Config = switch (comp.color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; - errors.renderToStdErr(ttyconf); + errors.renderToStdErr(get_tty_conf(comp.color)); const log_text = comp.getCompileLogOutput(); if (log_text.len != 0) { std.debug.print("\nCompile Log Output:\n{s}", .{log_text}); @@ -4714,14 +4709,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &all_modules, ); if (wip_errors.root_list.items.len > 0) { - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; var errors = try wip_errors.toOwnedBundle(); defer errors.deinit(gpa); - errors.renderToStdErr(ttyconf); + errors.renderToStdErr(get_tty_conf(color)); process.exit(1); } try fetch_result; @@ -4767,7 +4757,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi defer comp.destroy(); updateModule(gpa, comp, .none) catch |err| switch (err) { - error.SemanticAnalyzeFail => process.exit(1), + error.SemanticAnalyzeFail => process.exit(2), else => |e| return e, }; try comp.makeBinFileExecutable(); @@ -4982,14 +4972,9 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(get_tty_conf(color)); process.exit(2); } } else if (tree.errors.len != 0) { @@ -5193,14 +5178,9 @@ fn fmtPathFile( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - const ttyconf: std.debug.TTY.Config = switch (fmt.color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(get_tty_conf(fmt.color)); fmt.any_error = true; } } @@ -5235,14 +5215,9 @@ fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Co try putAstErrorsIntoBundle(gpa, tree, path, &wip_errors); - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(get_tty_conf(color)); } pub fn putAstErrorsIntoBundle( @@ -5848,11 +5823,6 @@ pub fn cmdAstCheck( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printAstErrorsToStderr(gpa, file.tree, file.sub_file_path, color); - if (file.tree.errors.len != 0) { - process.exit(1); - } - file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; defer file.zir.deinit(gpa); @@ -5862,14 +5832,9 @@ pub fn cmdAstCheck( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(std.io.getStdErr()), - .on => .escape_codes, - .off => .no_color, - }; var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(get_tty_conf(color)); process.exit(1); } @@ -5974,11 +5939,6 @@ pub fn cmdChangelist( file.tree_loaded = true; defer file.tree.deinit(gpa); - try printAstErrorsToStderr(gpa, file.tree, old_source_file, .auto); - if (file.tree.errors.len != 0) { - process.exit(1); - } - file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; defer file.zir.deinit(gpa); @@ -6013,11 +5973,6 @@ pub fn cmdChangelist( var new_tree = try Ast.parse(gpa, new_source, .zig); defer new_tree.deinit(gpa); - try printAstErrorsToStderr(gpa, new_tree, new_source_file, .auto); - if (new_tree.errors.len != 0) { - process.exit(1); - } - var old_zir = file.zir; defer old_zir.deinit(gpa); file.zir_loaded = false; @@ -6293,3 +6248,11 @@ const ClangSearchSanitizer = struct { iframework: bool = false, }; }; + +fn get_tty_conf(color: Color) std.debug.TTY.Config { + return switch (color) { + .auto => std.debug.detectTTYConfig(std.io.getStdErr()), + .on => .escape_codes, + .off => .no_color, + }; +} -- cgit v1.2.3 From 7cc4a6965c28e427cbfba57a985f837734d6257e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Mar 2023 18:13:55 -0700 Subject: build runner enhancements in preparation for test-cases * std.zig.ErrorBundle: support rendering options for whether to include the reference trace, whether to include the source line, and TTY configuration. * build runner: don't print progress in dumb terminals * std.Build.CompileStep: - add a way to expect compilation errors via the new `expect_errors` field. This is an advanced setting that can change the intent of the CompileStep. If this slice has nonzero length, it means that the CompileStep exists to check for compile errors and return *success* if they match, and failure otherwise. - remove the object format parameter from `checkObject`. The object format is known based on the CompileStep's target. - Avoid passing -L and -I flags for nonexistent directories within search_prefixes. This prevents a warning, that should probably be upgraded to an error in Zig's CLI parsing code, when the linker sees an -L directory that does not exist. * std.Build.Step: - When spawning the zig compiler process, takes advantage of the new `std.Progress.Node.setName` API to avoid ticking up a meaningless number at every progress update. --- lib/build_runner.zig | 30 +++++++---- lib/std/Build/CompileStep.zig | 113 ++++++++++++++++++++++++++++++++++++++---- lib/std/Build/Step.zig | 20 ++++++-- lib/std/zig/ErrorBundle.zig | 29 ++++++----- src/Sema.zig | 2 +- src/main.zig | 28 +++++++---- 6 files changed, 173 insertions(+), 49 deletions(-) (limited to 'src/main.zig') diff --git a/lib/build_runner.zig b/lib/build_runner.zig index bb04b3e132..8e0ebb4558 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -84,8 +84,6 @@ pub fn main() !void { ); defer builder.destroy(); - const Color = enum { auto, off, on }; - var targets = ArrayList([]const u8).init(arena); var debug_log_scopes = ArrayList([]const u8).init(arena); var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena }; @@ -273,13 +271,9 @@ pub fn main() !void { } const stderr = std.io.getStdErr(); - const ttyconf: std.debug.TTY.Config = switch (color) { - .auto => std.debug.detectTTYConfig(stderr), - .on => .escape_codes, - .off => .no_color, - }; + const ttyconf = get_tty_conf(color, stderr); - var progress: std.Progress = .{}; + var progress: std.Progress = .{ .dont_print_on_dumb = true }; const main_progress_node = progress.start("", 0); builder.debug_log_scopes = debug_log_scopes.items; @@ -498,7 +492,7 @@ fn runStepNames( if (total_compile_errors > 0) { for (compile_error_steps.items) |s| { if (s.result_error_bundle.errorMessageCount() > 0) { - s.result_error_bundle.renderToStdErr(ttyconf); + s.result_error_bundle.renderToStdErr(renderOptions(ttyconf)); } } @@ -961,3 +955,21 @@ fn cleanExit() void { // of calling exit. process.exit(0); } + +const Color = enum { auto, off, on }; + +fn get_tty_conf(color: Color, stderr: std.fs.File) std.debug.TTY.Config { + return switch (color) { + .auto => std.debug.detectTTYConfig(stderr), + .on => .escape_codes, + .off => .no_color, + }; +} + +fn renderOptions(ttyconf: std.debug.TTY.Config) std.zig.ErrorBundle.RenderOptions { + return .{ + .ttyconf = ttyconf, + .include_source_line = ttyconf != .no_color, + .include_reference_trace = ttyconf != .no_color, + }; +} diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 42cc63e8b4..5c44ab82d3 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -207,6 +207,12 @@ want_lto: ?bool = null, use_llvm: ?bool = null, use_lld: ?bool = null, +/// This is an advanced setting that can change the intent of this CompileStep. +/// If this slice has nonzero length, it means that this CompileStep exists to +/// check for compile errors and return *success* if they match, and failure +/// otherwise. +expect_errors: []const []const u8 = &.{}, + output_path_source: GeneratedFile, output_lib_path_source: GeneratedFile, output_h_path_source: GeneratedFile, @@ -552,8 +558,8 @@ pub fn run(cs: *CompileStep) *RunStep { return cs.step.owner.addRunArtifact(cs); } -pub fn checkObject(self: *CompileStep, obj_format: std.Target.ObjectFormat) *CheckObjectStep { - return CheckObjectStep.create(self.step.owner, self.getOutputSource(), obj_format); +pub fn checkObject(self: *CompileStep) *CheckObjectStep { + return CheckObjectStep.create(self.step.owner, self.getOutputSource(), self.target_info.target.ofmt); } pub fn setLinkerScriptPath(self: *CompileStep, source: FileSource) void { @@ -1838,14 +1844,38 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { } for (b.search_prefixes.items) |search_prefix| { - try zig_args.append("-L"); - try zig_args.append(b.pathJoin(&.{ - search_prefix, "lib", - })); - try zig_args.append("-I"); - try zig_args.append(b.pathJoin(&.{ - search_prefix, "include", - })); + var prefix_dir = fs.cwd().openDir(search_prefix, .{}) catch |err| { + return step.fail("unable to open prefix directory '{s}': {s}", .{ + search_prefix, @errorName(err), + }); + }; + defer prefix_dir.close(); + + // Avoid passing -L and -I flags for nonexistent directories. + // This prevents a warning, that should probably be upgraded to an error in Zig's + // CLI parsing code, when the linker sees an -L directory that does not exist. + + if (prefix_dir.accessZ("lib", .{})) |_| { + try zig_args.appendSlice(&.{ + "-L", try fs.path.join(b.allocator, &.{ search_prefix, "lib" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/lib' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } + + if (prefix_dir.accessZ("include", .{})) |_| { + try zig_args.appendSlice(&.{ + "-I", try fs.path.join(b.allocator, &.{ search_prefix, "include" }), + }); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return step.fail("unable to access '{s}/include' directory: {s}", .{ + search_prefix, @errorName(e), + }), + } } try addFlag(&zig_args, "valgrind", self.valgrind_support); @@ -1943,7 +1973,14 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append(resolved_args_file); } - const output_bin_path = try step.evalZigProcess(zig_args.items, prog_node); + const output_bin_path = step.evalZigProcess(zig_args.items, prog_node) catch |err| switch (err) { + error.NeedCompileErrorCheck => { + assert(self.expect_errors.len != 0); + try checkCompileErrors(self); + return; + }, + else => |e| return e, + }; const build_output_dir = fs.path.dirname(output_bin_path).?; if (self.output_dir) |output_dir| { @@ -2178,3 +2215,57 @@ const TransitiveDeps = struct { } } }; + +fn checkCompileErrors(self: *CompileStep) !void { + // Clear this field so that it does not get printed by the build runner. + const actual_eb = self.step.result_error_bundle; + self.step.result_error_bundle = std.zig.ErrorBundle.empty; + + const arena = self.step.owner.allocator; + + var actual_stderr_list = std.ArrayList(u8).init(arena); + try actual_eb.renderToWriter(.{ + .ttyconf = .no_color, + .include_reference_trace = false, + .include_source_line = false, + }, actual_stderr_list.writer()); + const actual_stderr = try actual_stderr_list.toOwnedSlice(); + + // Render the expected lines into a string that we can compare verbatim. + var expected_generated = std.ArrayList(u8).init(arena); + + var actual_line_it = mem.split(u8, actual_stderr, "\n"); + for (self.expect_errors) |expect_line| { + const actual_line = actual_line_it.next() orelse { + try expected_generated.appendSlice(expect_line); + try expected_generated.append('\n'); + continue; + }; + if (mem.endsWith(u8, actual_line, expect_line)) { + try expected_generated.appendSlice(actual_line); + try expected_generated.append('\n'); + continue; + } + if (mem.startsWith(u8, expect_line, ":?:?: ")) { + if (mem.endsWith(u8, actual_line, expect_line[":?:?: ".len..])) { + try expected_generated.appendSlice(actual_line); + try expected_generated.append('\n'); + continue; + } + } + try expected_generated.appendSlice(expect_line); + try expected_generated.append('\n'); + } + + if (mem.eql(u8, expected_generated.items, actual_stderr)) return; + + // TODO merge this with the testing.expectEqualStrings logic, and also CheckFile + return self.step.fail( + \\ + \\========= expected: ===================== + \\{s} + \\========= but found: ==================== + \\{s} + \\========================================= + , .{ expected_generated.items, actual_stderr }); +} diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index f1edab5881..45aa635972 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -295,8 +295,8 @@ pub fn evalZigProcess( var node_name: std.ArrayListUnmanaged(u8) = .{}; defer node_name.deinit(gpa); - var sub_prog_node: ?std.Progress.Node = null; - defer if (sub_prog_node) |*n| n.end(); + var sub_prog_node = prog_node.start("", 0); + defer sub_prog_node.end(); const stdout = poller.fifo(.stdout); @@ -336,11 +336,9 @@ pub fn evalZigProcess( }; }, .progress => { - if (sub_prog_node) |*n| n.end(); node_name.clearRetainingCapacity(); try node_name.appendSlice(gpa, body); - sub_prog_node = prog_node.start(node_name.items, 0); - sub_prog_node.?.activate(); + sub_prog_node.setName(node_name.items); }, .emit_bin_path => { const EbpHdr = std.zig.Server.Message.EmitBinPath; @@ -371,6 +369,18 @@ pub fn evalZigProcess( s.result_duration_ns = timer.read(); s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; + // Special handling for CompileStep that is expecting compile errors. + if (s.cast(Build.CompileStep)) |compile| switch (term) { + .Exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors.len != 0 and s.result_error_bundle.errorMessageCount() > 0) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + try handleChildProcessTerm(s, term, null, argv); if (s.result_error_bundle.errorMessageCount() > 0) { diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index 94f80797a8..845e9d8ff5 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -141,32 +141,35 @@ pub fn nullTerminatedString(eb: ErrorBundle, index: usize) [:0]const u8 { return string_bytes[index..end :0]; } -pub fn renderToStdErr(eb: ErrorBundle, ttyconf: std.debug.TTY.Config) void { +pub const RenderOptions = struct { + ttyconf: std.debug.TTY.Config, + include_reference_trace: bool = true, + include_source_line: bool = true, +}; + +pub fn renderToStdErr(eb: ErrorBundle, options: RenderOptions) void { std.debug.getStderrMutex().lock(); defer std.debug.getStderrMutex().unlock(); const stderr = std.io.getStdErr(); - return renderToWriter(eb, ttyconf, stderr.writer()) catch return; + return renderToWriter(eb, options, stderr.writer()) catch return; } -pub fn renderToWriter( - eb: ErrorBundle, - ttyconf: std.debug.TTY.Config, - writer: anytype, -) anyerror!void { +pub fn renderToWriter(eb: ErrorBundle, options: RenderOptions, writer: anytype) anyerror!void { for (eb.getMessages()) |err_msg| { - try renderErrorMessageToWriter(eb, err_msg, ttyconf, writer, "error", .Red, 0); + try renderErrorMessageToWriter(eb, options, err_msg, writer, "error", .Red, 0); } } fn renderErrorMessageToWriter( eb: ErrorBundle, + options: RenderOptions, err_msg_index: MessageIndex, - ttyconf: std.debug.TTY.Config, stderr: anytype, kind: []const u8, color: std.debug.TTY.Color, indent: usize, ) anyerror!void { + const ttyconf = options.ttyconf; var counting_writer = std.io.countingWriter(stderr); const counting_stderr = counting_writer.writer(); const err_msg = eb.getErrorMessage(err_msg_index); @@ -196,7 +199,7 @@ fn renderErrorMessageToWriter( try stderr.print(" ({d} times)\n", .{err_msg.count}); } try ttyconf.setColor(stderr, .Reset); - if (src.data.source_line != 0) { + if (src.data.source_line != 0 and options.include_source_line) { const line = eb.nullTerminatedString(src.data.source_line); for (line) |b| switch (b) { '\t' => try stderr.writeByte(' '), @@ -216,9 +219,9 @@ fn renderErrorMessageToWriter( try ttyconf.setColor(stderr, .Reset); } for (eb.getNotes(err_msg_index)) |note| { - try renderErrorMessageToWriter(eb, note, ttyconf, stderr, "note", .Cyan, indent); + try renderErrorMessageToWriter(eb, options, note, stderr, "note", .Cyan, indent); } - if (src.data.reference_trace_len > 0) { + if (src.data.reference_trace_len > 0 and options.include_reference_trace) { try ttyconf.setColor(stderr, .Reset); try ttyconf.setColor(stderr, .Dim); try stderr.print("referenced by:\n", .{}); @@ -266,7 +269,7 @@ fn renderErrorMessageToWriter( } try ttyconf.setColor(stderr, .Reset); for (eb.getNotes(err_msg_index)) |note| { - try renderErrorMessageToWriter(eb, note, ttyconf, stderr, "note", .Cyan, indent + 4); + try renderErrorMessageToWriter(eb, options, note, stderr, "note", .Cyan, indent + 4); } } } diff --git a/src/Sema.zig b/src/Sema.zig index 03ebbbdbac..c9f4f27fe2 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2220,7 +2220,7 @@ fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { Compilation.addModuleErrorMsg(&wip_errors, err_msg.*) catch unreachable; std.debug.print("compile error during Sema:\n", .{}); var error_bundle = wip_errors.toOwnedBundle() catch unreachable; - error_bundle.renderToStdErr(.no_color); + error_bundle.renderToStdErr(.{ .ttyconf = .no_color }); crash_report.compilerPanic("unexpected compile error occurred", null, null); } diff --git a/src/main.zig b/src/main.zig index 669f1afd0c..03d746af0c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4082,7 +4082,7 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void defer errors.deinit(comp.gpa); if (errors.errorMessageCount() > 0) { - errors.renderToStdErr(get_tty_conf(comp.color)); + errors.renderToStdErr(renderOptions(comp.color)); const log_text = comp.getCompileLogOutput(); if (log_text.len != 0) { std.debug.print("\nCompile Log Output:\n{s}", .{log_text}); @@ -4711,7 +4711,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi if (wip_errors.root_list.items.len > 0) { var errors = try wip_errors.toOwnedBundle(); defer errors.deinit(gpa); - errors.renderToStdErr(get_tty_conf(color)); + errors.renderToStdErr(renderOptions(color)); process.exit(1); } try fetch_result; @@ -4974,7 +4974,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void try Compilation.addZirErrorMessages(&wip_errors, &file); var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(get_tty_conf(color)); + error_bundle.renderToStdErr(renderOptions(color)); process.exit(2); } } else if (tree.errors.len != 0) { @@ -5180,7 +5180,7 @@ fn fmtPathFile( try Compilation.addZirErrorMessages(&wip_errors, &file); var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(get_tty_conf(fmt.color)); + error_bundle.renderToStdErr(renderOptions(fmt.color)); fmt.any_error = true; } } @@ -5217,7 +5217,7 @@ fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Co var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(get_tty_conf(color)); + error_bundle.renderToStdErr(renderOptions(color)); } pub fn putAstErrorsIntoBundle( @@ -5834,7 +5834,7 @@ pub fn cmdAstCheck( try Compilation.addZirErrorMessages(&wip_errors, &file); var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(get_tty_conf(color)); + error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); } @@ -5892,6 +5892,7 @@ pub fn cmdChangelist( arena: Allocator, args: []const []const u8, ) !void { + const color: Color = .auto; const Zir = @import("Zir.zig"); const old_source_file = args[0]; @@ -5948,10 +5949,9 @@ pub fn cmdChangelist( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); } @@ -5984,10 +5984,9 @@ pub fn cmdChangelist( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - const ttyconf = std.debug.detectTTYConfig(std.io.getStdErr()); var error_bundle = try wip_errors.toOwnedBundle(); defer error_bundle.deinit(gpa); - error_bundle.renderToStdErr(ttyconf); + error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); } @@ -6256,3 +6255,12 @@ fn get_tty_conf(color: Color) std.debug.TTY.Config { .off => .no_color, }; } + +fn renderOptions(color: Color) std.zig.ErrorBundle.RenderOptions { + const ttyconf = get_tty_conf(color); + return .{ + .ttyconf = ttyconf, + .include_source_line = ttyconf != .no_color, + .include_reference_trace = ttyconf != .no_color, + }; +} -- cgit v1.2.3 From a333bb91ffcf63fbe377ddc82fd7f1633e269430 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Mar 2023 21:17:56 -0700 Subject: zig objcopy: support the compiler protocol This commit extracts out server code into src/Server.zig and uses it both in the main CLI as well as `zig objcopy`. std.Build.ObjCopyStep now adds `--listen=-` to the CLI for `zig objcopy` and observes the protocol for progress and other kinds of integrations. This fixes the last two test failures of this branch when I run `zig build test` locally. --- CMakeLists.txt | 1 + lib/std/Build/ObjCopyStep.zig | 2 + src/Server.zig | 113 ++++++++++++++++++++++++++++++++++++++++++ src/main.zig | 111 +++++++---------------------------------- src/objcopy.zig | 47 ++++++++++++++++-- 5 files changed, 177 insertions(+), 97 deletions(-) create mode 100644 src/Server.zig (limited to 'src/main.zig') diff --git a/CMakeLists.txt b/CMakeLists.txt index c77c66add4..7ff8249f4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -623,6 +623,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/src/print_targets.zig" "${CMAKE_SOURCE_DIR}/src/print_zir.zig" "${CMAKE_SOURCE_DIR}/src/register_manager.zig" + "${CMAKE_SOURCE_DIR}/src/Server.zig" "${CMAKE_SOURCE_DIR}/src/target.zig" "${CMAKE_SOURCE_DIR}/src/tracy.zig" "${CMAKE_SOURCE_DIR}/src/translate_c.zig" diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/ObjCopyStep.zig index 5f675ed383..608c56591f 100644 --- a/lib/std/Build/ObjCopyStep.zig +++ b/lib/std/Build/ObjCopyStep.zig @@ -113,6 +113,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }; try argv.appendSlice(&.{ full_src_path, full_dest_path }); + + try argv.append("--listen=-"); _ = try step.evalZigProcess(argv.items, prog_node); self.output_file.path = full_dest_path; diff --git a/src/Server.zig b/src/Server.zig new file mode 100644 index 0000000000..a25dc93857 --- /dev/null +++ b/src/Server.zig @@ -0,0 +1,113 @@ +in: std.fs.File, +out: std.fs.File, +receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), + +pub const Options = struct { + gpa: Allocator, + in: std.fs.File, + out: std.fs.File, +}; + +pub fn init(options: Options) !Server { + var s: Server = .{ + .in = options.in, + .out = options.out, + .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), + }; + try s.serveStringMessage(.zig_version, build_options.version); + return s; +} + +pub fn deinit(s: *Server) void { + s.receive_fifo.deinit(); + s.* = undefined; +} + +pub fn receiveMessage(s: *Server) !InMessage.Header { + const Header = InMessage.Header; + const fifo = &s.receive_fifo; + + while (true) { + const buf = fifo.readableSlice(0); + assert(fifo.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + if (header.bytes_len != 0) + return error.InvalidClientMessage; + const result = header.*; + fifo.discard(@sizeOf(Header)); + return result; + } + + const write_buffer = try fifo.writableWithSize(256); + const amt = try s.in.read(write_buffer); + fifo.update(amt); + } +} + +pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { + return s.serveMessage(.{ + .tag = tag, + .bytes_len = @intCast(u32, msg.len), + }, &.{msg}); +} + +pub fn serveMessage( + s: *const Server, + header: OutMessage.Header, + bufs: []const []const u8, +) !void { + var iovecs: [10]std.os.iovec_const = undefined; + iovecs[0] = .{ + .iov_base = @ptrCast([*]const u8, &header), + .iov_len = @sizeOf(OutMessage.Header), + }; + for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { + iovec.* = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + try s.out.writevAll(iovecs[0 .. bufs.len + 1]); +} + +pub fn serveEmitBinPath( + s: *Server, + fs_path: []const u8, + header: std.zig.Server.Message.EmitBinPath, +) !void { + try s.serveMessage(.{ + .tag = .emit_bin_path, + .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)), + }, &.{ + std.mem.asBytes(&header), + fs_path, + }); +} + +pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { + const eb_hdr: std.zig.Server.Message.ErrorBundle = .{ + .extra_len = @intCast(u32, error_bundle.extra.len), + .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), + }; + const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) + + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; + try s.serveMessage(.{ + .tag = .error_bundle, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&eb_hdr), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(error_bundle.extra), + error_bundle.string_bytes, + }); +} + +const OutMessage = std.zig.Server.Message; +const InMessage = std.zig.Client.Message; + +const Server = @This(); +const std = @import("std"); +const build_options = @import("build_options"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; diff --git a/src/main.zig b/src/main.zig index 03d746af0c..e0283143d0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -26,6 +26,7 @@ const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); const Module = @import("Module.zig"); const AstGen = @import("AstGen.zig"); +const Server = @import("Server.zig"); pub const std_options = struct { pub const wasiCwd = wasi_cwd; @@ -3540,11 +3541,14 @@ fn serve( ) !void { const gpa = comp.gpa; - try serveStringMessage(out, .zig_version, build_options.version); + var server = try Server.init(.{ + .gpa = gpa, + .in = in, + .out = out, + }); + defer server.deinit(); var child_pid: ?std.ChildProcess.Id = null; - var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa); - defer receive_fifo.deinit(); var progress: std.Progress = .{ .terminal = null, @@ -3564,7 +3568,7 @@ fn serve( main_progress_node.context = &progress; while (true) { - const hdr = try receiveMessage(in, &receive_fifo); + const hdr = try server.receiveMessage(); switch (hdr.tag) { .exit => { @@ -3580,7 +3584,7 @@ fn serve( const arena = arena_instance.allocator(); var output: TranslateCOutput = undefined; try cmdTranslateC(comp, arena, &output); - try serveEmitBinPath(out, output.path, .{ + try server.serveEmitBinPath(output.path, .{ .flags = .{ .cache_hit = output.cache_hit }, }); continue; @@ -3594,7 +3598,7 @@ fn serve( var reset: std.Thread.ResetEvent = .{}; var progress_thread = try std.Thread.spawn(.{}, progressThread, .{ - &progress, out, &reset, + &progress, &server, &reset, }); defer { reset.set(); @@ -3605,7 +3609,7 @@ fn serve( } try comp.makeBinFileExecutable(); - try serveUpdateResults(out, comp); + try serveUpdateResults(&server, comp); }, .run => { if (child_pid != null) { @@ -3632,14 +3636,14 @@ fn serve( assert(main_progress_node.recently_updated_child == null); if (child_pid) |pid| { try comp.hotCodeSwap(main_progress_node, pid); - try serveUpdateResults(out, comp); + try serveUpdateResults(&server, comp); } else { if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } try comp.update(main_progress_node); try comp.makeBinFileExecutable(); - try serveUpdateResults(out, comp); + try serveUpdateResults(&server, comp); child_pid = try runOrTestHotSwap( comp, @@ -3659,7 +3663,7 @@ fn serve( } } -fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void { +fn progressThread(progress: *std.Progress, server: *const Server, reset: *std.Thread.ResetEvent) void { while (true) { if (reset.timedWait(500 * std.time.ns_per_ms)) |_| { // The Compilation update has completed. @@ -3705,7 +3709,7 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese const progress_string = buf.slice(); - serveMessage(out, .{ + server.serveMessage(.{ .tag = .progress, .bytes_len = @intCast(u32, progress_string.len), }, &.{ @@ -3716,100 +3720,21 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese } } -fn serveMessage( - out: fs.File, - header: std.zig.Server.Message.Header, - bufs: []const []const u8, -) !void { - var iovecs: [10]std.os.iovec_const = undefined; - iovecs[0] = .{ - .iov_base = @ptrCast([*]const u8, &header), - .iov_len = @sizeOf(std.zig.Server.Message.Header), - }; - for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { - iovec.* = .{ - .iov_base = buf.ptr, - .iov_len = buf.len, - }; - } - try out.writevAll(iovecs[0 .. bufs.len + 1]); -} - -fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void { - const eb_hdr: std.zig.Server.Message.ErrorBundle = .{ - .extra_len = @intCast(u32, error_bundle.extra.len), - .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), - }; - const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) + - 4 * error_bundle.extra.len + error_bundle.string_bytes.len; - try serveMessage(out, .{ - .tag = .error_bundle, - .bytes_len = @intCast(u32, bytes_len), - }, &.{ - std.mem.asBytes(&eb_hdr), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(error_bundle.extra), - error_bundle.string_bytes, - }); -} - -fn serveUpdateResults(out: fs.File, comp: *Compilation) !void { +fn serveUpdateResults(s: *Server, comp: *Compilation) !void { const gpa = comp.gpa; var error_bundle = try comp.getAllErrorsAlloc(); defer error_bundle.deinit(gpa); if (error_bundle.errorMessageCount() > 0) { - try serveErrorBundle(out, error_bundle); + try s.serveErrorBundle(error_bundle); } else if (comp.bin_file.options.emit) |emit| { const full_path = try emit.directory.join(gpa, &.{emit.sub_path}); defer gpa.free(full_path); - try serveEmitBinPath(out, full_path, .{ + try s.serveEmitBinPath(full_path, .{ .flags = .{ .cache_hit = comp.last_update_was_cache_hit }, }); } } -fn serveEmitBinPath( - out: fs.File, - fs_path: []const u8, - header: std.zig.Server.Message.EmitBinPath, -) !void { - try serveMessage(out, .{ - .tag = .emit_bin_path, - .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)), - }, &.{ - std.mem.asBytes(&header), - fs_path, - }); -} - -fn serveStringMessage(out: fs.File, tag: std.zig.Server.Message.Tag, s: []const u8) !void { - try serveMessage(out, .{ - .tag = tag, - .bytes_len = @intCast(u32, s.len), - }, &.{s}); -} - -fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header { - const Header = std.zig.Client.Message.Header; - - while (true) { - const buf = fifo.readableSlice(0); - assert(fifo.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); - if (header.bytes_len != 0) - return error.InvalidClientMessage; - const result = header.*; - fifo.discard(@sizeOf(Header)); - return result; - } - - const write_buffer = try fifo.writableWithSize(256); - const amt = try in.read(write_buffer); - fifo.update(amt); - } -} - const ModuleDepIterator = struct { split: mem.SplitIterator(u8), diff --git a/src/objcopy.zig b/src/objcopy.zig index 31e3d60d0d..e821a94b59 100644 --- a/src/objcopy.zig +++ b/src/objcopy.zig @@ -4,22 +4,25 @@ const fs = std.fs; const elf = std.elf; const Allocator = std.mem.Allocator; const File = std.fs.File; +const assert = std.debug.assert; + const main = @import("main.zig"); const fatal = main.fatal; const cleanExit = main.cleanExit; +const Server = @import("Server.zig"); pub fn cmdObjCopy( gpa: Allocator, arena: Allocator, args: []const []const u8, ) !void { - _ = gpa; var i: usize = 0; var opt_out_fmt: ?std.Target.ObjectFormat = null; var opt_input: ?[]const u8 = null; var opt_output: ?[]const u8 = null; var only_section: ?[]const u8 = null; var pad_to: ?u64 = null; + var listen = false; while (i < args.len) : (i += 1) { const arg = args[i]; if (!mem.startsWith(u8, arg, "-")) { @@ -54,6 +57,8 @@ pub fn cmdObjCopy( i += 1; if (i >= args.len) fatal("expected another argument after '{s}'", .{arg}); only_section = args[i]; + } else if (mem.eql(u8, arg, "--listen=-")) { + listen = true; } else if (mem.startsWith(u8, arg, "--only-section=")) { only_section = arg["--output-target=".len..]; } else if (mem.eql(u8, arg, "--pad-to")) { @@ -102,10 +107,44 @@ pub fn cmdObjCopy( .only_section = only_section, .pad_to = pad_to, }); - return cleanExit(); }, else => fatal("unsupported output object format: {s}", .{@tagName(out_fmt)}), } + + if (listen) { + var server = try Server.init(.{ + .gpa = gpa, + .in = std.io.getStdIn(), + .out = std.io.getStdOut(), + }); + defer server.deinit(); + + var seen_update = false; + while (true) { + const hdr = try server.receiveMessage(); + switch (hdr.tag) { + .exit => { + return cleanExit(); + }, + .update => { + if (seen_update) { + std.debug.print("zig objcopy only supports 1 update for now\n", .{}); + std.process.exit(1); + } + seen_update = true; + + try server.serveEmitBinPath(output, .{ + .flags = .{ .cache_hit = false }, + }); + }, + else => { + std.debug.print("unsupported message: {s}", .{@tagName(hdr.tag)}); + std.process.exit(1); + }, + } + } + } + return cleanExit(); } const usage = @@ -417,7 +456,7 @@ const HexWriter = struct { } fn Address(address: u32) Record { - std.debug.assert(address > 0xFFFF); + assert(address > 0xFFFF); const segment = @intCast(u16, address / 0x10000); if (address > 0xFFFFF) { return Record{ @@ -460,7 +499,7 @@ const HexWriter = struct { const BUFSIZE = 1 + (1 + 2 + 1 + MAX_PAYLOAD_LEN + 1) * 2 + linesep.len; var outbuf: [BUFSIZE]u8 = undefined; const payload_bytes = self.getPayloadBytes(); - std.debug.assert(payload_bytes.len <= MAX_PAYLOAD_LEN); + assert(payload_bytes.len <= MAX_PAYLOAD_LEN); const line = try std.fmt.bufPrint(&outbuf, ":{0X:0>2}{1X:0>4}{2X:0>2}{3s}{4X:0>2}" ++ linesep, .{ @intCast(u8, payload_bytes.len), -- cgit v1.2.3 From ede5dcffea5a3a5fc9fd14e4e180464633402fae Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 12 Mar 2023 00:39:21 -0700 Subject: make the build runner and test runner talk to each other std.Build.addTest creates a CompileStep as before, however, this kind of step no longer actually runs the unit tests. Instead it only compiles it, and one must additionally create a RunStep from the CompileStep in order to actually run the tests. RunStep gains integration with the default test runner, which now supports the standard --listen=- argument in order to communicate over stdin and stdout. It also reports test statistics; how many passed, failed, and leaked, as well as directly associating the relevant stderr with the particular test name that failed. This separation of CompileStep and RunStep means that `CompileStep.Kind.test_exe` is no longer needed, and therefore has been removed in this commit. * build runner: show unit test statistics in build summary * added Step.writeManifest since many steps want to treat it as a warning and emit the same message if it fails. * RunStep: fixed error message that prints the failed command printing the original argv and not the adjusted argv in case an interpreter was used. * RunStep: fixed not passing the command line arguments to the interpreter. * move src/Server.zig to std.zig.Server so that the default test runner can use it. * the simpler test runner function which is used by work-in-progress backends now no longer prints to stderr, which is necessary in order for the build runner to not print the stderr as a warning message. --- CMakeLists.txt | 2 +- lib/build_runner.zig | 59 +++- lib/std/Build.zig | 10 +- lib/std/Build/CompileStep.zig | 93 +----- lib/std/Build/InstallArtifactStep.zig | 5 +- lib/std/Build/RunStep.zig | 355 +++++++++++++++++---- lib/std/Build/Step.zig | 33 +- lib/std/Build/WriteFileStep.zig | 2 +- lib/std/zig/Client.zig | 7 + lib/std/zig/Server.zig | 200 ++++++++++++ lib/test_runner.zig | 172 +++++++--- src/Server.zig | 113 ------- src/main.zig | 21 +- src/objcopy.zig | 9 +- test/link/common_symbols/build.zig | 2 +- test/link/common_symbols_alignment/build.zig | 2 +- test/link/interdependent_static_c_libs/build.zig | 2 +- test/link/macho/tls/build.zig | 5 +- test/src/Cases.zig | 11 +- test/standalone/emit_asm_and_bin/build.zig | 2 +- test/standalone/global_linkage/build.zig | 2 +- test/standalone/issue_13970/build.zig | 6 +- test/standalone/main_pkg_path/build.zig | 2 +- test/standalone/options/build.zig | 2 +- test/standalone/pie/build.zig | 2 +- test/standalone/static_c_lib/build.zig | 2 +- .../test_runner_module_imports/build.zig | 2 +- test/standalone/test_runner_path/build.zig | 1 - test/standalone/use_alias/build.zig | 2 +- test/tests.zig | 27 +- 30 files changed, 780 insertions(+), 373 deletions(-) delete mode 100644 src/Server.zig (limited to 'src/main.zig') diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ff8249f4d..b0867c220b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -518,6 +518,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/zig/c_builtins.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/Parse.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/render.zig" + "${CMAKE_SOURCE_DIR}/lib/std/zig/Server.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/string_literal.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/system.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/system/NativePaths.zig" @@ -623,7 +624,6 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/src/print_targets.zig" "${CMAKE_SOURCE_DIR}/src/print_zir.zig" "${CMAKE_SOURCE_DIR}/src/register_manager.zig" - "${CMAKE_SOURCE_DIR}/src/Server.zig" "${CMAKE_SOURCE_DIR}/src/target.zig" "${CMAKE_SOURCE_DIR}/src/tracy.zig" "${CMAKE_SOURCE_DIR}/src/translate_c.zig" diff --git a/lib/build_runner.zig b/lib/build_runner.zig index e28be8274d..bb341574a7 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -416,6 +416,12 @@ fn runStepNames( } assert(run.memory_blocked_steps.items.len == 0); + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_pass_count: usize = 0; + var test_leak_count: usize = 0; + var test_count: usize = 0; + var success_count: usize = 0; var skipped_count: usize = 0; var failure_count: usize = 0; @@ -425,6 +431,12 @@ fn runStepNames( defer compile_error_steps.deinit(gpa); for (step_stack.keys()) |s| { + test_fail_count += s.test_results.fail_count; + test_skip_count += s.test_results.skip_count; + test_leak_count += s.test_results.leak_count; + test_pass_count += s.test_results.passCount(); + test_count += s.test_results.test_count; + switch (s.state) { .precheck_unstarted => unreachable, .precheck_started => unreachable, @@ -468,6 +480,11 @@ fn runStepNames( if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; + if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; + if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; + if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; + if (run.enable_summary == null) { ttyconf.setColor(stderr, .Dim) catch {}; stderr.writeAll(" (disable with -fno-summary)") catch {}; @@ -566,6 +583,13 @@ fn printTreeStep( try ttyconf.setColor(stderr, .Green); if (s.result_cached) { try stderr.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + try stderr.writer().print(" {d} passed", .{pass_count}); + if (s.test_results.skip_count > 0) { + try ttyconf.setColor(stderr, .Yellow); + try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); + } } else { try stderr.writeAll(" success"); } @@ -609,15 +633,46 @@ fn printTreeStep( }, .failure => { - try ttyconf.setColor(stderr, .Red); if (s.result_error_bundle.errorMessageCount() > 0) { + try ttyconf.setColor(stderr, .Red); try stderr.writer().print(" {d} errors\n", .{ s.result_error_bundle.errorMessageCount(), }); + try ttyconf.setColor(stderr, .Reset); + } else if (!s.test_results.isSuccess()) { + try stderr.writer().print(" {d}/{d} passed", .{ + s.test_results.passCount(), s.test_results.test_count, + }); + if (s.test_results.fail_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Red); + try stderr.writer().print("{d} failed", .{ + s.test_results.fail_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + if (s.test_results.skip_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Yellow); + try stderr.writer().print("{d} skipped", .{ + s.test_results.skip_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + if (s.test_results.leak_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Red); + try stderr.writer().print("{d} leaked", .{ + s.test_results.leak_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + try stderr.writeAll("\n"); } else { + try ttyconf.setColor(stderr, .Red); try stderr.writeAll(" failure\n"); + try ttyconf.setColor(stderr, .Reset); } - try ttyconf.setColor(stderr, .Reset); }, } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 5f74af3db6..279dd765b5 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -531,7 +531,6 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep { pub const TestOptions = struct { name: []const u8 = "test", - kind: CompileStep.Kind = .@"test", root_source_file: FileSource, target: CrossTarget = .{}, optimize: std.builtin.Mode = .Debug, @@ -542,7 +541,7 @@ pub const TestOptions = struct { pub fn addTest(b: *Build, options: TestOptions) *CompileStep { return CompileStep.create(b, .{ .name = options.name, - .kind = options.kind, + .kind = .@"test", .root_source_file = options.root_source_file, .target = options.target, .optimize = options.optimize, @@ -626,16 +625,15 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep { /// Creates a `RunStep` with an executable built with `addExecutable`. /// Add command line arguments with methods of `RunStep`. pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep { - assert(exe.kind == .exe or exe.kind == .test_exe); - // It doesn't have to be native. We catch that if you actually try to run it. // Consider that this is declarative; the run step may not be run unless a user // option is supplied. const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.name})); run_step.addArtifactArg(exe); - if (exe.kind == .test_exe) { - run_step.addArg(b.zig_exe); + if (exe.kind == .@"test") { + run_step.stdio = .zig_test; + run_step.addArgs(&.{"--listen=-"}); } if (exe.vcpkg_bin_path) |path| { diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 5c44ab82d3..5753641966 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -289,7 +289,6 @@ pub const Kind = enum { lib, obj, @"test", - test_exe, }; pub const Linkage = enum { dynamic, static }; @@ -328,7 +327,7 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep { .exe => "zig build-exe", .lib => "zig build-lib", .obj => "zig build-obj", - .test_exe, .@"test" => "zig test", + .@"test" => "zig test", }, name_adjusted, @tagName(options.optimize), @@ -410,7 +409,7 @@ fn computeOutFileNames(self: *CompileStep) void { .output_mode = switch (self.kind) { .lib => .Lib, .obj => .Obj, - .exe, .@"test", .test_exe => .Exe, + .exe, .@"test" => .Exe, }, .link_mode = if (self.linkage) |some| @as(std.builtin.LinkMode, switch (some) { .dynamic => .Dynamic, @@ -621,7 +620,7 @@ pub fn producesPdbFile(self: *CompileStep) bool { if (!self.target.isWindows() and !self.target.isUefi()) return false; if (self.target.getObjectFormat() == .c) return false; if (self.strip == true) return false; - return self.isDynamicLibrary() or self.kind == .exe or self.kind == .test_exe; + return self.isDynamicLibrary() or self.kind == .exe or self.kind == .@"test"; } pub fn linkLibC(self: *CompileStep) void { @@ -850,19 +849,19 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct { pub fn setNamePrefix(self: *CompileStep, text: []const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.name_prefix = b.dupe(text); } pub fn setFilter(self: *CompileStep, text: ?[]const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.filter = if (text) |t| b.dupe(t) else null; } pub fn setTestRunner(self: *CompileStep, path: ?[]const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.test_runner = if (path) |p| b.dupePath(p) else null; } @@ -938,7 +937,7 @@ pub fn getOutputLibSource(self: *CompileStep) FileSource { /// Returns the generated header file. /// This function can only be called for libraries or object files which have `emit_h` set. pub fn getOutputHSource(self: *CompileStep) FileSource { - assert(self.kind != .exe and self.kind != .test_exe and self.kind != .@"test"); + assert(self.kind != .exe and self.kind != .@"test"); assert(self.emit_h); return .{ .generated = &self.output_h_path_source }; } @@ -1243,7 +1242,6 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .exe => "build-exe", .obj => "build-obj", .@"test" => "test", - .test_exe => "test", }; try zig_args.append(cmd); @@ -1293,7 +1291,6 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .other_step => |other| switch (other.kind) { .exe => @panic("Cannot link with an executable build artifact"), - .test_exe => @panic("Cannot link with an executable build artifact"), .@"test" => @panic("Cannot link with a test"), .obj => { try zig_args.append(other.getOutputSource().getPath(b)); @@ -1661,83 +1658,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append("--test-cmd-bin"); } } - } else { - const need_cross_glibc = self.target.isGnuLibC() and transitive_deps.is_linking_libc; - - switch (b.host.getExternalExecutor(self.target_info, .{ - .qemu_fixes_dl = need_cross_glibc and b.glibc_runtimes_dir != null, - .link_libc = transitive_deps.is_linking_libc, - })) { - .native => {}, - .bad_dl, .bad_os_or_cpu => { - try zig_args.append("--test-no-exec"); - }, - .rosetta => if (b.enable_rosetta) { - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .qemu => |bin_name| ok: { - if (b.enable_qemu) qemu: { - const glibc_dir_arg = if (need_cross_glibc) - b.glibc_runtimes_dir orelse break :qemu - else - null; - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - if (glibc_dir_arg) |dir| { - // TODO look into making this a call to `linuxTriple`. This - // needs the directory to be called "i686" rather than - // "x86" which is why we do it manually here. - const fmt_str = "{s}" ++ fs.path.sep_str ++ "{s}-{s}-{s}"; - const cpu_arch = self.target.getCpuArch(); - const os_tag = self.target.getOsTag(); - const abi = self.target.getAbi(); - const cpu_arch_name: []const u8 = if (cpu_arch == .x86) - "i686" - else - @tagName(cpu_arch); - const full_dir = try std.fmt.allocPrint(b.allocator, fmt_str, .{ - dir, cpu_arch_name, @tagName(os_tag), @tagName(abi), - }); - - try zig_args.append("--test-cmd"); - try zig_args.append("-L"); - try zig_args.append("--test-cmd"); - try zig_args.append(full_dir); - } - try zig_args.append("--test-cmd-bin"); - break :ok; - } - try zig_args.append("--test-no-exec"); - }, - .wine => |bin_name| if (b.enable_wine) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .wasmtime => |bin_name| if (b.enable_wasmtime) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd"); - try zig_args.append("--dir=."); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .darling => |bin_name| if (b.enable_darling) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - } } - } else if (self.kind == .test_exe) { - try zig_args.append("--test-no-exec"); } try self.appendModuleArgs(&zig_args); diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/InstallArtifactStep.zig index 803998a619..445f1e8ea8 100644 --- a/lib/std/Build/InstallArtifactStep.zig +++ b/lib/std/Build/InstallArtifactStep.zig @@ -32,12 +32,11 @@ pub fn create(owner: *std.Build, artifact: *CompileStep) *InstallArtifactStep { .artifact = artifact, .dest_dir = artifact.override_dest_dir orelse switch (artifact.kind) { .obj => @panic("Cannot install a .obj build artifact."), - .@"test" => @panic("Cannot install a .test build artifact, use .test_exe instead."), - .exe, .test_exe => InstallDir{ .bin = {} }, + .exe, .@"test" => InstallDir{ .bin = {} }, .lib => InstallDir{ .lib = {} }, }, .pdb_dir = if (artifact.producesPdbFile()) blk: { - if (artifact.kind == .exe or artifact.kind == .test_exe) { + if (artifact.kind == .exe or artifact.kind == .@"test") { break :blk InstallDir{ .bin = {} }; } else { break :blk InstallDir{ .lib = {} }; diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 9a1c887d7d..484600bf9b 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -92,6 +92,9 @@ pub const StdIo = union(enum) { /// Note that an explicit check for exit code 0 needs to be added to this /// list if such a check is desireable. check: std.ArrayList(Check), + /// This RunStep is running a zig unit test binary and will communicate + /// extra metadata over the IPC protocol. + zig_test, pub const Check = union(enum) { expect_stderr_exact: []const u8, @@ -324,6 +327,7 @@ fn hasSideEffects(self: RunStep) bool { .infer_from_args => !self.hasAnyOutputArgs(), .inherit => true, .check => false, + .zig_test => false, }; } @@ -366,11 +370,6 @@ fn checksContainStderr(checks: []const StdIo.Check) bool { } fn make(step: *Step, prog_node: *std.Progress.Node) !void { - // Unfortunately we have no way to collect progress from arbitrary programs. - // Perhaps in the future Zig could offer some kind of opt-in IPC mechanism that - // processes could use to supply progress updates. - _ = prog_node; - const b = step.owner; const arena = b.allocator; const self = @fieldParentPtr(RunStep, "step", step); @@ -439,7 +438,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { hashStdIo(&man.hash, self.stdio); if (has_side_effects) { - try runCommand(self, argv_list.items, has_side_effects, null); + try runCommand(self, argv_list.items, has_side_effects, null, prog_node); return; } @@ -492,8 +491,9 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { argv_list.items[placeholder.index] = cli_arg; } - try runCommand(self, argv_list.items, has_side_effects, &digest); - try man.writeManifest(); + try runCommand(self, argv_list.items, has_side_effects, &digest, prog_node); + + try step.writeManifest(&man); } fn formatTerm( @@ -546,6 +546,7 @@ fn runCommand( argv: []const []const u8, has_side_effects: bool, digest: ?*const [std.Build.Cache.hex_digest_len]u8, + prog_node: *std.Progress.Node, ) !void { const step = &self.step; const b = step.owner; @@ -554,7 +555,15 @@ fn runCommand( try step.handleChildProcUnsupported(self.cwd, argv); try Step.handleVerbose(step.owner, self.cwd, argv); - const result = spawnChildAndCollect(self, argv, has_side_effects) catch |err| term: { + const allow_skip = switch (self.stdio) { + .check, .zig_test => self.skip_foreign_checks, + else => false, + }; + + var interp_argv = std.ArrayList([]const u8).init(b.allocator); + defer interp_argv.deinit(); + + const result = spawnChildAndCollect(self, argv, has_side_effects, prog_node) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -566,10 +575,10 @@ fn runCommand( .artifact => |exe| exe, else => break :interpret, }; - if (exe.kind != .exe) break :interpret; - - var interp_argv = std.ArrayList([]const u8).init(b.allocator); - defer interp_argv.deinit(); + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } const need_cross_glibc = exe.target.isGnuLibC() and exe.is_linking_libc; switch (b.host.getExternalExecutor(exe.target_info, .{ @@ -577,14 +586,13 @@ fn runCommand( .link_libc = exe.is_linking_libc, })) { .native, .rosetta => { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; - + if (allow_skip) return error.MakeSkipped; break :interpret; }, .wine => |bin_name| { if (b.enable_wine) { try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fwine", argv[0], exe); } @@ -617,6 +625,8 @@ fn runCommand( try interp_argv.append("-L"); try interp_argv.append(full_dir); } + + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fqemu", argv[0], exe); } @@ -624,6 +634,7 @@ fn runCommand( .darling => |bin_name| { if (b.enable_darling) { try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fdarling", argv[0], exe); } @@ -632,13 +643,15 @@ fn runCommand( if (b.enable_wasmtime) { try interp_argv.append(bin_name); try interp_argv.append("--dir=."); + try interp_argv.append(argv[0]); + try interp_argv.append("--"); + try interp_argv.appendSlice(argv[1..]); } else { return failForeign(self, "-fwasmtime", argv[0], exe); } }, .bad_dl => |foreign_dl| { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; + if (allow_skip) return error.MakeSkipped; const host_dl = b.host.dynamic_linker.get() orelse "(none)"; @@ -650,8 +663,7 @@ fn runCommand( , .{ host_dl, foreign_dl }); }, .bad_os_or_cpu => { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; + if (allow_skip) return error.MakeSkipped; const host_name = try b.host.target.zigTriple(b.allocator); const foreign_name = try exe.target.zigTriple(b.allocator); @@ -667,11 +679,9 @@ fn runCommand( RunStep.addPathForDynLibsInternal(&self.step, b, exe); } - try interp_argv.append(argv[0]); - try Step.handleVerbose(step.owner, self.cwd, interp_argv.items); - break :term spawnChildAndCollect(self, interp_argv.items, has_side_effects) catch |e| { + break :term spawnChildAndCollect(self, interp_argv.items, has_side_effects, prog_node) catch |e| { return step.fail("unable to spawn {s}: {s}", .{ interp_argv.items[0], @errorName(e), }); @@ -683,6 +693,7 @@ fn runCommand( step.result_duration_ns = result.elapsed_ns; step.result_peak_rss = result.peak_rss; + step.test_results = result.stdio.test_results; // Capture stdout and stderr to GeneratedFile objects. const Stream = struct { @@ -693,13 +704,13 @@ fn runCommand( for ([_]Stream{ .{ .captured = self.captured_stdout, - .is_null = result.stdout_null, - .bytes = result.stdout, + .is_null = result.stdio.stdout_null, + .bytes = result.stdio.stdout, }, .{ .captured = self.captured_stderr, - .is_null = result.stderr_null, - .bytes = result.stderr, + .is_null = result.stdio.stderr_null, + .bytes = result.stdio.stderr, }, }) |stream| { if (stream.captured) |output| { @@ -724,11 +735,13 @@ fn runCommand( } } + const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; + switch (self.stdio) { .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { - assert(!result.stderr_null); - if (!mem.eql(u8, expected_bytes, result.stderr)) { + assert(!result.stdio.stderr_null); + if (!mem.eql(u8, expected_bytes, result.stdio.stderr)) { return step.fail( \\ \\========= expected this stderr: ========= @@ -739,14 +752,14 @@ fn runCommand( \\{s} , .{ expected_bytes, - result.stderr, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stderr, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stderr_match => |match| { - assert(!result.stderr_null); - if (mem.indexOf(u8, result.stderr, match) == null) { + assert(!result.stdio.stderr_null); + if (mem.indexOf(u8, result.stdio.stderr, match) == null) { return step.fail( \\ \\========= expected to find in stderr: ========= @@ -757,14 +770,14 @@ fn runCommand( \\{s} , .{ match, - result.stderr, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stderr, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stdout_exact => |expected_bytes| { - assert(!result.stdout_null); - if (!mem.eql(u8, expected_bytes, result.stdout)) { + assert(!result.stdio.stdout_null); + if (!mem.eql(u8, expected_bytes, result.stdio.stdout)) { return step.fail( \\ \\========= expected this stdout: ========= @@ -775,14 +788,14 @@ fn runCommand( \\{s} , .{ expected_bytes, - result.stdout, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stdout, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stdout_match => |match| { - assert(!result.stdout_null); - if (mem.indexOf(u8, result.stdout, match) == null) { + assert(!result.stdio.stdout_null); + if (mem.indexOf(u8, result.stdio.stdout, match) == null) { return step.fail( \\ \\========= expected to find in stdout: ========= @@ -793,8 +806,8 @@ fn runCommand( \\{s} , .{ match, - result.stdout, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stdout, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, @@ -803,33 +816,46 @@ fn runCommand( return step.fail("the following command {} (expected {}):\n{s}", .{ fmtTerm(result.term), fmtTerm(expected_term), - try Step.allocPrintCmd(arena, self.cwd, argv), + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, }, + .zig_test => { + const expected_term: std.process.Child.Term = .{ .Exited = 0 }; + if (!termMatches(expected_term, result.term)) { + return step.fail("the following command {} (expected {}):\n{s}", .{ + fmtTerm(result.term), + fmtTerm(expected_term), + try Step.allocPrintCmd(arena, self.cwd, final_argv), + }); + } + if (!result.stdio.test_results.isSuccess()) { + return step.fail( + "the following test command failed:\n{s}", + .{try Step.allocPrintCmd(arena, self.cwd, final_argv)}, + ); + } + }, else => { - try step.handleChildProcessTerm(result.term, self.cwd, argv); + try step.handleChildProcessTerm(result.term, self.cwd, final_argv); }, } } const ChildProcResult = struct { - // These use boolean flags instead of optionals as a workaround for - // https://github.com/ziglang/zig/issues/14783 - stdout: []const u8, - stderr: []const u8, - stdout_null: bool, - stderr_null: bool, term: std.process.Child.Term, elapsed_ns: u64, peak_rss: usize, + + stdio: StdIoResult, }; fn spawnChildAndCollect( self: *RunStep, argv: []const []const u8, has_side_effects: bool, + prog_node: *std.Progress.Node, ) !ChildProcResult { const b = self.step.owner; const arena = b.allocator; @@ -848,16 +874,19 @@ fn spawnChildAndCollect( .infer_from_args => if (has_side_effects) .Inherit else .Close, .inherit => .Inherit, .check => .Close, + .zig_test => .Pipe, }; child.stdout_behavior = switch (self.stdio) { .infer_from_args => if (has_side_effects) .Inherit else .Ignore, .inherit => .Inherit, .check => |checks| if (checksContainStdout(checks.items)) .Pipe else .Ignore, + .zig_test => .Pipe, }; child.stderr_behavior = switch (self.stdio) { .infer_from_args => if (has_side_effects) .Inherit else .Pipe, .inherit => .Inherit, .check => .Pipe, + .zig_test => .Pipe, }; if (self.captured_stdout != null) child.stdout_behavior = .Pipe; if (self.captured_stderr != null) child.stderr_behavior = .Pipe; @@ -871,6 +900,219 @@ fn spawnChildAndCollect( }); var timer = try std.time.Timer.start(); + const result = if (self.stdio == .zig_test) + evalZigTest(self, &child, prog_node) + else + evalGeneric(self, &child); + + const term = try child.wait(); + const elapsed_ns = timer.read(); + + return .{ + .stdio = try result, + .term = term, + .elapsed_ns = elapsed_ns, + .peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0, + }; +} + +const StdIoResult = struct { + // These use boolean flags instead of optionals as a workaround for + // https://github.com/ziglang/zig/issues/14783 + stdout: []const u8, + stderr: []const u8, + stdout_null: bool, + stderr_null: bool, + test_results: Step.TestResults, +}; + +fn evalZigTest( + self: *RunStep, + child: *std.process.Child, + prog_node: *std.Progress.Node, +) !StdIoResult { + const gpa = self.step.owner.allocator; + const arena = self.step.owner.allocator; + + var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try sendMessage(child.stdin.?, .query_test_metadata); + + const Header = std.zig.Server.Message.Header; + + const stdout = poller.fifo(.stdout); + const stderr = poller.fifo(.stderr); + + var fail_count: u32 = 0; + var skip_count: u32 = 0; + var leak_count: u32 = 0; + var test_count: u32 = 0; + + var metadata: ?TestMetadata = null; + + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |*n| n.end(); + + poll: while (try poller.poll()) { + while (true) { + const buf = stdout.readableSlice(0); + assert(stdout.readableLength() == buf.len); + if (buf.len < @sizeOf(Header)) continue :poll; + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + const header_and_msg_len = header.bytes_len + @sizeOf(Header); + if (buf.len < header_and_msg_len) continue :poll; + const body = buf[@sizeOf(Header)..][0..header.bytes_len]; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return self.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .test_metadata => { + const TmHdr = std.zig.Server.Message.TestMetadata; + const tm_hdr = @ptrCast(*align(1) const TmHdr, body); + test_count = tm_hdr.tests_len; + + const names_bytes = body[@sizeOf(TmHdr)..][0 .. test_count * @sizeOf(u32)]; + const async_frame_lens_bytes = body[@sizeOf(TmHdr) + names_bytes.len ..][0 .. test_count * @sizeOf(u32)]; + const expected_panic_msgs_bytes = body[@sizeOf(TmHdr) + names_bytes.len + async_frame_lens_bytes.len ..][0 .. test_count * @sizeOf(u32)]; + const string_bytes = body[@sizeOf(TmHdr) + names_bytes.len + async_frame_lens_bytes.len + expected_panic_msgs_bytes.len ..][0..tm_hdr.string_bytes_len]; + + const names = std.mem.bytesAsSlice(u32, names_bytes); + const async_frame_lens = std.mem.bytesAsSlice(u32, async_frame_lens_bytes); + const expected_panic_msgs = std.mem.bytesAsSlice(u32, expected_panic_msgs_bytes); + const names_aligned = try arena.alloc(u32, names.len); + for (names_aligned, names) |*dest, src| dest.* = src; + + const async_frame_lens_aligned = try arena.alloc(u32, async_frame_lens.len); + for (async_frame_lens_aligned, async_frame_lens) |*dest, src| dest.* = src; + + const expected_panic_msgs_aligned = try arena.alloc(u32, expected_panic_msgs.len); + for (expected_panic_msgs_aligned, expected_panic_msgs) |*dest, src| dest.* = src; + + prog_node.setEstimatedTotalItems(names.len); + metadata = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .names = names_aligned, + .async_frame_lens = async_frame_lens_aligned, + .expected_panic_msgs = expected_panic_msgs_aligned, + .next_index = 0, + .prog_node = prog_node, + }; + + try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); + }, + .test_results => { + const md = metadata.?; + + const TrHdr = std.zig.Server.Message.TestResults; + const tr_hdr = @ptrCast(*align(1) const TrHdr, body); + fail_count += @boolToInt(tr_hdr.flags.fail); + skip_count += @boolToInt(tr_hdr.flags.skip); + leak_count += @boolToInt(tr_hdr.flags.leak); + + if (tr_hdr.flags.fail or tr_hdr.flags.leak) { + const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0); + const msg = std.mem.trim(u8, stderr.readableSlice(0), "\n"); + const label = if (tr_hdr.flags.fail) "failed" else "leaked"; + if (msg.len > 0) { + try self.step.addError("'{s}' {s}: {s}", .{ name, label, msg }); + } else { + try self.step.addError("'{s}' {s}", .{ name, label }); + } + stderr.discard(msg.len); + } + + try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); + }, + else => {}, // ignore other messages + } + stdout.discard(header_and_msg_len); + } + } + + if (stderr.readableLength() > 0) { + const msg = std.mem.trim(u8, try stderr.toOwnedSlice(), "\n"); + if (msg.len > 0) try self.step.result_error_msgs.append(arena, msg); + } + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + return .{ + .stdout = &.{}, + .stderr = &.{}, + .stdout_null = true, + .stderr_null = true, + .test_results = .{ + .test_count = test_count, + .fail_count = fail_count, + .skip_count = skip_count, + .leak_count = leak_count, + }, + }; +} + +const TestMetadata = struct { + names: []const u32, + async_frame_lens: []const u32, + expected_panic_msgs: []const u32, + string_bytes: []const u8, + next_index: u32, + prog_node: *std.Progress.Node, + + fn testName(tm: TestMetadata, index: u32) []const u8 { + return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); + } +}; + +fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { + while (metadata.next_index < metadata.names.len) { + const i = metadata.next_index; + metadata.next_index += 1; + + if (metadata.async_frame_lens[i] != 0) continue; + if (metadata.expected_panic_msgs[i] != 0) continue; + + const name = metadata.testName(i); + if (sub_prog_node.*) |*n| n.end(); + sub_prog_node.* = metadata.prog_node.start(name, 0); + + try sendRunTestMessage(in, i); + return; + } else { + try sendMessage(in, .exit); + } +} + +fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + try file.writeAll(std.mem.asBytes(&header)); +} + +fn sendRunTestMessage(file: std.fs.File, index: u32) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .run_test, + .bytes_len = 4, + }; + const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index); + try file.writeAll(full_msg); +} + +fn evalGeneric(self: *RunStep, child: *std.process.Child) !StdIoResult { + const arena = self.step.owner.allocator; + if (self.stdin) |stdin| { child.stdin.?.writeAll(stdin) catch |err| { return self.step.fail("unable to write stdin: {s}", .{@errorName(err)}); @@ -925,17 +1167,12 @@ fn spawnChildAndCollect( } } - const term = try child.wait(); - const elapsed_ns = timer.read(); - return .{ .stdout = stdout_bytes, .stderr = stderr_bytes, .stdout_null = stdout_null, .stderr_null = stderr_null, - .term = term, - .elapsed_ns = elapsed_ns, - .peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0, + .test_results = .{}, }; } @@ -966,7 +1203,7 @@ fn failForeign( exe: *CompileStep, ) error{ MakeFailed, MakeSkipped, OutOfMemory } { switch (self.stdio) { - .check => { + .check, .zig_test => { if (self.skip_foreign_checks) return error.MakeSkipped; @@ -987,7 +1224,7 @@ fn failForeign( fn hashStdIo(hh: *std.Build.Cache.HashHelper, stdio: StdIo) void { switch (stdio) { - .infer_from_args, .inherit => {}, + .infer_from_args, .inherit, .zig_test => {}, .check => |checks| for (checks.items) |check| { hh.add(@as(std.meta.Tag(StdIo.Check), check)); switch (check) { diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 45aa635972..05c4faa52d 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -35,11 +35,27 @@ result_cached: bool, result_duration_ns: ?u64, /// 0 means unavailable or not reported. result_peak_rss: usize, +test_results: TestResults, /// The return addresss associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: [n_debug_stack_frames]usize, +pub const TestResults = struct { + fail_count: u32 = 0, + skip_count: u32 = 0, + leak_count: u32 = 0, + test_count: u32 = 0, + + pub fn isSuccess(tr: TestResults) bool { + return tr.fail_count == 0 and tr.leak_count == 0; + } + + pub fn passCount(tr: TestResults) u32 { + return tr.test_count - tr.fail_count - tr.skip_count; + } +}; + pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void; const n_debug_stack_frames = 4; @@ -134,6 +150,7 @@ pub fn init(options: Options) Step { .result_cached = false, .result_duration_ns = null, .result_peak_rss = 0, + .test_results = .{}, }; } @@ -152,6 +169,10 @@ pub fn make(s: *Step, prog_node: *std.Progress.Node) error{ MakeFailed, MakeSkip }, }; + if (!s.test_results.isSuccess()) { + return error.MakeFailed; + } + if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { const msg = std.fmt.allocPrint(arena, "memory usage peaked at {d} bytes, exceeding the declared upper bound of {d}", .{ s.result_peak_rss, s.max_rss, @@ -346,9 +367,7 @@ pub fn evalZigProcess( s.result_cached = ebp_hdr.flags.cache_hit; result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); }, - _ => { - // Unrecognized message. - }, + else => {}, // ignore other messages } stdout.discard(header_and_msg_len); } @@ -475,3 +494,11 @@ fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyer const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path }); } + +pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void { + if (s.test_results.isSuccess()) { + man.writeManifest() catch |err| { + try s.addError("unable to write cache manifest: {s}", .{@errorName(err)}); + }; + } +} diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 9b033e5ae2..dee79af5be 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -282,7 +282,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }); } - try man.writeManifest(); + try step.writeManifest(&man); } const std = @import("../std.zig"); diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig index a68c189e57..af4c29d37d 100644 --- a/lib/std/zig/Client.zig +++ b/lib/std/zig/Client.zig @@ -26,6 +26,13 @@ pub const Message = struct { /// swap. /// No body. hot_update, + /// Ask the test runner for metadata about all the unit tests that can + /// be run. Server will respond with a `test_metadata` message. + /// No body. + query_test_metadata, + /// Ask the test runner to run a particular test. + /// The message body is a u32 test index. + run_test, _, }; diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index d34b2193e9..3238f22043 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -1,3 +1,7 @@ +in: std.fs.File, +out: std.fs.File, +receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), + pub const Message = struct { pub const Header = extern struct { tag: Tag, @@ -14,6 +18,11 @@ pub const Message = struct { progress, /// Body is a EmitBinPath. emit_bin_path, + /// Body is a TestMetadata + test_metadata, + /// Body is a TestResults + test_results, + _, }; @@ -26,6 +35,33 @@ pub const Message = struct { string_bytes_len: u32, }; + /// Trailing: + /// * name: [tests_len]u32 + /// - null-terminated string_bytes index + /// * async_frame_len: [tests_len]u32, + /// - 0 means not async + /// * expected_panic_msg: [tests_len]u32, + /// - null-terminated string_bytes index + /// - 0 means does not expect pani + /// * string_bytes: [string_bytes_len]u8, + pub const TestMetadata = extern struct { + string_bytes_len: u32, + tests_len: u32, + }; + + pub const TestResults = extern struct { + index: u32, + flags: Flags, + + pub const Flags = packed struct(u8) { + fail: bool, + skip: bool, + leak: bool, + + reserved: u5 = 0, + }; + }; + /// Trailing: /// * the file system path the emitted binary can be found pub const EmitBinPath = extern struct { @@ -37,3 +73,167 @@ pub const Message = struct { }; }; }; + +pub const Options = struct { + gpa: Allocator, + in: std.fs.File, + out: std.fs.File, + zig_version: []const u8, +}; + +pub fn init(options: Options) !Server { + var s: Server = .{ + .in = options.in, + .out = options.out, + .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), + }; + try s.serveStringMessage(.zig_version, options.zig_version); + return s; +} + +pub fn deinit(s: *Server) void { + s.receive_fifo.deinit(); + s.* = undefined; +} + +pub fn receiveMessage(s: *Server) !InMessage.Header { + const Header = InMessage.Header; + const fifo = &s.receive_fifo; + + while (true) { + const buf = fifo.readableSlice(0); + assert(fifo.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + + if (buf.len - @sizeOf(Header) >= header.bytes_len) { + const result = header.*; + fifo.discard(@sizeOf(Header)); + return result; + } else { + const needed = header.bytes_len - (buf.len - @sizeOf(Header)); + const write_buffer = try fifo.writableWithSize(needed); + const amt = try s.in.read(write_buffer); + fifo.update(amt); + continue; + } + } + + const write_buffer = try fifo.writableWithSize(256); + const amt = try s.in.read(write_buffer); + fifo.update(amt); + } +} + +pub fn receiveBody_u32(s: *Server) !u32 { + const fifo = &s.receive_fifo; + const buf = fifo.readableSlice(0); + const result = @ptrCast(*align(1) const u32, buf[0..4]).*; + fifo.discard(4); + return result; +} + +pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { + return s.serveMessage(.{ + .tag = tag, + .bytes_len = @intCast(u32, msg.len), + }, &.{msg}); +} + +pub fn serveMessage( + s: *const Server, + header: OutMessage.Header, + bufs: []const []const u8, +) !void { + var iovecs: [10]std.os.iovec_const = undefined; + iovecs[0] = .{ + .iov_base = @ptrCast([*]const u8, &header), + .iov_len = @sizeOf(OutMessage.Header), + }; + for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { + iovec.* = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + try s.out.writevAll(iovecs[0 .. bufs.len + 1]); +} + +pub fn serveEmitBinPath( + s: *Server, + fs_path: []const u8, + header: OutMessage.EmitBinPath, +) !void { + try s.serveMessage(.{ + .tag = .emit_bin_path, + .bytes_len = @intCast(u32, fs_path.len + @sizeOf(OutMessage.EmitBinPath)), + }, &.{ + std.mem.asBytes(&header), + fs_path, + }); +} + +pub fn serveTestResults( + s: *Server, + msg: OutMessage.TestResults, +) !void { + try s.serveMessage(.{ + .tag = .test_results, + .bytes_len = @intCast(u32, @sizeOf(OutMessage.TestResults)), + }, &.{ + std.mem.asBytes(&msg), + }); +} + +pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { + const eb_hdr: OutMessage.ErrorBundle = .{ + .extra_len = @intCast(u32, error_bundle.extra.len), + .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), + }; + const bytes_len = @sizeOf(OutMessage.ErrorBundle) + + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; + try s.serveMessage(.{ + .tag = .error_bundle, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&eb_hdr), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(error_bundle.extra), + error_bundle.string_bytes, + }); +} + +pub const TestMetadata = struct { + names: []const u32, + async_frame_sizes: []const u32, + expected_panic_msgs: []const u32, + string_bytes: []const u8, +}; + +pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { + const header: OutMessage.TestMetadata = .{ + .tests_len = @intCast(u32, test_metadata.names.len), + .string_bytes_len = @intCast(u32, test_metadata.string_bytes.len), + }; + const bytes_len = @sizeOf(OutMessage.TestMetadata) + + 3 * 4 * test_metadata.names.len + test_metadata.string_bytes.len; + return s.serveMessage(.{ + .tag = .test_metadata, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&header), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(test_metadata.names), + std.mem.sliceAsBytes(test_metadata.async_frame_sizes), + std.mem.sliceAsBytes(test_metadata.expected_panic_msgs), + test_metadata.string_bytes, + }); +} + +const OutMessage = std.zig.Server.Message; +const InMessage = std.zig.Client.Message; + +const Server = @This(); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; diff --git a/lib/test_runner.zig b/lib/test_runner.zig index 5968fdaa54..47ababbc2c 100644 --- a/lib/test_runner.zig +++ b/lib/test_runner.zig @@ -8,14 +8,130 @@ pub const std_options = struct { }; var log_err_count: usize = 0; +var cmdline_buffer: [4096]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer); pub fn main() void { - if (builtin.zig_backend != .stage1 and - builtin.zig_backend != .stage2_llvm and - builtin.zig_backend != .stage2_c) + if (builtin.zig_backend == .stage2_wasm or + builtin.zig_backend == .stage2_x86_64 or + builtin.zig_backend == .stage2_aarch64) { - return main2() catch @panic("test failure"); + return mainSimple() catch @panic("test failure"); + } + + const args = std.process.argsAlloc(fba.allocator()) catch + @panic("unable to parse command line args"); + + var listen = false; + + for (args[1..]) |arg| { + if (std.mem.eql(u8, arg, "--listen=-")) { + listen = true; + } else { + @panic("unrecognized command line argument"); + } + } + + if (listen) { + return mainServer(); + } else { + return mainTerminal(); + } +} + +fn mainServer() void { + return mainServerFallible() catch @panic("internal test runner failure"); +} + +fn mainServerFallible() !void { + var server = try std.zig.Server.init(.{ + .gpa = fba.allocator(), + .in = std.io.getStdIn(), + .out = std.io.getStdOut(), + .zig_version = builtin.zig_version_string, + }); + defer server.deinit(); + + while (true) { + const hdr = try server.receiveMessage(); + switch (hdr.tag) { + .exit => { + return std.process.exit(0); + }, + .query_test_metadata => { + std.testing.allocator_instance = .{}; + defer if (std.testing.allocator_instance.deinit()) { + @panic("internal test runner memory leak"); + }; + + var string_bytes: std.ArrayListUnmanaged(u8) = .{}; + defer string_bytes.deinit(std.testing.allocator); + try string_bytes.append(std.testing.allocator, 0); // Reserve 0 for null. + + const test_fns = builtin.test_functions; + const names = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(names); + const async_frame_sizes = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(async_frame_sizes); + const expected_panic_msgs = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(expected_panic_msgs); + + for (test_fns, names, async_frame_sizes, expected_panic_msgs) |test_fn, *name, *async_frame_size, *expected_panic_msg| { + name.* = @intCast(u32, string_bytes.items.len); + try string_bytes.ensureUnusedCapacity(std.testing.allocator, test_fn.name.len + 1); + string_bytes.appendSliceAssumeCapacity(test_fn.name); + string_bytes.appendAssumeCapacity(0); + + async_frame_size.* = @intCast(u32, test_fn.async_frame_size orelse 0); + expected_panic_msg.* = 0; + } + + try server.serveTestMetadata(.{ + .names = names, + .async_frame_sizes = async_frame_sizes, + .expected_panic_msgs = expected_panic_msgs, + .string_bytes = string_bytes.items, + }); + }, + + .run_test => { + std.testing.allocator_instance = .{}; + const index = try server.receiveBody_u32(); + const test_fn = builtin.test_functions[index]; + if (test_fn.async_frame_size != null) + @panic("TODO test runner implement async tests"); + var fail = false; + var skip = false; + var leak = false; + test_fn.func() catch |err| switch (err) { + error.SkipZigTest => skip = true, + else => { + fail = true; + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + }, + }; + leak = std.testing.allocator_instance.deinit(); + try server.serveTestResults(.{ + .index = index, + .flags = .{ + .fail = fail, + .skip = skip, + .leak = leak, + }, + }); + }, + + else => { + std.debug.print("unsupported message: {x}", .{@enumToInt(hdr.tag)}); + std.process.exit(1); + }, + } } +} + +fn mainTerminal() void { const test_fn_list = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; @@ -118,51 +234,17 @@ pub fn log( } } -pub fn main2() anyerror!void { - var skipped: usize = 0; - var failed: usize = 0; - // Simpler main(), exercising fewer language features, so that stage2 can handle it. +/// Simpler main(), exercising fewer language features, so that +/// work-in-progress backends can handle it. +pub fn mainSimple() anyerror!void { + //const stderr = std.io.getStdErr(); for (builtin.test_functions) |test_fn| { test_fn.func() catch |err| { if (err != error.SkipZigTest) { - failed += 1; - } else { - skipped += 1; + //stderr.writeAll(test_fn.name) catch {}; + //stderr.writeAll("\n") catch {}; + return err; } }; } - if (builtin.zig_backend == .stage2_wasm or - builtin.zig_backend == .stage2_x86_64 or - builtin.zig_backend == .stage2_aarch64 or - builtin.zig_backend == .stage2_llvm or - builtin.zig_backend == .stage2_c) - { - const passed = builtin.test_functions.len - skipped - failed; - const stderr = std.io.getStdErr(); - writeInt(stderr, passed) catch {}; - stderr.writeAll(" passed; ") catch {}; - writeInt(stderr, skipped) catch {}; - stderr.writeAll(" skipped; ") catch {}; - writeInt(stderr, failed) catch {}; - stderr.writeAll(" failed.\n") catch {}; - } - if (failed != 0) { - return error.TestsFailed; - } -} - -fn writeInt(stderr: std.fs.File, int: usize) anyerror!void { - const base = 10; - var buf: [100]u8 = undefined; - var a: usize = int; - var index: usize = buf.len; - while (true) { - const digit = a % base; - index -= 1; - buf[index] = std.fmt.digitToChar(@intCast(u8, digit), .lower); - a /= base; - if (a == 0) break; - } - const slice = buf[index..]; - try stderr.writeAll(slice); } diff --git a/src/Server.zig b/src/Server.zig deleted file mode 100644 index a25dc93857..0000000000 --- a/src/Server.zig +++ /dev/null @@ -1,113 +0,0 @@ -in: std.fs.File, -out: std.fs.File, -receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), - -pub const Options = struct { - gpa: Allocator, - in: std.fs.File, - out: std.fs.File, -}; - -pub fn init(options: Options) !Server { - var s: Server = .{ - .in = options.in, - .out = options.out, - .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), - }; - try s.serveStringMessage(.zig_version, build_options.version); - return s; -} - -pub fn deinit(s: *Server) void { - s.receive_fifo.deinit(); - s.* = undefined; -} - -pub fn receiveMessage(s: *Server) !InMessage.Header { - const Header = InMessage.Header; - const fifo = &s.receive_fifo; - - while (true) { - const buf = fifo.readableSlice(0); - assert(fifo.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); - if (header.bytes_len != 0) - return error.InvalidClientMessage; - const result = header.*; - fifo.discard(@sizeOf(Header)); - return result; - } - - const write_buffer = try fifo.writableWithSize(256); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - } -} - -pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { - return s.serveMessage(.{ - .tag = tag, - .bytes_len = @intCast(u32, msg.len), - }, &.{msg}); -} - -pub fn serveMessage( - s: *const Server, - header: OutMessage.Header, - bufs: []const []const u8, -) !void { - var iovecs: [10]std.os.iovec_const = undefined; - iovecs[0] = .{ - .iov_base = @ptrCast([*]const u8, &header), - .iov_len = @sizeOf(OutMessage.Header), - }; - for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { - iovec.* = .{ - .iov_base = buf.ptr, - .iov_len = buf.len, - }; - } - try s.out.writevAll(iovecs[0 .. bufs.len + 1]); -} - -pub fn serveEmitBinPath( - s: *Server, - fs_path: []const u8, - header: std.zig.Server.Message.EmitBinPath, -) !void { - try s.serveMessage(.{ - .tag = .emit_bin_path, - .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)), - }, &.{ - std.mem.asBytes(&header), - fs_path, - }); -} - -pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { - const eb_hdr: std.zig.Server.Message.ErrorBundle = .{ - .extra_len = @intCast(u32, error_bundle.extra.len), - .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), - }; - const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) + - 4 * error_bundle.extra.len + error_bundle.string_bytes.len; - try s.serveMessage(.{ - .tag = .error_bundle, - .bytes_len = @intCast(u32, bytes_len), - }, &.{ - std.mem.asBytes(&eb_hdr), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(error_bundle.extra), - error_bundle.string_bytes, - }); -} - -const OutMessage = std.zig.Server.Message; -const InMessage = std.zig.Client.Message; - -const Server = @This(); -const std = @import("std"); -const build_options = @import("build_options"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; diff --git a/src/main.zig b/src/main.zig index e0283143d0..9602a5cd31 100644 --- a/src/main.zig +++ b/src/main.zig @@ -10,6 +10,7 @@ const ArrayList = std.ArrayList; const Ast = std.zig.Ast; const warn = std.log.warn; const ThreadPool = std.Thread.Pool; +const cleanExit = std.process.cleanExit; const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); @@ -26,7 +27,7 @@ const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); const Module = @import("Module.zig"); const AstGen = @import("AstGen.zig"); -const Server = @import("Server.zig"); +const Server = std.zig.Server; pub const std_options = struct { pub const wasiCwd = wasi_cwd; @@ -3545,6 +3546,7 @@ fn serve( .gpa = gpa, .in = in, .out = out, + .zig_version = build_options.version, }); defer server.deinit(); @@ -3656,8 +3658,8 @@ fn serve( ); } }, - _ => { - @panic("TODO unrecognized message from client"); + else => { + fatal("unrecognized message from client: 0x{x}", .{@enumToInt(hdr.tag)}); }, } } @@ -5624,19 +5626,6 @@ fn detectNativeTargetInfo(cross_target: std.zig.CrossTarget) !std.zig.system.Nat return std.zig.system.NativeTargetInfo.detect(cross_target); } -/// Indicate that we are now terminating with a successful exit code. -/// In debug builds, this is a no-op, so that the calling code's -/// cleanup mechanisms are tested and so that external tools that -/// check for resource leaks can be accurate. In release builds, this -/// calls exit(0), and does not return. -pub fn cleanExit() void { - if (builtin.mode == .Debug) { - return; - } else { - process.exit(0); - } -} - const usage_ast_check = \\Usage: zig ast-check [file] \\ diff --git a/src/objcopy.zig b/src/objcopy.zig index e821a94b59..c3305e8c04 100644 --- a/src/objcopy.zig +++ b/src/objcopy.zig @@ -8,8 +8,8 @@ const assert = std.debug.assert; const main = @import("main.zig"); const fatal = main.fatal; -const cleanExit = main.cleanExit; -const Server = @import("Server.zig"); +const Server = std.zig.Server; +const build_options = @import("build_options"); pub fn cmdObjCopy( gpa: Allocator, @@ -116,6 +116,7 @@ pub fn cmdObjCopy( .gpa = gpa, .in = std.io.getStdIn(), .out = std.io.getStdOut(), + .zig_version = build_options.version, }); defer server.deinit(); @@ -124,7 +125,7 @@ pub fn cmdObjCopy( const hdr = try server.receiveMessage(); switch (hdr.tag) { .exit => { - return cleanExit(); + return std.process.cleanExit(); }, .update => { if (seen_update) { @@ -144,7 +145,7 @@ pub fn cmdObjCopy( } } } - return cleanExit(); + return std.process.cleanExit(); } const usage = diff --git a/test/link/common_symbols/build.zig b/test/link/common_symbols/build.zig index e3c302f0f7..a8c276d1f3 100644 --- a/test/link/common_symbols/build.zig +++ b/test/link/common_symbols/build.zig @@ -24,5 +24,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize }); test_exe.linkLibrary(lib_a); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/common_symbols_alignment/build.zig b/test/link/common_symbols_alignment/build.zig index 7d1d813447..83548f2d8a 100644 --- a/test/link/common_symbols_alignment/build.zig +++ b/test/link/common_symbols_alignment/build.zig @@ -24,5 +24,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize }); test_exe.linkLibrary(lib_a); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/interdependent_static_c_libs/build.zig b/test/link/interdependent_static_c_libs/build.zig index c4118c1ca4..0d06410a79 100644 --- a/test/link/interdependent_static_c_libs/build.zig +++ b/test/link/interdependent_static_c_libs/build.zig @@ -35,5 +35,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize test_exe.linkLibrary(lib_b); test_exe.addIncludePath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/macho/tls/build.zig b/test/link/macho/tls/build.zig index e91f40bd59..5981fea194 100644 --- a/test/link/macho/tls/build.zig +++ b/test/link/macho/tls/build.zig @@ -32,5 +32,8 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize test_exe.linkLibrary(lib); test_exe.linkLibC(); - test_step.dependOn(&test_exe.step); + const run = test_exe.run(); + run.skip_foreign_checks = true; + + test_step.dependOn(&run.step); } diff --git a/test/src/Cases.zig b/test/src/Cases.zig index 5007939b14..6affe968d1 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -547,15 +547,12 @@ pub fn lowerToBuildSteps( parent_step.dependOn(&artifact.step); }, .Execution => |expected_stdout| { - if (case.is_test) { - parent_step.dependOn(&artifact.step); - } else { - const run = b.addRunArtifact(artifact); - run.skip_foreign_checks = true; + const run = b.addRunArtifact(artifact); + run.skip_foreign_checks = true; + if (!case.is_test) { run.expectStdOutEqual(expected_stdout); - - parent_step.dependOn(&run.step); } + parent_step.dependOn(&run.step); }, .Header => @panic("TODO"), } diff --git a/test/standalone/emit_asm_and_bin/build.zig b/test/standalone/emit_asm_and_bin/build.zig index 9bdfadc33d..594bf6552e 100644 --- a/test/standalone/emit_asm_and_bin/build.zig +++ b/test/standalone/emit_asm_and_bin/build.zig @@ -11,5 +11,5 @@ pub fn build(b: *std.Build) void { main.emit_asm = .{ .emit_to = b.pathFromRoot("main.s") }; main.emit_bin = .{ .emit_to = b.pathFromRoot("main") }; - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/global_linkage/build.zig b/test/standalone/global_linkage/build.zig index 2cf1b248a5..ddcddd612a 100644 --- a/test/standalone/global_linkage/build.zig +++ b/test/standalone/global_linkage/build.zig @@ -28,5 +28,5 @@ pub fn build(b: *std.Build) void { main.linkLibrary(obj1); main.linkLibrary(obj2); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/issue_13970/build.zig b/test/standalone/issue_13970/build.zig index bbaaac5886..1eb8a5a121 100644 --- a/test/standalone/issue_13970/build.zig +++ b/test/standalone/issue_13970/build.zig @@ -17,7 +17,7 @@ pub fn build(b: *std.Build) void { test2.setTestRunner("src/main.zig"); test3.setTestRunner("src/main.zig"); - test_step.dependOn(&test1.step); - test_step.dependOn(&test2.step); - test_step.dependOn(&test3.step); + test_step.dependOn(&test1.run().step); + test_step.dependOn(&test2.run().step); + test_step.dependOn(&test3.run().step); } diff --git a/test/standalone/main_pkg_path/build.zig b/test/standalone/main_pkg_path/build.zig index cd49573692..a4dd301c43 100644 --- a/test/standalone/main_pkg_path/build.zig +++ b/test/standalone/main_pkg_path/build.zig @@ -9,5 +9,5 @@ pub fn build(b: *std.Build) void { }); test_exe.setMainPkgPath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/standalone/options/build.zig b/test/standalone/options/build.zig index 3f1e823359..5e894102a7 100644 --- a/test/standalone/options/build.zig +++ b/test/standalone/options/build.zig @@ -20,5 +20,5 @@ pub fn build(b: *std.Build) void { options.addOption([]const u8, "string", b.option([]const u8, "string", "s").?); const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/pie/build.zig b/test/standalone/pie/build.zig index 615111b6c2..546b4a922f 100644 --- a/test/standalone/pie/build.zig +++ b/test/standalone/pie/build.zig @@ -17,5 +17,5 @@ pub fn build(b: *std.Build) void { }); main.pie = true; - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/static_c_lib/build.zig b/test/standalone/static_c_lib/build.zig index 5996c978d8..794b813b75 100644 --- a/test/standalone/static_c_lib/build.zig +++ b/test/standalone/static_c_lib/build.zig @@ -21,5 +21,5 @@ pub fn build(b: *std.Build) void { test_exe.linkLibrary(foo); test_exe.addIncludePath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/standalone/test_runner_module_imports/build.zig b/test/standalone/test_runner_module_imports/build.zig index 973365e495..73c5536dc6 100644 --- a/test/standalone/test_runner_module_imports/build.zig +++ b/test/standalone/test_runner_module_imports/build.zig @@ -15,5 +15,5 @@ pub fn build(b: *std.Build) void { t.addModule("module2", module2); const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&t.step); + test_step.dependOn(&t.run().step); } diff --git a/test/standalone/test_runner_path/build.zig b/test/standalone/test_runner_path/build.zig index 40aad42b21..ce5b668054 100644 --- a/test/standalone/test_runner_path/build.zig +++ b/test/standalone/test_runner_path/build.zig @@ -8,7 +8,6 @@ pub fn build(b: *std.Build) void { const test_exe = b.addTest(.{ .root_source_file = .{ .path = "test.zig" }, - .kind = .test_exe, }); test_exe.test_runner = "test_runner.zig"; diff --git a/test/standalone/use_alias/build.zig b/test/standalone/use_alias/build.zig index 947db0828d..db47fe6692 100644 --- a/test/standalone/use_alias/build.zig +++ b/test/standalone/use_alias/build.zig @@ -12,5 +12,5 @@ pub fn build(b: *std.Build) void { }); main.addIncludePath("."); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/tests.zig b/test/tests.zig index 9ba9639e2a..76ea537bb2 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -596,7 +596,7 @@ pub fn addStandaloneTests( }); if (case.link_libc) exe.linkLibC(); - step.dependOn(&exe.step); + step.dependOn(&exe.run().step); } } } @@ -981,14 +981,6 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { }); const single_threaded_txt = if (test_target.single_threaded) "single" else "multi"; const backend_txt = if (test_target.backend) |backend| @tagName(backend) else "default"; - these_tests.setNamePrefix(b.fmt("{s}-{s}-{s}-{s}-{s}-{s} ", .{ - options.name, - triple_prefix, - @tagName(test_target.optimize_mode), - libc_prefix, - single_threaded_txt, - backend_txt, - })); these_tests.single_threaded = test_target.single_threaded; these_tests.setFilter(options.test_filter); if (test_target.link_libc) { @@ -1014,7 +1006,18 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { }, }; - step.dependOn(&these_tests.step); + const run = these_tests.run(); + run.skip_foreign_checks = true; + run.setName(b.fmt("run test {s}-{s}-{s}-{s}-{s}-{s}", .{ + options.name, + triple_prefix, + @tagName(test_target.optimize_mode), + libc_prefix, + single_threaded_txt, + backend_txt, + })); + + step.dependOn(&run.step); } return step; } @@ -1053,7 +1056,9 @@ pub fn addCAbiTests(b: *std.Build, skip_non_native: bool, skip_release: bool) *S @tagName(optimize_mode), })); - step.dependOn(&test_step.step); + const run = test_step.run(); + run.skip_foreign_checks = true; + step.dependOn(&run.step); } } return step; -- cgit v1.2.3 From bde12930939c50fbfa344c3701a01885e6dd06d4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 13 Mar 2023 19:11:36 -0700 Subject: CLI: remove the experimental --watch flag The compiler REPL will move to an external process that communicates with the compiler over the binary protocol. --- src/main.zig | 142 ++++------------------------------------------------------- 1 file changed, 9 insertions(+), 133 deletions(-) (limited to 'src/main.zig') diff --git a/src/main.zig b/src/main.zig index 9602a5cd31..19f1a6e52c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -365,7 +365,6 @@ const usage_build_generic = \\ \\General Options: \\ -h, --help Print this help and exit - \\ --watch Enable compiler REPL \\ --color [auto|off|on] Enable or disable colored error messages \\ -femit-bin[=path] (default) Output machine code \\ -fno-emit-bin Do not output machine code @@ -700,7 +699,6 @@ fn buildOutputType( var formatted_panics: ?bool = null; var function_sections = false; var no_builtin = false; - var watch = false; var listen: Listen = .none; var debug_compile_errors = false; var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK"); @@ -1163,7 +1161,6 @@ fn buildOutputType( const next_arg = args_iter.nextOrFatal(); if (mem.eql(u8, next_arg, "-")) { listen = .stdio; - watch = true; } else { if (build_options.omit_pkg_fetching_code) unreachable; // example: --listen 127.0.0.1:9000 @@ -1174,11 +1171,9 @@ fn buildOutputType( fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) }); listen = .{ .ip4 = std.net.Ip4Address.parse(host, port) catch |err| fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }) }; - watch = true; } } else if (mem.eql(u8, arg, "--listen=-")) { listen = .stdio; - watch = true; } else if (mem.eql(u8, arg, "--debug-link-snapshot")) { if (!build_options.enable_link_snapshots) { std.log.warn("Zig was compiled without linker snapshots enabled (-Dlink-snapshot). --debug-link-snapshot has no effect.", .{}); @@ -1207,8 +1202,6 @@ fn buildOutputType( test_evented_io = true; } else if (mem.eql(u8, arg, "--test-no-exec")) { test_no_exec = true; - } else if (mem.eql(u8, arg, "--watch")) { - watch = true; } else if (mem.eql(u8, arg, "-ftime-report")) { time_report = true; } else if (mem.eql(u8, arg, "-fstack-report")) { @@ -3355,7 +3348,7 @@ fn buildOutputType( }; updateModule(gpa, comp, hook) catch |err| switch (err) { - error.SemanticAnalyzeFail => if (!watch) process.exit(1), + error.SemanticAnalyzeFail => if (listen == .none) process.exit(1), else => |e| return e, }; if (build_options.only_c) return cleanExit(); @@ -3411,7 +3404,6 @@ fn buildOutputType( self_exe_path.?, arg_mode, target_info, - watch, &comp_destroyed, all_args, runtime_args_start, @@ -3419,113 +3411,6 @@ fn buildOutputType( ); } - // TODO move this REPL implementation to the standard library / build - // system and have it be a CLI abstraction layer on top of the real, actual - // binary protocol of the compiler. Make it actually interface through the - // server protocol. This way the REPL does not have any special powers that - // an IDE couldn't also have. - - const stdin = std.io.getStdIn().reader(); - const stderr = std.io.getStdErr().writer(); - var repl_buf: [1024]u8 = undefined; - - const ReplCmd = enum { - update, - help, - run, - update_and_run, - }; - - var last_cmd: ReplCmd = .help; - - while (watch) { - try stderr.print("(zig) ", .{}); - try comp.makeBinFileExecutable(); - if (stdin.readUntilDelimiterOrEof(&repl_buf, '\n') catch |err| { - try stderr.print("\nUnable to parse command: {s}\n", .{@errorName(err)}); - continue; - }) |line| { - const actual_line = mem.trimRight(u8, line, "\r\n "); - const cmd: ReplCmd = blk: { - if (mem.eql(u8, actual_line, "update")) { - break :blk .update; - } else if (mem.eql(u8, actual_line, "exit")) { - break; - } else if (mem.eql(u8, actual_line, "help")) { - break :blk .help; - } else if (mem.eql(u8, actual_line, "run")) { - break :blk .run; - } else if (mem.eql(u8, actual_line, "update-and-run")) { - break :blk .update_and_run; - } else if (actual_line.len == 0) { - break :blk last_cmd; - } else { - try stderr.print("unknown command: {s}\n", .{actual_line}); - continue; - } - }; - last_cmd = cmd; - switch (cmd) { - .update => { - tracy.frameMark(); - if (output_mode == .Exe) { - try comp.makeBinFileWritable(); - } - updateModule(gpa, comp, hook) catch |err| switch (err) { - error.SemanticAnalyzeFail => continue, - else => |e| return e, - }; - }, - .help => { - try stderr.writeAll(repl_help); - }, - .run => { - tracy.frameMark(); - try runOrTest( - comp, - gpa, - arena, - test_exec_args.items, - self_exe_path.?, - arg_mode, - target_info, - watch, - &comp_destroyed, - all_args, - runtime_args_start, - link_libc, - ); - }, - .update_and_run => { - tracy.frameMark(); - if (output_mode == .Exe) { - try comp.makeBinFileWritable(); - } - updateModule(gpa, comp, hook) catch |err| switch (err) { - error.SemanticAnalyzeFail => continue, - else => |e| return e, - }; - try comp.makeBinFileExecutable(); - try runOrTest( - comp, - gpa, - arena, - test_exec_args.items, - self_exe_path.?, - arg_mode, - target_info, - watch, - &comp_destroyed, - all_args, - runtime_args_start, - link_libc, - ); - }, - } - } else { - break; - } - } // Skip resource deallocation in release builds; let the OS do it. return cleanExit(); } @@ -3822,7 +3707,6 @@ fn runOrTest( self_exe_path: []const u8, arg_mode: ArgMode, target_info: std.zig.system.NativeTargetInfo, - watch: bool, comp_destroyed: *bool, all_args: []const []const u8, runtime_args_start: ?usize, @@ -3853,7 +3737,7 @@ fn runOrTest( // We do not execve for tests because if the test fails we want to print // the error message and invocation below. - if (std.process.can_execv and arg_mode == .run and !watch) { + if (std.process.can_execv and arg_mode == .run) { // execv releases the locks; no need to destroy the Compilation here. const err = std.process.execve(gpa, argv.items, &env_map); try warnAboutForeignBinaries(arena, arg_mode, target_info, link_libc); @@ -3866,12 +3750,10 @@ fn runOrTest( child.stdout_behavior = .Inherit; child.stderr_behavior = .Inherit; - if (!watch) { - // Here we release all the locks associated with the Compilation so - // that whatever this child process wants to do won't deadlock. - comp.destroy(); - comp_destroyed.* = true; - } + // Here we release all the locks associated with the Compilation so + // that whatever this child process wants to do won't deadlock. + comp.destroy(); + comp_destroyed.* = true; const term = child.spawnAndWait() catch |err| { try warnAboutForeignBinaries(arena, arg_mode, target_info, link_libc); @@ -3883,19 +3765,13 @@ fn runOrTest( switch (term) { .Exited => |code| { if (code == 0) { - if (!watch) return cleanExit(); - } else if (watch) { - warn("process exited with code {d}", .{code}); + return cleanExit(); } else { process.exit(code); } }, else => { - if (watch) { - warn("process aborted abnormally", .{}); - } else { - process.exit(1); - } + process.exit(1); }, } }, @@ -3903,7 +3779,7 @@ fn runOrTest( switch (term) { .Exited => |code| { if (code == 0) { - if (!watch) return cleanExit(); + return cleanExit(); } else { const cmd = try std.mem.join(arena, " ", argv.items); fatal("the following test command failed with exit code {d}:\n{s}", .{ code, cmd }); -- cgit v1.2.3 From 363d4a107de3af95620f82e851e8777aa9f576c9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 14 Mar 2023 16:12:24 -0700 Subject: add compile log output to build runner --- lib/std/zig/ErrorBundle.zig | 29 ++++++++++++++++++++++++++++- src/Compilation.zig | 8 ++------ src/Sema.zig | 2 +- src/main.zig | 18 +++++++----------- 4 files changed, 38 insertions(+), 19 deletions(-) (limited to 'src/main.zig') diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index 845e9d8ff5..ffe748203e 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -32,6 +32,8 @@ pub const SourceLocationIndex = enum(u32) { pub const ErrorMessageList = struct { len: u32, start: u32, + /// null-terminated string index. 0 means no compile log text. + compile_log_text: u32, }; /// Trailing: @@ -110,6 +112,10 @@ pub fn getNotes(eb: ErrorBundle, index: MessageIndex) []const MessageIndex { return @ptrCast([]const MessageIndex, eb.extra[start..][0..notes_len]); } +pub fn getCompileLogOutput(eb: ErrorBundle) [:0]const u8 { + return nullTerminatedString(eb, getErrorMessageList(eb).compile_log_text); +} + /// Returns the requested data, as well as the new index which is at the start of the /// trailers for the object. fn extraData(eb: ErrorBundle, comptime T: type, index: usize) struct { data: T, end: usize } { @@ -145,6 +151,7 @@ pub const RenderOptions = struct { ttyconf: std.debug.TTY.Config, include_reference_trace: bool = true, include_source_line: bool = true, + include_log_text: bool = true, }; pub fn renderToStdErr(eb: ErrorBundle, options: RenderOptions) void { @@ -158,6 +165,14 @@ pub fn renderToWriter(eb: ErrorBundle, options: RenderOptions, writer: anytype) for (eb.getMessages()) |err_msg| { try renderErrorMessageToWriter(eb, options, err_msg, writer, "error", .Red, 0); } + + if (options.include_log_text) { + const log_text = eb.getCompileLogOutput(); + if (log_text.len != 0) { + try writer.writeAll("\nCompile Log Output:\n"); + try writer.writeAll(log_text); + } + } } fn renderErrorMessageToWriter( @@ -314,6 +329,7 @@ pub const Wip = struct { assert(0 == try addExtra(wip, ErrorMessageList{ .len = 0, .start = 0, + .compile_log_text = 0, })); } @@ -325,9 +341,10 @@ pub const Wip = struct { wip.* = undefined; } - pub fn toOwnedBundle(wip: *Wip) !ErrorBundle { + pub fn toOwnedBundle(wip: *Wip, compile_log_text: []const u8) !ErrorBundle { const gpa = wip.gpa; if (wip.root_list.items.len == 0) { + assert(compile_log_text.len == 0); // Special encoding when there are no errors. wip.deinit(); wip.* = .{ @@ -338,9 +355,19 @@ pub const Wip = struct { }; return empty; } + + const compile_log_str_index = if (compile_log_text.len == 0) 0 else str: { + const str = @intCast(u32, wip.string_bytes.items.len); + try wip.string_bytes.ensureUnusedCapacity(gpa, compile_log_text.len + 1); + wip.string_bytes.appendSliceAssumeCapacity(compile_log_text); + wip.string_bytes.appendAssumeCapacity(0); + break :str str; + }; + wip.setExtra(0, ErrorMessageList{ .len = @intCast(u32, wip.root_list.items.len), .start = @intCast(u32, wip.extra.items.len), + .compile_log_text = compile_log_str_index, }); try wip.extra.appendSlice(gpa, @ptrCast([]const u32, wip.root_list.items)); wip.root_list.clearAndFree(gpa); diff --git a/src/Compilation.zig b/src/Compilation.zig index ec21d2c483..89512ce744 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2705,7 +2705,8 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { assert(self.totalErrorCount() == bundle.root_list.items.len); - return bundle.toOwnedBundle(); + const compile_log_text = if (self.bin_file.options.module) |m| m.compile_log_text.items else ""; + return bundle.toOwnedBundle(compile_log_text); } pub const ErrorNoteHashContext = struct { @@ -2954,11 +2955,6 @@ pub fn addZirErrorMessages(eb: *ErrorBundle.Wip, file: *Module.File) !void { } } -pub fn getCompileLogOutput(self: *Compilation) []const u8 { - const module = self.bin_file.options.module orelse return &[0]u8{}; - return module.compile_log_text.items; -} - pub fn performAllTheWork( comp: *Compilation, main_progress_node: *std.Progress.Node, diff --git a/src/Sema.zig b/src/Sema.zig index c9f4f27fe2..fc39fbb9fc 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2219,7 +2219,7 @@ fn failWithOwnedErrorMsg(sema: *Sema, err_msg: *Module.ErrorMsg) CompileError { wip_errors.init(gpa) catch unreachable; Compilation.addModuleErrorMsg(&wip_errors, err_msg.*) catch unreachable; std.debug.print("compile error during Sema:\n", .{}); - var error_bundle = wip_errors.toOwnedBundle() catch unreachable; + var error_bundle = wip_errors.toOwnedBundle("") catch unreachable; error_bundle.renderToStdErr(.{ .ttyconf = .no_color }); crash_report.compilerPanic("unexpected compile error occurred", null, null); } diff --git a/src/main.zig b/src/main.zig index 19f1a6e52c..e7d5c647b5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3886,10 +3886,6 @@ fn updateModule(gpa: Allocator, comp: *Compilation, hook: AfterUpdateHook) !void if (errors.errorMessageCount() > 0) { errors.renderToStdErr(renderOptions(comp.color)); - const log_text = comp.getCompileLogOutput(); - if (log_text.len != 0) { - std.debug.print("\nCompile Log Output:\n{s}", .{log_text}); - } return error.SemanticAnalyzeFail; } else switch (hook) { .none => {}, @@ -4512,7 +4508,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &all_modules, ); if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(); + var errors = try wip_errors.toOwnedBundle(""); defer errors.deinit(gpa); errors.renderToStdErr(renderOptions(color)); process.exit(1); @@ -4775,7 +4771,7 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(color)); process.exit(2); @@ -4981,7 +4977,7 @@ fn fmtPathFile( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(fmt.color)); fmt.any_error = true; @@ -5018,7 +5014,7 @@ fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Co try putAstErrorsIntoBundle(gpa, tree, path, &wip_errors); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(color)); } @@ -5622,7 +5618,7 @@ pub fn cmdAstCheck( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); @@ -5739,7 +5735,7 @@ pub fn cmdChangelist( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); @@ -5774,7 +5770,7 @@ pub fn cmdChangelist( try wip_errors.init(gpa); defer wip_errors.deinit(); try Compilation.addZirErrorMessages(&wip_errors, &file); - var error_bundle = try wip_errors.toOwnedBundle(); + var error_bundle = try wip_errors.toOwnedBundle(""); defer error_bundle.deinit(gpa); error_bundle.renderToStdErr(renderOptions(color)); process.exit(1); -- cgit v1.2.3