From abf895595189eb45df8c97f4029c58976815b450 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 14 Jul 2024 19:48:08 -0700 Subject: make zig compiler processes live across rebuilds Changes the `make` function signature to take an options struct, which additionally includes `watch: bool`. I intentionally am not exposing this information to configure phase logic. Also adds global zig cache to the compiler cache prefixes. Closes #20600 --- lib/std/Build/Step.zig | 157 ++++++++++++++++++++++--------- lib/std/Build/Step/CheckFile.zig | 4 +- lib/std/Build/Step/CheckObject.zig | 4 +- lib/std/Build/Step/Compile.zig | 14 ++- lib/std/Build/Step/ConfigHeader.zig | 4 +- lib/std/Build/Step/Fail.zig | 4 +- lib/std/Build/Step/Fmt.zig | 4 +- lib/std/Build/Step/InstallArtifact.zig | 4 +- lib/std/Build/Step/InstallDir.zig | 4 +- lib/std/Build/Step/InstallFile.zig | 4 +- lib/std/Build/Step/ObjCopy.zig | 5 +- lib/std/Build/Step/Options.zig | 6 +- lib/std/Build/Step/RemoveDir.zig | 6 +- lib/std/Build/Step/Run.zig | 3 +- lib/std/Build/Step/TranslateC.zig | 5 +- lib/std/Build/Step/UpdateSourceFiles.zig | 4 +- lib/std/Build/Step/WriteFile.zig | 4 +- 17 files changed, 158 insertions(+), 78 deletions(-) (limited to 'lib/std/Build') diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 3c6cd660ff..397ad6c55f 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -68,7 +68,13 @@ pub const TestResults = struct { } }; -pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerror!void; +pub const MakeOptions = struct { + progress_node: std.Progress.Node, + thread_pool: *std.Thread.Pool, + watch: bool, +}; + +pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; pub const State = enum { precheck_unstarted, @@ -219,10 +225,10 @@ pub fn init(options: StepOptions) Step { /// If the Step's `make` function reports `error.MakeFailed`, it indicates they /// have already reported the error. Otherwise, we add a simple error report /// here. -pub fn make(s: *Step, prog_node: std.Progress.Node) error{ MakeFailed, MakeSkipped }!void { +pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { const arena = s.owner.allocator; - s.makeFn(s, prog_node) catch |err| switch (err) { + s.makeFn(s, options) catch |err| switch (err) { error.MakeFailed => return error.MakeFailed, error.MakeSkipped => return error.MakeSkipped, else => { @@ -260,8 +266,8 @@ pub fn getStackTrace(s: *Step) ?std.builtin.StackTrace { }; } -fn makeNoOp(step: *Step, prog_node: std.Progress.Node) anyerror!void { - _ = prog_node; +fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { + _ = options; var all_cached = true; @@ -352,13 +358,25 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO try step.result_error_msgs.append(arena, msg); } +pub const ZigProcess = struct { + child: std.process.Child, + poller: std.io.Poller(StreamEnum), + + pub const StreamEnum = enum { stdout, stderr }; +}; + /// Assumes that argv contains `--listen=-` and that the process being spawned /// is the zig compiler - the same version that compiled the build runner. pub fn evalZigProcess( s: *Step, argv: []const []const u8, prog_node: std.Progress.Node, + watch: bool, ) !?[]const u8 { + if (s.getZigProcess()) |zp| { + assert(watch); + return zigProcessUpdate(s, zp, watch); + } assert(argv.len != 0); const b = s.owner; const arena = b.allocator; @@ -378,29 +396,76 @@ pub fn evalZigProcess( child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err), }); - var timer = try std.time.Timer.start(); - var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ - .stdout = child.stdout.?, - .stderr = child.stderr.?, - }); - defer poller.deinit(); + const zp = try arena.create(ZigProcess); + zp.* = .{ + .child = child, + .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }), + }; + if (watch) s.setZigProcess(zp); + defer if (!watch) zp.poller.deinit(); + + const result = try zigProcessUpdate(s, zp, watch); + + if (!watch) { + // Send EOF to stdin. + zp.child.stdin.?.close(); + zp.child.stdin = null; + + const term = zp.child.wait() catch |err| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + + // Special handling for Compile step that is expecting compile errors. + if (s.cast(Compile)) |compile| switch (term) { + .Exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors != null) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + + try handleChildProcessTerm(s, term, null, argv); + } + + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ + s.result_error_bundle.errorMessageCount(), + try allocPrintCmd(arena, null, argv), + }); + } + + return result; +} - try sendMessage(child.stdin.?, .update); - try sendMessage(child.stdin.?, .exit); +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 { + const b = s.owner; + const arena = b.allocator; + + var timer = try std.time.Timer.start(); + + try sendMessage(zp.child.stdin.?, .update); + if (!watch) try sendMessage(zp.child.stdin.?, .exit); const Header = std.zig.Server.Message.Header; var result: ?[]const u8 = null; - const stdout = poller.fifo(.stdout); + const stdout = zp.poller.fifo(.stdout); poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { - if (!(try poller.poll())) break :poll; + if (!(try zp.poller.poll())) break :poll; } const header = stdout.reader().readStruct(Header) catch unreachable; while (stdout.readableLength() < header.bytes_len) { - if (!(try poller.poll())) break :poll; + if (!(try zp.poller.poll())) break :poll; } const body = stdout.readableSliceOfLen(header.bytes_len); @@ -428,12 +493,22 @@ pub fn evalZigProcess( .string_bytes = try arena.dupe(u8, string_bytes), .extra = extra_array, }; + if (watch) { + // This message indicates the end of the update. + stdout.discard(body.len); + break; + } }, .emit_bin_path => { const EbpHdr = std.zig.Server.Message.EmitBinPath; const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body)); s.result_cached = ebp_hdr.flags.cache_hit; result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); + if (watch) { + // This message indicates the end of the update. + stdout.discard(body.len); + break; + } }, .file_system_inputs => { s.clearWatchInputs(); @@ -470,6 +545,13 @@ pub fn evalZigProcess( }; try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); }, + .global_cache => { + const path: Build.Cache.Path = .{ + .root_dir = s.owner.graph.global_cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, } } }, @@ -479,43 +561,28 @@ pub fn evalZigProcess( stdout.discard(body.len); } - const stderr = poller.fifo(.stderr); + s.result_duration_ns = timer.read(); + + const stderr = zp.poller.fifo(.stderr); if (stderr.readableLength() > 0) { try s.result_error_msgs.append(arena, try stderr.toOwnedSlice()); } - // Send EOF to stdin. - child.stdin.?.close(); - child.stdin = null; + return result; +} - const term = child.wait() catch |err| { - return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); +fn getZigProcess(s: *Step) ?*ZigProcess { + return switch (s.id) { + .compile => s.cast(Compile).?.zig_process, + else => null, }; - s.result_duration_ns = timer.read(); - s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; - - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .Exited => { - // Note that the exit code may be 0 in this case due to the - // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } - }, - else => {}, - }; - - try handleChildProcessTerm(s, term, null, argv); +} - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ - s.result_error_bundle.errorMessageCount(), - try allocPrintCmd(arena, null, argv), - }); +fn setZigProcess(s: *Step, zp: *ZigProcess) void { + switch (s.id) { + .compile => s.cast(Compile).?.zig_process = zp, + else => unreachable, } - - return result; } fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index c7a2046c1f..699e6d2e9d 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -46,8 +46,8 @@ pub fn setName(check_file: *CheckFile, name: []const u8) void { check_file.step.name = name; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const check_file: *CheckFile = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(check_file.source); diff --git a/lib/std/Build/Step/CheckObject.zig b/lib/std/Build/Step/CheckObject.zig index 93ee57e3b4..ce33003117 100644 --- a/lib/std/Build/Step/CheckObject.zig +++ b/lib/std/Build/Step/CheckObject.zig @@ -550,8 +550,8 @@ pub fn checkComputeCompare( check_object.checks.append(check) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, make_options: Step.MakeOptions) !void { + _ = make_options; const b = step.owner; const gpa = b.allocator; const check_object: *CheckObject = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 89862e7501..9af6ace9c0 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -213,6 +213,10 @@ is_linking_libcpp: bool = false, no_builtin: bool = false, +/// Populated during the make phase when there is a long-lived compiler process. +/// Managed by the build runner, not user build script. +zig_process: ?*Step.ZigProcess, + pub const ExpectedCompileErrors = union(enum) { contains: []const u8, exact: []const []const u8, @@ -398,6 +402,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, + + .zig_process = null, }; compile.root_module.init(owner, options.root_module, compile); @@ -1735,13 +1741,17 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { return try zig_args.toOwnedSlice(); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; const compile: *Compile = @fieldParentPtr("step", step); const zig_args = try getZigArgs(compile); - const maybe_output_bin_path = step.evalZigProcess(zig_args, prog_node) catch |err| switch (err) { + const maybe_output_bin_path = step.evalZigProcess( + zig_args, + options.progress_node, + options.watch, + ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); try checkCompileErrors(compile); diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index fd655125cf..512460a532 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -164,8 +164,8 @@ fn putValue(config_header: *ConfigHeader, field_name: []const u8, comptime T: ty } } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const config_header: *ConfigHeader = @fieldParentPtr("step", step); if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp); diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig index 09b8515f77..9236c2ac7b 100644 --- a/lib/std/Build/Step/Fail.zig +++ b/lib/std/Build/Step/Fail.zig @@ -24,8 +24,8 @@ pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { return fail; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; // No progress to report. +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; // No progress to report. const fail: *Fail = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index 1d3850a7ec..576aeb1d21 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -36,7 +36,9 @@ pub fn create(owner: *std.Build, options: Options) *Fmt { return fmt; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; + // TODO: if check=false, this means we are modifying source files in place, which // is an operation that could race against other operations also modifying source files // in place. In this case, this step should obtain a write lock while making those diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index bd1d5db4a9..4e778d897c 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -115,8 +115,8 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins return install_artifact; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); const b = step.owner; const cwd = fs.cwd(); diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 78281e56d9..4d4ff78cfc 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -55,8 +55,8 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { return install_dir; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const install_dir: *InstallDir = @fieldParentPtr("step", step); step.clearWatchInputs(); diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index d29ac21c1c..fb1c0ffc34 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -35,8 +35,8 @@ pub fn create( return install_file; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const install_file: *InstallFile = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(install_file.source); diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index d314550f60..6ed23d3fb4 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -90,7 +90,8 @@ pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath { return if (objcopy.output_file_debug) |*file| .{ .generated = .{ .file = file } } else null; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const objcopy: *ObjCopy = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(objcopy.input_file); @@ -158,7 +159,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { try argv.appendSlice(&.{ full_src_path, full_dest_path }); try argv.append("--listen=-"); - _ = try step.evalZigProcess(argv.items, prog_node); + _ = try step.evalZigProcess(argv.items, prog_node, false); objcopy.output_file.path = full_dest_path; if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug; diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index b67acd4086..7c872db170 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -410,9 +410,9 @@ pub fn getOutput(options: *Options) LazyPath { return .{ .generated = .{ .file = &options.generated_file } }; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - // This step completes so quickly that no progress is necessary. - _ = prog_node; +fn make(step: *Step, make_options: Step.MakeOptions) !void { + // This step completes so quickly that no progress reporting is necessary. + _ = make_options; const b = step.owner; const options: *Options = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/RemoveDir.zig b/lib/std/Build/Step/RemoveDir.zig index 1b7dc7feb8..e2d4c02abc 100644 --- a/lib/std/Build/Step/RemoveDir.zig +++ b/lib/std/Build/Step/RemoveDir.zig @@ -23,10 +23,8 @@ pub fn create(owner: *std.Build, doomed_path: LazyPath) *RemoveDir { return remove_dir; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - // TODO update progress node while walking file system. - // Should the standard library support this use case?? - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const remove_dir: *RemoveDir = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index bc89d1a12f..d79d4647b1 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -595,7 +595,8 @@ const IndexedOutput = struct { tag: @typeInfo(Arg).Union.tag_type.?, output: *Output, }; -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const arena = b.allocator; const run: *Run = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index d7c62ed194..ac4729abd0 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -116,7 +116,8 @@ pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) voi translate_c.c_macros.append(translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const translate_c: *TranslateC = @fieldParentPtr("step", step); @@ -154,7 +155,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { try argv_list.append(translate_c.source.getPath2(b, step)); - const output_path = try step.evalZigProcess(argv_list.items, prog_node); + const output_path = try step.evalZigProcess(argv_list.items, prog_node, false); translate_c.out_basename = fs.path.basename(output_path.?); const output_dir = fs.path.dirname(output_path.?).?; diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 9d1c8e20fe..d4a9565083 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -67,8 +67,8 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: [] }) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index c1488a23d2..29fba1c871 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -171,8 +171,8 @@ fn maybeUpdateName(write_file: *WriteFile) void { } } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const arena = b.allocator; const gpa = arena; -- cgit v1.2.3 From 716b128a24ffc44a8694f3d61e3d74b5297fd564 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 14 Jul 2024 21:18:09 -0700 Subject: frontend: add -fincremental, -fno-incremental flag Remove --debug-incremental This flag is also added to the build system. Importantly, this tells Compile step whether or not to keep the compiler running between rebuilds. It defaults off because it is currently crashing zirUpdateRefs. --- lib/compiler/build_runner.zig | 7 +++++++ lib/std/Build.zig | 1 + lib/std/Build/Step/Compile.zig | 4 +++- src/Compilation.zig | 6 +++--- src/Sema.zig | 12 ++++++------ src/Zcu.zig | 2 +- src/Zcu/PerThread.zig | 2 +- src/main.zig | 23 ++++++++++++++--------- 8 files changed, 36 insertions(+), 21 deletions(-) (limited to 'lib/std/Build') diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 5f5a991fd6..af65d10948 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -72,6 +72,7 @@ pub fn main() !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, + .incremental = null, }; graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); @@ -235,6 +236,10 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { builder.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { @@ -1216,6 +1221,8 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified \\ --debounce Delay before rebuilding after changed file detected + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index c5ff143aca..29621f95c9 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -120,6 +120,7 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, + incremental: ?bool, }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 9af6ace9c0..724656ce29 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1679,6 +1679,8 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { b.fmt("{}", .{err_limit}), }); + try addFlag(&zig_args, "incremental", b.graph.incremental); + try zig_args.append("--listen=-"); // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux @@ -1750,7 +1752,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { const maybe_output_bin_path = step.evalZigProcess( zig_args, options.progress_node, - options.watch, + (b.graph.incremental == true) and options.watch, ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); diff --git a/src/Compilation.zig b/src/Compilation.zig index 83a7501ffa..76dc42fc64 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -169,7 +169,7 @@ time_report: bool, stack_report: bool, debug_compiler_runtime_libs: bool, debug_compile_errors: bool, -debug_incremental: bool, +incremental: bool, job_queued_compiler_rt_lib: bool = false, job_queued_compiler_rt_obj: bool = false, job_queued_update_builtin_zig: bool, @@ -1134,7 +1134,7 @@ pub const CreateOptions = struct { verbose_llvm_cpu_features: bool = false, debug_compiler_runtime_libs: bool = false, debug_compile_errors: bool = false, - debug_incremental: bool = false, + incremental: bool = false, /// Normally when you create a `Compilation`, Zig will automatically build /// and link in required dependencies, such as compiler-rt and libc. When /// building such dependencies themselves, this flag must be set to avoid @@ -1516,7 +1516,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .test_name_prefix = options.test_name_prefix, .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs, .debug_compile_errors = options.debug_compile_errors, - .debug_incremental = options.debug_incremental, + .incremental = options.incremental, .libcxx_abi_version = options.libcxx_abi_version, .root_name = root_name, .sysroot = sysroot, diff --git a/src/Sema.zig b/src/Sema.zig index f5ee909caf..36694eef47 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2726,7 +2726,7 @@ fn maybeRemoveOutdatedType(sema: *Sema, ty: InternPool.Index) !bool { const pt = sema.pt; const zcu = pt.zcu; - if (!zcu.comp.debug_incremental) return false; + if (!zcu.comp.incremental) return false; const decl_index = Type.fromInterned(ty).getOwnerDecl(zcu); const decl_as_depender = AnalUnit.wrap(.{ .decl = decl_index }); @@ -2826,7 +2826,7 @@ fn zirStructDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try ip.addDependency( sema.gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3064,7 +3064,7 @@ fn zirEnumDecl( new_decl.owns_tv = true; errdefer if (!done) pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try mod.intern_pool.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3331,7 +3331,7 @@ fn zirUnionDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try mod.intern_pool.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3421,7 +3421,7 @@ fn zirOpaqueDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try ip.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -38098,7 +38098,7 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void { const zcu = sema.pt.zcu; - if (!zcu.comp.debug_incremental) return; + if (!zcu.comp.incremental) return; // Avoid creating dependencies on ourselves. This situation can arise when we analyze the fields // of a type and they use `@This()`. This dependency would be unnecessary, and in fact would diff --git a/src/Zcu.zig b/src/Zcu.zig index 15f418c6fe..36b4c95571 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -2679,7 +2679,7 @@ fn markTransitiveDependersPotentiallyOutdated(zcu: *Zcu, maybe_outdated: AnalUni } pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?AnalUnit { - if (!zcu.comp.debug_incremental) return null; + if (!zcu.comp.incremental) return null; if (zcu.outdated.count() == 0 and zcu.potentially_outdated.count() == 0) { log.debug("findOutdatedToAnalyze: no outdated depender", .{}); diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 73b4e6de5c..c3f569cc7d 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -888,7 +888,7 @@ fn getFileRootStruct( }; errdefer wip_ty.cancel(ip, pt.tid); - if (zcu.comp.debug_incremental) { + if (zcu.comp.incremental) { try ip.addDependency( gpa, InternPool.AnalUnit.wrap(.{ .decl = decl_index }), diff --git a/src/main.zig b/src/main.zig index 06c20d514c..2b36c31865 100644 --- a/src/main.zig +++ b/src/main.zig @@ -404,6 +404,8 @@ const usage_build_generic = \\ -h, --help Print this help and exit \\ --color [auto|off|on] Enable or disable colored error messages \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation \\ -femit-bin[=path] (default) Output machine code \\ -fno-emit-bin Do not output machine code \\ -femit-asm[=path] Output .s (assembly code) @@ -642,7 +644,6 @@ const usage_build_generic = \\ --debug-log [scope] Enable printing debug/info log messages for scope \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format - \\ --debug-incremental Enable experimental feature: incremental compilation \\ ; @@ -904,7 +905,7 @@ fn buildOutputType( var minor_subsystem_version: ?u16 = null; var mingw_unicode_entry_point: bool = false; var enable_link_snapshots: bool = false; - var debug_incremental: bool = false; + var opt_incremental: ?bool = null; var install_name: ?[]const u8 = null; var hash_style: link.File.Elf.HashStyle = .both; var entitlements: ?[]const u8 = null; @@ -1357,8 +1358,10 @@ fn buildOutputType( } else { enable_link_snapshots = true; } - } else if (mem.eql(u8, arg, "--debug-incremental")) { - debug_incremental = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + opt_incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + opt_incremental = false; } else if (mem.eql(u8, arg, "--entitlements")) { entitlements = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-fcompiler-rt")) { @@ -3225,6 +3228,8 @@ fn buildOutputType( break :b .incremental; }; + const incremental = opt_incremental orelse false; + process.raiseFileDescriptorLimit(); var file_system_inputs: std.ArrayListUnmanaged(u8) = .{}; @@ -3336,7 +3341,7 @@ fn buildOutputType( .cache_mode = cache_mode, .subsystem = subsystem, .debug_compile_errors = debug_compile_errors, - .debug_incremental = debug_incremental, + .incremental = incremental, .enable_link_snapshots = enable_link_snapshots, .install_name = install_name, .entitlements = entitlements, @@ -3443,7 +3448,7 @@ fn buildOutputType( updateModule(comp, color, root_prog_node) catch |err| switch (err) { error.SemanticAnalyzeFail => { assert(listen == .none); - saveState(comp, debug_incremental); + saveState(comp, incremental); process.exit(1); }, else => |e| return e, @@ -3451,7 +3456,7 @@ fn buildOutputType( } if (build_options.only_c) return cleanExit(); try comp.makeBinFileExecutable(); - saveState(comp, debug_incremental); + saveState(comp, incremental); if (test_exec_args.items.len == 0 and target.ofmt == .c) default_exec_args: { // Default to using `zig run` to execute the produced .c code from `zig test`. @@ -4032,8 +4037,8 @@ fn createModule( return mod; } -fn saveState(comp: *Compilation, debug_incremental: bool) void { - if (debug_incremental) { +fn saveState(comp: *Compilation, incremental: bool) void { + if (incremental) { comp.saveState() catch |err| { warn("unable to save incremental compilation state: {s}", .{@errorName(err)}); }; -- cgit v1.2.3 From f6c1b71c220406d60ca4bdf9c949e775f7fca466 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 14 Jul 2024 21:38:55 -0700 Subject: build system: update std.Progress.Node for long-lived children --- lib/std/Build/Step.zig | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib/std/Build') diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 397ad6c55f..fc40a680c4 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -361,6 +361,7 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO pub const ZigProcess = struct { child: std.process.Child, poller: std.io.Poller(StreamEnum), + progress_ipc_fd: if (std.Progress.have_ipc) ?std.posix.fd_t else void, pub const StreamEnum = enum { stdout, stderr }; }; @@ -375,6 +376,7 @@ pub fn evalZigProcess( ) !?[]const u8 { if (s.getZigProcess()) |zp| { assert(watch); + if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd); return zigProcessUpdate(s, zp, watch); } assert(argv.len != 0); @@ -404,6 +406,7 @@ pub fn evalZigProcess( .stdout = child.stdout.?, .stderr = child.stderr.?, }), + .progress_ipc_fd = if (std.Progress.have_ipc) child.progress_node.getIpcFd() else {}, }; if (watch) s.setZigProcess(zp); defer if (!watch) zp.poller.deinit(); @@ -435,6 +438,8 @@ pub fn evalZigProcess( try handleChildProcessTerm(s, term, null, argv); } + // This is intentionally printed for failure on the first build but not for + // subsequent rebuilds. if (s.result_error_bundle.errorMessageCount() > 0) { return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ s.result_error_bundle.errorMessageCount(), -- cgit v1.2.3 From 987f63208e4bdd044768465f1581d03ee8964e28 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 14 Jul 2024 22:17:29 -0700 Subject: build runner: handle compiler subprocess failures gracefully Compilation errors now report a failure on rebuilds triggered by file system watches. Compiler crashes now report failure correctly on rebuilds triggered by file system watches. The compiler subprocess is restarted if a broken pipe is encountered on a rebuild. --- lib/std/Build/Step.zig | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) (limited to 'lib/std/Build') diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index fc40a680c4..661ee58509 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -374,10 +374,37 @@ pub fn evalZigProcess( prog_node: std.Progress.Node, watch: bool, ) !?[]const u8 { - if (s.getZigProcess()) |zp| { + if (s.getZigProcess()) |zp| update: { assert(watch); if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd); - return zigProcessUpdate(s, zp, watch); + const result = zigProcessUpdate(s, zp, watch) catch |err| switch (err) { + error.BrokenPipe => { + // Process restart required. + const term = zp.child.wait() catch |e| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); + }; + _ = term; + s.clearZigProcess(); + break :update; + }, + else => |e| return e, + }; + + if (s.result_error_bundle.errorMessageCount() > 0) + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + + if (s.result_error_msgs.items.len > 0 and result == null) { + // Crash detected. + const term = zp.child.wait() catch |e| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + s.clearZigProcess(); + try handleChildProcessTerm(s, term, null, argv); + return error.MakeFailed; + } + + return result; } assert(argv.len != 0); const b = s.owner; @@ -399,7 +426,7 @@ pub fn evalZigProcess( argv[0], @errorName(err), }); - const zp = try arena.create(ZigProcess); + const zp = try gpa.create(ZigProcess); zp.* = .{ .child = child, .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{ @@ -590,6 +617,20 @@ fn setZigProcess(s: *Step, zp: *ZigProcess) void { } } +fn clearZigProcess(s: *Step) void { + const gpa = s.owner.allocator; + switch (s.id) { + .compile => { + const compile = s.cast(Compile).?; + if (compile.zig_process) |zp| { + gpa.destroy(zp); + compile.zig_process = null; + } + }, + else => unreachable, + } +} + fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { const header: std.zig.Client.Message.Header = .{ .tag = tag, -- cgit v1.2.3 From 445bd7a06fc34c9a59c6458774769bfaa2757a2f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 14 Jul 2024 22:27:51 -0700 Subject: build runner: update watch caption to include subprocesses --- lib/compiler/build_runner.zig | 13 ++++++++++--- lib/std/Build.zig | 2 +- lib/std/Build/Step.zig | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) (limited to 'lib/std/Build') diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index af65d10948..1b13881586 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -72,7 +72,6 @@ pub fn main() !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, - .incremental = null, }; graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); @@ -411,8 +410,8 @@ pub fn main() !void { // trigger a rebuild on all steps with modified inputs, as well as their // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{ - w.dir_table.entries.len, + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var debounce_timeout: Watch.Timeout = .none; @@ -445,6 +444,14 @@ fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { }; } +fn countSubProcesses(all_steps: []const *Step) usize { + var count: usize = 0; + for (all_steps) |s| { + count += @intFromBool(s.getZigProcess() != null); + } + return count; +} + const Run = struct { max_rss: u64, max_rss_is_default: bool, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 29621f95c9..06de7aa6f4 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -120,7 +120,7 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, - incremental: ?bool, + incremental: ?bool = null, }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 661ee58509..8f3236d867 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -603,7 +603,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 { return result; } -fn getZigProcess(s: *Step) ?*ZigProcess { +pub fn getZigProcess(s: *Step) ?*ZigProcess { return switch (s.id) { .compile => s.cast(Compile).?.zig_process, else => null, -- cgit v1.2.3