From 274555be21ea756d8480a586f771274a79a58d80 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 6 Dec 2021 16:38:12 -0700 Subject: stage2: improve handling of the generated file builtin.zig All Zig code is eligible to `@import("builtin")` which is mapped to a generated file, build.zig, based on the target and other settings. Zig invocations which share the same target settings will generate the same builtin.zig file and thus the path to builtin.zig is in a shared cache folder, and different projects can sometimes use the same file. Before this commit, this led to race conditions where multiple invocations of `zig` would race to write this file. If one process wanted to *read* the file while the other process *wrote* the file, the reading process could observe a truncated or partially written builtin.zig file. This commit makes the following improvements: - limitations: - avoid clobbering the inode, mtime in the hot path - avoid creating a partially written file - builtin.zig needs to be on disk for debug info / stack trace purposes - don't mark the task as complete until the file is finished being populated (possibly by an external process) - strategy: - create the `@import("builtin")` `Module.File` during the AstGen work, based on generating the contents in memory rather than loading from disk. - write builtin.zig in a separate task that doesn't have to complete until the end of the AstGen work queue so that it can be done in parallel with everything else. - when writing the file, first stat the file path. If it exists, we are done. - otherwise, write the file to a temp file in the same directory and atomically rename it into place (clobbering the inode, mtime in the cold path). - summary: - all limitations respected - hot path: one stat() syscall that happens in a worker thread This required adding a missing function to the standard library: `std.fs.Dir.statFile`. In this commit, it does open() and then fstat() which is two syscalls. It should be improved in a future commit to only make one. Fixes #9439. --- src/Compilation.zig | 69 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 27 deletions(-) (limited to 'src/Compilation.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index b07761e7cb..f0be3aac65 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2152,15 +2152,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor defer main_progress_node.end(); if (self.color == .off) progress.terminal = null; - // If we need to write out builtin.zig, it needs to be done before starting - // the AstGen tasks. - if (self.bin_file.options.module) |mod| { - if (mod.job_queued_update_builtin_zig) { - mod.job_queued_update_builtin_zig = false; - try self.updateBuiltinZigFile(mod); - } - } - // Here we queue up all the AstGen tasks first, followed by C object compilation. // We wait until the AstGen tasks are all completed before proceeding to the // (at least for now) single-threaded main work queue. However, C object compilation @@ -2185,6 +2176,21 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor self.astgen_wait_group.reset(); defer self.astgen_wait_group.wait(); + // builtin.zig is handled specially for two reasons: + // 1. to avoid race condition of zig processes truncating each other's builtin.zig files + // 2. optimization; in the hot path it only incurs a stat() syscall, which happens + // in the `astgen_wait_group`. + if (self.bin_file.options.module) |mod| { + if (mod.job_queued_update_builtin_zig) { + mod.job_queued_update_builtin_zig = false; + + self.astgen_wait_group.start(); + try self.thread_pool.spawn(workerUpdateBuiltinZigFile, .{ + self, mod, &self.astgen_wait_group, + }); + } + } + while (self.astgen_work_queue.readItem()) |file| { self.astgen_wait_group.start(); try self.thread_pool.spawn(workerAstGenFile, .{ @@ -2748,6 +2754,8 @@ fn workerAstGenFile( extra_index = item.end; const import_path = file.zir.nullTerminatedString(item.data.name); + // `@import("builtin")` is handled specially. + if (mem.eql(u8, import_path, "builtin")) continue; const import_result = blk: { comp.mutex.lock(); @@ -2775,6 +2783,29 @@ fn workerAstGenFile( } } +fn workerUpdateBuiltinZigFile( + comp: *Compilation, + mod: *Module, + wg: *WaitGroup, +) void { + defer wg.finish(); + + mod.populateBuiltinFile() catch |err| { + const dir_path: []const u8 = mod.zig_cache_artifact_directory.path orelse "."; + + comp.mutex.lock(); + defer comp.mutex.unlock(); + + comp.setMiscFailure(.write_builtin_zig, "unable to write builtin.zig to {s}: {s}", .{ + dir_path, @errorName(err), + }) catch |oom| switch (oom) { + error.OutOfMemory => log.err("unable to write builtin.zig to {s}: {s}", .{ + dir_path, @errorName(err), + }), + }; + }; +} + fn workerCheckEmbedFile( comp: *Compilation, embed_file: *Module.EmbedFile, @@ -4046,22 +4077,6 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool { comp.bin_file.options.object_format != .c; } -fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) Allocator.Error!void { - const tracy_trace = trace(@src()); - defer tracy_trace.end(); - - const source = try comp.generateBuiltinZigSource(comp.gpa); - defer comp.gpa.free(source); - - mod.zig_cache_artifact_directory.handle.writeFile("builtin.zig", source) catch |err| { - const dir_path: []const u8 = mod.zig_cache_artifact_directory.path orelse "."; - try comp.setMiscFailure(.write_builtin_zig, "unable to write builtin.zig to {s}: {s}", .{ - dir_path, - @errorName(err), - }); - }; -} - fn setMiscFailure( comp: *Compilation, tag: MiscTask, @@ -4084,7 +4099,7 @@ pub fn dump_argv(argv: []const []const u8) void { std.debug.print("{s}\n", .{argv[argv.len - 1]}); } -pub fn generateBuiltinZigSource(comp: *Compilation, allocator: Allocator) Allocator.Error![]u8 { +pub fn generateBuiltinZigSource(comp: *Compilation, allocator: Allocator) Allocator.Error![:0]u8 { const tracy_trace = trace(@src()); defer tracy_trace.end(); @@ -4290,7 +4305,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: Allocator) Alloca } } - return buffer.toOwnedSlice(); + return buffer.toOwnedSliceSentinel(0); } pub fn updateSubCompilation(sub_compilation: *Compilation) !void { -- cgit v1.2.3