From e563b166b2b70975899c84beb425c8739d05ed65 Mon Sep 17 00:00:00 2001 From: Stephen Gutekanst Date: Tue, 14 Dec 2021 01:10:43 -0700 Subject: Fix compilation cache updating bug leading to slow builds While investigating slow build times with [a large project](https://github.com/hexops/mach/issues/124), I found that the compiler was reading from disk nearly every C source file in my project when rebuilding despite no changes having been made. This accounted for several seconds of time (approx. 20-30% of running `zig build` without any changes to the sources.) The cause of this was that comparisons of file mtimes would _always_ fail (the mtime of the file on disk was always newer than that stored in the cache manifest), and so the cache logic would always fall back to byte-for-byte file content comparisons with what is on disk vs. in the cache-reading every C source file in my project from disk during each rebuild. Because file contents were the same, a cache hit occurred, and _despite the mtime being different the cache manifest would not be updated._ One can reproduce this by building a Zig project so the cache is populated, and then changing mtimes of their C source files to be newer than what is in the cache (without altering file contents.) The fix is rather simple: we should always write the updated cache manifest regardless of whether or not a cache hit occurred (a cache hit doesn't indicate if a manifest is dirty) Luckily, `writeManifest` already contains logic to determine if a manifest is dirty and becomes no-op if no change to the manifest file is necessary-so we merely need to ensure it is invoked. Signed-off-by: Stephen Gutekanst --- src/Compilation.zig | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'src/Compilation.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index f0be3aac65..81c3d0132f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2967,13 +2967,17 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { try out_zig_file.writeAll(formatted); - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)}); - }; - break :digest digest; } else man.final(); + // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is + // possible we had a hit and the manifest is dirty, for example if the file mtime changed but + // the contents were the same, we hit the cache but the manifest is dirty and we need to update + // it to prevent doing a full file content comparison the next time around. + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)}); + }; + const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ "o", &digest, cimport_zig_basename, }); @@ -3288,13 +3292,17 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P defer o_dir.close(); const tmp_basename = std.fs.path.basename(out_obj_path); try std.fs.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename); - - man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ c_object.src.src_path, @errorName(err) }); - }; break :blk digest; }; + // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is + // possible we had a hit and the manifest is dirty, for example if the file mtime changed but + // the contents were the same, we hit the cache but the manifest is dirty and we need to update + // it to prevent doing a full file content comparison the next time around. + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ c_object.src.src_path, @errorName(err) }); + }; + const o_basename = try std.fmt.allocPrint(arena, "{s}{s}", .{ o_basename_noext, o_ext }); c_object.status = .{ -- cgit v1.2.3