From 30b8b29f88362d18ea6523a859b29f7bc6dec622 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 30 Oct 2022 16:51:13 -0700 Subject: re-apply "Fix C include files not being in `whole` cache (#11365)" This reverts commit 06310e3d4eb47fed88b175891cb5865bb050f020, reapplying commit a430630002bf02162ccbf8d3eb10fd73e490cefd. I deeply apologize to @moosichu and those affected by this bug. The original fix was actually fine. When I reverted it, I misremembered how the Cache API works. I thought the fix was going to introduce nondeterminism into the hash, but I forgot that the order of files in the manifest doesn't actually matter when checking for a cache hit. Actually, it does matter a little bit. This fix has a subtle downside which is that it does introduce the possibility of false negatives when checking for cache hits of 2+ iterations ago. For example, if the code goes from "foo", to "bar", and then back to "foo", it may look like a cache miss when it should have been a hit because 2 iterations ago the code was the same. However, this is an uncommon use case, and all it does is cause a bit of wasted time and disk space. That said, my suggestion from earlier still applies and would be a nice follow-up enhancement to this fix: The proper solution to this is to, in whole cache mode, append the hash inputs to some data structure, and then after the compilation is complete, do some kind of sorting on the hash inputs so that they will be the same order every time, then apply them in sequence. No lock on the Cache object is needed for this scheme. closes #11063 --- src/Compilation.zig | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/Compilation.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index 2e8ba98a89..958aac5e1b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -48,6 +48,7 @@ bin_file: *link.File, c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{}, /// This is a pointer to a local variable inside `update()`. whole_cache_manifest: ?*Cache.Manifest = null, +whole_cache_manifest_mutex: std.Thread.Mutex = .{}, link_error_flags: link.File.ErrorFlags = .{}, @@ -2199,8 +2200,8 @@ pub fn update(comp: *Compilation) !void { // We are about to obtain this lock, so here we give other processes a chance first. comp.bin_file.releaseLock(); - comp.whole_cache_manifest = &man; man = comp.cache_parent.obtain(); + comp.whole_cache_manifest = &man; try comp.addNonIncrementalStuffToCacheManifest(&man); const is_hit = man.hit() catch |err| { @@ -3598,6 +3599,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult { const dep_basename = std.fs.path.basename(out_dep_path); try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); if (comp.whole_cache_manifest) |whole_cache_manifest| { + comp.whole_cache_manifest_mutex.lock(); + defer comp.whole_cache_manifest_mutex.unlock(); try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename); } @@ -4060,6 +4063,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P const dep_basename = std.fs.path.basename(dep_file_path); // Add the files depended on to the cache system. try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); + if (comp.whole_cache_manifest) |whole_cache_manifest| { + comp.whole_cache_manifest_mutex.lock(); + defer comp.whole_cache_manifest_mutex.unlock(); + try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename); + } // Just to save disk space, we delete the file because it is never needed again. zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) }); -- cgit v1.2.3