stage2: implement `@embedFile`

1 files changed, 100 insertions, 0 deletions

diff --git a/src/Compilation.zig b/src/Compilation.zig
index f997b53388..63a6b50d5b 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -55,6 +55,10 @@ c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
 /// since the last compilation, as well as scan for `@import` and queue up
 /// additional jobs corresponding to those new files.
 astgen_work_queue: std.fifo.LinearFifo(*Module.File, .Dynamic),
+/// These jobs are to inspect the file system stat() and if the embedded file has changed
+/// on disk, mark the corresponding Decl outdated and queue up an `analyze_decl`
+/// task for it.
+embed_file_work_queue: std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic),
 
 /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
 /// This data is accessed by multiple threads and is protected by `mutex`.
@@ -181,6 +185,10 @@ const Job = union(enum) {
     /// It may have already be analyzed, or it may have been determined
     /// to be outdated; in this case perform semantic analysis again.
     analyze_decl: *Module.Decl,
+    /// The file that was loaded with `@embedFile` has changed on disk
+    /// and has been re-loaded into memory. All Decls that depend on it
+    /// need to be re-analyzed.
+    update_embed_file: *Module.EmbedFile,
     /// The source file containing the Decl has been updated, and so the
     /// Decl may need its line number information updated in the debug info.
     update_line_number: *Module.Decl,
@@ -1447,6 +1455,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
             .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
             .astgen_work_queue = std.fifo.LinearFifo(*Module.File, .Dynamic).init(gpa),
+            .embed_file_work_queue = std.fifo.LinearFifo(*Module.EmbedFile, .Dynamic).init(gpa),
             .keep_source_files_loaded = options.keep_source_files_loaded,
             .use_clang = use_clang,
             .clang_argv = options.clang_argv,
@@ -1632,6 +1641,7 @@ pub fn destroy(self: *Compilation) void {
     self.work_queue.deinit();
     self.c_object_work_queue.deinit();
     self.astgen_work_queue.deinit();
+    self.embed_file_work_queue.deinit();
 
     {
         var it = self.crt_files.iterator();
@@ -1747,6 +1757,16 @@ pub fn update(self: *Compilation) !void {
         }
 
         if (!use_stage1) {
+            // Put a work item in for checking if any files used with `@embedFile` changed.
+            {
+                try self.embed_file_work_queue.ensureUnusedCapacity(module.embed_table.count());
+                var it = module.embed_table.iterator();
+                while (it.next()) |entry| {
+                    const embed_file = entry.value_ptr.*;
+                    self.embed_file_work_queue.writeItemAssumeCapacity(embed_file);
+                }
+            }
+
             try self.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
             if (self.bin_file.options.is_test) {
                 try self.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
@@ -1870,6 +1890,7 @@ pub fn totalErrorCount(self: *Compilation) usize {
 
     if (self.bin_file.options.module) |module| {
         total += module.failed_exports.count();
+        total += module.failed_embed_files.count();
 
         {
             var it = module.failed_files.iterator();
@@ -1967,6 +1988,13 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
             }
         }
         {
+            var it = module.failed_embed_files.iterator();
+            while (it.next()) |entry| {
+                const msg = entry.value_ptr.*;
+                try AllErrors.add(module, &arena, &errors, msg.*);
+            }
+        }
+        {
             var it = module.failed_decls.iterator();
             while (it.next()) |entry| {
                 // Skip errors for Decls within files that had a parse failure.
@@ -2065,6 +2093,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var c_obj_prog_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_obj_prog_node.end();
 
+    var embed_file_prog_node = main_progress_node.start("Detect @embedFile updates", self.embed_file_work_queue.count);
+    defer embed_file_prog_node.end();
+
     self.work_queue_wait_group.reset();
     defer self.work_queue_wait_group.wait();
 
@@ -2079,6 +2110,13 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             });
         }
 
+        while (self.embed_file_work_queue.readItem()) |embed_file| {
+            self.astgen_wait_group.start();
+            try self.thread_pool.spawn(workerCheckEmbedFile, .{
+                self, embed_file, &embed_file_prog_node, &self.astgen_wait_group,
+            });
+        }
+
         while (self.c_object_work_queue.readItem()) |c_object| {
             self.work_queue_wait_group.start();
             try self.thread_pool.spawn(workerUpdateCObject, .{
@@ -2260,6 +2298,15 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 error.AnalysisFail => continue,
             };
         },
+        .update_embed_file => |embed_file| {
+            if (build_options.omit_stage2)
+                @panic("sadly stage2 is omitted from this build to save memory on the CI server");
+            const module = self.bin_file.options.module.?;
+            module.updateEmbedFile(embed_file) catch |err| switch (err) {
+                error.OutOfMemory => return error.OutOfMemory,
+                error.AnalysisFail => continue,
+            };
+        },
         .update_line_number => |decl| {
             if (build_options.omit_stage2)
                 @panic("sadly stage2 is omitted from this build to save memory on the CI server");
@@ -2542,6 +2589,29 @@ fn workerAstGenFile(
     }
 }
 
+fn workerCheckEmbedFile(
+    comp: *Compilation,
+    embed_file: *Module.EmbedFile,
+    prog_node: *std.Progress.Node,
+    wg: *WaitGroup,
+) void {
+    defer wg.finish();
+
+    var child_prog_node = prog_node.start(embed_file.sub_file_path, 0);
+    child_prog_node.activate();
+    defer child_prog_node.end();
+
+    const mod = comp.bin_file.options.module.?;
+    mod.detectEmbedFileUpdate(embed_file) catch |err| {
+        comp.reportRetryableEmbedFileError(embed_file, err) catch |oom| switch (oom) {
+            // Swallowing this error is OK because it's implied to be OOM when
+            // there is a missing `failed_embed_files` error message.
+            error.OutOfMemory => {},
+        };
+        return;
+    };
+}
+
 pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
     var man = comp.cache_parent.obtain();
 
@@ -2790,6 +2860,36 @@ fn reportRetryableAstGenError(
     }
 }
 
+fn reportRetryableEmbedFileError(
+    comp: *Compilation,
+    embed_file: *Module.EmbedFile,
+    err: anyerror,
+) error{OutOfMemory}!void {
+    const mod = comp.bin_file.options.module.?;
+    const gpa = mod.gpa;
+
+    const src_loc: Module.SrcLoc = embed_file.owner_decl.srcLoc();
+
+    const err_msg = if (embed_file.pkg.root_src_directory.path) |dir_path|
+        try Module.ErrorMsg.create(
+            gpa,
+            src_loc,
+            "unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}",
+            .{ dir_path, embed_file.sub_file_path, @errorName(err) },
+        )
+    else
+        try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{
+            embed_file.sub_file_path, @errorName(err),
+        });
+    errdefer err_msg.destroy(gpa);
+
+    {
+        const lock = comp.mutex.acquire();
+        defer lock.release();
+        try mod.failed_embed_files.putNoClobber(gpa, embed_file, err_msg);
+    }
+}
+
 fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.Progress.Node) !void {
     if (!build_options.have_llvm) {
         return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});