stage2: implement error notes and regress -femit-zir

 * Implement error notes
   - note: other symbol exported here
   - note: previous else prong is here
   - note: previous '_' prong is here
 * Add Compilation.CObject.ErrorMsg. This object properly converts to
   AllErrors.Message when the time comes.
 * Add Compilation.CObject.failure_retryable. Properly handles
   out-of-memory and other transient failures.
 * Introduce Module.SrcLoc which has not only a byte offset but also
   references the file which the byte offset applies to.
 * Scope.Block now contains both a pointer to the "owner" Decl and the
   "source" Decl. As an example, during inline function call, the
   "owner" will be the Decl of the caller and the "source" will be the
   Decl of the callee.
 * Module.ErrorMsg now sports a `file_scope` field so that notes can
   refer to source locations in a file other than the parent error
   message.
 * Some instances where a `*Scope` was stored, now store a
   `*Scope.Container`.
 * Some methods in the `Scope` namespace were moved to the more specific
   type, since there was only an implementation for one particular tag.
   - `removeDecl` moved to `Scope.Container`
   - `destroy` moved to `Scope.File`
 * Two kinds of Scope deleted:
   - zir_module
   - decl
 * astgen: properly use DeclVal / DeclRef. DeclVal was incorrectly
   changed to be a reference; this commit fixes it. Fewer ZIR
   instructions processed as a result.
   - declval_in_module is renamed to declval
   - previous declval ZIR instruction is deleted; it was only for .zir
     files.
 * Test harness: friendlier diagnostics when an unexpected set of errors
   is encountered.
 * zir_sema: fix analyzeInstBlockFlat by properly calling resolvingInst
   on the last zir instruction in the block.

Compile log implementation:
 * Write to a buffer rather than directly to stderr.
 * Only keep track of 1 callsite per Decl.
 * No longer mutate the ZIR Inst struct data.
 * "Compile log statement found" errors are only emitted when there are
   no other compile errors.

-femit-zir and support for .zir source files is regressed. If we wanted
to support this again, outputting .zir would need to be done as yet
another backend rather than in the haphazard way it was previously
implemented.

For parsing .zir, it was implemented previously in a way that was not
helpful for debugging. We need tighter integration with the test harness
for it to be useful; so clearly a rewrite is needed. Given that a
rewrite is needed, and it was getting in the way of progress and
organization of the rest of stage2, I regressed the feature.

1 files changed, 179 insertions, 126 deletions

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 225a91e5d2..ad99e40541 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -51,7 +51,7 @@ c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
 
 /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
 /// This data is accessed by multiple threads and is protected by `mutex`.
-failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{},
+failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *CObject.ErrorMsg) = .{},
 
 keep_source_files_loaded: bool,
 use_clang: bool,
@@ -215,13 +215,29 @@ pub const CObject = struct {
         },
         /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects.
         failure,
+        /// A transient failure happened when trying to compile the C Object; it may
+        /// succeed if we try again. There may be a corresponding ErrorMsg in
+        /// Compilation.failed_c_objects. If there is not, the failure is out of memory.
+        failure_retryable,
     },
 
+    pub const ErrorMsg = struct {
+        msg: []const u8,
+        line: u32,
+        column: u32,
+
+        pub fn destroy(em: *ErrorMsg, gpa: *Allocator) void {
+            gpa.free(em.msg);
+            gpa.destroy(em);
+            em.* = undefined;
+        }
+    };
+
     /// Returns if there was failure.
     pub fn clearStatus(self: *CObject, gpa: *Allocator) bool {
         switch (self.status) {
             .new => return false,
-            .failure => {
+            .failure, .failure_retryable => {
                 self.status = .new;
                 return true;
             },
@@ -240,6 +256,11 @@ pub const CObject = struct {
     }
 };
 
+/// To support incremental compilation, errors are stored in various places
+/// so that they can be created and destroyed appropriately. This structure
+/// is used to collect all the errors from the various places into one
+/// convenient place for API users to consume. It is allocated into 1 heap
+/// and freed all at once.
 pub const AllErrors = struct {
     arena: std.heap.ArenaAllocator.State,
     list: []const Message,
@@ -251,23 +272,32 @@ pub const AllErrors = struct {
             column: usize,
             byte_offset: usize,
             msg: []const u8,
+            notes: []Message = &.{},
         },
         plain: struct {
             msg: []const u8,
         },
 
-        pub fn renderToStdErr(self: Message) void {
-            switch (self) {
+        pub fn renderToStdErr(msg: Message) void {
+            return msg.renderToStdErrInner("error");
+        }
+
+        fn renderToStdErrInner(msg: Message, kind: []const u8) void {
+            switch (msg) {
                 .src => |src| {
-                    std.debug.print("{s}:{d}:{d}: error: {s}\n", .{
+                    std.debug.print("{s}:{d}:{d}: {s}: {s}\n", .{
                         src.src_path,
                         src.line + 1,
                         src.column + 1,
+                        kind,
                         src.msg,
                     });
+                    for (src.notes) |note| {
+                        note.renderToStdErrInner("note");
+                    }
                 },
                 .plain => |plain| {
-                    std.debug.print("error: {s}\n", .{plain.msg});
+                    std.debug.print("{s}: {s}\n", .{ kind, plain.msg });
                 },
             }
         }
@@ -278,20 +308,38 @@ pub const AllErrors = struct {
     }
 
     fn add(
+        module: *Module,
         arena: *std.heap.ArenaAllocator,
         errors: *std.ArrayList(Message),
-        sub_file_path: []const u8,
-        source: []const u8,
-        simple_err_msg: ErrorMsg,
+        module_err_msg: Module.ErrorMsg,
     ) !void {
-        const loc = std.zig.findLineColumn(source, simple_err_msg.byte_offset);
+        const notes = try arena.allocator.alloc(Message, module_err_msg.notes.len);
+        for (notes) |*note, i| {
+            const module_note = module_err_msg.notes[i];
+            const source = try module_note.src_loc.file_scope.getSource(module);
+            const loc = std.zig.findLineColumn(source, module_note.src_loc.byte_offset);
+            const sub_file_path = module_note.src_loc.file_scope.sub_file_path;
+            note.* = .{
+                .src = .{
+                    .src_path = try arena.allocator.dupe(u8, sub_file_path),
+                    .msg = try arena.allocator.dupe(u8, module_note.msg),
+                    .byte_offset = module_note.src_loc.byte_offset,
+                    .line = loc.line,
+                    .column = loc.column,
+                },
+            };
+        }
+        const source = try module_err_msg.src_loc.file_scope.getSource(module);
+        const loc = std.zig.findLineColumn(source, module_err_msg.src_loc.byte_offset);
+        const sub_file_path = module_err_msg.src_loc.file_scope.sub_file_path;
         try errors.append(.{
             .src = .{
                 .src_path = try arena.allocator.dupe(u8, sub_file_path),
-                .msg = try arena.allocator.dupe(u8, simple_err_msg.msg),
-                .byte_offset = simple_err_msg.byte_offset,
+                .msg = try arena.allocator.dupe(u8, module_err_msg.msg),
+                .byte_offset = module_err_msg.src_loc.byte_offset,
                 .line = loc.line,
                 .column = loc.column,
+                .notes = notes,
             },
         });
     }
@@ -849,17 +897,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                             .ty = struct_ty,
                         },
                     };
-                    break :rs &root_scope.base;
+                    break :rs root_scope;
                 } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) {
-                    const root_scope = try gpa.create(Module.Scope.ZIRModule);
-                    root_scope.* = .{
-                        .sub_file_path = root_pkg.root_src_path,
-                        .source = .{ .unloaded = {} },
-                        .contents = .{ .not_available = {} },
-                        .status = .never_loaded,
-                        .decls = .{},
-                    };
-                    break :rs &root_scope.base;
+                    return error.ZirFilesUnsupported;
                 } else {
                     unreachable;
                 }
@@ -1258,32 +1298,23 @@ pub fn update(self: *Compilation) !void {
     const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
     if (!use_stage1) {
         if (self.bin_file.options.module) |module| {
+            module.compile_log_text.shrinkAndFree(module.gpa, 0);
             module.generation += 1;
 
             // TODO Detect which source files changed.
             // Until then we simulate a full cache miss. Source files could have been loaded for any reason;
             // to force a refresh we unload now.
-            if (module.root_scope.cast(Module.Scope.File)) |zig_file| {
-                zig_file.unload(module.gpa);
-                module.failed_root_src_file = null;
-                module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) {
-                    error.AnalysisFail => {
-                        assert(self.totalErrorCount() != 0);
-                    },
-                    error.OutOfMemory => return error.OutOfMemory,
-                    else => |e| {
-                        module.failed_root_src_file = e;
-                    },
-                };
-            } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| {
-                zir_module.unload(module.gpa);
-                module.analyzeRootZIRModule(zir_module) catch |err| switch (err) {
-                    error.AnalysisFail => {
-                        assert(self.totalErrorCount() != 0);
-                    },
-                    else => |e| return e,
-                };
-            }
+            module.root_scope.unload(module.gpa);
+            module.failed_root_src_file = null;
+            module.analyzeContainer(&module.root_scope.root_container) catch |err| switch (err) {
+                error.AnalysisFail => {
+                    assert(self.totalErrorCount() != 0);
+                },
+                error.OutOfMemory => return error.OutOfMemory,
+                else => |e| {
+                    module.failed_root_src_file = e;
+                },
+            };
 
             // TODO only analyze imports if they are still referenced
             for (module.import_table.items()) |entry| {
@@ -1359,14 +1390,18 @@ pub fn totalErrorCount(self: *Compilation) usize {
             module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
-        for (module.compile_log_decls.items()) |entry| {
-            total += entry.value.items.len;
-        }
     }
 
     // The "no entry point found" error only counts if there are no other errors.
     if (total == 0) {
-        return @boolToInt(self.link_error_flags.no_entry_point_found);
+        total += @boolToInt(self.link_error_flags.no_entry_point_found);
+    }
+
+    // Compile log errors only count if there are no other errors.
+    if (total == 0) {
+        if (self.bin_file.options.module) |module| {
+            total += @boolToInt(module.compile_log_decls.items().len != 0);
+        }
     }
 
     return total;
@@ -1382,32 +1417,32 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     for (self.failed_c_objects.items()) |entry| {
         const c_object = entry.key;
         const err_msg = entry.value;
-        try AllErrors.add(&arena, &errors, c_object.src.src_path, "", err_msg.*);
+        // TODO these fields will need to be adjusted when we have proper
+        // C error reporting bubbling up.
+        try errors.append(.{
+            .src = .{
+                .src_path = try arena.allocator.dupe(u8, c_object.src.src_path),
+                .msg = try std.fmt.allocPrint(&arena.allocator, "unable to build C object: {s}", .{
+                    err_msg.msg,
+                }),
+                .byte_offset = 0,
+                .line = err_msg.line,
+                .column = err_msg.column,
+            },
+        });
     }
     if (self.bin_file.options.module) |module| {
         for (module.failed_files.items()) |entry| {
-            const scope = entry.key;
-            const err_msg = entry.value;
-            const source = try scope.getSource(module);
-            try AllErrors.add(&arena, &errors, scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.failed_decls.items()) |entry| {
-            const decl = entry.key;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.emit_h_failed_decls.items()) |entry| {
-            const decl = entry.key;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.failed_exports.items()) |entry| {
-            const decl = entry.key.owner_decl;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         if (module.failed_root_src_file) |err| {
             const file_path = try module.root_pkg.root_src_directory.join(&arena.allocator, &[_][]const u8{
@@ -1418,15 +1453,6 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
             });
             try AllErrors.addPlain(&arena, &errors, msg);
         }
-        for (module.compile_log_decls.items()) |entry| {
-            const decl = entry.key;
-            const path = decl.scope.subFilePath();
-            const source = try decl.scope.getSource(module);
-            for (entry.value.items) |src_loc| {
-                const err_msg = ErrorMsg{ .byte_offset = src_loc, .msg = "found compile log statement" };
-                try AllErrors.add(&arena, &errors, path, source, err_msg);
-            }
-        }
     }
 
     if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) {
@@ -1437,6 +1463,28 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
         });
     }
 
+    if (self.bin_file.options.module) |module| {
+        const compile_log_items = module.compile_log_decls.items();
+        if (errors.items.len == 0 and compile_log_items.len != 0) {
+            // First one will be the error; subsequent ones will be notes.
+            const err_msg = Module.ErrorMsg{
+                .src_loc = compile_log_items[0].value,
+                .msg = "found compile log statement",
+                .notes = try self.gpa.alloc(Module.ErrorMsg, compile_log_items.len - 1),
+            };
+            defer self.gpa.free(err_msg.notes);
+
+            for (compile_log_items[1..]) |entry, i| {
+                err_msg.notes[i] = .{
+                    .src_loc = entry.value,
+                    .msg = "also here",
+                };
+            }
+
+            try AllErrors.add(module, &arena, &errors, err_msg);
+        }
+    }
+
     assert(errors.items.len == self.totalErrorCount());
 
     return AllErrors{
@@ -1445,6 +1493,11 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     };
 }
 
+pub fn getCompileLogOutput(self: *Compilation) []const u8 {
+    const module = self.bin_file.options.module orelse return &[0]u8{};
+    return module.compile_log_text.items;
+}
+
 pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
     var progress: std.Progress = .{};
     var main_progress_node = try progress.start("", 0);
@@ -1517,9 +1570,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     },
                     else => {
                         try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                        module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                        module.failed_decls.putAssumeCapacityNoClobber(decl, try Module.ErrorMsg.create(
                             module.gpa,
-                            decl.src(),
+                            decl.srcLoc(),
                             "unable to codegen: {s}",
                             .{@errorName(err)},
                         ));
@@ -1586,9 +1639,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             const module = self.bin_file.options.module.?;
             self.bin_file.updateDeclLineNumber(module, decl) catch |err| {
                 try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                module.failed_decls.putAssumeCapacityNoClobber(decl, try Module.ErrorMsg.create(
                     module.gpa,
-                    decl.src(),
+                    decl.srcLoc(),
                     "unable to update line number: {s}",
                     .{@errorName(err)},
                 ));
@@ -1858,26 +1911,38 @@ fn workerUpdateCObject(
     comp.updateCObject(c_object, progress_node) catch |err| switch (err) {
         error.AnalysisFail => return,
         else => {
-            {
-                const lock = comp.mutex.acquire();
-                defer lock.release();
-                comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
-                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
-                };
-                comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
-                    comp.gpa,
-                    0,
-                    "unable to build C object: {s}",
-                    .{@errorName(err)},
-                ) catch {
-                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
-                });
-            }
-            c_object.status = .{ .failure = {} };
+            comp.reportRetryableCObjectError(c_object, err) catch |oom| switch (oom) {
+                // Swallowing this error is OK because it's implied to be OOM when
+                // there is a missing failed_c_objects error message.
+                error.OutOfMemory => {},
+            };
         },
     };
 }
 
+fn reportRetryableCObjectError(
+    comp: *Compilation,
+    c_object: *CObject,
+    err: anyerror,
+) error{OutOfMemory}!void {
+    c_object.status = .failure_retryable;
+
+    const c_obj_err_msg = try comp.gpa.create(CObject.ErrorMsg);
+    errdefer comp.gpa.destroy(c_obj_err_msg);
+    const msg = try std.fmt.allocPrint(comp.gpa, "unable to build C object: {s}", .{@errorName(err)});
+    errdefer comp.gpa.free(msg);
+    c_obj_err_msg.* = .{
+        .msg = msg,
+        .line = 0,
+        .column = 0,
+    };
+    {
+        const lock = comp.mutex.acquire();
+        defer lock.release();
+        try comp.failed_c_objects.putNoClobber(comp.gpa, c_object, c_obj_err_msg);
+    }
+}
+
 fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *std.Progress.Node) !void {
     if (!build_options.have_llvm) {
         return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
@@ -1892,7 +1957,9 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
         // There was previous failure.
         const lock = comp.mutex.acquire();
         defer lock.release();
-        comp.failed_c_objects.removeAssertDiscard(c_object);
+        // If the failure was OOM, there will not be an entry here, so we do
+        // not assert discard.
+        _ = comp.failed_c_objects.swapRemove(c_object);
     }
 
     var man = comp.obtainCObjectCacheManifest();
@@ -2343,11 +2410,27 @@ pub fn addCCArgs(
 
 fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError {
     @setCold(true);
-    const err_msg = try ErrorMsg.create(comp.gpa, 0, "unable to build C object: " ++ format, args);
+    const err_msg = blk: {
+        const msg = try std.fmt.allocPrint(comp.gpa, format, args);
+        errdefer comp.gpa.free(msg);
+        const err_msg = try comp.gpa.create(CObject.ErrorMsg);
+        errdefer comp.gpa.destroy(err_msg);
+        err_msg.* = .{
+            .msg = msg,
+            .line = 0,
+            .column = 0,
+        };
+        break :blk err_msg;
+    };
     return comp.failCObjWithOwnedErrorMsg(c_object, err_msg);
 }
 
-fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError {
+fn failCObjWithOwnedErrorMsg(
+    comp: *Compilation,
+    c_object: *CObject,
+    err_msg: *CObject.ErrorMsg,
+) InnerError {
+    @setCold(true);
     {
         const lock = comp.mutex.acquire();
         defer lock.release();
@@ -2361,36 +2444,6 @@ fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *E
     return error.AnalysisFail;
 }
 
-pub const ErrorMsg = struct {
-    byte_offset: usize,
-    msg: []const u8,
-
-    pub fn create(gpa: *Allocator, byte_offset: usize, comptime format: []const u8, args: anytype) !*ErrorMsg {
-        const self = try gpa.create(ErrorMsg);
-        errdefer gpa.destroy(self);
-        self.* = try init(gpa, byte_offset, format, args);
-        return self;
-    }
-
-    /// Assumes the ErrorMsg struct and msg were both allocated with allocator.
-    pub fn destroy(self: *ErrorMsg, gpa: *Allocator) void {
-        self.deinit(gpa);
-        gpa.destroy(self);
-    }
-
-    pub fn init(gpa: *Allocator, byte_offset: usize, comptime format: []const u8, args: anytype) !ErrorMsg {
-        return ErrorMsg{
-            .byte_offset = byte_offset,
-            .msg = try std.fmt.allocPrint(gpa, format, args),
-        };
-    }
-
-    pub fn deinit(self: *ErrorMsg, gpa: *Allocator) void {
-        gpa.free(self.msg);
-        self.* = undefined;
-    }
-};
-
 pub const FileExt = enum {
     c,
     cpp,