incr-check: check compile errors against expected

Also modifies all incremental cases using `#expect_error` to include the
errors and notes which are expected.

13 files changed, 162 insertions, 35 deletions

diff --git a/test/incremental/add_decl b/test/incremental/add_decl
index b842593056..87f33b1c51 100644
--- a/test/incremental/add_decl
+++ b/test/incremental/add_decl
@@ -39,7 +39,7 @@ pub fn main() !void {
 }
 const foo = "good morning\n";
 const bar = "good evening\n";
-#expect_error=ignored
+#expect_error=main.zig:3:37: error: use of undeclared identifier 'qux'
 
 #update=add missing declaration
 #file=main.zig
diff --git a/test/incremental/add_decl_namespaced b/test/incremental/add_decl_namespaced
index a42e5600bc..84472effb5 100644
--- a/test/incremental/add_decl_namespaced
+++ b/test/incremental/add_decl_namespaced
@@ -39,7 +39,8 @@ pub fn main() !void {
 }
 const foo = "good morning\n";
 const bar = "good evening\n";
-#expect_error=ignored
+#expect_error=main.zig:3:44: error: root source file struct 'main' has no member named 'qux'
+#expect_error=main.zig:1:1: note: struct declared here
 
 #update=add missing declaration
 #file=main.zig
diff --git a/test/incremental/change_embed_file b/test/incremental/change_embed_file
index 3cb29be2ed..171e4d3178 100644
--- a/test/incremental/change_embed_file
+++ b/test/incremental/change_embed_file
@@ -20,7 +20,7 @@ Hello again, World!
 
 #update=delete file
 #rm_file=string.txt
-#expect_error=ignored
+#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound
 
 #update=remove reference to file
 #file=main.zig
@@ -38,7 +38,7 @@ const string = @embedFile("string.txt");
 pub fn main() !void {
     try std.io.getStdOut().writeAll(string);
 }
-#expect_error=ignore
+#expect_error=main.zig:2:27: error: unable to open 'string.txt': FileNotFound
 
 #update=recreate file
 #file=string.txt
diff --git a/test/incremental/change_enum_tag_type b/test/incremental/change_enum_tag_type
index e66728e08d..d3f6c85c37 100644
--- a/test/incremental/change_enum_tag_type
+++ b/test/incremental/change_enum_tag_type
@@ -39,7 +39,7 @@ comptime {
     std.debug.assert(@TypeOf(@intFromEnum(Foo.e)) == Tag);
 }
 const std = @import("std");
-#expect_error=ignored
+#expect_error=main.zig:7:5: error: enumeration value '4' too large for type 'u2'
 #update=increase tag size
 #file=main.zig
 const Tag = u3;
diff --git a/test/incremental/compile_error_then_log b/test/incremental/compile_error_then_log
index 9ab844bb4a..800fb92dfe 100644
--- a/test/incremental/compile_error_then_log
+++ b/test/incremental/compile_error_then_log
@@ -4,19 +4,22 @@
 #target=wasm32-wasi-selfhosted
 #update=initial version with compile error
 #file=main.zig
+pub fn main() void {}
 comptime {
     @compileError("this is an error");
 }
 comptime {
     @compileLog("this is a log");
 }
-#expect_error=ignored
+#expect_error=main.zig:3:5: error: this is an error
+
 #update=remove the compile error
 #file=main.zig
+pub fn main() void {}
 comptime {
     //@compileError("this is an error");
 }
 comptime {
     @compileLog("this is a log");
 }
-#expect_error=ignored
+#expect_error=main.zig:6:5: error: found compile log statement
diff --git a/test/incremental/delete_comptime_decls b/test/incremental/delete_comptime_decls
index 45c77048eb..b5ccd438a2 100644
--- a/test/incremental/delete_comptime_decls
+++ b/test/incremental/delete_comptime_decls
@@ -26,7 +26,10 @@ comptime {
     const slice = array[3..2];
     _ = slice;
 }
-#expect_error=ignored
+#expect_error=main.zig:5:32: error: end index 6 out of bounds for slice of length 4 +1 (sentinel)
+#expect_error=main.zig:10:28: error: end index 6 out of bounds for array of length 4 +1 (sentinel)
+#expect_error=main.zig:15:28: error: end index 5 out of bounds for array of length 4
+#expect_error=main.zig:20:25: error: start index 3 is larger than end index 2
 
 #update=delete and modify comptime decls
 #file=main.zig
@@ -38,4 +41,4 @@ comptime {
     const y = x[0..runtime_len];
     _ = y;
 }
-#expect_error=ignored
+#expect_error=main.zig:6:16: error: slice of null pointer
diff --git a/test/incremental/fix_astgen_failure b/test/incremental/fix_astgen_failure
index a57b1ebde3..51972e9232 100644
--- a/test/incremental/fix_astgen_failure
+++ b/test/incremental/fix_astgen_failure
@@ -11,7 +11,7 @@ pub fn main() !void {
 pub fn hello() !void {
     try std.io.getStdOut().writeAll("Hello, World!\n");
 }
-#expect_error=ignored
+#expect_error=foo.zig:2:9: error: use of undeclared identifier 'std'
 #update=fix the error
 #file=foo.zig
 const std = @import("std");
@@ -25,7 +25,7 @@ const std = @import("std");
 pub fn hello() !void {
     try std.io.getStdOut().writeAll(hello_str);
 }
-#expect_error=ignored
+#expect_error=foo.zig:3:37: error: use of undeclared identifier 'hello_str'
 #update=fix the new error
 #file=foo.zig
 const std = @import("std");
diff --git a/test/incremental/fix_many_errors b/test/incremental/fix_many_errors
index 0722cdfced..1d9446022c 100644
--- a/test/incremental/fix_many_errors
+++ b/test/incremental/fix_many_errors
@@ -24,7 +24,26 @@ export fn f6() void { @compileError("f6"); }
 export fn f7() void { @compileError("f7"); }
 export fn f8() void { @compileError("f8"); }
 export fn f9() void { @compileError("f9"); }
-#expect_error=ignored
+#expect_error=main.zig:2:12: error: c0
+#expect_error=main.zig:3:12: error: c1
+#expect_error=main.zig:4:12: error: c2
+#expect_error=main.zig:5:12: error: c3
+#expect_error=main.zig:6:12: error: c4
+#expect_error=main.zig:7:12: error: c5
+#expect_error=main.zig:8:12: error: c6
+#expect_error=main.zig:9:12: error: c7
+#expect_error=main.zig:10:12: error: c8
+#expect_error=main.zig:11:12: error: c9
+#expect_error=main.zig:12:23: error: f0
+#expect_error=main.zig:13:23: error: f1
+#expect_error=main.zig:14:23: error: f2
+#expect_error=main.zig:15:23: error: f3
+#expect_error=main.zig:16:23: error: f4
+#expect_error=main.zig:17:23: error: f5
+#expect_error=main.zig:18:23: error: f6
+#expect_error=main.zig:19:23: error: f7
+#expect_error=main.zig:20:23: error: f8
+#expect_error=main.zig:21:23: error: f9
 #update=fix all the errors
 #file=main.zig
 pub fn main() !void {}
diff --git a/test/incremental/remove_enum_field b/test/incremental/remove_enum_field
index a1e5e20fd3..8d3796b7c3 100644
--- a/test/incremental/remove_enum_field
+++ b/test/incremental/remove_enum_field
@@ -23,4 +23,5 @@ pub fn main() !void {
     try std.io.getStdOut().writer().print("{}\n", .{@intFromEnum(MyEnum.foo)});
 }
 const std = @import("std");
-#expect_error=ignored
+#expect_error=main.zig:6:73: error: enum 'main.MyEnum' has no member named 'foo'
+#expect_error=main.zig:1:16: note: enum declared here
diff --git a/test/incremental/remove_invalid_union_backing_enum b/test/incremental/remove_invalid_union_backing_enum
index 4308899f9a..1f7ee69b14 100644
--- a/test/incremental/remove_invalid_union_backing_enum
+++ b/test/incremental/remove_invalid_union_backing_enum
@@ -15,7 +15,8 @@ pub fn main() void {
     const u: U = .{ .a = 123 };
     _ = u;
 }
-#expect_error=ignored
+#expect_error=main.zig:6:5: error: no field named 'd' in enum 'main.E'
+#expect_error=main.zig:1:11: note: enum declared here
 #update=remove invalid backing enum
 #file=main.zig
 const U = union {
diff --git a/test/incremental/temporary_parse_error b/test/incremental/temporary_parse_error
index 675232ea94..0933546d33 100644
--- a/test/incremental/temporary_parse_error
+++ b/test/incremental/temporary_parse_error
@@ -11,7 +11,7 @@ pub fn main() !void {}
 #update=introduce parse error
 #file=main.zig
 pub fn main() !void {
-#expect_error=ignored
+#expect_error=main.zig:2:1: error: expected statement, found 'EOF'
 
 #update=fix parse error
 #file=main.zig
diff --git a/test/incremental/unreferenced_error b/test/incremental/unreferenced_error
index 29a9a34d97..6025f3fdae 100644
--- a/test/incremental/unreferenced_error
+++ b/test/incremental/unreferenced_error
@@ -18,7 +18,7 @@ pub fn main() !void {
     try std.io.getStdOut().writeAll(a);
 }
 const a = @compileError("bad a");
-#expect_error=ignored
+#expect_error=main.zig:5:11: error: bad a
 
 #update=remove error reference
 #file=main.zig
diff --git a/tools/incr-check.zig b/tools/incr-check.zig
index 8ef3fddc56..93d16e1e58 100644
--- a/tools/incr-check.zig
+++ b/tools/incr-check.zig
@@ -340,19 +340,63 @@ const Eval = struct {
     }
 
     fn checkErrorOutcome(eval: *Eval, update: Case.Update, error_bundle: std.zig.ErrorBundle) !void {
-        switch (update.outcome) {
+        const expected_errors = switch (update.outcome) {
             .unknown => return,
-            .compile_errors => |expected_errors| {
-                for (expected_errors) |expected_error| {
-                    _ = expected_error;
-                    @panic("TODO check if the expected error matches the compile errors");
-                }
-            },
+            .compile_errors => |expected_errors| expected_errors,
             .stdout, .exit_code => {
                 const color: std.zig.Color = .auto;
                 error_bundle.renderToStdErr(color.renderOptions());
                 eval.fatal("update '{s}': unexpected compile errors", .{update.name});
             },
+        };
+
+        var expected_idx: usize = 0;
+
+        for (error_bundle.getMessages()) |err_idx| {
+            if (expected_idx == expected_errors.len) {
+                const color: std.zig.Color = .auto;
+                error_bundle.renderToStdErr(color.renderOptions());
+                eval.fatal("update '{s}': more errors than expected", .{update.name});
+            }
+            eval.checkOneError(update, error_bundle, expected_errors[expected_idx], false, err_idx);
+            expected_idx += 1;
+
+            for (error_bundle.getNotes(err_idx)) |note_idx| {
+                if (expected_idx == expected_errors.len) {
+                    const color: std.zig.Color = .auto;
+                    error_bundle.renderToStdErr(color.renderOptions());
+                    eval.fatal("update '{s}': more error notes than expected", .{update.name});
+                }
+                eval.checkOneError(update, error_bundle, expected_errors[expected_idx], true, note_idx);
+                expected_idx += 1;
+            }
+        }
+    }
+
+    fn checkOneError(
+        eval: *Eval,
+        update: Case.Update,
+        eb: std.zig.ErrorBundle,
+        expected: Case.ExpectedError,
+        is_note: bool,
+        err_idx: std.zig.ErrorBundle.MessageIndex,
+    ) void {
+        const err = eb.getErrorMessage(err_idx);
+        if (err.src_loc == .none) @panic("TODO error message with no source location");
+        if (err.count != 1) @panic("TODO error message with count>1");
+        const msg = eb.nullTerminatedString(err.msg);
+        const src = eb.getSourceLocation(err.src_loc);
+        const filename = eb.nullTerminatedString(src.src_path);
+
+        if (expected.is_note != is_note or
+            !std.mem.eql(u8, expected.filename, filename) or
+            expected.line != src.line + 1 or
+            expected.column != src.column + 1 or
+            !std.mem.eql(u8, expected.msg, msg))
+        {
+            const color: std.zig.Color = .auto;
+            eb.renderToStdErr(color.renderOptions());
+            eval.fatal("update '{s}': compile error did not match expected error", .{update.name});
         }
     }
 
@@ -595,11 +639,11 @@ const Case = struct {
     };
 
     const ExpectedError = struct {
-        file_name: ?[]const u8 = null,
-        line: ?u32 = null,
-        column: ?u32 = null,
-        msg_exact: ?[]const u8 = null,
-        msg_substring: ?[]const u8 = null,
+        is_note: bool,
+        filename: []const u8,
+        line: u32,
+        column: u32,
+        msg: []const u8,
     };
 
     fn parse(arena: Allocator, bytes: []const u8) !Case {
@@ -665,13 +709,11 @@ const Case = struct {
                     var src: std.ArrayListUnmanaged(u8) = .empty;
 
                     while (true) {
-                        const old = it;
-                        const next_line_raw = it.next() orelse fatal("line {d}: unexpected EOF", .{line_n});
+                        const next_line_raw = it.peek() orelse fatal("line {d}: unexpected EOF", .{line_n});
                         const next_line = std.mem.trimRight(u8, next_line_raw, "\r");
-                        if (std.mem.startsWith(u8, next_line, "#")) {
-                            it = old;
-                            break;
-                        }
+                        if (std.mem.startsWith(u8, next_line, "#")) break;
+
+                        _ = it.next();
                         line_n += 1;
 
                         try src.ensureUnusedCapacity(arena, next_line.len + 1);
@@ -699,7 +741,24 @@ const Case = struct {
                     if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n});
                     const last_update = &updates.items[updates.items.len - 1];
                     if (last_update.outcome != .unknown) fatal("line {d}: conflicting expect directive", .{line_n});
-                    last_update.outcome = .{ .compile_errors = &.{} };
+
+                    var errors: std.ArrayListUnmanaged(ExpectedError) = .empty;
+                    try errors.append(arena, parseExpectedError(val, line_n));
+                    while (true) {
+                        const next_line = it.peek() orelse break;
+                        if (!std.mem.startsWith(u8, next_line, "#")) break;
+                        var new_line_it = std.mem.splitScalar(u8, next_line, '=');
+                        const new_key = new_line_it.first()[1..];
+                        const new_val = std.mem.trimRight(u8, new_line_it.rest(), "\r");
+                        if (new_val.len == 0) break;
+                        if (!std.mem.eql(u8, new_key, "expect_error")) break;
+
+                        _ = it.next();
+                        line_n += 1;
+                        try errors.append(arena, parseExpectedError(new_val, line_n));
+                    }
+
+                    last_update.outcome = .{ .compile_errors = errors.items };
                 } else {
                     fatal("line {d}: unrecognized key '{s}'", .{ line_n, key });
                 }
@@ -749,3 +808,43 @@ fn waitChild(child: *std.process.Child, eval: *Eval) void {
         .Signal, .Stopped, .Unknown => eval.fatal("compiler terminated unexpectedly", .{}),
     }
 }
+
+fn parseExpectedError(str: []const u8, l: usize) Case.ExpectedError {
+    // #expect_error=foo.zig:1:2: error: the error message
+    // #expect_error=foo.zig:1:2: note: and a note
+
+    const fatal = std.process.fatal;
+
+    var it = std.mem.splitScalar(u8, str, ':');
+    const filename = it.first();
+    const line_str = it.next() orelse fatal("line {d}: incomplete error specification", .{l});
+    const column_str = it.next() orelse fatal("line {d}: incomplete error specification", .{l});
+    const error_or_note_str = std.mem.trim(
+        u8,
+        it.next() orelse fatal("line {d}: incomplete error specification", .{l}),
+        " ",
+    );
+    const message = std.mem.trim(u8, it.rest(), " ");
+    if (filename.len == 0) fatal("line {d}: empty filename", .{l});
+    if (message.len == 0) fatal("line {d}: empty error message", .{l});
+    const is_note = if (std.mem.eql(u8, error_or_note_str, "error"))
+        false
+    else if (std.mem.eql(u8, error_or_note_str, "note"))
+        true
+    else
+        fatal("line {d}: expeted 'error' or 'note', found '{s}'", .{ l, error_or_note_str });
+
+    const line = std.fmt.parseInt(u32, line_str, 10) catch
+        fatal("line {d}: invalid line number '{s}'", .{ l, line_str });
+
+    const column = std.fmt.parseInt(u32, column_str, 10) catch
+        fatal("line {d}: invalid column number '{s}'", .{ l, column_str });
+
+    return .{
+        .is_note = is_note,
+        .filename = filename,
+        .line = line,
+        .column = column,
+        .msg = message,
+    };
+}