Check for duped error messages in compile tests

2 files changed, 81 insertions, 24 deletions

diff --git a/test/compile_errors.zig b/test/compile_errors.zig
index 7953eda2ab..a18884cc9a 100644
--- a/test/compile_errors.zig
+++ b/test/compile_errors.zig
@@ -137,19 +137,23 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
         ".tmp_source.zig:3:15: error: C pointers cannot point to non-C-ABI-compatible type 'Foo'",
     );
 
-    cases.addTest(
-        "compile log statement warning deduplication in generic fn",
-        \\export fn entry() void {
-        \\    inner(1);
-        \\    inner(2);
-        \\}
-        \\fn inner(comptime n: usize) void {
-        \\    comptime var i = 0;
-        \\    inline while (i < n) : (i += 1) { @compileLog("!@#$"); }
-        \\}
-    ,
-        ".tmp_source.zig:7:39: error: found compile log statement",
-    );
+    cases.addCase(x: {
+        var tc = cases.create(
+            "compile log statement warning deduplication in generic fn",
+            \\export fn entry() void {
+            \\    inner(1);
+            \\    inner(2);
+            \\}
+            \\fn inner(comptime n: usize) void {
+            \\    comptime var i = 0;
+            \\    inline while (i < n) : (i += 1) { @compileLog("!@#$"); }
+            \\}
+        ,
+            ".tmp_source.zig:7:39: error: found compile log statement",
+        );
+        tc.expect_exact = true;
+        break :x tc;
+    });
 
     cases.addTest(
         "@truncate undefined value",
diff --git a/test/tests.zig b/test/tests.zig
index 800ddc1ccd..656c05cb9b 100644
--- a/test/tests.zig
+++ b/test/tests.zig
@@ -536,6 +536,7 @@ pub const CompileErrorContext = struct {
         name: []const u8,
         sources: ArrayList(SourceFile),
         expected_errors: ArrayList([]const u8),
+        expect_exact: bool,
         link_libc: bool,
         is_exe: bool,
         is_test: bool,
@@ -565,6 +566,26 @@ pub const CompileErrorContext = struct {
         case: *const TestCase,
         build_mode: Mode,
 
+        const ErrLineIter = struct {
+            lines: mem.SplitIterator,
+
+            const source_file = ".tmp_source.zig";
+
+            fn init(input: []const u8) ErrLineIter {
+                return ErrLineIter {
+                    .lines = mem.separate(input, "\n"),
+                };
+            }
+
+            fn next(self: *ErrLineIter) ?[]const u8 {
+                while (self.lines.next()) |line| {
+                    if (mem.indexOf(u8, line, source_file) != null)
+                        return line;
+                }
+                return null;
+            }
+        };
+
         pub fn create(context: *CompileErrorContext, name: []const u8, case: *const TestCase, build_mode: Mode) *CompileCmpOutputStep {
             const allocator = context.b.allocator;
             const ptr = allocator.create(CompileCmpOutputStep) catch unreachable;
@@ -674,19 +695,50 @@ pub const CompileErrorContext = struct {
                 return error.TestFailed;
             }
 
-            for (self.case.expected_errors.toSliceConst()) |expected_error| {
-                if (mem.indexOf(u8, stderr, expected_error) == null) {
-                    warn(
-                        \\
-                        \\========= Expected this compile error: =========
-                        \\{}
-                        \\================================================
-                        \\{}
-                        \\
-                    , expected_error, stderr);
-                    return error.TestFailed;
+            var ok = true;
+            if (self.case.expect_exact) {
+                var err_iter = ErrLineIter.init(stderr);
+                var i: usize = 0;
+                ok = while (err_iter.next()) |line| : (i += 1) {
+                    if (i >= self.case.expected_errors.len) break false;
+                    const expected = self.case.expected_errors.at(i);
+                    if (mem.indexOf(u8, line, expected) == null) break false;
+                    continue;
+                } else true;
+
+                ok = ok and i == self.case.expected_errors.len;
+
+                if (!ok) {
+                    warn("\n======== Expected these compile errors: ========\n");
+                    for (self.case.expected_errors.toSliceConst()) |expected| {
+                        warn("{}\n", expected);
+                    }
+                }
+            } else {
+                for (self.case.expected_errors.toSliceConst()) |expected| {
+                    if (mem.indexOf(u8, stderr, expected) == null) {
+                        warn(
+                            \\=========== Expected compile error: ============
+                            \\{}
+                            \\
+                        , expected
+                        );
+                        ok = false;
+                        break;
+                    }
                 }
             }
+
+            if (!ok) {
+                warn(
+                    \\================= Full output: =================
+                    \\{}
+                    \\
+                , stderr
+                );
+                return error.TestFailed;
+            }
+
             warn("OK\n");
         }
     };
@@ -704,6 +756,7 @@ pub const CompileErrorContext = struct {
             .name = name,
             .sources = ArrayList(TestCase.SourceFile).init(self.b.allocator),
             .expected_errors = ArrayList([]const u8).init(self.b.allocator),
+            .expect_exact = false,
             .link_libc = false,
             .is_exe = false,
             .is_test = false,