From 0feacc2b81679514c0168a6ba4c0decafeb2e43e Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Wed, 24 Sep 2025 12:10:32 +0200 Subject: fuzzing: implement limited fuzzing Adds the limit option to `--fuzz=[limit]`. the limit expresses a number of iterations that *each fuzz test* will perform at maximum before exiting. The limit argument supports also 'K', 'M', and 'G' suffixeds (e.g. '10K'). Does not imply `--web-ui` (like unlimited fuzzing does) and prints a fuzzing report at the end. Closes #22900 but does not implement the time based limit, as after internal discussions we concluded to be problematic to both implement and use correctly. --- lib/fuzzer.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib/fuzzer.zig') diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 9171129427..4db2b4a2cd 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -600,9 +600,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void { } /// fuzzer_init_test must be called first -export fn fuzzer_main() void { - while (true) { - fuzzer.cycle(); +export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void { + switch (limit_kind) { + .forever => while (true) fuzzer.cycle(), + .iterations => for (0..amount -| 1) |_| fuzzer.cycle(), } } -- cgit v1.2.3 From 9bb0b43ea3ababb715a15bba8c09ba71e9c3ccc2 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Thu, 25 Sep 2025 17:02:43 +0200 Subject: implement review suggestions --- lib/compiler/build_runner.zig | 16 ++++--- lib/compiler/test_runner.zig | 85 ++++++++------------------------------ lib/fuzzer.zig | 19 ++++++++- lib/std/Build/abi.zig | 12 +++++- test/standalone/libfuzzer/main.zig | 2 +- 5 files changed, 56 insertions(+), 78 deletions(-) (limited to 'lib/fuzzer.zig') diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 4aeffe21a9..9648bdb6b1 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -280,21 +280,21 @@ pub fn main() !void { } } else if (mem.startsWith(u8, arg, "--fuzz=")) { const value = arg["--fuzz=".len..]; - if (value.len == 0) fatal("missing argument to --fuzz\n", .{}); + if (value.len == 0) fatal("missing argument to --fuzz", .{}); const unit: u8 = value[value.len - 1]; - const digits = switch (value[value.len - 1]) { + const digits = switch (unit) { '0'...'9' => value, 'K', 'M', 'G' => value[0 .. value.len - 1], else => fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]\n", + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", .{}, ), }; const amount = std.fmt.parseInt(u64, digits, 10) catch { fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]\n", + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", .{}, ); }; @@ -305,7 +305,7 @@ pub fn main() !void { 'K' => 1000, 'M' => 1_000_000, 'G' => 1_000_000_000, - }) catch fatal("fuzzing limit amount overflows u64\n", .{}); + }) catch fatal("fuzzing limit amount overflows u64", .{}); fuzz = .{ .limit = .{ @@ -520,7 +520,11 @@ pub fn main() !void { }; if (run.web_server) |*web_server| { - if (fuzz) |mode| assert(mode == .forever); + if (fuzz) |mode| if (mode != .forever) fatal( + "error: limited fuzzing is not implemented yet for --webui", + .{}, + ); + web_server.finishBuild(.{ .fuzz = fuzz != null }); } diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 5cdeb95d18..5e7bbd294c 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -56,20 +56,21 @@ pub fn main() void { } } - fba.reset(); if (builtin.fuzz) { const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument"); fuzz_abi.fuzzer_init(.fromSlice(cache_dir)); } + fba.reset(); + if (listen) { - return mainServer(opt_cache_dir) catch @panic("internal test runner failure"); + return mainServer() catch @panic("internal test runner failure"); } else { return mainTerminal(); } } -fn mainServer(opt_cache_dir: ?[]const u8) !void { +fn mainServer() !void { @disableInstrumentation(); var stdin_reader = std.fs.File.stdin().readerStreaming(&stdin_buffer); var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer); @@ -79,66 +80,14 @@ fn mainServer(opt_cache_dir: ?[]const u8) !void { .zig_version = builtin.zig_version_string, }); - if (builtin.fuzz) blk: { - const cache_dir = opt_cache_dir.?; - const coverage_id = fuzz_abi.fuzzer_coverage_id(); - const coverage_file_path: std.Build.Cache.Path = .{ - .root_dir = .{ - .path = cache_dir, - .handle = std.fs.cwd().openDir(cache_dir, .{}) catch |err| { - if (err == error.FileNotFound) { - try server.serveCoverageIdMessage(coverage_id, 0, 0, 0); - break :blk; - } - - fatal("failed to access cache dir '{s}': {s}", .{ - cache_dir, @errorName(err), - }); - }, - }, - .sub_path = "v/" ++ std.fmt.hex(coverage_id), - }; - - var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| { - if (err == error.FileNotFound) { - try server.serveCoverageIdMessage(coverage_id, 0, 0, 0); - break :blk; - } - - fatal("failed to load coverage file '{f}': {s}", .{ - coverage_file_path, @errorName(err), - }); - }; - defer coverage_file.close(); - - var rbuf: [0x1000]u8 = undefined; - var r = coverage_file.reader(&rbuf); - - var header: fuzz_abi.SeenPcsHeader = undefined; - r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { - fatal("failed to read from coverage file '{f}': {s}", .{ - coverage_file_path, @errorName(err), - }); - }; - - if (header.pcs_len == 0) { - fatal("corrupted coverage file '{f}': pcs_len was zero", .{ - coverage_file_path, - }); - } - - var seen_count: usize = 0; - const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); - for (0..chunk_count) |_| { - const seen = r.interface.takeInt(usize, .little) catch |err| { - fatal("failed to read from coverage file '{f}': {s}", .{ - coverage_file_path, @errorName(err), - }); - }; - seen_count += @popCount(seen); - } - - try server.serveCoverageIdMessage(coverage_id, header.n_runs, header.unique_runs, seen_count); + if (builtin.fuzz) { + const coverage = fuzz_abi.fuzzer_coverage(); + try server.serveCoverageIdMessage( + coverage.id, + coverage.runs, + coverage.unique, + coverage.seen, + ); } while (true) { @@ -235,7 +184,7 @@ fn mainServer(opt_cache_dir: ?[]const u8) !void { if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } - std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.debug.print("failed with error.{t}\n", .{err}); std.process.exit(1); }, }; @@ -305,11 +254,11 @@ fn mainTerminal() void { else => { fail_count += 1; if (have_tty) { - std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ - i + 1, test_fn_list.len, test_fn.name, @errorName(err), + std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{ + i + 1, test_fn_list.len, test_fn.name, err, }); } else { - std.debug.print("FAIL ({s})\n", .{@errorName(err)}); + std.debug.print("FAIL ({t})\n", .{err}); } if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); @@ -450,7 +399,7 @@ pub fn fuzz( else => { std.debug.lockStdErr(); if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*); - std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.debug.print("failed with error.{t}\n", .{err}); std.process.exit(1); }, }; diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 4db2b4a2cd..72815d42cd 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -1,5 +1,6 @@ const builtin = @import("builtin"); const std = @import("std"); +const fatal = std.process.fatal; const mem = std.mem; const math = std.math; const Allocator = mem.Allocator; @@ -105,6 +106,7 @@ const Executable = struct { const coverage_file_len = @sizeOf(abi.SeenPcsHeader) + pc_bitset_usizes * @sizeOf(usize) + pcs.len * @sizeOf(usize); + if (populate) { defer coverage_file.lock(.shared) catch |e| panic( "failed to demote lock for coverage file '{s}': {t}", @@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void { } /// Invalid until `fuzzer_init` is called. -export fn fuzzer_coverage_id() u64 { - return exec.pc_digest; +export fn fuzzer_coverage() abi.Coverage { + const coverage_id = exec.pc_digest; + const header: *const abi.SeenPcsHeader = @ptrCast(@volatileCast(exec.shared_seen_pcs.items.ptr)); + + var seen_count: usize = 0; + for (header.seenBits()) |chunk| { + seen_count += @popCount(chunk); + } + + return .{ + .id = coverage_id, + .runs = header.n_runs, + .unique = header.unique_runs, + .seen = seen_count, + }; } /// fuzzer_init must be called beforehand diff --git a/lib/std/Build/abi.zig b/lib/std/Build/abi.zig index e9482b257f..020e2ed032 100644 --- a/lib/std/Build/abi.zig +++ b/lib/std/Build/abi.zig @@ -140,7 +140,7 @@ pub const Rebuild = extern struct { pub const fuzz = struct { pub const TestOne = *const fn (Slice) callconv(.c) void; pub extern fn fuzzer_init(cache_dir_path: Slice) void; - pub extern fn fuzzer_coverage_id() u64; + pub extern fn fuzzer_coverage() Coverage; pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void; pub extern fn fuzzer_new_input(bytes: Slice) void; pub extern fn fuzzer_main(limit_kind: LimitKind, amount: u64) void; @@ -253,6 +253,16 @@ pub const fuzz = struct { return .{ .locs_len_raw = @bitCast(locs_len) }; } }; + + /// Sent by lib/fuzzer to test_runner to obtain information about the + /// active memory mapped input file and cumulative stats about previous + /// fuzzing runs. + pub const Coverage = extern struct { + id: u64, + runs: u64, + unique: u64, + seen: u64, + }; }; /// ABI bits specifically relating to the time report interface. diff --git a/test/standalone/libfuzzer/main.zig b/test/standalone/libfuzzer/main.zig index ae7b9941d5..b21e9be250 100644 --- a/test/standalone/libfuzzer/main.zig +++ b/test/standalone/libfuzzer/main.zig @@ -24,7 +24,7 @@ pub fn main() !void { abi.fuzzer_new_input(.fromSlice("")); abi.fuzzer_new_input(.fromSlice("hello")); - const pc_digest = abi.fuzzer_coverage_id(); + const pc_digest = abi.fuzzer_coverage().id; const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest); const coverage_file = try cache_dir.openFile(coverage_file_path, .{}); defer coverage_file.close(); -- cgit v1.2.3 From 2da8ec9865b6b086341b0f01334f27a488bc220a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 25 Sep 2025 17:16:10 -0700 Subject: fuzzing: fix off-by-one in limit count --- lib/fuzzer.zig | 88 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) (limited to 'lib/fuzzer.zig') diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 72815d42cd..6b7a846e4c 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -512,7 +512,7 @@ const Fuzzer = struct { self.corpus_pos = 0; const rng = self.rng.random(); - while (true) { + const m = while (true) { const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)]; if (!m.mutate( rng, @@ -524,53 +524,53 @@ const Fuzzer = struct { inst.const_vals8.items, inst.const_vals16.items, )) continue; + break m; + }; - self.run(); - if (inst.isFresh()) { - @branchHint(.unlikely); - - const header = mem.bytesAsValue( - abi.SeenPcsHeader, - exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)], - ); - _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic); + self.run(); - inst.setFresh(); - self.minimizeInput(); - inst.updateSeen(); - - // An empty-input has always been tried, so if an empty input is fresh then the - // test has to be non-deterministic. This has to be checked as duplicate empty - // entries are not allowed. - if (self.input.items.len - 8 == 0) { - std.log.warn("non-deterministic test (empty input produces different hits)", .{}); - _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic); - return; - } + if (inst.isFresh()) { + @branchHint(.unlikely); - const arena = self.arena_ctx.allocator(); - const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM"); - - self.corpus.append(gpa, bytes) catch @panic("OOM"); - self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM"); - - // Write new corpus to cache - var name_buf: [@sizeOf(usize) * 2]u8 = undefined; - self.corpus_dir.writeFile(.{ - .sub_path = std.fmt.bufPrint( - &name_buf, - "{x}", - .{self.corpus_dir_idx}, - ) catch unreachable, - .data = bytes, - }) catch |e| panic( - "failed to write corpus file '{x}': {t}", - .{ self.corpus_dir_idx, e }, - ); - self.corpus_dir_idx += 1; + const header = mem.bytesAsValue( + abi.SeenPcsHeader, + exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)], + ); + _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic); + + inst.setFresh(); + self.minimizeInput(); + inst.updateSeen(); + + // An empty-input has always been tried, so if an empty input is fresh then the + // test has to be non-deterministic. This has to be checked as duplicate empty + // entries are not allowed. + if (self.input.items.len - 8 == 0) { + std.log.warn("non-deterministic test (empty input produces different hits)", .{}); + _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic); + return; } - break; + const arena = self.arena_ctx.allocator(); + const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM"); + + self.corpus.append(gpa, bytes) catch @panic("OOM"); + self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM"); + + // Write new corpus to cache + var name_buf: [@sizeOf(usize) * 2]u8 = undefined; + self.corpus_dir.writeFile(.{ + .sub_path = std.fmt.bufPrint( + &name_buf, + "{x}", + .{self.corpus_dir_idx}, + ) catch unreachable, + .data = bytes, + }) catch |e| panic( + "failed to write corpus file '{x}': {t}", + .{ self.corpus_dir_idx, e }, + ); + self.corpus_dir_idx += 1; } } }; @@ -618,7 +618,7 @@ export fn fuzzer_new_input(bytes: abi.Slice) void { export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void { switch (limit_kind) { .forever => while (true) fuzzer.cycle(), - .iterations => for (0..amount -| 1) |_| fuzzer.cycle(), + .iterations => for (0..amount) |_| fuzzer.cycle(), } } -- cgit v1.2.3