fuzzing: progress towards web UI

* libfuzzer: close file after mmap
* fuzzer/main.js: connect with EventSource and debug dump the messages.
  currently this prints how many fuzzer runs have been attempted to
  console.log.
* extract some `std.debug.Info` logic into `std.debug.Coverage`.
  Prepares for consolidation across multiple different executables which
  share source files, and makes it possible to send all the
  PC/SourceLocation mapping data with 4 memcpy'd arrays.
* std.Build.Fuzz:
  - spawn a thread to watch the message queue and signal event
    subscribers.
  - track coverage map data
  - respond to /events URL with EventSource messages on a timer

2 files changed, 254 insertions, 143 deletions

diff --git a/lib/std/debug/Coverage.zig b/lib/std/debug/Coverage.zig
new file mode 100644
index 0000000000..d9cc7fdebd
--- /dev/null
+++ b/lib/std/debug/Coverage.zig
@@ -0,0 +1,244 @@
+const std = @import("../std.zig");
+const Allocator = std.mem.Allocator;
+const Hash = std.hash.Wyhash;
+const Dwarf = std.debug.Dwarf;
+const assert = std.debug.assert;
+
+const Coverage = @This();
+
+/// Provides a globally-scoped integer index for directories.
+///
+/// As opposed to, for example, a directory index that is compilation-unit
+/// scoped inside a single ELF module.
+///
+/// String memory references the memory-mapped debug information.
+///
+/// Protected by `mutex`.
+directories: std.ArrayHashMapUnmanaged(String, void, String.MapContext, false),
+/// Provides a globally-scoped integer index for files.
+///
+/// String memory references the memory-mapped debug information.
+///
+/// Protected by `mutex`.
+files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
+string_bytes: std.ArrayListUnmanaged(u8),
+/// Protects the other fields.
+mutex: std.Thread.Mutex,
+
+pub const init: Coverage = .{
+    .directories = .{},
+    .files = .{},
+    .mutex = .{},
+    .string_bytes = .{},
+};
+
+pub const String = enum(u32) {
+    _,
+
+    pub const MapContext = struct {
+        string_bytes: []const u8,
+
+        pub fn eql(self: @This(), a: String, b: String, b_index: usize) bool {
+            _ = b_index;
+            const a_slice = span(self.string_bytes[@intFromEnum(a)..]);
+            const b_slice = span(self.string_bytes[@intFromEnum(b)..]);
+            return std.mem.eql(u8, a_slice, b_slice);
+        }
+
+        pub fn hash(self: @This(), a: String) u32 {
+            return @truncate(Hash.hash(0, span(self.string_bytes[@intFromEnum(a)..])));
+        }
+    };
+
+    pub const SliceAdapter = struct {
+        string_bytes: []const u8,
+
+        pub fn eql(self: @This(), a_slice: []const u8, b: String, b_index: usize) bool {
+            _ = b_index;
+            const b_slice = span(self.string_bytes[@intFromEnum(b)..]);
+            return std.mem.eql(u8, a_slice, b_slice);
+        }
+        pub fn hash(self: @This(), a: []const u8) u32 {
+            _ = self;
+            return @truncate(Hash.hash(0, a));
+        }
+    };
+};
+
+pub const SourceLocation = struct {
+    file: File.Index,
+    line: u32,
+    column: u32,
+
+    pub const invalid: SourceLocation = .{
+        .file = .invalid,
+        .line = 0,
+        .column = 0,
+    };
+};
+
+pub const File = struct {
+    directory_index: u32,
+    basename: String,
+
+    pub const Index = enum(u32) {
+        invalid = std.math.maxInt(u32),
+        _,
+    };
+
+    pub const MapContext = struct {
+        string_bytes: []const u8,
+
+        pub fn hash(self: MapContext, a: File) u32 {
+            const a_basename = span(self.string_bytes[@intFromEnum(a.basename)..]);
+            return @truncate(Hash.hash(a.directory_index, a_basename));
+        }
+
+        pub fn eql(self: MapContext, a: File, b: File, b_index: usize) bool {
+            _ = b_index;
+            if (a.directory_index != b.directory_index) return false;
+            const a_basename = span(self.string_bytes[@intFromEnum(a.basename)..]);
+            const b_basename = span(self.string_bytes[@intFromEnum(b.basename)..]);
+            return std.mem.eql(u8, a_basename, b_basename);
+        }
+    };
+
+    pub const SliceAdapter = struct {
+        string_bytes: []const u8,
+
+        pub const Entry = struct {
+            directory_index: u32,
+            basename: []const u8,
+        };
+
+        pub fn hash(self: @This(), a: Entry) u32 {
+            _ = self;
+            return @truncate(Hash.hash(a.directory_index, a.basename));
+        }
+
+        pub fn eql(self: @This(), a: Entry, b: File, b_index: usize) bool {
+            _ = b_index;
+            if (a.directory_index != b.directory_index) return false;
+            const b_basename = span(self.string_bytes[@intFromEnum(b.basename)..]);
+            return std.mem.eql(u8, a.basename, b_basename);
+        }
+    };
+};
+
+pub fn deinit(cov: *Coverage, gpa: Allocator) void {
+    cov.directories.deinit(gpa);
+    cov.files.deinit(gpa);
+    cov.string_bytes.deinit(gpa);
+    cov.* = undefined;
+}
+
+pub fn fileAt(cov: *Coverage, index: File.Index) *File {
+    return &cov.files.keys()[@intFromEnum(index)];
+}
+
+pub fn stringAt(cov: *Coverage, index: String) [:0]const u8 {
+    return span(cov.string_bytes.items[@intFromEnum(index)..]);
+}
+
+pub const ResolveAddressesDwarfError = Dwarf.ScanError;
+
+pub fn resolveAddressesDwarf(
+    cov: *Coverage,
+    gpa: Allocator,
+    sorted_pc_addrs: []const u64,
+    /// Asserts its length equals length of `sorted_pc_addrs`.
+    output: []SourceLocation,
+    d: *Dwarf,
+) ResolveAddressesDwarfError!void {
+    assert(sorted_pc_addrs.len == output.len);
+    assert(d.compile_units_sorted);
+
+    var cu_i: usize = 0;
+    var line_table_i: usize = 0;
+    var cu: *Dwarf.CompileUnit = &d.compile_unit_list.items[0];
+    var range = cu.pc_range.?;
+    // Protects directories and files tables from other threads.
+    cov.mutex.lock();
+    defer cov.mutex.unlock();
+    next_pc: for (sorted_pc_addrs, output) |pc, *out| {
+        while (pc >= range.end) {
+            cu_i += 1;
+            if (cu_i >= d.compile_unit_list.items.len) {
+                out.* = SourceLocation.invalid;
+                continue :next_pc;
+            }
+            cu = &d.compile_unit_list.items[cu_i];
+            line_table_i = 0;
+            range = cu.pc_range orelse {
+                out.* = SourceLocation.invalid;
+                continue :next_pc;
+            };
+        }
+        if (pc < range.start) {
+            out.* = SourceLocation.invalid;
+            continue :next_pc;
+        }
+        if (line_table_i == 0) {
+            line_table_i = 1;
+            cov.mutex.unlock();
+            defer cov.mutex.lock();
+            d.populateSrcLocCache(gpa, cu) catch |err| switch (err) {
+                error.MissingDebugInfo, error.InvalidDebugInfo => {
+                    out.* = SourceLocation.invalid;
+                    cu_i += 1;
+                    if (cu_i < d.compile_unit_list.items.len) {
+                        cu = &d.compile_unit_list.items[cu_i];
+                        line_table_i = 0;
+                        if (cu.pc_range) |r| range = r;
+                    }
+                    continue :next_pc;
+                },
+                else => |e| return e,
+            };
+        }
+        const slc = &cu.src_loc_cache.?;
+        const table_addrs = slc.line_table.keys();
+        while (line_table_i < table_addrs.len and table_addrs[line_table_i] < pc) line_table_i += 1;
+
+        const entry = slc.line_table.values()[line_table_i - 1];
+        const corrected_file_index = entry.file - @intFromBool(slc.version < 5);
+        const file_entry = slc.files[corrected_file_index];
+        const dir_path = slc.directories[file_entry.dir_index].path;
+        try cov.string_bytes.ensureUnusedCapacity(gpa, dir_path.len + file_entry.path.len + 2);
+        const dir_gop = try cov.directories.getOrPutContextAdapted(gpa, dir_path, String.SliceAdapter{
+            .string_bytes = cov.string_bytes.items,
+        }, String.MapContext{
+            .string_bytes = cov.string_bytes.items,
+        });
+        if (!dir_gop.found_existing)
+            dir_gop.key_ptr.* = addStringAssumeCapacity(cov, dir_path);
+        const file_gop = try cov.files.getOrPutContextAdapted(gpa, File.SliceAdapter.Entry{
+            .directory_index = @intCast(dir_gop.index),
+            .basename = file_entry.path,
+        }, File.SliceAdapter{
+            .string_bytes = cov.string_bytes.items,
+        }, File.MapContext{
+            .string_bytes = cov.string_bytes.items,
+        });
+        if (!file_gop.found_existing) file_gop.key_ptr.* = .{
+            .directory_index = @intCast(dir_gop.index),
+            .basename = addStringAssumeCapacity(cov, file_entry.path),
+        };
+        out.* = .{
+            .file = @enumFromInt(file_gop.index),
+            .line = entry.line,
+            .column = entry.column,
+        };
+    }
+}
+
+pub fn addStringAssumeCapacity(cov: *Coverage, s: []const u8) String {
+    const result: String = @enumFromInt(cov.string_bytes.items.len);
+    cov.string_bytes.appendSliceAssumeCapacity(s);
+    cov.string_bytes.appendAssumeCapacity(0);
+    return result;
+}
+
+fn span(s: []const u8) [:0]const u8 {
+    return std.mem.sliceTo(@as([:0]const u8, @ptrCast(s)), 0);
+}
diff --git a/lib/std/debug/Info.zig b/lib/std/debug/Info.zig
index a52de6549b..ee191d2c12 100644
--- a/lib/std/debug/Info.zig
+++ b/lib/std/debug/Info.zig
@@ -12,85 +12,31 @@ const Path = std.Build.Cache.Path;
 const Dwarf = std.debug.Dwarf;
 const page_size = std.mem.page_size;
 const assert = std.debug.assert;
-const Hash = std.hash.Wyhash;
+const Coverage = std.debug.Coverage;
+const SourceLocation = std.debug.Coverage.SourceLocation;
 
 const Info = @This();
 
 /// Sorted by key, ascending.
 address_map: std.AutoArrayHashMapUnmanaged(u64, Dwarf.ElfModule),
-
-/// Provides a globally-scoped integer index for directories.
-///
-/// As opposed to, for example, a directory index that is compilation-unit
-/// scoped inside a single ELF module.
-///
-/// String memory references the memory-mapped debug information.
-///
-/// Protected by `mutex`.
-directories: std.StringArrayHashMapUnmanaged(void),
-/// Provides a globally-scoped integer index for files.
-///
-/// String memory references the memory-mapped debug information.
-///
-/// Protected by `mutex`.
-files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
-/// Protects `directories` and `files`.
-mutex: std.Thread.Mutex,
-
-pub const SourceLocation = struct {
-    file: File.Index,
-    line: u32,
-    column: u32,
-
-    pub const invalid: SourceLocation = .{
-        .file = .invalid,
-        .line = 0,
-        .column = 0,
-    };
-};
-
-pub const File = struct {
-    directory_index: u32,
-    basename: []const u8,
-
-    pub const Index = enum(u32) {
-        invalid = std.math.maxInt(u32),
-        _,
-    };
-
-    pub const MapContext = struct {
-        pub fn hash(ctx: MapContext, a: File) u32 {
-            _ = ctx;
-            return @truncate(Hash.hash(a.directory_index, a.basename));
-        }
-
-        pub fn eql(ctx: MapContext, a: File, b: File, b_index: usize) bool {
-            _ = ctx;
-            _ = b_index;
-            return a.directory_index == b.directory_index and std.mem.eql(u8, a.basename, b.basename);
-        }
-    };
-};
+/// Externally managed, outlives this `Info` instance.
+coverage: *Coverage,
 
 pub const LoadError = Dwarf.ElfModule.LoadError;
 
-pub fn load(gpa: Allocator, path: Path) LoadError!Info {
+pub fn load(gpa: Allocator, path: Path, coverage: *Coverage) LoadError!Info {
     var sections: Dwarf.SectionArray = Dwarf.null_section_array;
     var elf_module = try Dwarf.ElfModule.loadPath(gpa, path, null, null, &sections, null);
     try elf_module.dwarf.sortCompileUnits();
     var info: Info = .{
         .address_map = .{},
-        .directories = .{},
-        .files = .{},
-        .mutex = .{},
+        .coverage = coverage,
     };
     try info.address_map.put(gpa, elf_module.base_address, elf_module);
     return info;
 }
 
 pub fn deinit(info: *Info, gpa: Allocator) void {
-    info.directories.deinit(gpa);
-    info.files.deinit(gpa);
     for (info.address_map.values()) |*elf_module| {
         elf_module.dwarf.deinit(gpa);
     }
@@ -98,98 +44,19 @@ pub fn deinit(info: *Info, gpa: Allocator) void {
     info.* = undefined;
 }
 
-pub fn fileAt(info: *Info, index: File.Index) *File {
-    return &info.files.keys()[@intFromEnum(index)];
-}
-
-pub const ResolveSourceLocationsError = Dwarf.ScanError;
+pub const ResolveAddressesError = Coverage.ResolveAddressesDwarfError;
 
 /// Given an array of virtual memory addresses, sorted ascending, outputs a
 /// corresponding array of source locations.
-pub fn resolveSourceLocations(
+pub fn resolveAddresses(
     info: *Info,
     gpa: Allocator,
     sorted_pc_addrs: []const u64,
     /// Asserts its length equals length of `sorted_pc_addrs`.
     output: []SourceLocation,
-) ResolveSourceLocationsError!void {
+) ResolveAddressesError!void {
     assert(sorted_pc_addrs.len == output.len);
     if (info.address_map.entries.len != 1) @panic("TODO");
     const elf_module = &info.address_map.values()[0];
-    return resolveSourceLocationsDwarf(info, gpa, sorted_pc_addrs, output, &elf_module.dwarf);
-}
-
-pub fn resolveSourceLocationsDwarf(
-    info: *Info,
-    gpa: Allocator,
-    sorted_pc_addrs: []const u64,
-    /// Asserts its length equals length of `sorted_pc_addrs`.
-    output: []SourceLocation,
-    d: *Dwarf,
-) ResolveSourceLocationsError!void {
-    assert(sorted_pc_addrs.len == output.len);
-    assert(d.compile_units_sorted);
-
-    var cu_i: usize = 0;
-    var line_table_i: usize = 0;
-    var cu: *Dwarf.CompileUnit = &d.compile_unit_list.items[0];
-    var range = cu.pc_range.?;
-    // Protects directories and files tables from other threads.
-    info.mutex.lock();
-    defer info.mutex.unlock();
-    next_pc: for (sorted_pc_addrs, output) |pc, *out| {
-        while (pc >= range.end) {
-            cu_i += 1;
-            if (cu_i >= d.compile_unit_list.items.len) {
-                out.* = SourceLocation.invalid;
-                continue :next_pc;
-            }
-            cu = &d.compile_unit_list.items[cu_i];
-            line_table_i = 0;
-            range = cu.pc_range orelse {
-                out.* = SourceLocation.invalid;
-                continue :next_pc;
-            };
-        }
-        if (pc < range.start) {
-            out.* = SourceLocation.invalid;
-            continue :next_pc;
-        }
-        if (line_table_i == 0) {
-            line_table_i = 1;
-            info.mutex.unlock();
-            defer info.mutex.lock();
-            d.populateSrcLocCache(gpa, cu) catch |err| switch (err) {
-                error.MissingDebugInfo, error.InvalidDebugInfo => {
-                    out.* = SourceLocation.invalid;
-                    cu_i += 1;
-                    if (cu_i < d.compile_unit_list.items.len) {
-                        cu = &d.compile_unit_list.items[cu_i];
-                        line_table_i = 0;
-                        if (cu.pc_range) |r| range = r;
-                    }
-                    continue :next_pc;
-                },
-                else => |e| return e,
-            };
-        }
-        const slc = &cu.src_loc_cache.?;
-        const table_addrs = slc.line_table.keys();
-        while (line_table_i < table_addrs.len and table_addrs[line_table_i] < pc) line_table_i += 1;
-
-        const entry = slc.line_table.values()[line_table_i - 1];
-        const corrected_file_index = entry.file - @intFromBool(slc.version < 5);
-        const file_entry = slc.files[corrected_file_index];
-        const dir_path = slc.directories[file_entry.dir_index].path;
-        const dir_gop = try info.directories.getOrPut(gpa, dir_path);
-        const file_gop = try info.files.getOrPut(gpa, .{
-            .directory_index = @intCast(dir_gop.index),
-            .basename = file_entry.path,
-        });
-        out.* = .{
-            .file = @enumFromInt(file_gop.index),
-            .line = entry.line,
-            .column = entry.column,
-        };
-    }
+    return info.coverage.resolveAddressesDwarf(gpa, sorted_pc_addrs, output, &elf_module.dwarf);
 }