From 11cc1c16fa36a7eb13cba1c43fb153ee6aca7b58 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 4 Feb 2023 16:14:43 -0700 Subject: make `@embedFile` support module-mapped names the same way as `@import` closes #14553 --- src/Module.zig | 54 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/Module.zig b/src/Module.zig index 3243c36ee5..e4cf0189cc 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4865,14 +4865,31 @@ pub fn importFile( }; } -pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*EmbedFile { +pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*EmbedFile { const gpa = mod.gpa; - // The resolved path is used as the key in the table, to detect if - // a file refers to the same as another, despite different relative paths. + if (cur_file.pkg.table.get(import_string)) |pkg| { + const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ + pkg.root_src_directory.path orelse ".", pkg.root_src_path, + }); + var keep_resolved_path = false; + defer if (!keep_resolved_path) gpa.free(resolved_path); + + const gop = try mod.embed_table.getOrPut(gpa, resolved_path); + errdefer assert(mod.embed_table.remove(resolved_path)); + if (gop.found_existing) return gop.value_ptr.*; + + const sub_file_path = try gpa.dupe(u8, pkg.root_src_path); + errdefer gpa.free(sub_file_path); + + return newEmbedFile(mod, pkg, sub_file_path, resolved_path, &keep_resolved_path, gop); + } + + // The resolved path is used as the key in the table, to detect if a file + // refers to the same as another, despite different relative paths. const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse "."; const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{ - cur_pkg_dir_path, cur_file.sub_file_path, "..", rel_file_path, + cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string, }); var keep_resolved_path = false; defer if (!keep_resolved_path) gpa.free(resolved_path); @@ -4881,9 +4898,6 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb errdefer assert(mod.embed_table.remove(resolved_path)); if (gop.found_existing) return gop.value_ptr.*; - const new_file = try gpa.create(EmbedFile); - errdefer gpa.destroy(new_file); - const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path}); defer gpa.free(resolved_root_path); @@ -4902,7 +4916,23 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb }; errdefer gpa.free(sub_file_path); - var file = try cur_file.pkg.root_src_directory.handle.openFile(sub_file_path, .{}); + return newEmbedFile(mod, cur_file.pkg, sub_file_path, resolved_path, &keep_resolved_path, gop); +} + +fn newEmbedFile( + mod: *Module, + pkg: *Package, + sub_file_path: []const u8, + resolved_path: []const u8, + keep_resolved_path: *bool, + gop: std.StringHashMapUnmanaged(*EmbedFile).GetOrPutResult, +) !*EmbedFile { + const gpa = mod.gpa; + + const new_file = try gpa.create(EmbedFile); + errdefer gpa.destroy(new_file); + + var file = try pkg.root_src_directory.handle.openFile(sub_file_path, .{}); defer file.close(); const actual_stat = try file.stat(); @@ -4915,10 +4945,6 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb const bytes = try file.readToEndAllocOptions(gpa, std.math.maxInt(u32), size_usize, 1, 0); errdefer gpa.free(bytes); - log.debug("new embedFile. resolved_root_path={s}, resolved_path={s}, sub_file_path={s}, rel_file_path={s}", .{ - resolved_root_path, resolved_path, sub_file_path, rel_file_path, - }); - if (mod.comp.whole_cache_manifest) |whole_cache_manifest| { const copied_resolved_path = try gpa.dupe(u8, resolved_path); errdefer gpa.free(copied_resolved_path); @@ -4927,13 +4953,13 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb try whole_cache_manifest.addFilePostContents(copied_resolved_path, bytes, stat); } - keep_resolved_path = true; // It's now owned by embed_table. + keep_resolved_path.* = true; // It's now owned by embed_table. gop.value_ptr.* = new_file; new_file.* = .{ .sub_file_path = sub_file_path, .bytes = bytes, .stat = stat, - .pkg = cur_file.pkg, + .pkg = pkg, .owner_decl = undefined, // Set by Sema immediately after this function returns. }; return new_file; -- cgit v1.2.3 From b32f5ee93283f7794c611ebd4a1fbc579b78d8ab Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 6 Feb 2023 13:23:03 +0100 Subject: macho: downgrade alignment requirements for symtab in object files Parse and sort relocations by address descending. --- src/link/MachO/Object.zig | 120 +++++++++++++++++++++++++++--------------- src/link/MachO/UnwindInfo.zig | 8 +-- src/link/MachO/ZldAtom.zig | 31 +++++------ src/link/MachO/dead_strip.zig | 7 +-- src/link/MachO/eh_frame.zig | 11 ++-- src/link/MachO/zld.zig | 2 +- 6 files changed, 104 insertions(+), 75 deletions(-) (limited to 'src') diff --git a/src/link/MachO/Object.zig b/src/link/MachO/Object.zig index 13219084b6..771af9b8b3 100644 --- a/src/link/MachO/Object.zig +++ b/src/link/MachO/Object.zig @@ -60,14 +60,20 @@ globals_lookup: []i64 = undefined, /// Can be undefined as set together with in_symtab. relocs_lookup: []RelocEntry = undefined, +/// All relocations sorted and flatened. +relocations: std.ArrayListUnmanaged(macho.relocation_info) = .{}, +/// Beginning index to the relocations array for each input section +/// defined within this Object file. +section_relocs_lookup: std.ArrayListUnmanaged(u32) = .{}, + atoms: std.ArrayListUnmanaged(AtomIndex) = .{}, exec_atoms: std.ArrayListUnmanaged(AtomIndex) = .{}, -eh_frame_sect: ?macho.section_64 = null, +eh_frame_sect_id: ?u8 = null, eh_frame_relocs_lookup: std.AutoArrayHashMapUnmanaged(u32, Record) = .{}, eh_frame_records_lookup: std.AutoArrayHashMapUnmanaged(AtomIndex, u32) = .{}, -unwind_info_sect: ?macho.section_64 = null, +unwind_info_sect_id: ?u8 = null, unwind_relocs_lookup: []Record = undefined, unwind_records_lookup: std.AutoHashMapUnmanaged(AtomIndex, u32) = .{}, @@ -100,6 +106,8 @@ pub fn deinit(self: *Object, gpa: Allocator) void { gpa.free(self.unwind_relocs_lookup); } self.unwind_records_lookup.deinit(gpa); + self.relocations.deinit(gpa); + self.section_relocs_lookup.deinit(gpa); } pub fn parse(self: *Object, allocator: Allocator, cpu_arch: std.Target.Cpu.Arch) !void { @@ -137,15 +145,18 @@ pub fn parse(self: *Object, allocator: Allocator, cpu_arch: std.Target.Cpu.Arch) .buffer = self.contents[@sizeOf(macho.mach_header_64)..][0..self.header.sizeofcmds], }; const nsects = self.getSourceSections().len; + + // Prepopulate relocations per section lookup table. + try self.section_relocs_lookup.resize(allocator, nsects); + mem.set(u32, self.section_relocs_lookup.items, 0); + + // Parse symtab. const symtab = while (it.next()) |cmd| switch (cmd.cmd()) { .SYMTAB => break cmd.cast(macho.symtab_command).?, else => {}, } else return; - self.in_symtab = @ptrCast( - [*]const macho.nlist_64, - @alignCast(@alignOf(macho.nlist_64), &self.contents[symtab.symoff]), - )[0..symtab.nsyms]; + self.in_symtab = @ptrCast([*]align(1) const macho.nlist_64, self.contents.ptr + symtab.symoff)[0..symtab.nsyms]; self.in_strtab = self.contents[symtab.stroff..][0..symtab.strsize]; self.symtab = try allocator.alloc(macho.nlist_64, self.in_symtab.?.len + nsects); @@ -212,10 +223,10 @@ pub fn parse(self: *Object, allocator: Allocator, cpu_arch: std.Target.Cpu.Arch) } // Parse __TEXT,__eh_frame header if one exists - self.eh_frame_sect = self.getSourceSectionByName("__TEXT", "__eh_frame"); + self.eh_frame_sect_id = self.getSourceSectionIndexByName("__TEXT", "__eh_frame"); // Parse __LD,__compact_unwind header if one exists - self.unwind_info_sect = self.getSourceSectionByName("__LD", "__compact_unwind"); + self.unwind_info_sect_id = self.getSourceSectionIndexByName("__LD", "__compact_unwind"); if (self.hasUnwindRecords()) { self.unwind_relocs_lookup = try allocator.alloc(Record, self.getUnwindRecords().len); mem.set(Record, self.unwind_relocs_lookup, .{ @@ -452,6 +463,8 @@ pub fn splitRegularSections(self: *Object, zld: *Zld, object_id: u32) !void { zld.sections.items(.header)[out_sect_id].sectName(), }); + try self.parseRelocs(gpa, section.id); + const cpu_arch = zld.options.target.cpu.arch; const sect_loc = filterSymbolsBySection(symtab[sect_sym_index..], sect_id + 1); const sect_start_index = sect_sym_index + sect_loc.index; @@ -623,25 +636,36 @@ fn filterRelocs( return .{ .start = @intCast(u32, start), .len = @intCast(u32, len) }; } +/// Parse all relocs for the input section, and sort in descending order. +/// Previously, I have wrongly assumed the compilers output relocations for each +/// section in a sorted manner which is simply not true. +fn parseRelocs(self: *Object, gpa: Allocator, sect_id: u8) !void { + const section = self.getSourceSection(sect_id); + const start = @intCast(u32, self.relocations.items.len); + if (self.getSourceRelocs(section)) |relocs| { + try self.relocations.ensureUnusedCapacity(gpa, relocs.len); + self.relocations.appendUnalignedSliceAssumeCapacity(relocs); + std.sort.sort(macho.relocation_info, self.relocations.items[start..], {}, relocGreaterThan); + } + self.section_relocs_lookup.items[sect_id] = start; +} + fn cacheRelocs(self: *Object, zld: *Zld, atom_index: AtomIndex) !void { const atom = zld.getAtom(atom_index); - const source_sect = if (self.getSourceSymbol(atom.sym_index)) |source_sym| blk: { - const source_sect = self.getSourceSection(source_sym.n_sect - 1); - assert(!source_sect.isZerofill()); - break :blk source_sect; + const source_sect_id = if (self.getSourceSymbol(atom.sym_index)) |source_sym| blk: { + break :blk source_sym.n_sect - 1; } else blk: { // If there was no matching symbol present in the source symtab, this means // we are dealing with either an entire section, or part of it, but also // starting at the beginning. const nbase = @intCast(u32, self.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); - const source_sect = self.getSourceSection(sect_id); - assert(!source_sect.isZerofill()); - break :blk source_sect; + const sect_id = @intCast(u8, atom.sym_index - nbase); + break :blk sect_id; }; - - const relocs = self.getRelocs(source_sect); + const source_sect = self.getSourceSection(source_sect_id); + assert(!source_sect.isZerofill()); + const relocs = self.getRelocs(source_sect_id); self.relocs_lookup[atom.sym_index] = if (self.getSourceSymbol(atom.sym_index)) |source_sym| blk: { const offset = source_sym.n_value - source_sect.addr; @@ -649,8 +673,14 @@ fn cacheRelocs(self: *Object, zld: *Zld, atom_index: AtomIndex) !void { } else filterRelocs(relocs, 0, atom.size); } +fn relocGreaterThan(ctx: void, lhs: macho.relocation_info, rhs: macho.relocation_info) bool { + _ = ctx; + return lhs.r_address > rhs.r_address; +} + fn parseEhFrameSection(self: *Object, zld: *Zld, object_id: u32) !void { - const sect = self.eh_frame_sect orelse return; + const sect_id = self.eh_frame_sect_id orelse return; + const sect = self.getSourceSection(sect_id); log.debug("parsing __TEXT,__eh_frame section", .{}); @@ -660,7 +690,8 @@ fn parseEhFrameSection(self: *Object, zld: *Zld, object_id: u32) !void { const gpa = zld.gpa; const cpu_arch = zld.options.target.cpu.arch; - const relocs = self.getRelocs(sect); + try self.parseRelocs(gpa, sect_id); + const relocs = self.getRelocs(sect_id); var it = self.getEhFrameRecordsIterator(); var record_count: u32 = 0; @@ -728,12 +759,12 @@ fn parseEhFrameSection(self: *Object, zld: *Zld, object_id: u32) !void { } fn parseUnwindInfo(self: *Object, zld: *Zld, object_id: u32) !void { - const sect = self.unwind_info_sect orelse { + const sect_id = self.unwind_info_sect_id orelse { // If it so happens that the object had `__eh_frame` section defined but no `__compact_unwind`, // we will try fully synthesising unwind info records to somewhat match Apple ld's // approach. However, we will only synthesise DWARF records and nothing more. For this reason, // we still create the output `__TEXT,__unwind_info` section. - if (self.eh_frame_sect != null) { + if (self.hasEhFrameRecords()) { if (zld.getSectionByName("__TEXT", "__unwind_info") == null) { _ = try zld.initSection("__TEXT", "__unwind_info", .{}); } @@ -758,15 +789,15 @@ fn parseUnwindInfo(self: *Object, zld: *Zld, object_id: u32) !void { if (UnwindInfo.UnwindEncoding.isDwarf(record.compactUnwindEncoding, cpu_arch)) break true; } else false; - if (needs_eh_frame) { - if (self.eh_frame_sect == null) { - log.err("missing __TEXT,__eh_frame section", .{}); - log.err(" in object {s}", .{self.name}); - return error.MissingSection; - } + if (needs_eh_frame and !self.hasEhFrameRecords()) { + log.err("missing __TEXT,__eh_frame section", .{}); + log.err(" in object {s}", .{self.name}); + return error.MissingSection; } - const relocs = self.getRelocs(sect); + try self.parseRelocs(gpa, sect_id); + const relocs = self.getRelocs(sect_id); + for (unwind_records) |record, record_id| { const offset = record_id * @sizeOf(macho.compact_unwind_entry); const rel_pos = filterRelocs( @@ -806,25 +837,23 @@ pub fn getSourceSymbol(self: Object, index: u32) ?macho.nlist_64 { return symtab[mapped_index]; } -pub fn getSourceSection(self: Object, index: u16) macho.section_64 { +pub fn getSourceSection(self: Object, index: u8) macho.section_64 { const sections = self.getSourceSections(); assert(index < sections.len); return sections[index]; } pub fn getSourceSectionByName(self: Object, segname: []const u8, sectname: []const u8) ?macho.section_64 { + const index = self.getSourceSectionIndexByName(segname, sectname) orelse return null; const sections = self.getSourceSections(); - for (sections) |sect| { - if (mem.eql(u8, segname, sect.segName()) and mem.eql(u8, sectname, sect.sectName())) - return sect; - } else return null; + return sections[index]; } pub fn getSourceSectionIndexByName(self: Object, segname: []const u8, sectname: []const u8) ?u8 { const sections = self.getSourceSections(); for (sections) |sect, i| { if (mem.eql(u8, segname, sect.segName()) and mem.eql(u8, sectname, sect.sectName())) - return @intCast(u8, i + 1); + return @intCast(u8, i); } else return null; } @@ -914,11 +943,18 @@ pub fn getSectionAliasSymbolPtr(self: *Object, sect_id: u8) *macho.nlist_64 { return &self.symtab[self.getSectionAliasSymbolIndex(sect_id)]; } -pub fn getRelocs(self: Object, sect: macho.section_64) []align(1) const macho.relocation_info { - if (sect.nreloc == 0) return &[0]macho.relocation_info{}; +fn getSourceRelocs(self: Object, sect: macho.section_64) ?[]align(1) const macho.relocation_info { + if (sect.nreloc == 0) return null; return @ptrCast([*]align(1) const macho.relocation_info, self.contents.ptr + sect.reloff)[0..sect.nreloc]; } +pub fn getRelocs(self: Object, sect_id: u8) []const macho.relocation_info { + const sect = self.getSourceSection(sect_id); + const start = self.section_relocs_lookup.items[sect_id]; + const len = sect.nreloc; + return self.relocations.items[start..][0..len]; +} + pub fn getSymbolName(self: Object, index: u32) []const u8 { const strtab = self.in_strtab.?; const sym = self.symtab[index]; @@ -976,22 +1012,24 @@ pub fn getAtomIndexForSymbol(self: Object, sym_index: u32) ?AtomIndex { } pub fn hasUnwindRecords(self: Object) bool { - return self.unwind_info_sect != null; + return self.unwind_info_sect_id != null; } pub fn getUnwindRecords(self: Object) []align(1) const macho.compact_unwind_entry { - const sect = self.unwind_info_sect orelse return &[0]macho.compact_unwind_entry{}; + const sect_id = self.unwind_info_sect_id orelse return &[0]macho.compact_unwind_entry{}; + const sect = self.getSourceSection(sect_id); const data = self.getSectionContents(sect); const num_entries = @divExact(data.len, @sizeOf(macho.compact_unwind_entry)); return @ptrCast([*]align(1) const macho.compact_unwind_entry, data)[0..num_entries]; } pub fn hasEhFrameRecords(self: Object) bool { - return self.eh_frame_sect != null; + return self.eh_frame_sect_id != null; } pub fn getEhFrameRecordsIterator(self: Object) eh_frame.Iterator { - const sect = self.eh_frame_sect orelse return .{ .data = &[0]u8{} }; + const sect_id = self.eh_frame_sect_id orelse return .{ .data = &[0]u8{} }; + const sect = self.getSourceSection(sect_id); const data = self.getSectionContents(sect); return .{ .data = data }; } diff --git a/src/link/MachO/UnwindInfo.zig b/src/link/MachO/UnwindInfo.zig index 07091af08a..25bd667610 100644 --- a/src/link/MachO/UnwindInfo.zig +++ b/src/link/MachO/UnwindInfo.zig @@ -703,15 +703,11 @@ pub fn parseRelocTarget( } else return sym_loc; } -fn getRelocs( - zld: *Zld, - object_id: u32, - record_id: usize, -) []align(1) const macho.relocation_info { +fn getRelocs(zld: *Zld, object_id: u32, record_id: usize) []const macho.relocation_info { const object = &zld.objects.items[object_id]; assert(object.hasUnwindRecords()); const rel_pos = object.unwind_relocs_lookup[record_id].reloc; - const relocs = object.getRelocs(object.unwind_info_sect.?); + const relocs = object.getRelocs(object.unwind_info_sect_id.?); return relocs[rel_pos.start..][0..rel_pos.len]; } diff --git a/src/link/MachO/ZldAtom.zig b/src/link/MachO/ZldAtom.zig index bb8b7cffb3..e3d5f62a12 100644 --- a/src/link/MachO/ZldAtom.zig +++ b/src/link/MachO/ZldAtom.zig @@ -143,7 +143,7 @@ pub fn calcInnerSymbolOffset(zld: *Zld, atom_index: AtomIndex, sym_index: u32) u sym.n_value else blk: { const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); + const sect_id = @intCast(u8, atom.sym_index - nbase); const source_sect = object.getSourceSection(sect_id); break :blk source_sect.addr; }; @@ -180,7 +180,7 @@ pub fn getRelocContext(zld: *Zld, atom_index: AtomIndex) RelocContext { }; } const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); + const sect_id = @intCast(u8, atom.sym_index - nbase); const source_sect = object.getSourceSection(sect_id); return .{ .base_addr = source_sect.addr, @@ -724,7 +724,7 @@ fn resolveRelocsArm64( if (rel.r_extern == 0) { const base_addr = if (target.sym_index > object.source_address_lookup.len) - @intCast(i64, object.getSourceSection(@intCast(u16, rel.r_symbolnum - 1)).addr) + @intCast(i64, object.getSourceSection(@intCast(u8, rel.r_symbolnum - 1)).addr) else object.source_address_lookup[target.sym_index]; ptr_addend -= base_addr; @@ -861,7 +861,7 @@ fn resolveRelocsX86( if (rel.r_extern == 0) { const base_addr = if (target.sym_index > object.source_address_lookup.len) - @intCast(i64, object.getSourceSection(@intCast(u16, rel.r_symbolnum - 1)).addr) + @intCast(i64, object.getSourceSection(@intCast(u8, rel.r_symbolnum - 1)).addr) else object.source_address_lookup[target.sym_index]; addend += @intCast(i32, @intCast(i64, context.base_addr) + rel.r_address + 4 - @@ -884,7 +884,7 @@ fn resolveRelocsX86( if (rel.r_extern == 0) { const base_addr = if (target.sym_index > object.source_address_lookup.len) - @intCast(i64, object.getSourceSection(@intCast(u16, rel.r_symbolnum - 1)).addr) + @intCast(i64, object.getSourceSection(@intCast(u8, rel.r_symbolnum - 1)).addr) else object.source_address_lookup[target.sym_index]; addend -= base_addr; @@ -928,7 +928,7 @@ pub fn getAtomCode(zld: *Zld, atom_index: AtomIndex) []const u8 { // we are dealing with either an entire section, or part of it, but also // starting at the beginning. const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); + const sect_id = @intCast(u8, atom.sym_index - nbase); const source_sect = object.getSourceSection(sect_id); assert(!source_sect.isZerofill()); const code = object.getSectionContents(source_sect); @@ -943,28 +943,25 @@ pub fn getAtomCode(zld: *Zld, atom_index: AtomIndex) []const u8 { return code[offset..][0..code_len]; } -pub fn getAtomRelocs(zld: *Zld, atom_index: AtomIndex) []align(1) const macho.relocation_info { +pub fn getAtomRelocs(zld: *Zld, atom_index: AtomIndex) []const macho.relocation_info { const atom = zld.getAtom(atom_index); assert(atom.getFile() != null); // Synthetic atom shouldn't need to unique for relocs. const object = zld.objects.items[atom.getFile().?]; const cache = object.relocs_lookup[atom.sym_index]; - const source_sect = if (object.getSourceSymbol(atom.sym_index)) |source_sym| blk: { - const source_sect = object.getSourceSection(source_sym.n_sect - 1); - assert(!source_sect.isZerofill()); - break :blk source_sect; + const source_sect_id = if (object.getSourceSymbol(atom.sym_index)) |source_sym| blk: { + break :blk source_sym.n_sect - 1; } else blk: { // If there was no matching symbol present in the source symtab, this means // we are dealing with either an entire section, or part of it, but also // starting at the beginning. const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); - const source_sect = object.getSourceSection(sect_id); - assert(!source_sect.isZerofill()); - break :blk source_sect; + const sect_id = @intCast(u8, atom.sym_index - nbase); + break :blk sect_id; }; - - const relocs = object.getRelocs(source_sect); + const source_sect = object.getSourceSection(source_sect_id); + assert(!source_sect.isZerofill()); + const relocs = object.getRelocs(source_sect_id); return relocs[cache.start..][0..cache.len]; } diff --git a/src/link/MachO/dead_strip.zig b/src/link/MachO/dead_strip.zig index 1f8def96f3..6ba70acbfd 100644 --- a/src/link/MachO/dead_strip.zig +++ b/src/link/MachO/dead_strip.zig @@ -88,7 +88,7 @@ fn collectRoots(zld: *Zld, roots: *AtomTable) !void { source_sym.n_sect - 1 else sect_id: { const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); + const sect_id = @intCast(u8, atom.sym_index - nbase); break :sect_id sect_id; }; const source_sect = object.getSourceSection(sect_id); @@ -223,7 +223,7 @@ fn mark(zld: *Zld, roots: AtomTable, alive: *AtomTable) !void { source_sym.n_sect - 1 else blk: { const nbase = @intCast(u32, object.in_symtab.?.len); - const sect_id = @intCast(u16, atom.sym_index - nbase); + const sect_id = @intCast(u8, atom.sym_index - nbase); break :blk sect_id; }; const source_sect = object.getSourceSection(sect_id); @@ -350,8 +350,9 @@ fn markEhFrameRecord(zld: *Zld, object_id: u32, atom_index: AtomIndex, alive: *A } }, .x86_64 => { + const sect = object.getSourceSection(object.eh_frame_sect_id.?); const lsda_ptr = try fde.getLsdaPointer(cie, .{ - .base_addr = object.eh_frame_sect.?.addr, + .base_addr = sect.addr, .base_offset = fde_offset, }); if (lsda_ptr) |lsda_address| { diff --git a/src/link/MachO/eh_frame.zig b/src/link/MachO/eh_frame.zig index 8345a8e4f3..3867b15a96 100644 --- a/src/link/MachO/eh_frame.zig +++ b/src/link/MachO/eh_frame.zig @@ -171,8 +171,9 @@ pub fn write(zld: *Zld, unwind_info: *UnwindInfo) !void { const cie_record = eh_records.get( eh_frame_offset + 4 - fde_record.getCiePointer(), ).?; + const eh_frame_sect = object.getSourceSection(object.eh_frame_sect_id.?); const source_lsda_ptr = try fde_record.getLsdaPointer(cie_record, .{ - .base_addr = object.eh_frame_sect.?.addr, + .base_addr = eh_frame_sect.addr, .base_offset = fde_record_offset, }); if (source_lsda_ptr) |ptr| { @@ -552,16 +553,12 @@ pub fn EhFrameRecord(comptime is_mutable: bool) type { }; } -pub fn getRelocs( - zld: *Zld, - object_id: u32, - source_offset: u32, -) []align(1) const macho.relocation_info { +pub fn getRelocs(zld: *Zld, object_id: u32, source_offset: u32) []const macho.relocation_info { const object = &zld.objects.items[object_id]; assert(object.hasEhFrameRecords()); const urel = object.eh_frame_relocs_lookup.get(source_offset) orelse return &[0]macho.relocation_info{}; - const all_relocs = object.getRelocs(object.eh_frame_sect.?); + const all_relocs = object.getRelocs(object.eh_frame_sect_id.?); return all_relocs[urel.reloc.start..][0..urel.reloc.len]; } diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 81fae399ef..8619163ef2 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -2406,7 +2406,7 @@ pub const Zld = struct { source_sym.n_value else blk: { const nbase = @intCast(u32, object.in_symtab.?.len); - const source_sect_id = @intCast(u16, atom.sym_index - nbase); + const source_sect_id = @intCast(u8, atom.sym_index - nbase); break :blk object.getSourceSection(source_sect_id).addr; }; const filtered_dice = filterDataInCode(dice, source_addr, source_addr + atom.size); -- cgit v1.2.3 From f63eda3f6ae4369e12b68bb13e36b23957a9d809 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 6 Feb 2023 16:08:42 +0100 Subject: macho: parse and sort data-in-code entries ahead of time --- src/link/MachO/Object.zig | 37 +++++++++++++++++++++++++------------ src/link/MachO/zld.zig | 10 ++++------ 2 files changed, 29 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/link/MachO/Object.zig b/src/link/MachO/Object.zig index 771af9b8b3..4d24b2ed6a 100644 --- a/src/link/MachO/Object.zig +++ b/src/link/MachO/Object.zig @@ -60,12 +60,16 @@ globals_lookup: []i64 = undefined, /// Can be undefined as set together with in_symtab. relocs_lookup: []RelocEntry = undefined, -/// All relocations sorted and flatened. +/// All relocations sorted and flatened, sorted by address descending +/// per section. relocations: std.ArrayListUnmanaged(macho.relocation_info) = .{}, /// Beginning index to the relocations array for each input section /// defined within this Object file. section_relocs_lookup: std.ArrayListUnmanaged(u32) = .{}, +/// Data-in-code records sorted by address. +data_in_code: std.ArrayListUnmanaged(macho.data_in_code_entry) = .{}, + atoms: std.ArrayListUnmanaged(AtomIndex) = .{}, exec_atoms: std.ArrayListUnmanaged(AtomIndex) = .{}, @@ -108,6 +112,7 @@ pub fn deinit(self: *Object, gpa: Allocator) void { self.unwind_records_lookup.deinit(gpa); self.relocations.deinit(gpa); self.section_relocs_lookup.deinit(gpa); + self.data_in_code.deinit(gpa); } pub fn parse(self: *Object, allocator: Allocator, cpu_arch: std.Target.Cpu.Arch) !void { @@ -365,6 +370,7 @@ pub fn splitIntoAtoms(self: *Object, zld: *Zld, object_id: u32) !void { try self.splitRegularSections(zld, object_id); try self.parseEhFrameSection(zld, object_id); try self.parseUnwindInfo(zld, object_id); + try self.parseDataInCode(zld.gpa); } /// Splits input regular sections into Atoms. @@ -870,24 +876,27 @@ pub fn getSourceSections(self: Object) []const macho.section_64 { } else unreachable; } -pub fn parseDataInCode(self: Object) ?[]const macho.data_in_code_entry { +pub fn parseDataInCode(self: *Object, gpa: Allocator) !void { var it = LoadCommandIterator{ .ncmds = self.header.ncmds, .buffer = self.contents[@sizeOf(macho.mach_header_64)..][0..self.header.sizeofcmds], }; - while (it.next()) |cmd| { + const cmd = while (it.next()) |cmd| { switch (cmd.cmd()) { - .DATA_IN_CODE => { - const dice = cmd.cast(macho.linkedit_data_command).?; - const ndice = @divExact(dice.datasize, @sizeOf(macho.data_in_code_entry)); - return @ptrCast( - [*]const macho.data_in_code_entry, - @alignCast(@alignOf(macho.data_in_code_entry), &self.contents[dice.dataoff]), - )[0..ndice]; - }, + .DATA_IN_CODE => break cmd.cast(macho.linkedit_data_command).?, else => {}, } - } else return null; + } else return; + const ndice = @divExact(cmd.datasize, @sizeOf(macho.data_in_code_entry)); + const dice = @ptrCast([*]align(1) const macho.data_in_code_entry, self.contents.ptr + cmd.dataoff)[0..ndice]; + try self.data_in_code.ensureTotalCapacityPrecise(gpa, dice.len); + self.data_in_code.appendUnalignedSliceAssumeCapacity(dice); + std.sort.sort(macho.data_in_code_entry, self.data_in_code.items, {}, diceLessThan); +} + +fn diceLessThan(ctx: void, lhs: macho.data_in_code_entry, rhs: macho.data_in_code_entry) bool { + _ = ctx; + return lhs.offset < rhs.offset; } fn parseDysymtab(self: Object) ?macho.dysymtab_command { @@ -1033,3 +1042,7 @@ pub fn getEhFrameRecordsIterator(self: Object) eh_frame.Iterator { const data = self.getSectionContents(sect); return .{ .data = data }; } + +pub fn hasDataInCode(self: Object) bool { + return self.data_in_code.items.len > 0; +} diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 8619163ef2..6a83de0c19 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -2391,16 +2391,14 @@ pub const Zld = struct { const text_sect_header = self.sections.items(.header)[text_sect_id]; for (self.objects.items) |object| { - const dice = object.parseDataInCode() orelse continue; + if (!object.hasDataInCode()) continue; + const dice = object.data_in_code.items; try out_dice.ensureUnusedCapacity(dice.len); - for (object.atoms.items) |atom_index| { + for (object.exec_atoms.items) |atom_index| { const atom = self.getAtom(atom_index); const sym = self.getSymbol(atom.getSymbolWithLoc()); - const sect_id = sym.n_sect - 1; - if (sect_id != text_sect_id) { - continue; - } + if (sym.n_desc == N_DEAD) continue; const source_addr = if (object.getSourceSymbol(atom.sym_index)) |source_sym| source_sym.n_value -- cgit v1.2.3 From 4b1a883d35565766d30f587d2cb2ccfe8c065c8b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 7 Feb 2023 02:54:18 +0100 Subject: macho: ensure local syms buffer is nlist_64 aligned when re-reading from file --- src/link/MachO/Dylib.zig | 2 ++ src/link/MachO/zld.zig | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/link/MachO/Dylib.zig b/src/link/MachO/Dylib.zig index 1c1842aaed..a2c4bad942 100644 --- a/src/link/MachO/Dylib.zig +++ b/src/link/MachO/Dylib.zig @@ -166,6 +166,8 @@ pub fn parseFromBinary( const symtab_cmd = cmd.cast(macho.symtab_command).?; const symtab = @ptrCast( [*]const macho.nlist_64, + // Alignment is guaranteed as a dylib is a final linked image and has to have sections + // properly aligned in order to be correctly loaded by the loader. @alignCast(@alignOf(macho.nlist_64), &data[symtab_cmd.symoff]), )[0..symtab_cmd.nsyms]; const strtab = data[symtab_cmd.stroff..][0..symtab_cmd.strsize]; diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 6a83de0c19..095ac9b5ce 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -2697,12 +2697,12 @@ pub const Zld = struct { // Exclude region comprising all symbol stabs. const nlocals = self.dysymtab_cmd.nlocalsym; - const locals_buf = try self.gpa.alloc(u8, nlocals * @sizeOf(macho.nlist_64)); - defer self.gpa.free(locals_buf); + const locals = try self.gpa.alloc(macho.nlist_64, nlocals); + defer self.gpa.free(locals); + const locals_buf = @ptrCast([*]u8, locals.ptr)[0 .. @sizeOf(macho.nlist_64) * nlocals]; const amt = try self.file.preadAll(locals_buf, self.symtab_cmd.symoff); if (amt != locals_buf.len) return error.InputOutput; - const locals = @ptrCast([*]macho.nlist_64, @alignCast(@alignOf(macho.nlist_64), locals_buf))[0..nlocals]; const istab: usize = for (locals) |local, i| { if (local.stab()) break i; -- cgit v1.2.3 From 1f7390f3999e80f775dbc0e62f1dcb071c3bed77 Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Wed, 8 Feb 2023 13:56:03 +1100 Subject: fix custom test runner package path resolution Fixes #13970. This fix makes test runners resolve package paths relative to the directory the test runner is in. This means it is not possible to import a file from outside the file tree root at the directory containing the test runner. --- src/Compilation.zig | 7 +++++-- test/standalone.zig | 1 + test/standalone/issue_13970/build.zig | 21 +++++++++++++++++++++ test/standalone/issue_13970/empty.zig | 0 test/standalone/issue_13970/src/empty.zig | 0 test/standalone/issue_13970/src/main.zig | 8 ++++++++ test/standalone/issue_13970/src/package.zig | 1 + test/standalone/issue_13970/test_root/empty.zig | 0 8 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test/standalone/issue_13970/build.zig create mode 100644 test/standalone/issue_13970/empty.zig create mode 100644 test/standalone/issue_13970/src/empty.zig create mode 100644 test/standalone/issue_13970/src/main.zig create mode 100644 test/standalone/issue_13970/src/package.zig create mode 100644 test/standalone/issue_13970/test_root/empty.zig (limited to 'src') diff --git a/src/Compilation.zig b/src/Compilation.zig index e09b8f18ab..8f265f9eb6 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1621,8 +1621,11 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const root_pkg = if (options.is_test) root_pkg: { // TODO: we currently have two packages named 'root' here, which is weird. This // should be changed as part of the resolution of #12201 - const test_pkg = if (options.test_runner_path) |test_runner| - try Package.create(gpa, "root", null, test_runner) + const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { + const test_dir = std.fs.path.dirname(test_runner); + const basename = std.fs.path.basename(test_runner); + break :test_pkg try Package.create(gpa, "root", test_dir, basename); + } else try Package.createWithDir( gpa, diff --git a/test/standalone.zig b/test/standalone.zig index af972ccb86..81eb1b0042 100644 --- a/test/standalone.zig +++ b/test/standalone.zig @@ -27,6 +27,7 @@ pub fn addCases(cases: *tests.StandaloneContext) void { cases.add("test/standalone/noreturn_call/inline.zig"); cases.add("test/standalone/noreturn_call/as_arg.zig"); cases.addBuildFile("test/standalone/test_runner_path/build.zig", .{ .requires_stage2 = true }); + cases.addBuildFile("test/standalone/issue_13970/build.zig", .{}); cases.addBuildFile("test/standalone/main_pkg_path/build.zig", .{}); cases.addBuildFile("test/standalone/shared_library/build.zig", .{}); cases.addBuildFile("test/standalone/mix_o_files/build.zig", .{}); diff --git a/test/standalone/issue_13970/build.zig b/test/standalone/issue_13970/build.zig new file mode 100644 index 0000000000..f5e07d8903 --- /dev/null +++ b/test/standalone/issue_13970/build.zig @@ -0,0 +1,21 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const test1 = b.addTest(.{ + .root_source_file = .{ .path = "test_root/empty.zig" }, + }); + const test2 = b.addTest(.{ + .root_source_file = .{ .path = "src/empty.zig" }, + }); + const test3 = b.addTest(.{ + .root_source_file = .{ .path = "empty.zig" }, + }); + test1.setTestRunner("src/main.zig"); + test2.setTestRunner("src/main.zig"); + test3.setTestRunner("src/main.zig"); + + const test_step = b.step("test", "Test package path resolution of custom test runner"); + test_step.dependOn(&test1.step); + test_step.dependOn(&test2.step); + test_step.dependOn(&test3.step); +} diff --git a/test/standalone/issue_13970/empty.zig b/test/standalone/issue_13970/empty.zig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/standalone/issue_13970/src/empty.zig b/test/standalone/issue_13970/src/empty.zig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/standalone/issue_13970/src/main.zig b/test/standalone/issue_13970/src/main.zig new file mode 100644 index 0000000000..d71320d46a --- /dev/null +++ b/test/standalone/issue_13970/src/main.zig @@ -0,0 +1,8 @@ +const std = @import("std"); +const package = @import("package.zig"); +const root = @import("root"); +const builtin = @import("builtin"); + +pub fn main() !void { + _ = package.decl; +} diff --git a/test/standalone/issue_13970/src/package.zig b/test/standalone/issue_13970/src/package.zig new file mode 100644 index 0000000000..4f6037b38d --- /dev/null +++ b/test/standalone/issue_13970/src/package.zig @@ -0,0 +1 @@ +pub const decl = 0; diff --git a/test/standalone/issue_13970/test_root/empty.zig b/test/standalone/issue_13970/test_root/empty.zig new file mode 100644 index 0000000000..e69de29bb2 -- cgit v1.2.3 From 94c68c1f9ed74cd5e2b4ab4329166ba5e5d7abc3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 7 Feb 2023 19:27:11 +0100 Subject: macho: fix incorrect representation of encodings count per page There can be a maximum of 256 compact encodings per page in compact unwind info, and we were using `u8` to represent the count which is insufficient. This commit bumps it to `u9`. --- src/link/MachO/UnwindInfo.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/link/MachO/UnwindInfo.zig b/src/link/MachO/UnwindInfo.zig index 25bd667610..5e61834bbc 100644 --- a/src/link/MachO/UnwindInfo.zig +++ b/src/link/MachO/UnwindInfo.zig @@ -68,7 +68,7 @@ const Page = struct { start: RecordIndex, count: u16, page_encodings: [max_compact_encodings]RecordIndex = undefined, - page_encodings_count: u8 = 0, + page_encodings_count: u9 = 0, fn appendPageEncoding(page: *Page, record_id: RecordIndex) void { assert(page.page_encodings_count <= max_compact_encodings); @@ -81,13 +81,13 @@ const Page = struct { info: *const UnwindInfo, enc: macho.compact_unwind_encoding_t, ) ?u8 { - comptime var index: u8 = 0; + comptime var index: u9 = 0; inline while (index < max_compact_encodings) : (index += 1) { if (index >= page.page_encodings_count) return null; const record_id = page.page_encodings[index]; const record = info.records.items[record_id]; if (record.compactUnwindEncoding == enc) { - return index; + return @intCast(u8, index); } } return null; -- cgit v1.2.3 From edc0e84270c04acf432096c4772bfe199afdaf15 Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Wed, 8 Feb 2023 15:04:29 +1100 Subject: allow custom test runners to import modules --- src/Compilation.zig | 25 +++++++++++++--------- .../test_runner_module_imports/build.zig | 19 ++++++++++++++++ .../test_runner_module_imports/module1/main.zig | 1 + .../test_runner_module_imports/module2/main.zig | 1 + .../test_runner_module_imports/src/main.zig | 6 ++++++ .../test_runner/main.zig | 9 ++++++++ 6 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 test/standalone/test_runner_module_imports/build.zig create mode 100644 test/standalone/test_runner_module_imports/module1/main.zig create mode 100644 test/standalone/test_runner_module_imports/module2/main.zig create mode 100644 test/standalone/test_runner_module_imports/src/main.zig create mode 100644 test/standalone/test_runner_module_imports/test_runner/main.zig (limited to 'src') diff --git a/src/Compilation.zig b/src/Compilation.zig index 8f265f9eb6..f19e83680d 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1624,16 +1624,21 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { const test_dir = std.fs.path.dirname(test_runner); const basename = std.fs.path.basename(test_runner); - break :test_pkg try Package.create(gpa, "root", test_dir, basename); - } - else - try Package.createWithDir( - gpa, - "root", - options.zig_lib_directory, - null, - "test_runner.zig", - ); + const pkg = try Package.create(gpa, "root", test_dir, basename); + + // copy package table from main_pkg to root_pkg + var iter = main_pkg.table.valueIterator(); + while (iter.next()) |v| { + try pkg.add(gpa, v.*); + } + break :test_pkg pkg; + } else try Package.createWithDir( + gpa, + "root", + options.zig_lib_directory, + null, + "test_runner.zig", + ); errdefer test_pkg.destroy(gpa); break :root_pkg test_pkg; diff --git a/test/standalone/test_runner_module_imports/build.zig b/test/standalone/test_runner_module_imports/build.zig new file mode 100644 index 0000000000..973365e495 --- /dev/null +++ b/test/standalone/test_runner_module_imports/build.zig @@ -0,0 +1,19 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const t = b.addTest(.{ + .root_source_file = .{ .path = "src/main.zig" }, + }); + t.setTestRunner("test_runner/main.zig"); + + const module1 = b.createModule(.{ .source_file = .{ .path = "module1/main.zig" } }); + const module2 = b.createModule(.{ + .source_file = .{ .path = "module2/main.zig" }, + .dependencies = &.{.{ .name = "module1", .module = module1 }}, + }); + + t.addModule("module2", module2); + + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&t.step); +} diff --git a/test/standalone/test_runner_module_imports/module1/main.zig b/test/standalone/test_runner_module_imports/module1/main.zig new file mode 100644 index 0000000000..883c61ac7e --- /dev/null +++ b/test/standalone/test_runner_module_imports/module1/main.zig @@ -0,0 +1 @@ +pub const decl: usize = 1234567890; diff --git a/test/standalone/test_runner_module_imports/module2/main.zig b/test/standalone/test_runner_module_imports/module2/main.zig new file mode 100644 index 0000000000..f8bb21d7df --- /dev/null +++ b/test/standalone/test_runner_module_imports/module2/main.zig @@ -0,0 +1 @@ +pub const mod1 = @import("module1"); diff --git a/test/standalone/test_runner_module_imports/src/main.zig b/test/standalone/test_runner_module_imports/src/main.zig new file mode 100644 index 0000000000..9d5d5795b6 --- /dev/null +++ b/test/standalone/test_runner_module_imports/src/main.zig @@ -0,0 +1,6 @@ +const mod2 = @import("module2"); +const std = @import("std"); + +test { + try std.testing.expectEqual(@as(usize, 1234567890), mod2.mod1.decl); +} diff --git a/test/standalone/test_runner_module_imports/test_runner/main.zig b/test/standalone/test_runner_module_imports/test_runner/main.zig new file mode 100644 index 0000000000..13ce91b88f --- /dev/null +++ b/test/standalone/test_runner_module_imports/test_runner/main.zig @@ -0,0 +1,9 @@ +const std = @import("std"); +const mod2 = @import("module2"); + +pub fn main() !void { + try std.testing.expectEqual(@as(usize, 1234567890), mod2.mod1.decl); + for (@import("builtin").test_functions) |test_fn| { + try test_fn.func(); + } +} -- cgit v1.2.3 From 948754c5d4b707c991c4db9cdfd968e6f4866ea4 Mon Sep 17 00:00:00 2001 From: Dominic <4678790+dweiller@users.noreply.github.com> Date: Wed, 8 Feb 2023 22:39:38 +1100 Subject: clone package table into custom test runner Co-authored-by: Veikka Tuominen --- src/Compilation.zig | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src') diff --git a/src/Compilation.zig b/src/Compilation.zig index f19e83680d..18d0e46892 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1627,10 +1627,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const pkg = try Package.create(gpa, "root", test_dir, basename); // copy package table from main_pkg to root_pkg - var iter = main_pkg.table.valueIterator(); - while (iter.next()) |v| { - try pkg.add(gpa, v.*); - } + pkg.table = try main_pkg.table.clone(gpa); break :test_pkg pkg; } else try Package.createWithDir( gpa, -- cgit v1.2.3 From a5d25fabdaf7e68f375874b9bda402acaeb9545d Mon Sep 17 00:00:00 2001 From: John Schmidt <3405586+schmee@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:46:29 +0100 Subject: translate_c: fix typedeffed pointer subtraction Closes #14560. --- src/translate_c.zig | 3 ++- test/translate_c.zig | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/translate_c.zig b/src/translate_c.zig index a6715d161c..7fd4b93821 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -1748,7 +1748,8 @@ fn transBinaryOperator( const lhs_expr = stmt.getLHS(); const lhs_qt = getExprQualType(c, lhs_expr); const lhs_qt_translated = try transQualType(c, scope, lhs_qt, lhs_expr.getBeginLoc()); - const elem_type = lhs_qt_translated.castTag(.c_pointer).?.data.elem_type; + const c_pointer = getContainer(c, lhs_qt_translated).?; + const elem_type = c_pointer.castTag(.c_pointer).?.data.elem_type; const sizeof = try Tag.sizeof.create(c.arena, elem_type); const bitcast = try Tag.bit_cast.create(c.arena, .{ .lhs = ptrdiff_type, .rhs = infixOpNode }); diff --git a/test/translate_c.zig b/test/translate_c.zig index d2db895a5a..92dc3038c0 100644 --- a/test/translate_c.zig +++ b/test/translate_c.zig @@ -3916,4 +3916,36 @@ pub fn addCases(cases: *tests.TranslateCContext) void { \\ }) != 0) {} \\} }); + + if (builtin.os.tag == .windows) { + cases.add("Pointer subtraction with typedef", + \\typedef char* S; + \\void foo() { + \\ S a, b; + \\ long long c = a - b; + \\} + , &[_][]const u8{ + \\pub export fn foo() void { + \\ var a: S = undefined; + \\ var b: S = undefined; + \\ var c: c_longlong = @divExact(@bitCast(c_longlong, @ptrToInt(a) -% @ptrToInt(b)), @sizeOf(u8)); + \\ _ = @TypeOf(c); + \\} + }); + } else { + cases.add("Pointer subtraction with typedef", + \\typedef char* S; + \\void foo() { + \\ S a, b; + \\ long c = a - b; + \\} + , &[_][]const u8{ + \\pub export fn foo() void { + \\ var a: S = undefined; + \\ var b: S = undefined; + \\ var c: c_long = @divExact(@bitCast(c_long, @ptrToInt(a) -% @ptrToInt(b)), @sizeOf(u8)); + \\ _ = @TypeOf(c); + \\} + }); + } } -- cgit v1.2.3 From b9c2837c1c9eddc903aeeb168b8d2daa48837ddb Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Thu, 2 Feb 2023 14:16:15 +0200 Subject: Sema: validate inferred error set payload type This was missed in b0a55e1b3be3a274546f9c18016e9609d546bdb0 --- src/Sema.zig | 29 +++++++++++++--------- .../function_returning_opaque_type.zig | 12 ++++----- .../invalid_error_union_payload_type.zig | 7 ++++++ 3 files changed, 30 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/Sema.zig b/src/Sema.zig index b7b3a55063..2dba678931 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -7669,17 +7669,21 @@ fn zirErrorUnionType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileEr error_set.fmt(sema.mod), }); } - if (payload.zigTypeTag() == .Opaque) { - return sema.fail(block, rhs_src, "error union with payload of opaque type '{}' not allowed", .{ - payload.fmt(sema.mod), + try sema.validateErrorUnionPayloadType(block, payload, rhs_src); + const err_union_ty = try Type.errorUnion(sema.arena, error_set, payload, sema.mod); + return sema.addType(err_union_ty); +} + +fn validateErrorUnionPayloadType(sema: *Sema, block: *Block, payload_ty: Type, payload_src: LazySrcLoc) !void { + if (payload_ty.zigTypeTag() == .Opaque) { + return sema.fail(block, payload_src, "error union with payload of opaque type '{}' not allowed", .{ + payload_ty.fmt(sema.mod), }); - } else if (payload.zigTypeTag() == .ErrorSet) { - return sema.fail(block, rhs_src, "error union with payload of error set type '{}' not allowed", .{ - payload.fmt(sema.mod), + } else if (payload_ty.zigTypeTag() == .ErrorSet) { + return sema.fail(block, payload_src, "error union with payload of error set type '{}' not allowed", .{ + payload_ty.fmt(sema.mod), }); } - const err_union_ty = try Type.errorUnion(sema.arena, error_set, payload, sema.mod); - return sema.addType(err_union_ty); } fn zirErrorValue(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref { @@ -8639,6 +8643,7 @@ fn funcCommon( const return_type = if (!inferred_error_set or ret_poison) bare_return_type else blk: { + try sema.validateErrorUnionPayloadType(block, bare_return_type, ret_ty_src); const node = try sema.gpa.create(Module.Fn.InferredErrorSetListNode); node.data = .{ .func = new_func }; maybe_inferred_error_set_node = node; @@ -8650,15 +8655,15 @@ fn funcCommon( }); }; - if (!bare_return_type.isValidReturnType()) { - const opaque_str = if (bare_return_type.zigTypeTag() == .Opaque) "opaque " else ""; + if (!return_type.isValidReturnType()) { + const opaque_str = if (return_type.zigTypeTag() == .Opaque) "opaque " else ""; const msg = msg: { const msg = try sema.errMsg(block, ret_ty_src, "{s}return type '{}' not allowed", .{ - opaque_str, bare_return_type.fmt(sema.mod), + opaque_str, return_type.fmt(sema.mod), }); errdefer msg.destroy(sema.gpa); - try sema.addDeclaredHereNote(msg, bare_return_type); + try sema.addDeclaredHereNote(msg, return_type); break :msg msg; }; return sema.failWithOwnedErrorMsg(msg); diff --git a/test/cases/compile_errors/function_returning_opaque_type.zig b/test/cases/compile_errors/function_returning_opaque_type.zig index caf5d74d40..26d70c86c1 100644 --- a/test/cases/compile_errors/function_returning_opaque_type.zig +++ b/test/cases/compile_errors/function_returning_opaque_type.zig @@ -1,11 +1,11 @@ const FooType = opaque {}; -export fn bar() !FooType { +export fn bar() FooType { return error.InvalidValue; } -export fn bav() !@TypeOf(null) { +export fn bav() @TypeOf(null) { return error.InvalidValue; } -export fn baz() !@TypeOf(undefined) { +export fn baz() @TypeOf(undefined) { return error.InvalidValue; } @@ -13,7 +13,7 @@ export fn baz() !@TypeOf(undefined) { // backend=stage2 // target=native // -// :2:18: error: opaque return type 'tmp.FooType' not allowed +// :2:17: error: opaque return type 'tmp.FooType' not allowed // :1:17: note: opaque declared here -// :5:18: error: return type '@TypeOf(null)' not allowed -// :8:18: error: return type '@TypeOf(undefined)' not allowed +// :5:17: error: return type '@TypeOf(null)' not allowed +// :8:17: error: return type '@TypeOf(undefined)' not allowed diff --git a/test/cases/compile_errors/invalid_error_union_payload_type.zig b/test/cases/compile_errors/invalid_error_union_payload_type.zig index f8646d9450..244eb765ec 100644 --- a/test/cases/compile_errors/invalid_error_union_payload_type.zig +++ b/test/cases/compile_errors/invalid_error_union_payload_type.zig @@ -4,6 +4,12 @@ comptime { comptime { _ = anyerror!anyerror; } +fn someFunction() !anyerror { + return error.C; +} +comptime { + _ = someFunction; +} // error // backend=stage2 @@ -11,3 +17,4 @@ comptime { // // :2:18: error: error union with payload of opaque type 'anyopaque' not allowed // :5:18: error: error union with payload of error set type 'anyerror' not allowed +// :7:20: error: error union with payload of error set type 'anyerror' not allowed -- cgit v1.2.3 From 31ed8d293ddb79d34306e0b23e76d8a361c8856f Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Thu, 2 Feb 2023 14:39:13 +0200 Subject: Sema: add missing peer type resolution for error unions Closes #14077 --- src/Sema.zig | 19 +++++++++++++++++++ test/behavior/cast.zig | 12 ++++++++++++ 2 files changed, 31 insertions(+) (limited to 'src') diff --git a/src/Sema.zig b/src/Sema.zig index 2dba678931..ddcdecb6b4 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -29761,6 +29761,25 @@ fn resolvePeerTypes( continue; } }, + .ErrorSet => { + chosen = candidate; + chosen_i = candidate_i + 1; + if (err_set_ty) |chosen_set_ty| { + if (.ok == try sema.coerceInMemoryAllowedErrorSets(block, chosen_set_ty, chosen_ty, src, src)) { + continue; + } + if (.ok == try sema.coerceInMemoryAllowedErrorSets(block, chosen_ty, chosen_set_ty, src, src)) { + err_set_ty = chosen_ty; + continue; + } + + err_set_ty = try chosen_set_ty.errorSetMerge(sema.arena, chosen_ty); + continue; + } else { + err_set_ty = chosen_ty; + continue; + } + }, else => {}, } diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig index dbb4c07f64..30889aef04 100644 --- a/test/behavior/cast.zig +++ b/test/behavior/cast.zig @@ -1541,3 +1541,15 @@ test "single item pointer to pointer to array to slice" { const z1 = @as([]const i32, @as(*[1]i32, &x)); try expect(z1[0] == 1234); } + +test "peer type resolution forms error union" { + if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO + + var foo: i32 = 123; + const result = if (foo < 0) switch (-foo) { + 0 => unreachable, + 42 => error.AccessDenied, + else => unreachable, + } else @intCast(u32, foo); + try expect(try result == 123); +} -- cgit v1.2.3 From 8127a27eb11cec26203a8fa15afdee046d3ff507 Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Tue, 7 Feb 2023 15:20:11 +0200 Subject: zig fmt: do not consider tuples blocks Closes #14056 --- lib/std/zig/render.zig | 8 -------- src/AstGen.zig | 3 ++- src/codegen/llvm.zig | 4 +++- 3 files changed, 5 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 1145007ff3..97bc85efac 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -2807,14 +2807,6 @@ fn nodeIsBlock(tag: Ast.Node.Tag) bool { .block_semicolon, .block_two, .block_two_semicolon, - .struct_init_dot, - .struct_init_dot_comma, - .struct_init_dot_two, - .struct_init_dot_two_comma, - .array_init_dot, - .array_init_dot_comma, - .array_init_dot_two, - .array_init_dot_two_comma, => true, else => false, }; diff --git a/src/AstGen.zig b/src/AstGen.zig index 10673a2b37..76a681abd3 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -1730,7 +1730,8 @@ fn structInitExprRlNone( .container_type = ty_inst, .name_start = str_index, }) } } - else .{ .rl = .none }; + else + .{ .rl = .none }; setExtra(astgen, extra_index, Zir.Inst.StructInitAnon.Item{ .field_name = str_index, .init = try expr(gz, scope, sub_ri, field_init), diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index e19c70f322..9c042120ad 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -2273,7 +2273,9 @@ pub const Object = struct { const full_di_fields: [2]*llvm.DIType = if (layout.tag_align >= layout.payload_align) - .{ tag_di, payload_di } else .{ payload_di, tag_di }; + .{ tag_di, payload_di } + else + .{ payload_di, tag_di }; const full_di_ty = dib.createStructType( compile_unit_scope, -- cgit v1.2.3 From 0328c9cbebd1ff907e9936c0986a9664c4509527 Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Fri, 10 Feb 2023 15:43:28 +0200 Subject: llvm: fix lowerDeclRefValue for extern function aliases Same as 0577069af5f5deb859762725736537d60c324453 for extern functions. Closes #14610 --- src/codegen/llvm.zig | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 9c042120ad..8c6943bfdb 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -4164,6 +4164,10 @@ pub const DeclGen = struct { if (func.data.owner_decl != decl_index) { return self.lowerDeclRefValue(tv, func.data.owner_decl); } + } else if (decl.val.castTag(.extern_fn)) |func| { + if (func.data.owner_decl != decl_index) { + return self.lowerDeclRefValue(tv, func.data.owner_decl); + } } const is_fn_body = decl.ty.zigTypeTag() == .Fn; -- cgit v1.2.3 From 28413ffcbad7817c57fd62d5b9908eabdd43147d Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Sat, 11 Feb 2023 13:45:07 +0200 Subject: Sema: fix typo in `zirCUndef` Closes #14617 --- src/Sema.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/Sema.zig b/src/Sema.zig index ddcdecb6b4..9eef07648d 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -21935,7 +21935,7 @@ fn zirCUndef( const src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node }; const name = try sema.resolveConstString(block, src, extra.operand, "name of macro being undefined must be comptime-known"); - try block.c_import_buf.?.writer().print("#undefine {s}\n", .{name}); + try block.c_import_buf.?.writer().print("#undef {s}\n", .{name}); return Air.Inst.Ref.void_value; } -- cgit v1.2.3 From a8edd67d3d87b585ebbea226bb916ee1f7263458 Mon Sep 17 00:00:00 2001 From: Ali Chraghi Date: Tue, 7 Feb 2023 16:34:03 +0330 Subject: zig fmt: additionally format .zon files --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index f634c259ff..00a7b126c8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4523,7 +4523,7 @@ fn fmtPathDir( if (is_dir and (mem.eql(u8, entry.name, "zig-cache") or mem.eql(u8, entry.name, "zig-out"))) continue; - if (is_dir or entry.kind == .File and mem.endsWith(u8, entry.name, ".zig")) { + if (is_dir or entry.kind == .File and (mem.endsWith(u8, entry.name, ".zig") or mem.endsWith(u8, entry.name, ".zon"))) { const full_path = try fs.path.join(fmt.gpa, &[_][]const u8{ file_path, entry.name }); defer fmt.gpa.free(full_path); -- cgit v1.2.3 From 9cb52ca6ce7043ba0ce08d5650ac542075f10685 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 19:39:04 -0700 Subject: move the cache system from compiler to std lib --- CMakeLists.txt | 5 +- lib/std/Build.zig | 2 + lib/std/Build/Cache.zig | 1276 ++++++++++++++++++++++++++++++++++ lib/std/Build/Cache/DepTokenizer.zig | 1069 ++++++++++++++++++++++++++++ src/Cache.zig | 1265 --------------------------------- src/Compilation.zig | 92 ++- src/DepTokenizer.zig | 1069 ---------------------------- src/Module.zig | 2 +- src/Package.zig | 2 +- src/glibc.zig | 2 +- src/link.zig | 2 +- src/link/Coff/lld.zig | 2 +- src/link/Elf.zig | 2 +- src/link/MachO.zig | 2 +- src/link/MachO/zld.zig | 2 +- src/link/Wasm.zig | 2 +- src/main.zig | 4 +- src/mingw.zig | 2 +- 18 files changed, 2404 insertions(+), 2398 deletions(-) create mode 100644 lib/std/Build/Cache.zig create mode 100644 lib/std/Build/Cache/DepTokenizer.zig delete mode 100644 src/Cache.zig delete mode 100644 src/DepTokenizer.zig (limited to 'src') diff --git a/CMakeLists.txt b/CMakeLists.txt index f8029fdcde..448fb400b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig" "${CMAKE_SOURCE_DIR}/lib/std/base64.zig" "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig" "${CMAKE_SOURCE_DIR}/lib/std/builtin.zig" "${CMAKE_SOURCE_DIR}/lib/std/c.zig" "${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig" @@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig" "${CMAKE_SOURCE_DIR}/src/Air.zig" "${CMAKE_SOURCE_DIR}/src/AstGen.zig" - "${CMAKE_SOURCE_DIR}/src/Cache.zig" "${CMAKE_SOURCE_DIR}/src/Compilation.zig" - "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig" "${CMAKE_SOURCE_DIR}/src/Liveness.zig" "${CMAKE_SOURCE_DIR}/src/Module.zig" "${CMAKE_SOURCE_DIR}/src/Package.zig" diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a67ded03e8..428a11948d 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo; const Sha256 = std.crypto.hash.sha2.Sha256; const Build = @This(); +pub const Cache = @import("Build/Cache.zig"); + /// deprecated: use `CompileStep`. pub const LibExeObjStep = CompileStep; /// deprecated: use `Build`. diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig new file mode 100644 index 0000000000..e316322c68 --- /dev/null +++ b/lib/std/Build/Cache.zig @@ -0,0 +1,1276 @@ +//! Manages `zig-cache` directories. +//! This is not a general-purpose cache. It is designed to be fast and simple, +//! not to withstand attacks using specially-crafted input. + +pub const Directory = struct { + /// This field is redundant for operations that can act on the open directory handle + /// directly, but it is needed when passing the directory to a child process. + /// `null` means cwd. + path: ?[]const u8, + handle: std.fs.Dir, + + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { + if (self.path) |p| { + // TODO clean way to do this with only 1 allocation + const part2 = try std.fs.path.join(allocator, paths); + defer allocator.free(part2); + return std.fs.path.join(allocator, &[_][]const u8{ p, part2 }); + } else { + return std.fs.path.join(allocator, paths); + } + } + + pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 { + if (self.path) |p| { + // TODO clean way to do this with only 1 allocation + const part2 = try std.fs.path.join(allocator, paths); + defer allocator.free(part2); + return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 }); + } else { + return std.fs.path.joinZ(allocator, paths); + } + } + + /// Whether or not the handle should be closed, or the path should be freed + /// is determined by usage, however this function is provided for convenience + /// if it happens to be what the caller needs. + pub fn closeAndFree(self: *Directory, gpa: Allocator) void { + self.handle.close(); + if (self.path) |p| gpa.free(p); + self.* = undefined; + } +}; + +gpa: Allocator, +manifest_dir: fs.Dir, +hash: HashHelper = .{}, +/// This value is accessed from multiple threads, protected by mutex. +recent_problematic_timestamp: i128 = 0, +mutex: std.Thread.Mutex = .{}, + +/// A set of strings such as the zig library directory or project source root, which +/// are stripped from the file paths before putting into the cache. They +/// are replaced with single-character indicators. This is not to save +/// space but to eliminate absolute file paths. This improves portability +/// and usefulness of the cache for advanced use cases. +prefixes_buffer: [3]Directory = undefined, +prefixes_len: usize = 0, + +pub const DepTokenizer = @import("Cache/DepTokenizer.zig"); + +const Cache = @This(); +const std = @import("std"); +const builtin = @import("builtin"); +const crypto = std.crypto; +const fs = std.fs; +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const fmt = std.fmt; +const Allocator = std.mem.Allocator; +const log = std.log.scoped(.cache); + +pub fn addPrefix(cache: *Cache, directory: Directory) void { + if (directory.path) |p| { + log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p }); + } + cache.prefixes_buffer[cache.prefixes_len] = directory; + cache.prefixes_len += 1; +} + +/// Be sure to call `Manifest.deinit` after successful initialization. +pub fn obtain(cache: *Cache) Manifest { + return Manifest{ + .cache = cache, + .hash = cache.hash, + .manifest_file = null, + .manifest_dirty = false, + .hex_digest = undefined, + }; +} + +pub fn prefixes(cache: *const Cache) []const Directory { + return cache.prefixes_buffer[0..cache.prefixes_len]; +} + +const PrefixedPath = struct { + prefix: u8, + sub_path: []u8, +}; + +fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { + const gpa = cache.gpa; + const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path}); + errdefer gpa.free(resolved_path); + return findPrefixResolved(cache, resolved_path); +} + +/// Takes ownership of `resolved_path` on success. +fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { + const gpa = cache.gpa; + const prefixes_slice = cache.prefixes(); + var i: u8 = 1; // Start at 1 to skip over checking the null prefix. + while (i < prefixes_slice.len) : (i += 1) { + const p = prefixes_slice[i].path.?; + if (mem.startsWith(u8, resolved_path, p)) { + // +1 to skip over the path separator here + const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]); + gpa.free(resolved_path); + return PrefixedPath{ + .prefix = @intCast(u8, i), + .sub_path = sub_path, + }; + } else { + log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p }); + } + } + + return PrefixedPath{ + .prefix = 0, + .sub_path = resolved_path, + }; +} + +/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 +pub const bin_digest_len = 16; +pub const hex_digest_len = bin_digest_len * 2; +pub const BinDigest = [bin_digest_len]u8; + +const manifest_file_size_max = 50 * 1024 * 1024; + +/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it +/// provides enough collision resistance for the Manifest use cases, while being one of our +/// fastest options right now. +pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); + +/// Initial state, that can be copied. +pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length); + +pub const File = struct { + prefixed_path: ?PrefixedPath, + max_file_size: ?usize, + stat: Stat, + bin_digest: BinDigest, + contents: ?[]const u8, + + pub const Stat = struct { + inode: fs.File.INode, + size: u64, + mtime: i128, + }; + + pub fn deinit(self: *File, gpa: Allocator) void { + if (self.prefixed_path) |pp| { + gpa.free(pp.sub_path); + self.prefixed_path = null; + } + if (self.contents) |contents| { + gpa.free(contents); + self.contents = null; + } + self.* = undefined; + } +}; + +pub const HashHelper = struct { + hasher: Hasher = hasher_init, + + /// Record a slice of bytes as an dependency of the process being cached + pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { + hh.hasher.update(mem.asBytes(&bytes.len)); + hh.hasher.update(bytes); + } + + pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { + hh.add(optional_bytes != null); + hh.addBytes(optional_bytes orelse return); + } + + pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { + hh.add(list_of_bytes.len); + for (list_of_bytes) |bytes| hh.addBytes(bytes); + } + + /// Convert the input value into bytes and record it as a dependency of the process being cached. + pub fn add(hh: *HashHelper, x: anytype) void { + switch (@TypeOf(x)) { + std.builtin.Version => { + hh.add(x.major); + hh.add(x.minor); + hh.add(x.patch); + }, + std.Target.Os.TaggedVersionRange => { + switch (x) { + .linux => |linux| { + hh.add(linux.range.min); + hh.add(linux.range.max); + hh.add(linux.glibc); + }, + .windows => |windows| { + hh.add(windows.min); + hh.add(windows.max); + }, + .semver => |semver| { + hh.add(semver.min); + hh.add(semver.max); + }, + .none => {}, + } + }, + else => switch (@typeInfo(@TypeOf(x))) { + .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), + else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), + }, + } + } + + pub fn addOptional(hh: *HashHelper, optional: anytype) void { + hh.add(optional != null); + hh.add(optional orelse return); + } + + /// Returns a hex encoded hash of the inputs, without modifying state. + pub fn peek(hh: HashHelper) [hex_digest_len]u8 { + var copy = hh; + return copy.final(); + } + + pub fn peekBin(hh: HashHelper) BinDigest { + var copy = hh; + var bin_digest: BinDigest = undefined; + copy.hasher.final(&bin_digest); + return bin_digest; + } + + /// Returns a hex encoded hash of the inputs, mutating the state of the hasher. + pub fn final(hh: *HashHelper) [hex_digest_len]u8 { + var bin_digest: BinDigest = undefined; + hh.hasher.final(&bin_digest); + + var out_digest: [hex_digest_len]u8 = undefined; + _ = std.fmt.bufPrint( + &out_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; + return out_digest; + } +}; + +pub const Lock = struct { + manifest_file: fs.File, + + pub fn release(lock: *Lock) void { + if (builtin.os.tag == .windows) { + // Windows does not guarantee that locks are immediately unlocked when + // the file handle is closed. See LockFileEx documentation. + lock.manifest_file.unlock(); + } + + lock.manifest_file.close(); + lock.* = undefined; + } +}; + +pub const Manifest = struct { + cache: *Cache, + /// Current state for incremental hashing. + hash: HashHelper, + manifest_file: ?fs.File, + manifest_dirty: bool, + /// Set this flag to true before calling hit() in order to indicate that + /// upon a cache hit, the code using the cache will not modify the files + /// within the cache directory. This allows multiple processes to utilize + /// the same cache directory at the same time. + want_shared_lock: bool = true, + have_exclusive_lock: bool = false, + // Indicate that we want isProblematicTimestamp to perform a filesystem write in + // order to obtain a problematic timestamp for the next call. Calls after that + // will then use the same timestamp, to avoid unnecessary filesystem writes. + want_refresh_timestamp: bool = true, + files: std.ArrayListUnmanaged(File) = .{}, + hex_digest: [hex_digest_len]u8, + /// Populated when hit() returns an error because of one + /// of the files listed in the manifest. + failed_file_index: ?usize = null, + /// Keeps track of the last time we performed a file system write to observe + /// what time the file system thinks it is, according to its own granularity. + recent_problematic_timestamp: i128 = 0, + + /// Add a file as a dependency of process being cached. When `hit` is + /// called, the file's contents will be checked to ensure that it matches + /// the contents from previous times. + /// + /// Max file size will be used to determine the amount of space the file contents + /// are allowed to take up in memory. If max_file_size is null, then the contents + /// will not be loaded into memory. + /// + /// Returns the index of the entry in the `files` array list. You can use it + /// to access the contents of the file after calling `hit()` like so: + /// + /// ``` + /// var file_contents = cache_hash.files.items[file_index].contents.?; + /// ``` + pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { + assert(self.manifest_file == null); + + const gpa = self.cache.gpa; + try self.files.ensureUnusedCapacity(gpa, 1); + const prefixed_path = try self.cache.findPrefix(file_path); + errdefer gpa.free(prefixed_path.sub_path); + + log.debug("Manifest.addFile {s} -> {d} {s}", .{ + file_path, prefixed_path.prefix, prefixed_path.sub_path, + }); + + self.files.addOneAssumeCapacity().* = .{ + .prefixed_path = prefixed_path, + .contents = null, + .max_file_size = max_file_size, + .stat = undefined, + .bin_digest = undefined, + }; + + self.hash.add(prefixed_path.prefix); + self.hash.addBytes(prefixed_path.sub_path); + + return self.files.items.len - 1; + } + + pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { + self.hash.add(optional_file_path != null); + const file_path = optional_file_path orelse return; + _ = try self.addFile(file_path, null); + } + + pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void { + self.hash.add(list_of_files.len); + for (list_of_files) |file_path| { + _ = try self.addFile(file_path, null); + } + } + + /// Check the cache to see if the input exists in it. If it exists, returns `true`. + /// A hex encoding of its hash is available by calling `final`. + /// + /// This function will also acquire an exclusive lock to the manifest file. This means + /// that a process holding a Manifest will block any other process attempting to + /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the + /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest + /// file to be locked in exclusive mode. + /// + /// The lock on the manifest file is released when `deinit` is called. As another + /// option, one may call `toOwnedLock` to obtain a smaller object which can represent + /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. + pub fn hit(self: *Manifest) !bool { + const gpa = self.cache.gpa; + assert(self.manifest_file == null); + + self.failed_file_index = null; + + const ext = ".txt"; + var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined; + + var bin_digest: BinDigest = undefined; + self.hash.hasher.final(&bin_digest); + + _ = std.fmt.bufPrint( + &self.hex_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; + + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); + + mem.copy(u8, &manifest_file_path, &self.hex_digest); + manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; + + if (self.files.items.len == 0) { + // If there are no file inputs, we check if the manifest file exists instead of + // comparing the hashes on the files used for the cached item + while (true) { + if (self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .mode = .read_write, + .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.have_exclusive_lock = true; + break; + } else |open_err| switch (open_err) { + error.WouldBlock => { + self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .lock = .Shared, + }); + break; + }, + error.FileNotFound => { + if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ + .read = true, + .truncate = false, + .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.manifest_dirty = true; + self.have_exclusive_lock = true; + return false; // cache miss; exclusive lock already held + } else |err| switch (err) { + error.WouldBlock => continue, + else => |e| return e, + } + }, + else => |e| return e, + } + } + } else { + if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ + .read = true, + .truncate = false, + .lock = .Exclusive, + .lock_nonblocking = self.want_shared_lock, + })) |manifest_file| { + self.manifest_file = manifest_file; + self.have_exclusive_lock = true; + } else |err| switch (err) { + error.WouldBlock => { + self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .lock = .Shared, + }); + }, + else => |e| return e, + } + } + + self.want_refresh_timestamp = true; + + const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max); + defer gpa.free(file_contents); + + const input_file_count = self.files.items.len; + var any_file_changed = false; + var line_iter = mem.tokenize(u8, file_contents, "\n"); + var idx: usize = 0; + while (line_iter.next()) |line| { + defer idx += 1; + + const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { + const new = try self.files.addOne(gpa); + new.* = .{ + .prefixed_path = null, + .contents = null, + .max_file_size = null, + .stat = undefined, + .bin_digest = undefined, + }; + break :blk new; + }; + + var iter = mem.tokenize(u8, line, " "); + const size = iter.next() orelse return error.InvalidFormat; + const inode = iter.next() orelse return error.InvalidFormat; + const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; + const digest_str = iter.next() orelse return error.InvalidFormat; + const prefix_str = iter.next() orelse return error.InvalidFormat; + const file_path = iter.rest(); + + cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; + cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; + cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; + _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; + const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat; + if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; + + if (file_path.len == 0) { + return error.InvalidFormat; + } + if (cache_hash_file.prefixed_path) |pp| { + if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) { + return error.InvalidFormat; + } + } + + if (cache_hash_file.prefixed_path == null) { + cache_hash_file.prefixed_path = .{ + .prefix = prefix, + .sub_path = try gpa.dupe(u8, file_path), + }; + } + + const pp = cache_hash_file.prefixed_path.?; + const dir = self.cache.prefixes()[pp.prefix].handle; + const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => { + try self.upgradeToExclusiveLock(); + return false; + }, + else => return error.CacheUnavailable, + }; + defer this_file.close(); + + const actual_stat = this_file.stat() catch |err| { + self.failed_file_index = idx; + return err; + }; + const size_match = actual_stat.size == cache_hash_file.stat.size; + const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime; + const inode_match = actual_stat.inode == cache_hash_file.stat.inode; + + if (!size_match or !mtime_match or !inode_match) { + self.manifest_dirty = true; + + cache_hash_file.stat = .{ + .size = actual_stat.size, + .mtime = actual_stat.mtime, + .inode = actual_stat.inode, + }; + + if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) { + // The actual file has an unreliable timestamp, force it to be hashed + cache_hash_file.stat.mtime = 0; + cache_hash_file.stat.inode = 0; + } + + var actual_digest: BinDigest = undefined; + hashFile(this_file, &actual_digest) catch |err| { + self.failed_file_index = idx; + return err; + }; + + if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { + cache_hash_file.bin_digest = actual_digest; + // keep going until we have the input file digests + any_file_changed = true; + } + } + + if (!any_file_changed) { + self.hash.hasher.update(&cache_hash_file.bin_digest); + } + } + + if (any_file_changed) { + // cache miss + // keep the manifest file open + self.unhit(bin_digest, input_file_count); + try self.upgradeToExclusiveLock(); + return false; + } + + if (idx < input_file_count) { + self.manifest_dirty = true; + while (idx < input_file_count) : (idx += 1) { + const ch_file = &self.files.items[idx]; + self.populateFileHash(ch_file) catch |err| { + self.failed_file_index = idx; + return err; + }; + } + try self.upgradeToExclusiveLock(); + return false; + } + + if (self.want_shared_lock) { + try self.downgradeToSharedLock(); + } + + return true; + } + + pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void { + // Reset the hash. + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); + + // Remove files not in the initial hash. + for (self.files.items[input_file_count..]) |*file| { + file.deinit(self.cache.gpa); + } + self.files.shrinkRetainingCapacity(input_file_count); + + for (self.files.items) |file| { + self.hash.hasher.update(&file.bin_digest); + } + } + + fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool { + // If the file_time is prior to the most recent problematic timestamp + // then we don't need to access the filesystem. + if (file_time < man.recent_problematic_timestamp) + return false; + + // Next we will check the globally shared Cache timestamp, which is accessed + // from multiple threads. + man.cache.mutex.lock(); + defer man.cache.mutex.unlock(); + + // Save the global one to our local one to avoid locking next time. + man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp; + if (file_time < man.recent_problematic_timestamp) + return false; + + // This flag prevents multiple filesystem writes for the same hit() call. + if (man.want_refresh_timestamp) { + man.want_refresh_timestamp = false; + + var file = man.cache.manifest_dir.createFile("timestamp", .{ + .read = true, + .truncate = true, + }) catch return true; + defer file.close(); + + // Save locally and also save globally (we still hold the global lock). + man.recent_problematic_timestamp = (file.stat() catch return true).mtime; + man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp; + } + + return file_time >= man.recent_problematic_timestamp; + } + + fn populateFileHash(self: *Manifest, ch_file: *File) !void { + const pp = ch_file.prefixed_path.?; + const dir = self.cache.prefixes()[pp.prefix].handle; + const file = try dir.openFile(pp.sub_path, .{}); + defer file.close(); + + const actual_stat = try file.stat(); + ch_file.stat = .{ + .size = actual_stat.size, + .mtime = actual_stat.mtime, + .inode = actual_stat.inode, + }; + + if (self.isProblematicTimestamp(ch_file.stat.mtime)) { + // The actual file has an unreliable timestamp, force it to be hashed + ch_file.stat.mtime = 0; + ch_file.stat.inode = 0; + } + + if (ch_file.max_file_size) |max_file_size| { + if (ch_file.stat.size > max_file_size) { + return error.FileTooBig; + } + + const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size)); + errdefer self.cache.gpa.free(contents); + + // Hash while reading from disk, to keep the contents in the cpu cache while + // doing hashing. + var hasher = hasher_init; + var off: usize = 0; + while (true) { + // give me everything you've got, captain + const bytes_read = try file.read(contents[off..]); + if (bytes_read == 0) break; + hasher.update(contents[off..][0..bytes_read]); + off += bytes_read; + } + hasher.final(&ch_file.bin_digest); + + ch_file.contents = contents; + } else { + try hashFile(file, &ch_file.bin_digest); + } + + self.hash.hasher.update(&ch_file.bin_digest); + } + + /// Add a file as a dependency of process being cached, after the initial hash has been + /// calculated. This is useful for processes that don't know all the files that + /// are depended on ahead of time. For example, a source file that can import other files + /// will need to be recompiled if the imported file is changed. + pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 { + assert(self.manifest_file != null); + + const gpa = self.cache.gpa; + const prefixed_path = try self.cache.findPrefix(file_path); + errdefer gpa.free(prefixed_path.sub_path); + + log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{ + file_path, prefixed_path.prefix, prefixed_path.sub_path, + }); + + const new_ch_file = try self.files.addOne(gpa); + new_ch_file.* = .{ + .prefixed_path = prefixed_path, + .max_file_size = max_file_size, + .stat = undefined, + .bin_digest = undefined, + .contents = null, + }; + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); + + try self.populateFileHash(new_ch_file); + + return new_ch_file.contents.?; + } + + /// Add a file as a dependency of process being cached, after the initial hash has been + /// calculated. This is useful for processes that don't know the all the files that + /// are depended on ahead of time. For example, a source file that can import other files + /// will need to be recompiled if the imported file is changed. + pub fn addFilePost(self: *Manifest, file_path: []const u8) !void { + assert(self.manifest_file != null); + + const gpa = self.cache.gpa; + const prefixed_path = try self.cache.findPrefix(file_path); + errdefer gpa.free(prefixed_path.sub_path); + + log.debug("Manifest.addFilePost {s} -> {d} {s}", .{ + file_path, prefixed_path.prefix, prefixed_path.sub_path, + }); + + const new_ch_file = try self.files.addOne(gpa); + new_ch_file.* = .{ + .prefixed_path = prefixed_path, + .max_file_size = null, + .stat = undefined, + .bin_digest = undefined, + .contents = null, + }; + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); + + try self.populateFileHash(new_ch_file); + } + + /// Like `addFilePost` but when the file contents have already been loaded from disk. + /// On success, cache takes ownership of `resolved_path`. + pub fn addFilePostContents( + self: *Manifest, + resolved_path: []u8, + bytes: []const u8, + stat: File.Stat, + ) error{OutOfMemory}!void { + assert(self.manifest_file != null); + const gpa = self.cache.gpa; + + const ch_file = try self.files.addOne(gpa); + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); + + log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path}); + + const prefixed_path = try self.cache.findPrefixResolved(resolved_path); + errdefer gpa.free(prefixed_path.sub_path); + + log.debug("Manifest.addFilePostContents -> {d} {s}", .{ + prefixed_path.prefix, prefixed_path.sub_path, + }); + + ch_file.* = .{ + .prefixed_path = prefixed_path, + .max_file_size = null, + .stat = stat, + .bin_digest = undefined, + .contents = null, + }; + + if (self.isProblematicTimestamp(ch_file.stat.mtime)) { + // The actual file has an unreliable timestamp, force it to be hashed + ch_file.stat.mtime = 0; + ch_file.stat.inode = 0; + } + + { + var hasher = hasher_init; + hasher.update(bytes); + hasher.final(&ch_file.bin_digest); + } + + self.hash.hasher.update(&ch_file.bin_digest); + } + + pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void { + assert(self.manifest_file != null); + + const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max); + defer self.cache.gpa.free(dep_file_contents); + + var error_buf = std.ArrayList(u8).init(self.cache.gpa); + defer error_buf.deinit(); + + var it: DepTokenizer = .{ .bytes = dep_file_contents }; + + // Skip first token: target. + switch (it.next() orelse return) { // Empty dep file OK. + .target, .target_must_resolve, .prereq => {}, + else => |err| { + try err.printError(error_buf.writer()); + log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items }); + return error.InvalidDepFile; + }, + } + // Process 0+ preqreqs. + // Clang is invoked in single-source mode so we never get more targets. + while (true) { + switch (it.next() orelse return) { + .target, .target_must_resolve => return, + .prereq => |file_path| try self.addFilePost(file_path), + else => |err| { + try err.printError(error_buf.writer()); + log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items }); + return error.InvalidDepFile; + }, + } + } + } + + /// Returns a hex encoded hash of the inputs. + pub fn final(self: *Manifest) [hex_digest_len]u8 { + assert(self.manifest_file != null); + + // We don't close the manifest file yet, because we want to + // keep it locked until the API user is done using it. + // We also don't write out the manifest yet, because until + // cache_release is called we still might be working on creating + // the artifacts to cache. + + var bin_digest: BinDigest = undefined; + self.hash.hasher.final(&bin_digest); + + var out_digest: [hex_digest_len]u8 = undefined; + _ = std.fmt.bufPrint( + &out_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; + + return out_digest; + } + + /// If `want_shared_lock` is true, this function automatically downgrades the + /// lock from exclusive to shared. + pub fn writeManifest(self: *Manifest) !void { + assert(self.have_exclusive_lock); + + const manifest_file = self.manifest_file.?; + if (self.manifest_dirty) { + self.manifest_dirty = false; + + var contents = std.ArrayList(u8).init(self.cache.gpa); + defer contents.deinit(); + + const writer = contents.writer(); + var encoded_digest: [hex_digest_len]u8 = undefined; + + for (self.files.items) |file| { + _ = std.fmt.bufPrint( + &encoded_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, + ) catch unreachable; + try writer.print("{d} {d} {d} {s} {d} {s}\n", .{ + file.stat.size, + file.stat.inode, + file.stat.mtime, + &encoded_digest, + file.prefixed_path.?.prefix, + file.prefixed_path.?.sub_path, + }); + } + + try manifest_file.setEndPos(contents.items.len); + try manifest_file.pwriteAll(contents.items, 0); + } + + if (self.want_shared_lock) { + try self.downgradeToSharedLock(); + } + } + + fn downgradeToSharedLock(self: *Manifest) !void { + if (!self.have_exclusive_lock) return; + + // WASI does not currently support flock, so we bypass it here. + // TODO: If/when flock is supported on WASI, this check should be removed. + // See https://github.com/WebAssembly/wasi-filesystem/issues/2 + if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) { + const manifest_file = self.manifest_file.?; + try manifest_file.downgradeLock(); + } + + self.have_exclusive_lock = false; + } + + fn upgradeToExclusiveLock(self: *Manifest) !void { + if (self.have_exclusive_lock) return; + assert(self.manifest_file != null); + + // WASI does not currently support flock, so we bypass it here. + // TODO: If/when flock is supported on WASI, this check should be removed. + // See https://github.com/WebAssembly/wasi-filesystem/issues/2 + if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) { + const manifest_file = self.manifest_file.?; + // Here we intentionally have a period where the lock is released, in case there are + // other processes holding a shared lock. + manifest_file.unlock(); + try manifest_file.lock(.Exclusive); + } + self.have_exclusive_lock = true; + } + + /// Obtain only the data needed to maintain a lock on the manifest file. + /// The `Manifest` remains safe to deinit. + /// Don't forget to call `writeManifest` before this! + pub fn toOwnedLock(self: *Manifest) Lock { + const lock: Lock = .{ + .manifest_file = self.manifest_file.?, + }; + + self.manifest_file = null; + return lock; + } + + /// Releases the manifest file and frees any memory the Manifest was using. + /// `Manifest.hit` must be called first. + /// Don't forget to call `writeManifest` before this! + pub fn deinit(self: *Manifest) void { + if (self.manifest_file) |file| { + if (builtin.os.tag == .windows) { + // See Lock.release for why this is required on Windows + file.unlock(); + } + + file.close(); + } + for (self.files.items) |*file| { + file.deinit(self.cache.gpa); + } + self.files.deinit(self.cache.gpa); + } +}; + +/// On operating systems that support symlinks, does a readlink. On other operating systems, +/// uses the file contents. Windows supports symlinks but only with elevated privileges, so +/// it is treated as not supporting symlinks. +pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 { + if (builtin.os.tag == .windows) { + return dir.readFile(sub_path, buffer); + } else { + return dir.readLink(sub_path, buffer); + } +} + +/// On operating systems that support symlinks, does a symlink. On other operating systems, +/// uses the file contents. Windows supports symlinks but only with elevated privileges, so +/// it is treated as not supporting symlinks. +/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer. +pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void { + assert(data.len <= 255); + if (builtin.os.tag == .windows) { + return dir.writeFile(sub_path, data); + } else { + return dir.symLink(data, sub_path, .{}); + } +} + +fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void { + var buf: [1024]u8 = undefined; + + var hasher = hasher_init; + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + hasher.update(buf[0..bytes_read]); + } + + hasher.final(bin_digest); +} + +// Create/Write a file, close it, then grab its stat.mtime timestamp. +fn testGetCurrentFileTimestamp() !i128 { + var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{ + .read = true, + .truncate = true, + }); + defer file.close(); + + return (try file.stat()).mtime; +} + +test "cache file and then recall it" { + if (builtin.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + + const cwd = fs.cwd(); + + const temp_file = "test.txt"; + const temp_manifest_dir = "temp_manifest_dir"; + + try cwd.writeFile(temp_file, "Hello, world!\n"); + + // Wait for file timestamps to tick + const initial_time = try testGetCurrentFileTimestamp(); + while ((try testGetCurrentFileTimestamp()) == initial_time) { + std.time.sleep(1); + } + + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; + + { + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); + + // There should be nothing in the cache + try testing.expectEqual(false, try ch.hit()); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); + + // Cache hit! We just "built" the same file + try testing.expect(try ch.hit()); + digest2 = ch.final(); + + try testing.expectEqual(false, ch.have_exclusive_lock); + } + + try testing.expectEqual(digest1, digest2); + } + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteFile(temp_file); +} + +test "check that changing a file makes cache fail" { + if (builtin.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + + const temp_file = "cache_hash_change_file_test.txt"; + const temp_manifest_dir = "cache_hash_change_file_manifest_dir"; + const original_temp_file_contents = "Hello, world!\n"; + const updated_temp_file_contents = "Hello, world; but updated!\n"; + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteTree(temp_file); + + try cwd.writeFile(temp_file, original_temp_file_contents); + + // Wait for file timestamps to tick + const initial_time = try testGetCurrentFileTimestamp(); + while ((try testGetCurrentFileTimestamp()) == initial_time) { + std.time.sleep(1); + } + + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; + + { + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); + + // There should be nothing in the cache + try testing.expectEqual(false, try ch.hit()); + + try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); + + digest1 = ch.final(); + + try ch.writeManifest(); + } + + try cwd.writeFile(temp_file, updated_temp_file_contents); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + try testing.expectEqual(false, try ch.hit()); + + // The cache system does not keep the contents of re-hashed input files. + try testing.expect(ch.files.items[temp_file_idx].contents == null); + + digest2 = ch.final(); + + try ch.writeManifest(); + } + + try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); + } + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteTree(temp_file); +} + +test "no file inputs" { + if (builtin.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + const temp_manifest_dir = "no_file_inputs_manifest_dir"; + defer cwd.deleteTree(temp_manifest_dir) catch {}; + + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; + + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); + defer cache.manifest_dir.close(); + + { + var man = cache.obtain(); + defer man.deinit(); + + man.hash.addBytes("1234"); + + // There should be nothing in the cache + try testing.expectEqual(false, try man.hit()); + + digest1 = man.final(); + + try man.writeManifest(); + } + { + var man = cache.obtain(); + defer man.deinit(); + + man.hash.addBytes("1234"); + + try testing.expect(try man.hit()); + digest2 = man.final(); + try testing.expectEqual(false, man.have_exclusive_lock); + } + + try testing.expectEqual(digest1, digest2); +} + +test "Manifest with files added after initial hash work" { + if (builtin.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + + const temp_file1 = "cache_hash_post_file_test1.txt"; + const temp_file2 = "cache_hash_post_file_test2.txt"; + const temp_manifest_dir = "cache_hash_post_file_manifest_dir"; + + try cwd.writeFile(temp_file1, "Hello, world!\n"); + try cwd.writeFile(temp_file2, "Hello world the second!\n"); + + // Wait for file timestamps to tick + const initial_time = try testGetCurrentFileTimestamp(); + while ((try testGetCurrentFileTimestamp()) == initial_time) { + std.time.sleep(1); + } + + var digest1: [hex_digest_len]u8 = undefined; + var digest2: [hex_digest_len]u8 = undefined; + var digest3: [hex_digest_len]u8 = undefined; + + { + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + // There should be nothing in the cache + try testing.expectEqual(false, try ch.hit()); + + _ = try ch.addFilePost(temp_file2); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + try testing.expect(try ch.hit()); + digest2 = ch.final(); + + try testing.expectEqual(false, ch.have_exclusive_lock); + } + try testing.expect(mem.eql(u8, &digest1, &digest2)); + + // Modify the file added after initial hash + try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); + + // Wait for file timestamps to tick + const initial_time2 = try testGetCurrentFileTimestamp(); + while ((try testGetCurrentFileTimestamp()) == initial_time2) { + std.time.sleep(1); + } + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + try testing.expectEqual(false, try ch.hit()); + + _ = try ch.addFilePost(temp_file2); + + digest3 = ch.final(); + + try ch.writeManifest(); + } + + try testing.expect(!mem.eql(u8, &digest1, &digest3)); + } + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteFile(temp_file1); + try cwd.deleteFile(temp_file2); +} diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig new file mode 100644 index 0000000000..8f9f2f81cd --- /dev/null +++ b/lib/std/Build/Cache/DepTokenizer.zig @@ -0,0 +1,1069 @@ +const Tokenizer = @This(); + +index: usize = 0, +bytes: []const u8, +state: State = .lhs, + +const std = @import("std"); +const testing = std.testing; +const assert = std.debug.assert; + +pub fn next(self: *Tokenizer) ?Token { + var start = self.index; + var must_resolve = false; + while (self.index < self.bytes.len) { + const char = self.bytes[self.index]; + switch (self.state) { + .lhs => switch (char) { + '\t', '\n', '\r', ' ' => { + // silently ignore whitespace + self.index += 1; + }, + else => { + start = self.index; + self.state = .target; + }, + }, + .target => switch (char) { + '\t', '\n', '\r', ' ' => { + return errorIllegalChar(.invalid_target, self.index, char); + }, + '$' => { + self.state = .target_dollar_sign; + self.index += 1; + }, + '\\' => { + self.state = .target_reverse_solidus; + self.index += 1; + }, + ':' => { + self.state = .target_colon; + self.index += 1; + }, + else => { + self.index += 1; + }, + }, + .target_reverse_solidus => switch (char) { + '\t', '\n', '\r' => { + return errorIllegalChar(.bad_target_escape, self.index, char); + }, + ' ', '#', '\\' => { + must_resolve = true; + self.state = .target; + self.index += 1; + }, + '$' => { + self.state = .target_dollar_sign; + self.index += 1; + }, + else => { + self.state = .target; + self.index += 1; + }, + }, + .target_dollar_sign => switch (char) { + '$' => { + must_resolve = true; + self.state = .target; + self.index += 1; + }, + else => { + return errorIllegalChar(.expected_dollar_sign, self.index, char); + }, + }, + .target_colon => switch (char) { + '\n', '\r' => { + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { + self.state = .lhs; + return finishTarget(must_resolve, bytes); + } + // silently ignore null target + self.state = .lhs; + }, + '/', '\\' => { + self.state = .target_colon_reverse_solidus; + self.index += 1; + }, + else => { + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { + self.state = .rhs; + return finishTarget(must_resolve, bytes); + } + // silently ignore null target + self.state = .lhs; + }, + }, + .target_colon_reverse_solidus => switch (char) { + '\n', '\r' => { + const bytes = self.bytes[start .. self.index - 2]; + if (bytes.len != 0) { + self.state = .lhs; + return finishTarget(must_resolve, bytes); + } + // silently ignore null target + self.state = .lhs; + }, + else => { + self.state = .target; + }, + }, + .rhs => switch (char) { + '\t', ' ' => { + // silently ignore horizontal whitespace + self.index += 1; + }, + '\n', '\r' => { + self.state = .lhs; + }, + '\\' => { + self.state = .rhs_continuation; + self.index += 1; + }, + '"' => { + self.state = .prereq_quote; + self.index += 1; + start = self.index; + }, + else => { + start = self.index; + self.state = .prereq; + }, + }, + .rhs_continuation => switch (char) { + '\n' => { + self.state = .rhs; + self.index += 1; + }, + '\r' => { + self.state = .rhs_continuation_linefeed; + self.index += 1; + }, + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, + .rhs_continuation_linefeed => switch (char) { + '\n' => { + self.state = .rhs; + self.index += 1; + }, + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, + .prereq_quote => switch (char) { + '"' => { + self.index += 1; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 1] }; + }, + else => { + self.index += 1; + }, + }, + .prereq => switch (char) { + '\t', ' ' => { + self.state = .rhs; + return Token{ .prereq = self.bytes[start..self.index] }; + }, + '\n', '\r' => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start..self.index] }; + }, + '\\' => { + self.state = .prereq_continuation; + self.index += 1; + }, + else => { + self.index += 1; + }, + }, + .prereq_continuation => switch (char) { + '\n' => { + self.index += 1; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 2] }; + }, + '\r' => { + self.state = .prereq_continuation_linefeed; + self.index += 1; + }, + else => { + // not continuation + self.state = .prereq; + self.index += 1; + }, + }, + .prereq_continuation_linefeed => switch (char) { + '\n' => { + self.index += 1; + self.state = .rhs; + return Token{ .prereq = self.bytes[start .. self.index - 1] }; + }, + else => { + return errorIllegalChar(.continuation_eol, self.index, char); + }, + }, + } + } else { + switch (self.state) { + .lhs, + .rhs, + .rhs_continuation, + .rhs_continuation_linefeed, + => return null, + .target => { + return errorPosition(.incomplete_target, start, self.bytes[start..]); + }, + .target_reverse_solidus, + .target_dollar_sign, + => { + const idx = self.index - 1; + return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]); + }, + .target_colon => { + const bytes = self.bytes[start .. self.index - 1]; + if (bytes.len != 0) { + self.index += 1; + self.state = .rhs; + return finishTarget(must_resolve, bytes); + } + // silently ignore null target + self.state = .lhs; + return null; + }, + .target_colon_reverse_solidus => { + const bytes = self.bytes[start .. self.index - 2]; + if (bytes.len != 0) { + self.index += 1; + self.state = .rhs; + return finishTarget(must_resolve, bytes); + } + // silently ignore null target + self.state = .lhs; + return null; + }, + .prereq_quote => { + return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]); + }, + .prereq => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start..] }; + }, + .prereq_continuation => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start .. self.index - 1] }; + }, + .prereq_continuation_linefeed => { + self.state = .lhs; + return Token{ .prereq = self.bytes[start .. self.index - 2] }; + }, + } + } + unreachable; +} + +fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token { + return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes }); +} + +fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token { + return @unionInit(Token, @tagName(id), .{ .index = index, .char = char }); +} + +fn finishTarget(must_resolve: bool, bytes: []const u8) Token { + return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes }; +} + +const State = enum { + lhs, + target, + target_reverse_solidus, + target_dollar_sign, + target_colon, + target_colon_reverse_solidus, + rhs, + rhs_continuation, + rhs_continuation_linefeed, + prereq_quote, + prereq, + prereq_continuation, + prereq_continuation_linefeed, +}; + +pub const Token = union(enum) { + target: []const u8, + target_must_resolve: []const u8, + prereq: []const u8, + + incomplete_quoted_prerequisite: IndexAndBytes, + incomplete_target: IndexAndBytes, + + invalid_target: IndexAndChar, + bad_target_escape: IndexAndChar, + expected_dollar_sign: IndexAndChar, + continuation_eol: IndexAndChar, + incomplete_escape: IndexAndChar, + + pub const IndexAndChar = struct { + index: usize, + char: u8, + }; + + pub const IndexAndBytes = struct { + index: usize, + bytes: []const u8, + }; + + /// Resolve escapes in target. Only valid with .target_must_resolve. + pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void { + const bytes = self.target_must_resolve; // resolve called on incorrect token + + var state: enum { start, escape, dollar } = .start; + for (bytes) |c| { + switch (state) { + .start => { + switch (c) { + '\\' => state = .escape, + '$' => state = .dollar, + else => try writer.writeByte(c), + } + }, + .escape => { + switch (c) { + ' ', '#', '\\' => {}, + '$' => { + try writer.writeByte('\\'); + state = .dollar; + continue; + }, + else => try writer.writeByte('\\'), + } + try writer.writeByte(c); + state = .start; + }, + .dollar => { + try writer.writeByte('$'); + switch (c) { + '$' => {}, + else => try writer.writeByte(c), + } + state = .start; + }, + } + } + } + + pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void { + switch (self) { + .target, .target_must_resolve, .prereq => unreachable, // not an error + .incomplete_quoted_prerequisite, + .incomplete_target, + => |index_and_bytes| { + try writer.print("{s} '", .{self.errStr()}); + if (self == .incomplete_target) { + const tmp = Token{ .target_must_resolve = index_and_bytes.bytes }; + try tmp.resolve(writer); + } else { + try printCharValues(writer, index_and_bytes.bytes); + } + try writer.print("' at position {d}", .{index_and_bytes.index}); + }, + .invalid_target, + .bad_target_escape, + .expected_dollar_sign, + .continuation_eol, + .incomplete_escape, + => |index_and_char| { + try writer.writeAll("illegal char "); + try printUnderstandableChar(writer, index_and_char.char); + try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() }); + }, + } + } + + fn errStr(self: Token) []const u8 { + return switch (self) { + .target, .target_must_resolve, .prereq => unreachable, // not an error + .incomplete_quoted_prerequisite => "incomplete quoted prerequisite", + .incomplete_target => "incomplete target", + .invalid_target => "invalid target", + .bad_target_escape => "bad target escape", + .expected_dollar_sign => "expecting '$'", + .continuation_eol => "continuation expecting end-of-line", + .incomplete_escape => "incomplete escape", + }; + } +}; + +test "empty file" { + try depTokenizer("", ""); +} + +test "empty whitespace" { + try depTokenizer("\n", ""); + try depTokenizer("\r", ""); + try depTokenizer("\r\n", ""); + try depTokenizer(" ", ""); +} + +test "empty colon" { + try depTokenizer(":", ""); + try depTokenizer("\n:", ""); + try depTokenizer("\r:", ""); + try depTokenizer("\r\n:", ""); + try depTokenizer(" :", ""); +} + +test "empty target" { + try depTokenizer("foo.o:", "target = {foo.o}"); + try depTokenizer( + \\foo.o: + \\bar.o: + \\abcd.o: + , + \\target = {foo.o} + \\target = {bar.o} + \\target = {abcd.o} + ); +} + +test "whitespace empty target" { + try depTokenizer("\nfoo.o:", "target = {foo.o}"); + try depTokenizer("\rfoo.o:", "target = {foo.o}"); + try depTokenizer("\r\nfoo.o:", "target = {foo.o}"); + try depTokenizer(" foo.o:", "target = {foo.o}"); +} + +test "escape empty target" { + try depTokenizer("\\ foo.o:", "target = { foo.o}"); + try depTokenizer("\\#foo.o:", "target = {#foo.o}"); + try depTokenizer("\\\\foo.o:", "target = {\\foo.o}"); + try depTokenizer("$$foo.o:", "target = {$foo.o}"); +} + +test "empty target linefeeds" { + try depTokenizer("\n", ""); + try depTokenizer("\r\n", ""); + + const expect = "target = {foo.o}"; + try depTokenizer( + \\foo.o: + , expect); + try depTokenizer( + \\foo.o: + \\ + , expect); + try depTokenizer( + \\foo.o: + , expect); + try depTokenizer( + \\foo.o: + \\ + , expect); +} + +test "empty target linefeeds + continuations" { + const expect = "target = {foo.o}"; + try depTokenizer( + \\foo.o:\ + , expect); + try depTokenizer( + \\foo.o:\ + \\ + , expect); + try depTokenizer( + \\foo.o:\ + , expect); + try depTokenizer( + \\foo.o:\ + \\ + , expect); +} + +test "empty target linefeeds + hspace + continuations" { + const expect = "target = {foo.o}"; + try depTokenizer( + \\foo.o: \ + , expect); + try depTokenizer( + \\foo.o: \ + \\ + , expect); + try depTokenizer( + \\foo.o: \ + , expect); + try depTokenizer( + \\foo.o: \ + \\ + , expect); +} + +test "prereq" { + const expect = + \\target = {foo.o} + \\prereq = {foo.c} + ; + try depTokenizer("foo.o: foo.c", expect); + try depTokenizer( + \\foo.o: \ + \\foo.c + , expect); + try depTokenizer( + \\foo.o: \ + \\ foo.c + , expect); + try depTokenizer( + \\foo.o: \ + \\ foo.c + , expect); +} + +test "prereq continuation" { + const expect = + \\target = {foo.o} + \\prereq = {foo.h} + \\prereq = {bar.h} + ; + try depTokenizer( + \\foo.o: foo.h\ + \\bar.h + , expect); + try depTokenizer( + \\foo.o: foo.h\ + \\bar.h + , expect); +} + +test "multiple prereqs" { + const expect = + \\target = {foo.o} + \\prereq = {foo.c} + \\prereq = {foo.h} + \\prereq = {bar.h} + ; + try depTokenizer("foo.o: foo.c foo.h bar.h", expect); + try depTokenizer( + \\foo.o: \ + \\foo.c foo.h bar.h + , expect); + try depTokenizer( + \\foo.o: foo.c foo.h bar.h\ + , expect); + try depTokenizer( + \\foo.o: foo.c foo.h bar.h\ + \\ + , expect); + try depTokenizer( + \\foo.o: \ + \\foo.c \ + \\ foo.h\ + \\bar.h + \\ + , expect); + try depTokenizer( + \\foo.o: \ + \\foo.c \ + \\ foo.h\ + \\bar.h\ + \\ + , expect); + try depTokenizer( + \\foo.o: \ + \\foo.c \ + \\ foo.h\ + \\bar.h\ + , expect); +} + +test "multiple targets and prereqs" { + try depTokenizer( + \\foo.o: foo.c + \\bar.o: bar.c a.h b.h c.h + \\abc.o: abc.c \ + \\ one.h two.h \ + \\ three.h four.h + , + \\target = {foo.o} + \\prereq = {foo.c} + \\target = {bar.o} + \\prereq = {bar.c} + \\prereq = {a.h} + \\prereq = {b.h} + \\prereq = {c.h} + \\target = {abc.o} + \\prereq = {abc.c} + \\prereq = {one.h} + \\prereq = {two.h} + \\prereq = {three.h} + \\prereq = {four.h} + ); + try depTokenizer( + \\ascii.o: ascii.c + \\base64.o: base64.c stdio.h + \\elf.o: elf.c a.h b.h c.h + \\macho.o: \ + \\ macho.c\ + \\ a.h b.h c.h + , + \\target = {ascii.o} + \\prereq = {ascii.c} + \\target = {base64.o} + \\prereq = {base64.c} + \\prereq = {stdio.h} + \\target = {elf.o} + \\prereq = {elf.c} + \\prereq = {a.h} + \\prereq = {b.h} + \\prereq = {c.h} + \\target = {macho.o} + \\prereq = {macho.c} + \\prereq = {a.h} + \\prereq = {b.h} + \\prereq = {c.h} + ); + try depTokenizer( + \\a$$scii.o: ascii.c + \\\\base64.o: "\base64.c" "s t#dio.h" + \\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$" + \\macho.o: \ + \\ "macho!.c" \ + \\ a.h b.h c.h + , + \\target = {a$scii.o} + \\prereq = {ascii.c} + \\target = {\base64.o} + \\prereq = {\base64.c} + \\prereq = {s t#dio.h} + \\target = {e\lf.o} + \\prereq = {e\lf.c} + \\prereq = {a.h$$} + \\prereq = {$$b.h c.h$$} + \\target = {macho.o} + \\prereq = {macho!.c} + \\prereq = {a.h} + \\prereq = {b.h} + \\prereq = {c.h} + ); +} + +test "windows quoted prereqs" { + try depTokenizer( + \\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c" + \\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h" + , + \\target = {c:\foo.o} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c} + \\target = {c:\foo2.o} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h} + ); +} + +test "windows mixed prereqs" { + try depTokenizer( + \\cimport.o: \ + \\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \ + \\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \ + \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \ + \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h" + , + \\target = {cimport.o} + \\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h} + \\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h} + \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h} + \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h} + ); +} + +test "windows funky targets" { + try depTokenizer( + \\C:\Users\anon\foo.o: + \\C:\Users\anon\foo\ .o: + \\C:\Users\anon\foo\#.o: + \\C:\Users\anon\foo$$.o: + \\C:\Users\anon\\\ foo.o: + \\C:\Users\anon\\#foo.o: + \\C:\Users\anon\$$foo.o: + \\C:\Users\anon\\\ \ \ \ \ foo.o: + , + \\target = {C:\Users\anon\foo.o} + \\target = {C:\Users\anon\foo .o} + \\target = {C:\Users\anon\foo#.o} + \\target = {C:\Users\anon\foo$.o} + \\target = {C:\Users\anon\ foo.o} + \\target = {C:\Users\anon\#foo.o} + \\target = {C:\Users\anon\$foo.o} + \\target = {C:\Users\anon\ foo.o} + ); +} + +test "windows drive and forward slashes" { + try depTokenizer( + \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \ + \\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c + , + \\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj} + \\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c} + ); +} + +test "error incomplete escape - reverse_solidus" { + try depTokenizer("\\", + \\ERROR: illegal char '\' at position 0: incomplete escape + ); + try depTokenizer("\t\\", + \\ERROR: illegal char '\' at position 1: incomplete escape + ); + try depTokenizer("\n\\", + \\ERROR: illegal char '\' at position 1: incomplete escape + ); + try depTokenizer("\r\\", + \\ERROR: illegal char '\' at position 1: incomplete escape + ); + try depTokenizer("\r\n\\", + \\ERROR: illegal char '\' at position 2: incomplete escape + ); + try depTokenizer(" \\", + \\ERROR: illegal char '\' at position 1: incomplete escape + ); +} + +test "error incomplete escape - dollar_sign" { + try depTokenizer("$", + \\ERROR: illegal char '$' at position 0: incomplete escape + ); + try depTokenizer("\t$", + \\ERROR: illegal char '$' at position 1: incomplete escape + ); + try depTokenizer("\n$", + \\ERROR: illegal char '$' at position 1: incomplete escape + ); + try depTokenizer("\r$", + \\ERROR: illegal char '$' at position 1: incomplete escape + ); + try depTokenizer("\r\n$", + \\ERROR: illegal char '$' at position 2: incomplete escape + ); + try depTokenizer(" $", + \\ERROR: illegal char '$' at position 1: incomplete escape + ); +} + +test "error incomplete target" { + try depTokenizer("foo.o", + \\ERROR: incomplete target 'foo.o' at position 0 + ); + try depTokenizer("\tfoo.o", + \\ERROR: incomplete target 'foo.o' at position 1 + ); + try depTokenizer("\nfoo.o", + \\ERROR: incomplete target 'foo.o' at position 1 + ); + try depTokenizer("\rfoo.o", + \\ERROR: incomplete target 'foo.o' at position 1 + ); + try depTokenizer("\r\nfoo.o", + \\ERROR: incomplete target 'foo.o' at position 2 + ); + try depTokenizer(" foo.o", + \\ERROR: incomplete target 'foo.o' at position 1 + ); + + try depTokenizer("\\ foo.o", + \\ERROR: incomplete target ' foo.o' at position 0 + ); + try depTokenizer("\\#foo.o", + \\ERROR: incomplete target '#foo.o' at position 0 + ); + try depTokenizer("\\\\foo.o", + \\ERROR: incomplete target '\foo.o' at position 0 + ); + try depTokenizer("$$foo.o", + \\ERROR: incomplete target '$foo.o' at position 0 + ); +} + +test "error illegal char at position - bad target escape" { + try depTokenizer("\\\t", + \\ERROR: illegal char \x09 at position 1: bad target escape + ); + try depTokenizer("\\\n", + \\ERROR: illegal char \x0A at position 1: bad target escape + ); + try depTokenizer("\\\r", + \\ERROR: illegal char \x0D at position 1: bad target escape + ); + try depTokenizer("\\\r\n", + \\ERROR: illegal char \x0D at position 1: bad target escape + ); +} + +test "error illegal char at position - execting dollar_sign" { + try depTokenizer("$\t", + \\ERROR: illegal char \x09 at position 1: expecting '$' + ); + try depTokenizer("$\n", + \\ERROR: illegal char \x0A at position 1: expecting '$' + ); + try depTokenizer("$\r", + \\ERROR: illegal char \x0D at position 1: expecting '$' + ); + try depTokenizer("$\r\n", + \\ERROR: illegal char \x0D at position 1: expecting '$' + ); +} + +test "error illegal char at position - invalid target" { + try depTokenizer("foo\t.o", + \\ERROR: illegal char \x09 at position 3: invalid target + ); + try depTokenizer("foo\n.o", + \\ERROR: illegal char \x0A at position 3: invalid target + ); + try depTokenizer("foo\r.o", + \\ERROR: illegal char \x0D at position 3: invalid target + ); + try depTokenizer("foo\r\n.o", + \\ERROR: illegal char \x0D at position 3: invalid target + ); +} + +test "error target - continuation expecting end-of-line" { + try depTokenizer("foo.o: \\\t", + \\target = {foo.o} + \\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line + ); + try depTokenizer("foo.o: \\ ", + \\target = {foo.o} + \\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line + ); + try depTokenizer("foo.o: \\x", + \\target = {foo.o} + \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line + ); + try depTokenizer("foo.o: \\\x0dx", + \\target = {foo.o} + \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line + ); +} + +test "error prereq - continuation expecting end-of-line" { + try depTokenizer("foo.o: foo.h\\\x0dx", + \\target = {foo.o} + \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line + ); +} + +// - tokenize input, emit textual representation, and compare to expect +fn depTokenizer(input: []const u8, expect: []const u8) !void { + var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); + const arena = arena_allocator.allocator(); + defer arena_allocator.deinit(); + + var it: Tokenizer = .{ .bytes = input }; + var buffer = std.ArrayList(u8).init(arena); + var resolve_buf = std.ArrayList(u8).init(arena); + var i: usize = 0; + while (it.next()) |token| { + if (i != 0) try buffer.appendSlice("\n"); + switch (token) { + .target, .prereq => |bytes| { + try buffer.appendSlice(@tagName(token)); + try buffer.appendSlice(" = {"); + for (bytes) |b| { + try buffer.append(printable_char_tab[b]); + } + try buffer.appendSlice("}"); + }, + .target_must_resolve => { + try buffer.appendSlice("target = {"); + try token.resolve(resolve_buf.writer()); + for (resolve_buf.items) |b| { + try buffer.append(printable_char_tab[b]); + } + resolve_buf.items.len = 0; + try buffer.appendSlice("}"); + }, + else => { + try buffer.appendSlice("ERROR: "); + try token.printError(buffer.writer()); + break; + }, + } + i += 1; + } + + if (std.mem.eql(u8, expect, buffer.items)) { + try testing.expect(true); + return; + } + + const out = std.io.getStdErr().writer(); + + try out.writeAll("\n"); + try printSection(out, "<<<< input", input); + try printSection(out, "==== expect", expect); + try printSection(out, ">>>> got", buffer.items); + try printRuler(out); + + try testing.expect(false); +} + +fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void { + try printLabel(out, label, bytes); + try hexDump(out, bytes); + try printRuler(out); + try out.writeAll(bytes); + try out.writeAll("\n"); +} + +fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void { + var buf: [80]u8 = undefined; + var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len }); + try out.writeAll(text); + var i: usize = text.len; + const end = 79; + while (i < end) : (i += 1) { + try out.writeAll(&[_]u8{label[0]}); + } + try out.writeAll("\n"); +} + +fn printRuler(out: anytype) !void { + var i: usize = 0; + const end = 79; + while (i < end) : (i += 1) { + try out.writeAll("-"); + } + try out.writeAll("\n"); +} + +fn hexDump(out: anytype, bytes: []const u8) !void { + const n16 = bytes.len >> 4; + var line: usize = 0; + var offset: usize = 0; + while (line < n16) : (line += 1) { + try hexDump16(out, offset, bytes[offset .. offset + 16]); + offset += 16; + } + + const n = bytes.len & 0x0f; + if (n > 0) { + try printDecValue(out, offset, 8); + try out.writeAll(":"); + try out.writeAll(" "); + var end1 = std.math.min(offset + n, offset + 8); + for (bytes[offset..end1]) |b| { + try out.writeAll(" "); + try printHexValue(out, b, 2); + } + var end2 = offset + n; + if (end2 > end1) { + try out.writeAll(" "); + for (bytes[end1..end2]) |b| { + try out.writeAll(" "); + try printHexValue(out, b, 2); + } + } + const short = 16 - n; + var i: usize = 0; + while (i < short) : (i += 1) { + try out.writeAll(" "); + } + if (end2 > end1) { + try out.writeAll(" |"); + } else { + try out.writeAll(" |"); + } + try printCharValues(out, bytes[offset..end2]); + try out.writeAll("|\n"); + offset += n; + } + + try printDecValue(out, offset, 8); + try out.writeAll(":"); + try out.writeAll("\n"); +} + +fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void { + try printDecValue(out, offset, 8); + try out.writeAll(":"); + try out.writeAll(" "); + for (bytes[0..8]) |b| { + try out.writeAll(" "); + try printHexValue(out, b, 2); + } + try out.writeAll(" "); + for (bytes[8..16]) |b| { + try out.writeAll(" "); + try printHexValue(out, b, 2); + } + try out.writeAll(" |"); + try printCharValues(out, bytes); + try out.writeAll("|\n"); +} + +fn printDecValue(out: anytype, value: u64, width: u8) !void { + var buffer: [20]u8 = undefined; + const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' }); + try out.writeAll(buffer[0..len]); +} + +fn printHexValue(out: anytype, value: u64, width: u8) !void { + var buffer: [16]u8 = undefined; + const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' }); + try out.writeAll(buffer[0..len]); +} + +fn printCharValues(out: anytype, bytes: []const u8) !void { + for (bytes) |b| { + try out.writeAll(&[_]u8{printable_char_tab[b]}); + } +} + +fn printUnderstandableChar(out: anytype, char: u8) !void { + if (std.ascii.isPrint(char)) { + try out.print("'{c}'", .{char}); + } else { + try out.print("\\x{X:0>2}", .{char}); + } +} + +// zig fmt: off +const printable_char_tab: [256]u8 = ( + "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++ + "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++ + "................................................................" ++ + "................................................................" +).*; diff --git a/src/Cache.zig b/src/Cache.zig deleted file mode 100644 index 3020f8e8c6..0000000000 --- a/src/Cache.zig +++ /dev/null @@ -1,1265 +0,0 @@ -//! Manages `zig-cache` directories. -//! This is not a general-purpose cache. It is designed to be fast and simple, -//! not to withstand attacks using specially-crafted input. - -gpa: Allocator, -manifest_dir: fs.Dir, -hash: HashHelper = .{}, -/// This value is accessed from multiple threads, protected by mutex. -recent_problematic_timestamp: i128 = 0, -mutex: std.Thread.Mutex = .{}, - -/// A set of strings such as the zig library directory or project source root, which -/// are stripped from the file paths before putting into the cache. They -/// are replaced with single-character indicators. This is not to save -/// space but to eliminate absolute file paths. This improves portability -/// and usefulness of the cache for advanced use cases. -prefixes_buffer: [3]Compilation.Directory = undefined, -prefixes_len: usize = 0, - -const Cache = @This(); -const std = @import("std"); -const builtin = @import("builtin"); -const crypto = std.crypto; -const fs = std.fs; -const assert = std.debug.assert; -const testing = std.testing; -const mem = std.mem; -const fmt = std.fmt; -const Allocator = std.mem.Allocator; -const Compilation = @import("Compilation.zig"); -const log = std.log.scoped(.cache); - -pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void { - if (directory.path) |p| { - log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p }); - } - cache.prefixes_buffer[cache.prefixes_len] = directory; - cache.prefixes_len += 1; -} - -/// Be sure to call `Manifest.deinit` after successful initialization. -pub fn obtain(cache: *Cache) Manifest { - return Manifest{ - .cache = cache, - .hash = cache.hash, - .manifest_file = null, - .manifest_dirty = false, - .hex_digest = undefined, - }; -} - -pub fn prefixes(cache: *const Cache) []const Compilation.Directory { - return cache.prefixes_buffer[0..cache.prefixes_len]; -} - -const PrefixedPath = struct { - prefix: u8, - sub_path: []u8, -}; - -fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { - const gpa = cache.gpa; - const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path}); - errdefer gpa.free(resolved_path); - return findPrefixResolved(cache, resolved_path); -} - -/// Takes ownership of `resolved_path` on success. -fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { - const gpa = cache.gpa; - const prefixes_slice = cache.prefixes(); - var i: u8 = 1; // Start at 1 to skip over checking the null prefix. - while (i < prefixes_slice.len) : (i += 1) { - const p = prefixes_slice[i].path.?; - if (mem.startsWith(u8, resolved_path, p)) { - // +1 to skip over the path separator here - const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]); - gpa.free(resolved_path); - return PrefixedPath{ - .prefix = @intCast(u8, i), - .sub_path = sub_path, - }; - } else { - log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p }); - } - } - - return PrefixedPath{ - .prefix = 0, - .sub_path = resolved_path, - }; -} - -/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -pub const bin_digest_len = 16; -pub const hex_digest_len = bin_digest_len * 2; -pub const BinDigest = [bin_digest_len]u8; - -const manifest_file_size_max = 50 * 1024 * 1024; - -/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it -/// provides enough collision resistance for the Manifest use cases, while being one of our -/// fastest options right now. -pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); - -/// Initial state, that can be copied. -pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length); - -pub const File = struct { - prefixed_path: ?PrefixedPath, - max_file_size: ?usize, - stat: Stat, - bin_digest: BinDigest, - contents: ?[]const u8, - - pub const Stat = struct { - inode: fs.File.INode, - size: u64, - mtime: i128, - }; - - pub fn deinit(self: *File, gpa: Allocator) void { - if (self.prefixed_path) |pp| { - gpa.free(pp.sub_path); - self.prefixed_path = null; - } - if (self.contents) |contents| { - gpa.free(contents); - self.contents = null; - } - self.* = undefined; - } -}; - -pub const HashHelper = struct { - hasher: Hasher = hasher_init, - - const EmitLoc = Compilation.EmitLoc; - - /// Record a slice of bytes as an dependency of the process being cached - pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { - hh.hasher.update(mem.asBytes(&bytes.len)); - hh.hasher.update(bytes); - } - - pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { - hh.add(optional_bytes != null); - hh.addBytes(optional_bytes orelse return); - } - - pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void { - hh.addBytes(emit_loc.basename); - } - - pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void { - hh.add(optional_emit_loc != null); - hh.addEmitLoc(optional_emit_loc orelse return); - } - - pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { - hh.add(list_of_bytes.len); - for (list_of_bytes) |bytes| hh.addBytes(bytes); - } - - /// Convert the input value into bytes and record it as a dependency of the process being cached. - pub fn add(hh: *HashHelper, x: anytype) void { - switch (@TypeOf(x)) { - std.builtin.Version => { - hh.add(x.major); - hh.add(x.minor); - hh.add(x.patch); - }, - std.Target.Os.TaggedVersionRange => { - switch (x) { - .linux => |linux| { - hh.add(linux.range.min); - hh.add(linux.range.max); - hh.add(linux.glibc); - }, - .windows => |windows| { - hh.add(windows.min); - hh.add(windows.max); - }, - .semver => |semver| { - hh.add(semver.min); - hh.add(semver.max); - }, - .none => {}, - } - }, - else => switch (@typeInfo(@TypeOf(x))) { - .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), - else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), - }, - } - } - - pub fn addOptional(hh: *HashHelper, optional: anytype) void { - hh.add(optional != null); - hh.add(optional orelse return); - } - - /// Returns a hex encoded hash of the inputs, without modifying state. - pub fn peek(hh: HashHelper) [hex_digest_len]u8 { - var copy = hh; - return copy.final(); - } - - pub fn peekBin(hh: HashHelper) BinDigest { - var copy = hh; - var bin_digest: BinDigest = undefined; - copy.hasher.final(&bin_digest); - return bin_digest; - } - - /// Returns a hex encoded hash of the inputs, mutating the state of the hasher. - pub fn final(hh: *HashHelper) [hex_digest_len]u8 { - var bin_digest: BinDigest = undefined; - hh.hasher.final(&bin_digest); - - var out_digest: [hex_digest_len]u8 = undefined; - _ = std.fmt.bufPrint( - &out_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&bin_digest)}, - ) catch unreachable; - return out_digest; - } -}; - -pub const Lock = struct { - manifest_file: fs.File, - - pub fn release(lock: *Lock) void { - if (builtin.os.tag == .windows) { - // Windows does not guarantee that locks are immediately unlocked when - // the file handle is closed. See LockFileEx documentation. - lock.manifest_file.unlock(); - } - - lock.manifest_file.close(); - lock.* = undefined; - } -}; - -pub const Manifest = struct { - cache: *Cache, - /// Current state for incremental hashing. - hash: HashHelper, - manifest_file: ?fs.File, - manifest_dirty: bool, - /// Set this flag to true before calling hit() in order to indicate that - /// upon a cache hit, the code using the cache will not modify the files - /// within the cache directory. This allows multiple processes to utilize - /// the same cache directory at the same time. - want_shared_lock: bool = true, - have_exclusive_lock: bool = false, - // Indicate that we want isProblematicTimestamp to perform a filesystem write in - // order to obtain a problematic timestamp for the next call. Calls after that - // will then use the same timestamp, to avoid unnecessary filesystem writes. - want_refresh_timestamp: bool = true, - files: std.ArrayListUnmanaged(File) = .{}, - hex_digest: [hex_digest_len]u8, - /// Populated when hit() returns an error because of one - /// of the files listed in the manifest. - failed_file_index: ?usize = null, - /// Keeps track of the last time we performed a file system write to observe - /// what time the file system thinks it is, according to its own granularity. - recent_problematic_timestamp: i128 = 0, - - /// Add a file as a dependency of process being cached. When `hit` is - /// called, the file's contents will be checked to ensure that it matches - /// the contents from previous times. - /// - /// Max file size will be used to determine the amount of space the file contents - /// are allowed to take up in memory. If max_file_size is null, then the contents - /// will not be loaded into memory. - /// - /// Returns the index of the entry in the `files` array list. You can use it - /// to access the contents of the file after calling `hit()` like so: - /// - /// ``` - /// var file_contents = cache_hash.files.items[file_index].contents.?; - /// ``` - pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { - assert(self.manifest_file == null); - - const gpa = self.cache.gpa; - try self.files.ensureUnusedCapacity(gpa, 1); - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - log.debug("Manifest.addFile {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - - self.files.addOneAssumeCapacity().* = .{ - .prefixed_path = prefixed_path, - .contents = null, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - }; - - self.hash.add(prefixed_path.prefix); - self.hash.addBytes(prefixed_path.sub_path); - - return self.files.items.len - 1; - } - - pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void { - _ = try self.addFile(c_source.src_path, null); - // Hash the extra flags, with special care to call addFile for file parameters. - // TODO this logic can likely be improved by utilizing clang_options_data.zig. - const file_args = [_][]const u8{"-include"}; - var arg_i: usize = 0; - while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { - const arg = c_source.extra_flags[arg_i]; - self.hash.addBytes(arg); - for (file_args) |file_arg| { - if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { - arg_i += 1; - _ = try self.addFile(c_source.extra_flags[arg_i], null); - } - } - } - } - - pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { - self.hash.add(optional_file_path != null); - const file_path = optional_file_path orelse return; - _ = try self.addFile(file_path, null); - } - - pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void { - self.hash.add(list_of_files.len); - for (list_of_files) |file_path| { - _ = try self.addFile(file_path, null); - } - } - - /// Check the cache to see if the input exists in it. If it exists, returns `true`. - /// A hex encoding of its hash is available by calling `final`. - /// - /// This function will also acquire an exclusive lock to the manifest file. This means - /// that a process holding a Manifest will block any other process attempting to - /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the - /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest - /// file to be locked in exclusive mode. - /// - /// The lock on the manifest file is released when `deinit` is called. As another - /// option, one may call `toOwnedLock` to obtain a smaller object which can represent - /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(self: *Manifest) !bool { - const gpa = self.cache.gpa; - assert(self.manifest_file == null); - - self.failed_file_index = null; - - const ext = ".txt"; - var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined; - - var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); - - _ = std.fmt.bufPrint( - &self.hex_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&bin_digest)}, - ) catch unreachable; - - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - mem.copy(u8, &manifest_file_path, &self.hex_digest); - manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; - - if (self.files.items.len == 0) { - // If there are no file inputs, we check if the manifest file exists instead of - // comparing the hashes on the files used for the cached item - while (true) { - if (self.cache.manifest_dir.openFile(&manifest_file_path, .{ - .mode = .read_write, - .lock = .Exclusive, - .lock_nonblocking = self.want_shared_lock, - })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; - break; - } else |open_err| switch (open_err) { - error.WouldBlock => { - self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ - .lock = .Shared, - }); - break; - }, - error.FileNotFound => { - if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ - .read = true, - .truncate = false, - .lock = .Exclusive, - .lock_nonblocking = self.want_shared_lock, - })) |manifest_file| { - self.manifest_file = manifest_file; - self.manifest_dirty = true; - self.have_exclusive_lock = true; - return false; // cache miss; exclusive lock already held - } else |err| switch (err) { - error.WouldBlock => continue, - else => |e| return e, - } - }, - else => |e| return e, - } - } - } else { - if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ - .read = true, - .truncate = false, - .lock = .Exclusive, - .lock_nonblocking = self.want_shared_lock, - })) |manifest_file| { - self.manifest_file = manifest_file; - self.have_exclusive_lock = true; - } else |err| switch (err) { - error.WouldBlock => { - self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ - .lock = .Shared, - }); - }, - else => |e| return e, - } - } - - self.want_refresh_timestamp = true; - - const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max); - defer gpa.free(file_contents); - - const input_file_count = self.files.items.len; - var any_file_changed = false; - var line_iter = mem.tokenize(u8, file_contents, "\n"); - var idx: usize = 0; - while (line_iter.next()) |line| { - defer idx += 1; - - const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { - const new = try self.files.addOne(gpa); - new.* = .{ - .prefixed_path = null, - .contents = null, - .max_file_size = null, - .stat = undefined, - .bin_digest = undefined, - }; - break :blk new; - }; - - var iter = mem.tokenize(u8, line, " "); - const size = iter.next() orelse return error.InvalidFormat; - const inode = iter.next() orelse return error.InvalidFormat; - const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; - const digest_str = iter.next() orelse return error.InvalidFormat; - const prefix_str = iter.next() orelse return error.InvalidFormat; - const file_path = iter.rest(); - - cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; - cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; - cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; - _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; - const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat; - if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; - - if (file_path.len == 0) { - return error.InvalidFormat; - } - if (cache_hash_file.prefixed_path) |pp| { - if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) { - return error.InvalidFormat; - } - } - - if (cache_hash_file.prefixed_path == null) { - cache_hash_file.prefixed_path = .{ - .prefix = prefix, - .sub_path = try gpa.dupe(u8, file_path), - }; - } - - const pp = cache_hash_file.prefixed_path.?; - const dir = self.cache.prefixes()[pp.prefix].handle; - const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) { - error.FileNotFound => { - try self.upgradeToExclusiveLock(); - return false; - }, - else => return error.CacheUnavailable, - }; - defer this_file.close(); - - const actual_stat = this_file.stat() catch |err| { - self.failed_file_index = idx; - return err; - }; - const size_match = actual_stat.size == cache_hash_file.stat.size; - const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime; - const inode_match = actual_stat.inode == cache_hash_file.stat.inode; - - if (!size_match or !mtime_match or !inode_match) { - self.manifest_dirty = true; - - cache_hash_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, - }; - - if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - cache_hash_file.stat.mtime = 0; - cache_hash_file.stat.inode = 0; - } - - var actual_digest: BinDigest = undefined; - hashFile(this_file, &actual_digest) catch |err| { - self.failed_file_index = idx; - return err; - }; - - if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { - cache_hash_file.bin_digest = actual_digest; - // keep going until we have the input file digests - any_file_changed = true; - } - } - - if (!any_file_changed) { - self.hash.hasher.update(&cache_hash_file.bin_digest); - } - } - - if (any_file_changed) { - // cache miss - // keep the manifest file open - self.unhit(bin_digest, input_file_count); - try self.upgradeToExclusiveLock(); - return false; - } - - if (idx < input_file_count) { - self.manifest_dirty = true; - while (idx < input_file_count) : (idx += 1) { - const ch_file = &self.files.items[idx]; - self.populateFileHash(ch_file) catch |err| { - self.failed_file_index = idx; - return err; - }; - } - try self.upgradeToExclusiveLock(); - return false; - } - - if (self.want_shared_lock) { - try self.downgradeToSharedLock(); - } - - return true; - } - - pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void { - // Reset the hash. - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - // Remove files not in the initial hash. - for (self.files.items[input_file_count..]) |*file| { - file.deinit(self.cache.gpa); - } - self.files.shrinkRetainingCapacity(input_file_count); - - for (self.files.items) |file| { - self.hash.hasher.update(&file.bin_digest); - } - } - - fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool { - // If the file_time is prior to the most recent problematic timestamp - // then we don't need to access the filesystem. - if (file_time < man.recent_problematic_timestamp) - return false; - - // Next we will check the globally shared Cache timestamp, which is accessed - // from multiple threads. - man.cache.mutex.lock(); - defer man.cache.mutex.unlock(); - - // Save the global one to our local one to avoid locking next time. - man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp; - if (file_time < man.recent_problematic_timestamp) - return false; - - // This flag prevents multiple filesystem writes for the same hit() call. - if (man.want_refresh_timestamp) { - man.want_refresh_timestamp = false; - - var file = man.cache.manifest_dir.createFile("timestamp", .{ - .read = true, - .truncate = true, - }) catch return true; - defer file.close(); - - // Save locally and also save globally (we still hold the global lock). - man.recent_problematic_timestamp = (file.stat() catch return true).mtime; - man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp; - } - - return file_time >= man.recent_problematic_timestamp; - } - - fn populateFileHash(self: *Manifest, ch_file: *File) !void { - const pp = ch_file.prefixed_path.?; - const dir = self.cache.prefixes()[pp.prefix].handle; - const file = try dir.openFile(pp.sub_path, .{}); - defer file.close(); - - const actual_stat = try file.stat(); - ch_file.stat = .{ - .size = actual_stat.size, - .mtime = actual_stat.mtime, - .inode = actual_stat.inode, - }; - - if (self.isProblematicTimestamp(ch_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - ch_file.stat.mtime = 0; - ch_file.stat.inode = 0; - } - - if (ch_file.max_file_size) |max_file_size| { - if (ch_file.stat.size > max_file_size) { - return error.FileTooBig; - } - - const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size)); - errdefer self.cache.gpa.free(contents); - - // Hash while reading from disk, to keep the contents in the cpu cache while - // doing hashing. - var hasher = hasher_init; - var off: usize = 0; - while (true) { - // give me everything you've got, captain - const bytes_read = try file.read(contents[off..]); - if (bytes_read == 0) break; - hasher.update(contents[off..][0..bytes_read]); - off += bytes_read; - } - hasher.final(&ch_file.bin_digest); - - ch_file.contents = contents; - } else { - try hashFile(file, &ch_file.bin_digest); - } - - self.hash.hasher.update(&ch_file.bin_digest); - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 { - assert(self.manifest_file != null); - - const gpa = self.cache.gpa; - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - - const new_ch_file = try self.files.addOne(gpa); - new_ch_file.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - - try self.populateFileHash(new_ch_file); - - return new_ch_file.contents.?; - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know the all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePost(self: *Manifest, file_path: []const u8) !void { - assert(self.manifest_file != null); - - const gpa = self.cache.gpa; - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - log.debug("Manifest.addFilePost {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - - const new_ch_file = try self.files.addOne(gpa); - new_ch_file.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - - try self.populateFileHash(new_ch_file); - } - - /// Like `addFilePost` but when the file contents have already been loaded from disk. - /// On success, cache takes ownership of `resolved_path`. - pub fn addFilePostContents( - self: *Manifest, - resolved_path: []u8, - bytes: []const u8, - stat: File.Stat, - ) error{OutOfMemory}!void { - assert(self.manifest_file != null); - const gpa = self.cache.gpa; - - const ch_file = try self.files.addOne(gpa); - errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - - log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path}); - - const prefixed_path = try self.cache.findPrefixResolved(resolved_path); - errdefer gpa.free(prefixed_path.sub_path); - - log.debug("Manifest.addFilePostContents -> {d} {s}", .{ - prefixed_path.prefix, prefixed_path.sub_path, - }); - - ch_file.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .stat = stat, - .bin_digest = undefined, - .contents = null, - }; - - if (self.isProblematicTimestamp(ch_file.stat.mtime)) { - // The actual file has an unreliable timestamp, force it to be hashed - ch_file.stat.mtime = 0; - ch_file.stat.inode = 0; - } - - { - var hasher = hasher_init; - hasher.update(bytes); - hasher.final(&ch_file.bin_digest); - } - - self.hash.hasher.update(&ch_file.bin_digest); - } - - pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void { - assert(self.manifest_file != null); - - const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max); - defer self.cache.gpa.free(dep_file_contents); - - var error_buf = std.ArrayList(u8).init(self.cache.gpa); - defer error_buf.deinit(); - - var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents }; - - // Skip first token: target. - switch (it.next() orelse return) { // Empty dep file OK. - .target, .target_must_resolve, .prereq => {}, - else => |err| { - try err.printError(error_buf.writer()); - log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items }); - return error.InvalidDepFile; - }, - } - // Process 0+ preqreqs. - // Clang is invoked in single-source mode so we never get more targets. - while (true) { - switch (it.next() orelse return) { - .target, .target_must_resolve => return, - .prereq => |file_path| try self.addFilePost(file_path), - else => |err| { - try err.printError(error_buf.writer()); - log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items }); - return error.InvalidDepFile; - }, - } - } - } - - /// Returns a hex encoded hash of the inputs. - pub fn final(self: *Manifest) [hex_digest_len]u8 { - assert(self.manifest_file != null); - - // We don't close the manifest file yet, because we want to - // keep it locked until the API user is done using it. - // We also don't write out the manifest yet, because until - // cache_release is called we still might be working on creating - // the artifacts to cache. - - var bin_digest: BinDigest = undefined; - self.hash.hasher.final(&bin_digest); - - var out_digest: [hex_digest_len]u8 = undefined; - _ = std.fmt.bufPrint( - &out_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&bin_digest)}, - ) catch unreachable; - - return out_digest; - } - - /// If `want_shared_lock` is true, this function automatically downgrades the - /// lock from exclusive to shared. - pub fn writeManifest(self: *Manifest) !void { - assert(self.have_exclusive_lock); - - const manifest_file = self.manifest_file.?; - if (self.manifest_dirty) { - self.manifest_dirty = false; - - var contents = std.ArrayList(u8).init(self.cache.gpa); - defer contents.deinit(); - - const writer = contents.writer(); - var encoded_digest: [hex_digest_len]u8 = undefined; - - for (self.files.items) |file| { - _ = std.fmt.bufPrint( - &encoded_digest, - "{s}", - .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, - ) catch unreachable; - try writer.print("{d} {d} {d} {s} {d} {s}\n", .{ - file.stat.size, - file.stat.inode, - file.stat.mtime, - &encoded_digest, - file.prefixed_path.?.prefix, - file.prefixed_path.?.sub_path, - }); - } - - try manifest_file.setEndPos(contents.items.len); - try manifest_file.pwriteAll(contents.items, 0); - } - - if (self.want_shared_lock) { - try self.downgradeToSharedLock(); - } - } - - fn downgradeToSharedLock(self: *Manifest) !void { - if (!self.have_exclusive_lock) return; - - // WASI does not currently support flock, so we bypass it here. - // TODO: If/when flock is supported on WASI, this check should be removed. - // See https://github.com/WebAssembly/wasi-filesystem/issues/2 - if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) { - const manifest_file = self.manifest_file.?; - try manifest_file.downgradeLock(); - } - - self.have_exclusive_lock = false; - } - - fn upgradeToExclusiveLock(self: *Manifest) !void { - if (self.have_exclusive_lock) return; - assert(self.manifest_file != null); - - // WASI does not currently support flock, so we bypass it here. - // TODO: If/when flock is supported on WASI, this check should be removed. - // See https://github.com/WebAssembly/wasi-filesystem/issues/2 - if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) { - const manifest_file = self.manifest_file.?; - // Here we intentionally have a period where the lock is released, in case there are - // other processes holding a shared lock. - manifest_file.unlock(); - try manifest_file.lock(.Exclusive); - } - self.have_exclusive_lock = true; - } - - /// Obtain only the data needed to maintain a lock on the manifest file. - /// The `Manifest` remains safe to deinit. - /// Don't forget to call `writeManifest` before this! - pub fn toOwnedLock(self: *Manifest) Lock { - const lock: Lock = .{ - .manifest_file = self.manifest_file.?, - }; - - self.manifest_file = null; - return lock; - } - - /// Releases the manifest file and frees any memory the Manifest was using. - /// `Manifest.hit` must be called first. - /// Don't forget to call `writeManifest` before this! - pub fn deinit(self: *Manifest) void { - if (self.manifest_file) |file| { - if (builtin.os.tag == .windows) { - // See Lock.release for why this is required on Windows - file.unlock(); - } - - file.close(); - } - for (self.files.items) |*file| { - file.deinit(self.cache.gpa); - } - self.files.deinit(self.cache.gpa); - } -}; - -/// On operating systems that support symlinks, does a readlink. On other operating systems, -/// uses the file contents. Windows supports symlinks but only with elevated privileges, so -/// it is treated as not supporting symlinks. -pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 { - if (builtin.os.tag == .windows) { - return dir.readFile(sub_path, buffer); - } else { - return dir.readLink(sub_path, buffer); - } -} - -/// On operating systems that support symlinks, does a symlink. On other operating systems, -/// uses the file contents. Windows supports symlinks but only with elevated privileges, so -/// it is treated as not supporting symlinks. -/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer. -pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void { - assert(data.len <= 255); - if (builtin.os.tag == .windows) { - return dir.writeFile(sub_path, data); - } else { - return dir.symLink(data, sub_path, .{}); - } -} - -fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void { - var buf: [1024]u8 = undefined; - - var hasher = hasher_init; - while (true) { - const bytes_read = try file.read(&buf); - if (bytes_read == 0) break; - hasher.update(buf[0..bytes_read]); - } - - hasher.final(bin_digest); -} - -// Create/Write a file, close it, then grab its stat.mtime timestamp. -fn testGetCurrentFileTimestamp() !i128 { - var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{ - .read = true, - .truncate = true, - }); - defer file.close(); - - return (try file.stat()).mtime; -} - -test "cache file and then recall it" { - if (builtin.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - - const cwd = fs.cwd(); - - const temp_file = "test.txt"; - const temp_manifest_dir = "temp_manifest_dir"; - - try cwd.writeFile(temp_file, "Hello, world!\n"); - - // Wait for file timestamps to tick - const initial_time = try testGetCurrentFileTimestamp(); - while ((try testGetCurrentFileTimestamp()) == initial_time) { - std.time.sleep(1); - } - - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; - - { - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); - - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); - - digest1 = ch.final(); - try ch.writeManifest(); - } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); - - // Cache hit! We just "built" the same file - try testing.expect(try ch.hit()); - digest2 = ch.final(); - - try testing.expectEqual(false, ch.have_exclusive_lock); - } - - try testing.expectEqual(digest1, digest2); - } - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteFile(temp_file); -} - -test "check that changing a file makes cache fail" { - if (builtin.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - - const temp_file = "cache_hash_change_file_test.txt"; - const temp_manifest_dir = "cache_hash_change_file_manifest_dir"; - const original_temp_file_contents = "Hello, world!\n"; - const updated_temp_file_contents = "Hello, world; but updated!\n"; - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteTree(temp_file); - - try cwd.writeFile(temp_file, original_temp_file_contents); - - // Wait for file timestamps to tick - const initial_time = try testGetCurrentFileTimestamp(); - while ((try testGetCurrentFileTimestamp()) == initial_time) { - std.time.sleep(1); - } - - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; - - { - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); - - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); - - try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); - - digest1 = ch.final(); - - try ch.writeManifest(); - } - - try cwd.writeFile(temp_file, updated_temp_file_contents); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit()); - - // The cache system does not keep the contents of re-hashed input files. - try testing.expect(ch.files.items[temp_file_idx].contents == null); - - digest2 = ch.final(); - - try ch.writeManifest(); - } - - try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); - } - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteTree(temp_file); -} - -test "no file inputs" { - if (builtin.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - const temp_manifest_dir = "no_file_inputs_manifest_dir"; - defer cwd.deleteTree(temp_manifest_dir) catch {}; - - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; - - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); - defer cache.manifest_dir.close(); - - { - var man = cache.obtain(); - defer man.deinit(); - - man.hash.addBytes("1234"); - - // There should be nothing in the cache - try testing.expectEqual(false, try man.hit()); - - digest1 = man.final(); - - try man.writeManifest(); - } - { - var man = cache.obtain(); - defer man.deinit(); - - man.hash.addBytes("1234"); - - try testing.expect(try man.hit()); - digest2 = man.final(); - try testing.expectEqual(false, man.have_exclusive_lock); - } - - try testing.expectEqual(digest1, digest2); -} - -test "Manifest with files added after initial hash work" { - if (builtin.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - - const temp_file1 = "cache_hash_post_file_test1.txt"; - const temp_file2 = "cache_hash_post_file_test2.txt"; - const temp_manifest_dir = "cache_hash_post_file_manifest_dir"; - - try cwd.writeFile(temp_file1, "Hello, world!\n"); - try cwd.writeFile(temp_file2, "Hello world the second!\n"); - - // Wait for file timestamps to tick - const initial_time = try testGetCurrentFileTimestamp(); - while ((try testGetCurrentFileTimestamp()) == initial_time) { - std.time.sleep(1); - } - - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; - var digest3: [hex_digest_len]u8 = undefined; - - { - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - cache.addPrefix(.{ .path = null, .handle = fs.cwd() }); - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - // There should be nothing in the cache - try testing.expectEqual(false, try ch.hit()); - - _ = try ch.addFilePost(temp_file2); - - digest1 = ch.final(); - try ch.writeManifest(); - } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - try testing.expect(try ch.hit()); - digest2 = ch.final(); - - try testing.expectEqual(false, ch.have_exclusive_lock); - } - try testing.expect(mem.eql(u8, &digest1, &digest2)); - - // Modify the file added after initial hash - try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); - - // Wait for file timestamps to tick - const initial_time2 = try testGetCurrentFileTimestamp(); - while ((try testGetCurrentFileTimestamp()) == initial_time2) { - std.time.sleep(1); - } - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - try testing.expectEqual(false, try ch.hit()); - - _ = try ch.addFilePost(temp_file2); - - digest3 = ch.final(); - - try ch.writeManifest(); - } - - try testing.expect(!mem.eql(u8, &digest1, &digest3)); - } - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteFile(temp_file1); - try cwd.deleteFile(temp_file2); -} diff --git a/src/Compilation.zig b/src/Compilation.zig index 18d0e46892..ea83d82109 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig"); const fatal = @import("main.zig").fatal; const clangMain = @import("main.zig").clangMain; const Module = @import("Module.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const translate_c = @import("translate_c.zig"); const clang = @import("clang.zig"); const c_codegen = @import("codegen/c.zig"); @@ -807,44 +807,7 @@ pub const AllErrors = struct { } }; -pub const Directory = struct { - /// This field is redundant for operations that can act on the open directory handle - /// directly, but it is needed when passing the directory to a child process. - /// `null` means cwd. - path: ?[]const u8, - handle: std.fs.Dir, - - pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { - if (self.path) |p| { - // TODO clean way to do this with only 1 allocation - const part2 = try std.fs.path.join(allocator, paths); - defer allocator.free(part2); - return std.fs.path.join(allocator, &[_][]const u8{ p, part2 }); - } else { - return std.fs.path.join(allocator, paths); - } - } - - pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 { - if (self.path) |p| { - // TODO clean way to do this with only 1 allocation - const part2 = try std.fs.path.join(allocator, paths); - defer allocator.free(part2); - return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 }); - } else { - return std.fs.path.joinZ(allocator, paths); - } - } - - /// Whether or not the handle should be closed, or the path should be freed - /// is determined by usage, however this function is provided for convenience - /// if it happens to be what the caller needs. - pub fn closeAndFree(self: *Directory, gpa: Allocator) void { - self.handle.close(); - if (self.path) |p| gpa.free(p); - self.* = undefined; - } -}; +pub const Directory = Cache.Directory; pub const EmitLoc = struct { /// If this is `null` it means the file will be output to the cache directory. @@ -854,6 +817,35 @@ pub const EmitLoc = struct { basename: []const u8, }; +pub const cache_helpers = struct { + pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void { + hh.addBytes(emit_loc.basename); + } + + pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void { + hh.add(optional_emit_loc != null); + addEmitLoc(hh, optional_emit_loc orelse return); + } + + pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void { + _ = try self.addFile(c_source.src_path, null); + // Hash the extra flags, with special care to call addFile for file parameters. + // TODO this logic can likely be improved by utilizing clang_options_data.zig. + const file_args = [_][]const u8{"-include"}; + var arg_i: usize = 0; + while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { + const arg = c_source.extra_flags[arg_i]; + self.hash.addBytes(arg); + for (file_args) |file_arg| { + if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { + arg_i += 1; + _ = try self.addFile(c_source.extra_flags[arg_i], null); + } + } + } + } +}; + pub const ClangPreprocessorMode = enum { no, /// This means we are doing `zig cc -E -o `. @@ -1522,8 +1514,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { cache.hash.add(link_libunwind); cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); - cache.hash.addOptionalEmitLoc(options.emit_bin); - cache.hash.addOptionalEmitLoc(options.emit_implib); + cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin); + cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib); cache.hash.addBytes(options.root_name); if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model); // TODO audit this and make sure everything is in it @@ -2636,11 +2628,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes man.hash.addListOfBytes(key.src.extra_flags); } - man.hash.addOptionalEmitLoc(comp.emit_asm); - man.hash.addOptionalEmitLoc(comp.emit_llvm_ir); - man.hash.addOptionalEmitLoc(comp.emit_llvm_bc); - man.hash.addOptionalEmitLoc(comp.emit_analysis); - man.hash.addOptionalEmitLoc(comp.emit_docs); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs); man.hash.addListOfBytes(comp.clang_argv); @@ -3959,11 +3951,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P defer man.deinit(); man.hash.add(comp.clang_preprocessor_mode); - man.hash.addOptionalEmitLoc(comp.emit_asm); - man.hash.addOptionalEmitLoc(comp.emit_llvm_ir); - man.hash.addOptionalEmitLoc(comp.emit_llvm_bc); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc); - try man.hashCSource(c_object.src); + try cache_helpers.hashCSource(&man, c_object.src); var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); diff --git a/src/DepTokenizer.zig b/src/DepTokenizer.zig deleted file mode 100644 index 8f9f2f81cd..0000000000 --- a/src/DepTokenizer.zig +++ /dev/null @@ -1,1069 +0,0 @@ -const Tokenizer = @This(); - -index: usize = 0, -bytes: []const u8, -state: State = .lhs, - -const std = @import("std"); -const testing = std.testing; -const assert = std.debug.assert; - -pub fn next(self: *Tokenizer) ?Token { - var start = self.index; - var must_resolve = false; - while (self.index < self.bytes.len) { - const char = self.bytes[self.index]; - switch (self.state) { - .lhs => switch (char) { - '\t', '\n', '\r', ' ' => { - // silently ignore whitespace - self.index += 1; - }, - else => { - start = self.index; - self.state = .target; - }, - }, - .target => switch (char) { - '\t', '\n', '\r', ' ' => { - return errorIllegalChar(.invalid_target, self.index, char); - }, - '$' => { - self.state = .target_dollar_sign; - self.index += 1; - }, - '\\' => { - self.state = .target_reverse_solidus; - self.index += 1; - }, - ':' => { - self.state = .target_colon; - self.index += 1; - }, - else => { - self.index += 1; - }, - }, - .target_reverse_solidus => switch (char) { - '\t', '\n', '\r' => { - return errorIllegalChar(.bad_target_escape, self.index, char); - }, - ' ', '#', '\\' => { - must_resolve = true; - self.state = .target; - self.index += 1; - }, - '$' => { - self.state = .target_dollar_sign; - self.index += 1; - }, - else => { - self.state = .target; - self.index += 1; - }, - }, - .target_dollar_sign => switch (char) { - '$' => { - must_resolve = true; - self.state = .target; - self.index += 1; - }, - else => { - return errorIllegalChar(.expected_dollar_sign, self.index, char); - }, - }, - .target_colon => switch (char) { - '\n', '\r' => { - const bytes = self.bytes[start .. self.index - 1]; - if (bytes.len != 0) { - self.state = .lhs; - return finishTarget(must_resolve, bytes); - } - // silently ignore null target - self.state = .lhs; - }, - '/', '\\' => { - self.state = .target_colon_reverse_solidus; - self.index += 1; - }, - else => { - const bytes = self.bytes[start .. self.index - 1]; - if (bytes.len != 0) { - self.state = .rhs; - return finishTarget(must_resolve, bytes); - } - // silently ignore null target - self.state = .lhs; - }, - }, - .target_colon_reverse_solidus => switch (char) { - '\n', '\r' => { - const bytes = self.bytes[start .. self.index - 2]; - if (bytes.len != 0) { - self.state = .lhs; - return finishTarget(must_resolve, bytes); - } - // silently ignore null target - self.state = .lhs; - }, - else => { - self.state = .target; - }, - }, - .rhs => switch (char) { - '\t', ' ' => { - // silently ignore horizontal whitespace - self.index += 1; - }, - '\n', '\r' => { - self.state = .lhs; - }, - '\\' => { - self.state = .rhs_continuation; - self.index += 1; - }, - '"' => { - self.state = .prereq_quote; - self.index += 1; - start = self.index; - }, - else => { - start = self.index; - self.state = .prereq; - }, - }, - .rhs_continuation => switch (char) { - '\n' => { - self.state = .rhs; - self.index += 1; - }, - '\r' => { - self.state = .rhs_continuation_linefeed; - self.index += 1; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, - }, - .rhs_continuation_linefeed => switch (char) { - '\n' => { - self.state = .rhs; - self.index += 1; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, - }, - .prereq_quote => switch (char) { - '"' => { - self.index += 1; - self.state = .rhs; - return Token{ .prereq = self.bytes[start .. self.index - 1] }; - }, - else => { - self.index += 1; - }, - }, - .prereq => switch (char) { - '\t', ' ' => { - self.state = .rhs; - return Token{ .prereq = self.bytes[start..self.index] }; - }, - '\n', '\r' => { - self.state = .lhs; - return Token{ .prereq = self.bytes[start..self.index] }; - }, - '\\' => { - self.state = .prereq_continuation; - self.index += 1; - }, - else => { - self.index += 1; - }, - }, - .prereq_continuation => switch (char) { - '\n' => { - self.index += 1; - self.state = .rhs; - return Token{ .prereq = self.bytes[start .. self.index - 2] }; - }, - '\r' => { - self.state = .prereq_continuation_linefeed; - self.index += 1; - }, - else => { - // not continuation - self.state = .prereq; - self.index += 1; - }, - }, - .prereq_continuation_linefeed => switch (char) { - '\n' => { - self.index += 1; - self.state = .rhs; - return Token{ .prereq = self.bytes[start .. self.index - 1] }; - }, - else => { - return errorIllegalChar(.continuation_eol, self.index, char); - }, - }, - } - } else { - switch (self.state) { - .lhs, - .rhs, - .rhs_continuation, - .rhs_continuation_linefeed, - => return null, - .target => { - return errorPosition(.incomplete_target, start, self.bytes[start..]); - }, - .target_reverse_solidus, - .target_dollar_sign, - => { - const idx = self.index - 1; - return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]); - }, - .target_colon => { - const bytes = self.bytes[start .. self.index - 1]; - if (bytes.len != 0) { - self.index += 1; - self.state = .rhs; - return finishTarget(must_resolve, bytes); - } - // silently ignore null target - self.state = .lhs; - return null; - }, - .target_colon_reverse_solidus => { - const bytes = self.bytes[start .. self.index - 2]; - if (bytes.len != 0) { - self.index += 1; - self.state = .rhs; - return finishTarget(must_resolve, bytes); - } - // silently ignore null target - self.state = .lhs; - return null; - }, - .prereq_quote => { - return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]); - }, - .prereq => { - self.state = .lhs; - return Token{ .prereq = self.bytes[start..] }; - }, - .prereq_continuation => { - self.state = .lhs; - return Token{ .prereq = self.bytes[start .. self.index - 1] }; - }, - .prereq_continuation_linefeed => { - self.state = .lhs; - return Token{ .prereq = self.bytes[start .. self.index - 2] }; - }, - } - } - unreachable; -} - -fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token { - return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes }); -} - -fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token { - return @unionInit(Token, @tagName(id), .{ .index = index, .char = char }); -} - -fn finishTarget(must_resolve: bool, bytes: []const u8) Token { - return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes }; -} - -const State = enum { - lhs, - target, - target_reverse_solidus, - target_dollar_sign, - target_colon, - target_colon_reverse_solidus, - rhs, - rhs_continuation, - rhs_continuation_linefeed, - prereq_quote, - prereq, - prereq_continuation, - prereq_continuation_linefeed, -}; - -pub const Token = union(enum) { - target: []const u8, - target_must_resolve: []const u8, - prereq: []const u8, - - incomplete_quoted_prerequisite: IndexAndBytes, - incomplete_target: IndexAndBytes, - - invalid_target: IndexAndChar, - bad_target_escape: IndexAndChar, - expected_dollar_sign: IndexAndChar, - continuation_eol: IndexAndChar, - incomplete_escape: IndexAndChar, - - pub const IndexAndChar = struct { - index: usize, - char: u8, - }; - - pub const IndexAndBytes = struct { - index: usize, - bytes: []const u8, - }; - - /// Resolve escapes in target. Only valid with .target_must_resolve. - pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void { - const bytes = self.target_must_resolve; // resolve called on incorrect token - - var state: enum { start, escape, dollar } = .start; - for (bytes) |c| { - switch (state) { - .start => { - switch (c) { - '\\' => state = .escape, - '$' => state = .dollar, - else => try writer.writeByte(c), - } - }, - .escape => { - switch (c) { - ' ', '#', '\\' => {}, - '$' => { - try writer.writeByte('\\'); - state = .dollar; - continue; - }, - else => try writer.writeByte('\\'), - } - try writer.writeByte(c); - state = .start; - }, - .dollar => { - try writer.writeByte('$'); - switch (c) { - '$' => {}, - else => try writer.writeByte(c), - } - state = .start; - }, - } - } - } - - pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void { - switch (self) { - .target, .target_must_resolve, .prereq => unreachable, // not an error - .incomplete_quoted_prerequisite, - .incomplete_target, - => |index_and_bytes| { - try writer.print("{s} '", .{self.errStr()}); - if (self == .incomplete_target) { - const tmp = Token{ .target_must_resolve = index_and_bytes.bytes }; - try tmp.resolve(writer); - } else { - try printCharValues(writer, index_and_bytes.bytes); - } - try writer.print("' at position {d}", .{index_and_bytes.index}); - }, - .invalid_target, - .bad_target_escape, - .expected_dollar_sign, - .continuation_eol, - .incomplete_escape, - => |index_and_char| { - try writer.writeAll("illegal char "); - try printUnderstandableChar(writer, index_and_char.char); - try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() }); - }, - } - } - - fn errStr(self: Token) []const u8 { - return switch (self) { - .target, .target_must_resolve, .prereq => unreachable, // not an error - .incomplete_quoted_prerequisite => "incomplete quoted prerequisite", - .incomplete_target => "incomplete target", - .invalid_target => "invalid target", - .bad_target_escape => "bad target escape", - .expected_dollar_sign => "expecting '$'", - .continuation_eol => "continuation expecting end-of-line", - .incomplete_escape => "incomplete escape", - }; - } -}; - -test "empty file" { - try depTokenizer("", ""); -} - -test "empty whitespace" { - try depTokenizer("\n", ""); - try depTokenizer("\r", ""); - try depTokenizer("\r\n", ""); - try depTokenizer(" ", ""); -} - -test "empty colon" { - try depTokenizer(":", ""); - try depTokenizer("\n:", ""); - try depTokenizer("\r:", ""); - try depTokenizer("\r\n:", ""); - try depTokenizer(" :", ""); -} - -test "empty target" { - try depTokenizer("foo.o:", "target = {foo.o}"); - try depTokenizer( - \\foo.o: - \\bar.o: - \\abcd.o: - , - \\target = {foo.o} - \\target = {bar.o} - \\target = {abcd.o} - ); -} - -test "whitespace empty target" { - try depTokenizer("\nfoo.o:", "target = {foo.o}"); - try depTokenizer("\rfoo.o:", "target = {foo.o}"); - try depTokenizer("\r\nfoo.o:", "target = {foo.o}"); - try depTokenizer(" foo.o:", "target = {foo.o}"); -} - -test "escape empty target" { - try depTokenizer("\\ foo.o:", "target = { foo.o}"); - try depTokenizer("\\#foo.o:", "target = {#foo.o}"); - try depTokenizer("\\\\foo.o:", "target = {\\foo.o}"); - try depTokenizer("$$foo.o:", "target = {$foo.o}"); -} - -test "empty target linefeeds" { - try depTokenizer("\n", ""); - try depTokenizer("\r\n", ""); - - const expect = "target = {foo.o}"; - try depTokenizer( - \\foo.o: - , expect); - try depTokenizer( - \\foo.o: - \\ - , expect); - try depTokenizer( - \\foo.o: - , expect); - try depTokenizer( - \\foo.o: - \\ - , expect); -} - -test "empty target linefeeds + continuations" { - const expect = "target = {foo.o}"; - try depTokenizer( - \\foo.o:\ - , expect); - try depTokenizer( - \\foo.o:\ - \\ - , expect); - try depTokenizer( - \\foo.o:\ - , expect); - try depTokenizer( - \\foo.o:\ - \\ - , expect); -} - -test "empty target linefeeds + hspace + continuations" { - const expect = "target = {foo.o}"; - try depTokenizer( - \\foo.o: \ - , expect); - try depTokenizer( - \\foo.o: \ - \\ - , expect); - try depTokenizer( - \\foo.o: \ - , expect); - try depTokenizer( - \\foo.o: \ - \\ - , expect); -} - -test "prereq" { - const expect = - \\target = {foo.o} - \\prereq = {foo.c} - ; - try depTokenizer("foo.o: foo.c", expect); - try depTokenizer( - \\foo.o: \ - \\foo.c - , expect); - try depTokenizer( - \\foo.o: \ - \\ foo.c - , expect); - try depTokenizer( - \\foo.o: \ - \\ foo.c - , expect); -} - -test "prereq continuation" { - const expect = - \\target = {foo.o} - \\prereq = {foo.h} - \\prereq = {bar.h} - ; - try depTokenizer( - \\foo.o: foo.h\ - \\bar.h - , expect); - try depTokenizer( - \\foo.o: foo.h\ - \\bar.h - , expect); -} - -test "multiple prereqs" { - const expect = - \\target = {foo.o} - \\prereq = {foo.c} - \\prereq = {foo.h} - \\prereq = {bar.h} - ; - try depTokenizer("foo.o: foo.c foo.h bar.h", expect); - try depTokenizer( - \\foo.o: \ - \\foo.c foo.h bar.h - , expect); - try depTokenizer( - \\foo.o: foo.c foo.h bar.h\ - , expect); - try depTokenizer( - \\foo.o: foo.c foo.h bar.h\ - \\ - , expect); - try depTokenizer( - \\foo.o: \ - \\foo.c \ - \\ foo.h\ - \\bar.h - \\ - , expect); - try depTokenizer( - \\foo.o: \ - \\foo.c \ - \\ foo.h\ - \\bar.h\ - \\ - , expect); - try depTokenizer( - \\foo.o: \ - \\foo.c \ - \\ foo.h\ - \\bar.h\ - , expect); -} - -test "multiple targets and prereqs" { - try depTokenizer( - \\foo.o: foo.c - \\bar.o: bar.c a.h b.h c.h - \\abc.o: abc.c \ - \\ one.h two.h \ - \\ three.h four.h - , - \\target = {foo.o} - \\prereq = {foo.c} - \\target = {bar.o} - \\prereq = {bar.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {abc.o} - \\prereq = {abc.c} - \\prereq = {one.h} - \\prereq = {two.h} - \\prereq = {three.h} - \\prereq = {four.h} - ); - try depTokenizer( - \\ascii.o: ascii.c - \\base64.o: base64.c stdio.h - \\elf.o: elf.c a.h b.h c.h - \\macho.o: \ - \\ macho.c\ - \\ a.h b.h c.h - , - \\target = {ascii.o} - \\prereq = {ascii.c} - \\target = {base64.o} - \\prereq = {base64.c} - \\prereq = {stdio.h} - \\target = {elf.o} - \\prereq = {elf.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - \\target = {macho.o} - \\prereq = {macho.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - ); - try depTokenizer( - \\a$$scii.o: ascii.c - \\\\base64.o: "\base64.c" "s t#dio.h" - \\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$" - \\macho.o: \ - \\ "macho!.c" \ - \\ a.h b.h c.h - , - \\target = {a$scii.o} - \\prereq = {ascii.c} - \\target = {\base64.o} - \\prereq = {\base64.c} - \\prereq = {s t#dio.h} - \\target = {e\lf.o} - \\prereq = {e\lf.c} - \\prereq = {a.h$$} - \\prereq = {$$b.h c.h$$} - \\target = {macho.o} - \\prereq = {macho!.c} - \\prereq = {a.h} - \\prereq = {b.h} - \\prereq = {c.h} - ); -} - -test "windows quoted prereqs" { - try depTokenizer( - \\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c" - \\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h" - , - \\target = {c:\foo.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c} - \\target = {c:\foo2.o} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h} - ); -} - -test "windows mixed prereqs" { - try depTokenizer( - \\cimport.o: \ - \\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \ - \\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \ - \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \ - \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h" - , - \\target = {cimport.o} - \\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h} - \\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h} - \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h} - \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h} - ); -} - -test "windows funky targets" { - try depTokenizer( - \\C:\Users\anon\foo.o: - \\C:\Users\anon\foo\ .o: - \\C:\Users\anon\foo\#.o: - \\C:\Users\anon\foo$$.o: - \\C:\Users\anon\\\ foo.o: - \\C:\Users\anon\\#foo.o: - \\C:\Users\anon\$$foo.o: - \\C:\Users\anon\\\ \ \ \ \ foo.o: - , - \\target = {C:\Users\anon\foo.o} - \\target = {C:\Users\anon\foo .o} - \\target = {C:\Users\anon\foo#.o} - \\target = {C:\Users\anon\foo$.o} - \\target = {C:\Users\anon\ foo.o} - \\target = {C:\Users\anon\#foo.o} - \\target = {C:\Users\anon\$foo.o} - \\target = {C:\Users\anon\ foo.o} - ); -} - -test "windows drive and forward slashes" { - try depTokenizer( - \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \ - \\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c - , - \\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj} - \\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c} - ); -} - -test "error incomplete escape - reverse_solidus" { - try depTokenizer("\\", - \\ERROR: illegal char '\' at position 0: incomplete escape - ); - try depTokenizer("\t\\", - \\ERROR: illegal char '\' at position 1: incomplete escape - ); - try depTokenizer("\n\\", - \\ERROR: illegal char '\' at position 1: incomplete escape - ); - try depTokenizer("\r\\", - \\ERROR: illegal char '\' at position 1: incomplete escape - ); - try depTokenizer("\r\n\\", - \\ERROR: illegal char '\' at position 2: incomplete escape - ); - try depTokenizer(" \\", - \\ERROR: illegal char '\' at position 1: incomplete escape - ); -} - -test "error incomplete escape - dollar_sign" { - try depTokenizer("$", - \\ERROR: illegal char '$' at position 0: incomplete escape - ); - try depTokenizer("\t$", - \\ERROR: illegal char '$' at position 1: incomplete escape - ); - try depTokenizer("\n$", - \\ERROR: illegal char '$' at position 1: incomplete escape - ); - try depTokenizer("\r$", - \\ERROR: illegal char '$' at position 1: incomplete escape - ); - try depTokenizer("\r\n$", - \\ERROR: illegal char '$' at position 2: incomplete escape - ); - try depTokenizer(" $", - \\ERROR: illegal char '$' at position 1: incomplete escape - ); -} - -test "error incomplete target" { - try depTokenizer("foo.o", - \\ERROR: incomplete target 'foo.o' at position 0 - ); - try depTokenizer("\tfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 - ); - try depTokenizer("\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 - ); - try depTokenizer("\rfoo.o", - \\ERROR: incomplete target 'foo.o' at position 1 - ); - try depTokenizer("\r\nfoo.o", - \\ERROR: incomplete target 'foo.o' at position 2 - ); - try depTokenizer(" foo.o", - \\ERROR: incomplete target 'foo.o' at position 1 - ); - - try depTokenizer("\\ foo.o", - \\ERROR: incomplete target ' foo.o' at position 0 - ); - try depTokenizer("\\#foo.o", - \\ERROR: incomplete target '#foo.o' at position 0 - ); - try depTokenizer("\\\\foo.o", - \\ERROR: incomplete target '\foo.o' at position 0 - ); - try depTokenizer("$$foo.o", - \\ERROR: incomplete target '$foo.o' at position 0 - ); -} - -test "error illegal char at position - bad target escape" { - try depTokenizer("\\\t", - \\ERROR: illegal char \x09 at position 1: bad target escape - ); - try depTokenizer("\\\n", - \\ERROR: illegal char \x0A at position 1: bad target escape - ); - try depTokenizer("\\\r", - \\ERROR: illegal char \x0D at position 1: bad target escape - ); - try depTokenizer("\\\r\n", - \\ERROR: illegal char \x0D at position 1: bad target escape - ); -} - -test "error illegal char at position - execting dollar_sign" { - try depTokenizer("$\t", - \\ERROR: illegal char \x09 at position 1: expecting '$' - ); - try depTokenizer("$\n", - \\ERROR: illegal char \x0A at position 1: expecting '$' - ); - try depTokenizer("$\r", - \\ERROR: illegal char \x0D at position 1: expecting '$' - ); - try depTokenizer("$\r\n", - \\ERROR: illegal char \x0D at position 1: expecting '$' - ); -} - -test "error illegal char at position - invalid target" { - try depTokenizer("foo\t.o", - \\ERROR: illegal char \x09 at position 3: invalid target - ); - try depTokenizer("foo\n.o", - \\ERROR: illegal char \x0A at position 3: invalid target - ); - try depTokenizer("foo\r.o", - \\ERROR: illegal char \x0D at position 3: invalid target - ); - try depTokenizer("foo\r\n.o", - \\ERROR: illegal char \x0D at position 3: invalid target - ); -} - -test "error target - continuation expecting end-of-line" { - try depTokenizer("foo.o: \\\t", - \\target = {foo.o} - \\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line - ); - try depTokenizer("foo.o: \\ ", - \\target = {foo.o} - \\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line - ); - try depTokenizer("foo.o: \\x", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line - ); - try depTokenizer("foo.o: \\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line - ); -} - -test "error prereq - continuation expecting end-of-line" { - try depTokenizer("foo.o: foo.h\\\x0dx", - \\target = {foo.o} - \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line - ); -} - -// - tokenize input, emit textual representation, and compare to expect -fn depTokenizer(input: []const u8, expect: []const u8) !void { - var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); - const arena = arena_allocator.allocator(); - defer arena_allocator.deinit(); - - var it: Tokenizer = .{ .bytes = input }; - var buffer = std.ArrayList(u8).init(arena); - var resolve_buf = std.ArrayList(u8).init(arena); - var i: usize = 0; - while (it.next()) |token| { - if (i != 0) try buffer.appendSlice("\n"); - switch (token) { - .target, .prereq => |bytes| { - try buffer.appendSlice(@tagName(token)); - try buffer.appendSlice(" = {"); - for (bytes) |b| { - try buffer.append(printable_char_tab[b]); - } - try buffer.appendSlice("}"); - }, - .target_must_resolve => { - try buffer.appendSlice("target = {"); - try token.resolve(resolve_buf.writer()); - for (resolve_buf.items) |b| { - try buffer.append(printable_char_tab[b]); - } - resolve_buf.items.len = 0; - try buffer.appendSlice("}"); - }, - else => { - try buffer.appendSlice("ERROR: "); - try token.printError(buffer.writer()); - break; - }, - } - i += 1; - } - - if (std.mem.eql(u8, expect, buffer.items)) { - try testing.expect(true); - return; - } - - const out = std.io.getStdErr().writer(); - - try out.writeAll("\n"); - try printSection(out, "<<<< input", input); - try printSection(out, "==== expect", expect); - try printSection(out, ">>>> got", buffer.items); - try printRuler(out); - - try testing.expect(false); -} - -fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void { - try printLabel(out, label, bytes); - try hexDump(out, bytes); - try printRuler(out); - try out.writeAll(bytes); - try out.writeAll("\n"); -} - -fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void { - var buf: [80]u8 = undefined; - var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len }); - try out.writeAll(text); - var i: usize = text.len; - const end = 79; - while (i < end) : (i += 1) { - try out.writeAll(&[_]u8{label[0]}); - } - try out.writeAll("\n"); -} - -fn printRuler(out: anytype) !void { - var i: usize = 0; - const end = 79; - while (i < end) : (i += 1) { - try out.writeAll("-"); - } - try out.writeAll("\n"); -} - -fn hexDump(out: anytype, bytes: []const u8) !void { - const n16 = bytes.len >> 4; - var line: usize = 0; - var offset: usize = 0; - while (line < n16) : (line += 1) { - try hexDump16(out, offset, bytes[offset .. offset + 16]); - offset += 16; - } - - const n = bytes.len & 0x0f; - if (n > 0) { - try printDecValue(out, offset, 8); - try out.writeAll(":"); - try out.writeAll(" "); - var end1 = std.math.min(offset + n, offset + 8); - for (bytes[offset..end1]) |b| { - try out.writeAll(" "); - try printHexValue(out, b, 2); - } - var end2 = offset + n; - if (end2 > end1) { - try out.writeAll(" "); - for (bytes[end1..end2]) |b| { - try out.writeAll(" "); - try printHexValue(out, b, 2); - } - } - const short = 16 - n; - var i: usize = 0; - while (i < short) : (i += 1) { - try out.writeAll(" "); - } - if (end2 > end1) { - try out.writeAll(" |"); - } else { - try out.writeAll(" |"); - } - try printCharValues(out, bytes[offset..end2]); - try out.writeAll("|\n"); - offset += n; - } - - try printDecValue(out, offset, 8); - try out.writeAll(":"); - try out.writeAll("\n"); -} - -fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void { - try printDecValue(out, offset, 8); - try out.writeAll(":"); - try out.writeAll(" "); - for (bytes[0..8]) |b| { - try out.writeAll(" "); - try printHexValue(out, b, 2); - } - try out.writeAll(" "); - for (bytes[8..16]) |b| { - try out.writeAll(" "); - try printHexValue(out, b, 2); - } - try out.writeAll(" |"); - try printCharValues(out, bytes); - try out.writeAll("|\n"); -} - -fn printDecValue(out: anytype, value: u64, width: u8) !void { - var buffer: [20]u8 = undefined; - const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' }); - try out.writeAll(buffer[0..len]); -} - -fn printHexValue(out: anytype, value: u64, width: u8) !void { - var buffer: [16]u8 = undefined; - const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' }); - try out.writeAll(buffer[0..len]); -} - -fn printCharValues(out: anytype, bytes: []const u8) !void { - for (bytes) |b| { - try out.writeAll(&[_]u8{printable_char_tab[b]}); - } -} - -fn printUnderstandableChar(out: anytype, char: u8) !void { - if (std.ascii.isPrint(char)) { - try out.print("'{c}'", .{char}); - } else { - try out.print("\\x{X:0>2}", .{char}); - } -} - -// zig fmt: off -const printable_char_tab: [256]u8 = ( - "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++ - "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++ - "................................................................" ++ - "................................................................" -).*; diff --git a/src/Module.zig b/src/Module.zig index e4cf0189cc..a129cb0cb6 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -16,7 +16,7 @@ const Ast = std.zig.Ast; const Module = @This(); const Compilation = @import("Compilation.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); diff --git a/src/Package.zig b/src/Package.zig index 401eef2121..a3afe21009 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); const ThreadPool = @import("ThreadPool.zig"); const WaitGroup = @import("WaitGroup.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const build_options = @import("build_options"); const Manifest = @import("Manifest.zig"); diff --git a/src/glibc.zig b/src/glibc.zig index 8dce1c5132..2a2887c334 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -11,7 +11,7 @@ const target_util = @import("target.zig"); const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Package = @import("Package.zig"); pub const Lib = struct { diff --git a/src/link.zig b/src/link.zig index 2b3ce51667..5650e0679a 100644 --- a/src/link.zig +++ b/src/link.zig @@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig"); const Air = @import("Air.zig"); const Allocator = std.mem.Allocator; -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Compilation = @import("Compilation.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const Liveness = @import("Liveness.zig"); diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index d705f62f5c..c308ff5989 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -5,6 +5,7 @@ const assert = std.debug.assert; const fs = std.fs; const log = std.log.scoped(.link); const mem = std.mem; +const Cache = std.Build.Cache; const mingw = @import("../../mingw.zig"); const link = @import("../../link.zig"); @@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace; const Allocator = mem.Allocator; -const Cache = @import("../../Cache.zig"); const Coff = @import("../Coff.zig"); const Compilation = @import("../../Compilation.zig"); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 45952da6c0..37ebfdc0dc 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace; const Air = @import("../Air.zig"); const Allocator = std.mem.Allocator; pub const Atom = @import("Elf/Atom.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const Compilation = @import("../Compilation.zig"); const Dwarf = @import("Dwarf.zig"); const File = link.File; diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 24ef275c5b..35f5f1b562 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -28,7 +28,7 @@ const Air = @import("../Air.zig"); const Allocator = mem.Allocator; const Archive = @import("MachO/Archive.zig"); pub const Atom = @import("MachO/Atom.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const CodeSignature = @import("MachO/CodeSignature.zig"); const Compilation = @import("../Compilation.zig"); const Dwarf = File.Dwarf; diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 095ac9b5ce..785fa71445 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace; const Allocator = mem.Allocator; const Archive = @import("Archive.zig"); const Atom = @import("ZldAtom.zig"); -const Cache = @import("../../Cache.zig"); +const Cache = std.Build.Cache; const CodeSignature = @import("CodeSignature.zig"); const Compilation = @import("../../Compilation.zig"); const DwarfInfo = @import("DwarfInfo.zig"); diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 9d20412788..e62a2050d7 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain; const trace = @import("../tracy.zig").trace; const build_options = @import("build_options"); const wasi_libc = @import("../wasi_libc.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const Type = @import("../type.zig").Type; const TypedValue = @import("../TypedValue.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; diff --git a/src/main.zig b/src/main.zig index 00a7b126c8..2add2f9165 100644 --- a/src/main.zig +++ b/src/main.zig @@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const wasi_libc = @import("wasi_libc.zig"); const translate_c = @import("translate_c.zig"); const clang = @import("clang.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const target_util = @import("target.zig"); const ThreadPool = @import("ThreadPool.zig"); const crash_report = @import("crash_report.zig"); @@ -3607,7 +3607,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void defer if (enable_cache) man.deinit(); man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects - man.hashCSource(c_source_file) catch |err| { + Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| { fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) }); }; diff --git a/src/mingw.zig b/src/mingw.zig index 06880743c6..4f94e26a98 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw); const builtin = @import("builtin"); const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; pub const CRTFile = enum { crt2_o, -- cgit v1.2.3 From c8dc00086e6622f3f29d7e18deb92ea102cf1074 Mon Sep 17 00:00:00 2001 From: AdamGoertz <36753247+AdamGoertz@users.noreply.github.com> Date: Mon, 13 Feb 2023 09:23:13 -0500 Subject: Add -ferror-tracing and -fno-error-tracing compile options --- lib/std/Build.zig | 2 +- src/Compilation.zig | 6 ++++-- src/main.zig | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 817073f867..f481290fa2 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -906,7 +906,7 @@ pub fn standardOptimizeOption(self: *Build, options: StandardOptimizeOptionOptio return self.option( std.builtin.Mode, "optimize", - "prioritize performance, safety, or binary size (-O flag)", + "Prioritize performance, safety, or binary size (-O flag)", ) orelse .Debug; } } diff --git a/src/Compilation.zig b/src/Compilation.zig index 18d0e46892..7508b13d24 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1029,6 +1029,7 @@ pub const InitOptions = struct { /// This is for stage1 and should be deleted upon completion of self-hosting. color: Color = .auto, reference_trace: ?u32 = null, + error_tracing: ?bool = null, test_filter: ?[]const u8 = null, test_name_prefix: ?[]const u8 = null, test_runner_path: ?[]const u8 = null, @@ -1715,8 +1716,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const error_return_tracing = !strip and switch (options.optimize_mode) { .Debug, .ReleaseSafe => (!options.target.isWasm() or options.target.os.tag == .emscripten) and - !options.target.cpu.arch.isBpf(), - .ReleaseFast, .ReleaseSmall => false, + !options.target.cpu.arch.isBpf() and (options.error_tracing orelse true), + .ReleaseFast => options.error_tracing orelse false, + .ReleaseSmall => false, }; // For resource management purposes. diff --git a/src/main.zig b/src/main.zig index 00a7b126c8..10b3de399d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -432,6 +432,8 @@ const usage_build_generic = \\ -fno-Clang Prevent using Clang as the C/C++ compilation backend \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error \\ -fno-reference-trace Disable reference trace + \\ -ferror-tracing Enable error tracing in ReleaseFast mode + \\ -fno-error-tracing Disable error tracing in Debug and ReleaseSafe mode \\ -fsingle-threaded Code assumes there is only one thread \\ -fno-single-threaded Code may not assume there is only one thread \\ -fbuiltin Enable implicit builtin knowledge of functions @@ -797,6 +799,7 @@ fn buildOutputType( var headerpad_max_install_names: bool = false; var dead_strip_dylibs: bool = false; var reference_trace: ?u32 = null; + var error_tracing: ?bool = null; var pdb_out_path: ?[]const u8 = null; // e.g. -m3dnow or -mno-outline-atomics. They correspond to std.Target llvm cpu feature names. @@ -1218,6 +1221,10 @@ fn buildOutputType( }; } else if (mem.eql(u8, arg, "-fno-reference-trace")) { reference_trace = null; + } else if (mem.eql(u8, arg, "-ferror-tracing")) { + error_tracing = true; + } else if (mem.eql(u8, arg, "-fno-error-tracing")) { + error_tracing = false; } else if (mem.eql(u8, arg, "-rdynamic")) { rdynamic = true; } else if (mem.eql(u8, arg, "-fsoname")) { @@ -3136,6 +3143,7 @@ fn buildOutputType( .headerpad_max_install_names = headerpad_max_install_names, .dead_strip_dylibs = dead_strip_dylibs, .reference_trace = reference_trace, + .error_tracing = error_tracing, .pdb_out_path = pdb_out_path, }) catch |err| switch (err) { error.LibCUnavailable => { -- cgit v1.2.3 From 47e14b7ffbe02c800ac4c2b4f181ab6c7f022988 Mon Sep 17 00:00:00 2001 From: Tom Read Cutting Date: Sun, 12 Feb 2023 10:48:39 +0000 Subject: Zld: Report archive file with cpu arch mismatch This is just a simple/hacky feature to report the source of a linking error. I found this helpful in fixing-up some of my libs when recently switching from an x86_64 to aarch64 device, so thought it might be useful to others a well before zld has a fully featured error reporting system. --- src/link/MachO/zld.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 785fa71445..a94a0828fc 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -1065,7 +1065,13 @@ pub const Zld = struct { assert(offsets.items.len > 0); const object_id = @intCast(u16, self.objects.items.len); - const object = try archive.parseObject(gpa, cpu_arch, offsets.items[0]); + const object = archive.parseObject(gpa, cpu_arch, offsets.items[0]) catch |e| switch (e) { + error.MismatchedCpuArchitecture => { + log.err("CPU architecture mismatch found in {s}", .{archive.name}); + return e; + }, + else => return e, + }; try self.objects.append(gpa, object); try self.resolveSymbolsInObject(object_id, resolver); -- cgit v1.2.3 From 4e6f21e2cb2c557b5c019f4acf445665a26edcba Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 14 Feb 2023 11:42:49 +0100 Subject: comp: reinstate -fcompiler-rt when used with build-obj as output When the following is specified ``` $ zig build-obj -fcompiler-rt example.zig ``` the resulting relocatable object file will have the compiler-rt unconditionally embedded inside. ``` $ nm example.o ... 0000000012345678 W __truncsfhf2 ... ``` --- src/Compilation.zig | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'src') diff --git a/src/Compilation.zig b/src/Compilation.zig index 5e408554d5..c6737ed3eb 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1635,10 +1635,25 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { } else main_pkg; errdefer if (options.is_test) root_pkg.destroy(gpa); + const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: { + break :compiler_rt_pkg try Package.createWithDir( + gpa, + "compiler_rt", + options.zig_lib_directory, + null, + "compiler_rt.zig", + ); + } else null; + errdefer if (compiler_rt_pkg) |p| p.destroy(gpa); + try main_pkg.addAndAdopt(gpa, builtin_pkg); try main_pkg.add(gpa, root_pkg); try main_pkg.addAndAdopt(gpa, std_pkg); + if (compiler_rt_pkg) |p| { + try main_pkg.addAndAdopt(gpa, p); + } + const main_pkg_is_std = m: { const std_path = try std.fs.path.resolve(arena, &[_][]const u8{ std_pkg.root_src_directory.path orelse ".", @@ -2355,6 +2370,10 @@ pub fn update(comp: *Compilation) !void { _ = try module.importPkg(module.main_pkg); } + if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { + _ = try module.importPkg(compiler_rt_pkg); + } + // Put a work item in for every known source file to detect if // it changed, and, if so, re-compute ZIR and then queue the job // to update it. @@ -2379,6 +2398,10 @@ pub fn update(comp: *Compilation) !void { if (comp.bin_file.options.is_test) { try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg }); } + + if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { + try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg }); + } } // If the terminal is dumb, we dont want to show the user all the output. -- cgit v1.2.3 From 7199d7c77715fe06606c5c89595e6852b3fa8c20 Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Mon, 13 Feb 2023 16:19:17 +0200 Subject: split `@qualCast` into `@constCast` and `@volatileCast` --- doc/langref.html.in | 24 ++++++--- lib/docs/main.js | 8 ++- lib/std/child_process.zig | 4 +- lib/std/fs.zig | 2 +- lib/std/mem/Allocator.zig | 4 +- lib/std/os.zig | 2 +- lib/std/os/windows.zig | 14 ++--- lib/std/zig/c_translation.zig | 6 ++- src/AstGen.zig | 18 ++++++- src/Autodoc.zig | 3 +- src/BuiltinFn.zig | 24 ++++++--- src/Sema.zig | 60 ++++++++++----------- src/Zir.zig | 12 ++--- src/print_zir.zig | 3 +- stage1/zig1.wasm | Bin 2373912 -> 2377691 bytes test/behavior/cast.zig | 15 ++++++ test/cases/compile_errors/invalid_qualcast.zig | 12 ----- .../ptrCast_discards_const_qualifier.zig | 2 +- 18 files changed, 126 insertions(+), 87 deletions(-) delete mode 100644 test/cases/compile_errors/invalid_qualcast.zig (limited to 'src') diff --git a/doc/langref.html.in b/doc/langref.html.in index 9490c4e69d..a74d06ccbf 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -8129,6 +8129,13 @@ test "main" { {#code_end#} {#header_close#} + {#header_open|@constCast#} + 
{#syntax#}@constCast(value: anytype) DestType{#endsyntax#}
+

+ Remove {#syntax#}const{#endsyntax#} qualifier from a pointer. +

+ {#header_close#} + {#header_open|@ctz#} 
{#syntax#}@ctz(operand: anytype){#endsyntax#}

{#syntax#}@TypeOf(operand){#endsyntax#} must be an integer type or an integer vector type.

@@ -8814,7 +8821,8 @@ pub const PrefetchOptions = struct { {#syntax#}@ptrCast{#endsyntax#} cannot be used for:

    -
  • Removing {#syntax#}const{#endsyntax#} or {#syntax#}volatile{#endsyntax#} qualifier, use {#link|@qualCast#}.
    • +
  • Removing {#syntax#}const{#endsyntax#} qualifier, use {#link|@constCast#}.
    • +
  • Removing {#syntax#}volatile{#endsyntax#} qualifier, use {#link|@volatileCast#}.
  • Changing pointer address space, use {#link|@addrSpaceCast#}.
  • Increasing pointer alignment, use {#link|@alignCast#}.
  • Casting a non-slice pointer to a slice, use slicing syntax {#syntax#}ptr[start..end]{#endsyntax#}.
    • @@ -8831,13 +8839,6 @@ pub const PrefetchOptions = struct { {#header_close#} - {#header_open|@qualCast#} - 
    {#syntax#}@qualCast(comptime DestType: type, value: anytype) DestType{#endsyntax#}
    -

    - Remove {#syntax#}const{#endsyntax#} or {#syntax#}volatile{#endsyntax#} qualifier from a pointer. -

    - {#header_close#} - {#header_open|@rem#} 
    {#syntax#}@rem(numerator: T, denominator: T) T{#endsyntax#}

    @@ -9525,6 +9526,13 @@ fn foo(comptime T: type, ptr: *T) T { 

    {#syntax#}@Vector(len: comptime_int, Element: type) type{#endsyntax#}

    Creates {#link|Vectors#}.

    {#header_close#} + + {#header_open|@volatileCast#} + 
    {#syntax#}@volatileCast(value: anytype) DestType{#endsyntax#}
    +

    + Remove {#syntax#}volatile{#endsyntax#} qualifier from a pointer. +

    + {#header_close#} {#header_close#} {#header_open|Build Mode#} diff --git a/lib/docs/main.js b/lib/docs/main.js index fae39c5fba..7a27f9db4f 100644 --- a/lib/docs/main.js +++ b/lib/docs/main.js @@ -1354,8 +1354,12 @@ const NAV_MODES = { payloadHtml += "ptrCast"; break; } - case "qual_cast": { - payloadHtml += "qualCast"; + case "const_cast": { + payloadHtml += "constCast"; + break; + } + case "volatile_cast": { + payloadHtml += "volatileCast"; break; } case "truncate": { diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 21d7b4fe3e..003e37d76b 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -1164,7 +1164,7 @@ fn windowsCreateProcessPathExt( var app_name_unicode_string = windows.UNICODE_STRING{ .Length = app_name_len_bytes, .MaximumLength = app_name_len_bytes, - .Buffer = @qualCast([*:0]u16, app_name_wildcard.ptr), + .Buffer = @constCast(app_name_wildcard.ptr), }; const rc = windows.ntdll.NtQueryDirectoryFile( dir.fd, @@ -1261,7 +1261,7 @@ fn windowsCreateProcessPathExt( var app_name_unicode_string = windows.UNICODE_STRING{ .Length = app_name_len_bytes, .MaximumLength = app_name_len_bytes, - .Buffer = @qualCast([*:0]u16, app_name_appended.ptr), + .Buffer = @constCast(app_name_appended.ptr), }; // Re-use the directory handle but this time we call with the appended app name diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 2300ad044a..52a93a498f 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1763,7 +1763,7 @@ pub const Dir = struct { var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @qualCast([*:0]u16, sub_path_w), + .Buffer = @constCast(sub_path_w), }; var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), diff --git a/lib/std/mem/Allocator.zig b/lib/std/mem/Allocator.zig index 6e56607865..63e5e9a1d5 100644 --- a/lib/std/mem/Allocator.zig +++ b/lib/std/mem/Allocator.zig @@ -112,7 +112,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void { const info = @typeInfo(@TypeOf(ptr)).Pointer; const T = info.child; if (@sizeOf(T) == 0) return; - const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr)); + const non_const_ptr = @ptrCast([*]u8, @constCast(ptr)); self.rawFree(non_const_ptr[0..@sizeOf(T)], math.log2(info.alignment), @returnAddress()); } @@ -297,7 +297,7 @@ pub fn free(self: Allocator, memory: anytype) void { const bytes = mem.sliceAsBytes(memory); const bytes_len = bytes.len + if (Slice.sentinel != null) @sizeOf(Slice.child) else 0; if (bytes_len == 0) return; - const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr)); + const non_const_ptr = @constCast(bytes.ptr); // TODO: https://github.com/ziglang/zig/issues/4298 @memset(non_const_ptr, undefined, bytes_len); self.rawFree(non_const_ptr[0..bytes_len], log2a(Slice.alignment), @returnAddress()); diff --git a/lib/std/os.zig b/lib/std/os.zig index 3cee30c32d..c5eeb34b1c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4513,7 +4513,7 @@ pub fn faccessatW(dirfd: fd_t, sub_path_w: [*:0]const u16, mode: u32, flags: u32 var nt_name = windows.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @qualCast([*:0]u16, sub_path_w), + .Buffer = @constCast(sub_path_w), }; var attr = windows.OBJECT_ATTRIBUTES{ .Length = @sizeOf(windows.OBJECT_ATTRIBUTES), diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 93e762827b..711bc9f349 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -85,7 +85,7 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN var nt_name = UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @qualCast([*]u16, sub_path_w.ptr), + .Buffer = @constCast(sub_path_w.ptr), }; var attr = OBJECT_ATTRIBUTES{ .Length = @sizeOf(OBJECT_ATTRIBUTES), @@ -634,7 +634,7 @@ pub fn SetCurrentDirectory(path_name: []const u16) SetCurrentDirectoryError!void var nt_name = UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @qualCast([*]u16, path_name.ptr), + .Buffer = @constCast(path_name.ptr), }; const rc = ntdll.RtlSetCurrentDirectory_U(&nt_name); @@ -766,7 +766,7 @@ pub fn ReadLink(dir: ?HANDLE, sub_path_w: []const u16, out_buffer: []u8) ReadLin var nt_name = UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, - .Buffer = @qualCast([*]u16, sub_path_w.ptr), + .Buffer = @constCast(sub_path_w.ptr), }; var attr = OBJECT_ATTRIBUTES{ .Length = @sizeOf(OBJECT_ATTRIBUTES), @@ -876,7 +876,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil .Length = path_len_bytes, .MaximumLength = path_len_bytes, // The Windows API makes this mutable, but it will not mutate here. - .Buffer = @qualCast([*]u16, sub_path_w.ptr), + .Buffer = @constCast(sub_path_w.ptr), }; if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { @@ -1414,7 +1414,7 @@ pub fn sendmsg( } pub fn sendto(s: ws2_32.SOCKET, buf: [*]const u8, len: usize, flags: u32, to: ?*const ws2_32.sockaddr, to_len: ws2_32.socklen_t) i32 { - var buffer = ws2_32.WSABUF{ .len = @truncate(u31, len), .buf = @qualCast([*]u8, buf) }; + var buffer = ws2_32.WSABUF{ .len = @truncate(u31, len), .buf = @constCast(buf) }; var bytes_send: DWORD = undefined; if (ws2_32.WSASendTo(s, @ptrCast([*]ws2_32.WSABUF, &buffer), 1, &bytes_send, flags, to, @intCast(i32, to_len), null, null) == ws2_32.SOCKET_ERROR) { return ws2_32.SOCKET_ERROR; @@ -1876,13 +1876,13 @@ pub fn eqlIgnoreCaseWTF16(a: []const u16, b: []const u16) bool { const a_string = UNICODE_STRING{ .Length = a_bytes, .MaximumLength = a_bytes, - .Buffer = @qualCast([*]u16, a.ptr), + .Buffer = @constCast(a.ptr), }; const b_bytes = @intCast(u16, b.len * 2); const b_string = UNICODE_STRING{ .Length = b_bytes, .MaximumLength = b_bytes, - .Buffer = @qualCast([*]u16, b.ptr), + .Buffer = @constCast(b.ptr), }; return ntdll.RtlEqualUnicodeString(&a_string, &b_string, TRUE) == TRUE; } diff --git a/lib/std/zig/c_translation.zig b/lib/std/zig/c_translation.zig index d33c74d777..6e95ab53ab 100644 --- a/lib/std/zig/c_translation.zig +++ b/lib/std/zig/c_translation.zig @@ -74,8 +74,10 @@ fn castPtr(comptime DestType: type, target: anytype) DestType { const dest = ptrInfo(DestType); const source = ptrInfo(@TypeOf(target)); - if (source.is_const and !dest.is_const or source.is_volatile and !dest.is_volatile) - return @qualCast(DestType, target) + if (source.is_const and !dest.is_const) + return @constCast(target) + else if (source.is_volatile and !dest.is_volatile) + return @volatileCast(target) else if (@typeInfo(dest.child) == .Opaque) // dest.alignment would error out return @ptrCast(DestType, target) diff --git a/src/AstGen.zig b/src/AstGen.zig index 76a681abd3..40eef32d4e 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -2531,7 +2531,6 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As .bit_size_of, .typeof_log2_int_type, .ptr_to_int, - .qual_cast, .align_of, .bool_to_int, .embed_file, @@ -8039,7 +8038,6 @@ fn builtinCall( .float_cast => return typeCast(gz, scope, ri, node, params[0], params[1], .float_cast), .int_cast => return typeCast(gz, scope, ri, node, params[0], params[1], .int_cast), .ptr_cast => return typeCast(gz, scope, ri, node, params[0], params[1], .ptr_cast), - .qual_cast => return typeCast(gz, scope, ri, node, params[0], params[1], .qual_cast), .truncate => return typeCast(gz, scope, ri, node, params[0], params[1], .truncate), // zig fmt: on @@ -8115,6 +8113,22 @@ fn builtinCall( }); return rvalue(gz, ri, result, node); }, + .const_cast => { + const operand = try expr(gz, scope, .{ .rl = .none }, params[0]); + const result = try gz.addExtendedPayload(.const_cast, Zir.Inst.UnNode{ + .node = gz.nodeIndexToRelative(node), + .operand = operand, + }); + return rvalue(gz, ri, result, node); + }, + .volatile_cast => { + const operand = try expr(gz, scope, .{ .rl = .none }, params[0]); + const result = try gz.addExtendedPayload(.volatile_cast, Zir.Inst.UnNode{ + .node = gz.nodeIndexToRelative(node), + .operand = operand, + }); + return rvalue(gz, ri, result, node); + }, // zig fmt: off .has_decl => return hasDeclOrField(gz, scope, ri, node, params[0], params[1], .has_decl), diff --git a/src/Autodoc.zig b/src/Autodoc.zig index 0c2c39bbcc..44325e3836 100644 --- a/src/Autodoc.zig +++ b/src/Autodoc.zig @@ -1400,7 +1400,6 @@ fn walkInstruction( .float_cast, .int_cast, .ptr_cast, - .qual_cast, .truncate, .align_cast, .has_decl, @@ -2983,6 +2982,8 @@ fn walkInstruction( .error_to_int, .int_to_error, .reify, + .const_cast, + .volatile_cast, => { const extra = file.zir.extraData(Zir.Inst.UnNode, extended.operand).data; const bin_index = self.exprs.items.len; diff --git a/src/BuiltinFn.zig b/src/BuiltinFn.zig index 80eb739185..20edbabe47 100644 --- a/src/BuiltinFn.zig +++ b/src/BuiltinFn.zig @@ -28,6 +28,7 @@ pub const Tag = enum { cmpxchg_weak, compile_error, compile_log, + const_cast, ctz, c_undef, c_va_arg, @@ -75,7 +76,6 @@ pub const Tag = enum { prefetch, ptr_cast, ptr_to_int, - qual_cast, rem, return_address, select, @@ -116,6 +116,7 @@ pub const Tag = enum { TypeOf, union_init, Vector, + volatile_cast, }; pub const MemLocRequirement = enum { @@ -345,6 +346,13 @@ pub const list = list: { .param_count = null, }, }, + .{ + "@constCast", + .{ + .tag = .const_cast, + .param_count = 1, + }, + }, .{ "@ctz", .{ @@ -675,13 +683,6 @@ pub const list = list: { .param_count = 1, }, }, - .{ - "@qualCast", - .{ - .tag = .qual_cast, - .param_count = 2, - }, - }, .{ "@rem", .{ @@ -964,5 +965,12 @@ pub const list = list: { .param_count = 2, }, }, + .{ + "@volatileCast", + .{ + .tag = .volatile_cast, + .param_count = 1, + }, + }, }); }; diff --git a/src/Sema.zig b/src/Sema.zig index 9eef07648d..cf6350e35f 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -1015,7 +1015,6 @@ fn analyzeBodyInner( .float_cast => try sema.zirFloatCast(block, inst), .int_cast => try sema.zirIntCast(block, inst), .ptr_cast => try sema.zirPtrCast(block, inst), - .qual_cast => try sema.zirQualCast(block, inst), .truncate => try sema.zirTruncate(block, inst), .align_cast => try sema.zirAlignCast(block, inst), .has_decl => try sema.zirHasDecl(block, inst), @@ -1147,6 +1146,8 @@ fn analyzeBodyInner( .c_va_copy => try sema.zirCVaCopy( block, extended), .c_va_end => try sema.zirCVaEnd( block, extended), .c_va_start => try sema.zirCVaStart( block, extended), + .const_cast, => try sema.zirConstCast( block, extended), + .volatile_cast, => try sema.zirVolatileCast( block, extended), // zig fmt: on .fence => { @@ -19545,7 +19546,7 @@ fn zirPtrCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air const msg = try sema.errMsg(block, src, "cast discards const qualifier", .{}); errdefer msg.destroy(sema.gpa); - try sema.errNote(block, src, msg, "consider using '@qualCast'", .{}); + try sema.errNote(block, src, msg, "consider using '@constCast'", .{}); break :msg msg; }; return sema.failWithOwnedErrorMsg(msg); @@ -19555,7 +19556,7 @@ fn zirPtrCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air const msg = try sema.errMsg(block, src, "cast discards volatile qualifier", .{}); errdefer msg.destroy(sema.gpa); - try sema.errNote(block, src, msg, "consider using '@qualCast'", .{}); + try sema.errNote(block, src, msg, "consider using '@volatileCast'", .{}); break :msg msg; }; return sema.failWithOwnedErrorMsg(msg); @@ -19660,40 +19661,37 @@ fn zirPtrCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air return block.addBitCast(aligned_dest_ty, ptr); } -fn zirQualCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref { - const inst_data = sema.code.instructions.items(.data)[inst].pl_node; - const src = inst_data.src(); - const dest_ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node }; - const operand_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node }; - const extra = sema.code.extraData(Zir.Inst.Bin, inst_data.payload_index).data; - const dest_ty = try sema.resolveType(block, dest_ty_src, extra.lhs); - const operand = try sema.resolveInst(extra.rhs); +fn zirConstCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref { + const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data; + const src = LazySrcLoc.nodeOffset(extra.node); + const operand_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node }; + const operand = try sema.resolveInst(extra.operand); const operand_ty = sema.typeOf(operand); - - try sema.checkPtrType(block, dest_ty_src, dest_ty); try sema.checkPtrOperand(block, operand_src, operand_ty); - var operand_payload = operand_ty.ptrInfo(); - var dest_info = dest_ty.ptrInfo(); + var ptr_info = operand_ty.ptrInfo().data; + ptr_info.mutable = true; + const dest_ty = try Type.ptr(sema.arena, sema.mod, ptr_info); + + if (try sema.resolveMaybeUndefVal(operand)) |operand_val| { + return sema.addConstant(dest_ty, operand_val); + } - operand_payload.data.mutable = dest_info.data.mutable; - operand_payload.data.@"volatile" = dest_info.data.@"volatile"; + try sema.requireRuntimeBlock(block, src, null); + return block.addBitCast(dest_ty, operand); +} - const altered_operand_ty = Type.initPayload(&operand_payload.base); - if (!altered_operand_ty.eql(dest_ty, sema.mod)) { - const msg = msg: { - const msg = try sema.errMsg(block, src, "'@qualCast' can only modify 'const' and 'volatile' qualifiers", .{}); - errdefer msg.destroy(sema.gpa); +fn zirVolatileCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref { + const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data; + const src = LazySrcLoc.nodeOffset(extra.node); + const operand_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node }; + const operand = try sema.resolveInst(extra.operand); + const operand_ty = sema.typeOf(operand); + try sema.checkPtrOperand(block, operand_src, operand_ty); - dest_info.data.mutable = !operand_ty.isConstPtr(); - dest_info.data.@"volatile" = operand_ty.isVolatilePtr(); - const altered_dest_ty = Type.initPayload(&dest_info.base); - try sema.errNote(block, src, msg, "expected type '{}'", .{altered_dest_ty.fmt(sema.mod)}); - try sema.errNote(block, src, msg, "got type '{}'", .{operand_ty.fmt(sema.mod)}); - break :msg msg; - }; - return sema.failWithOwnedErrorMsg(msg); - } + var ptr_info = operand_ty.ptrInfo().data; + ptr_info.@"volatile" = false; + const dest_ty = try Type.ptr(sema.arena, sema.mod, ptr_info); if (try sema.resolveMaybeUndefVal(operand)) |operand_val| { return sema.addConstant(dest_ty, operand_val); diff --git a/src/Zir.zig b/src/Zir.zig index b93422177e..58f9fdff14 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -857,9 +857,6 @@ pub const Inst = struct { /// Implements the `@ptrCast` builtin. /// Uses `pl_node` with payload `Bin`. `lhs` is dest type, `rhs` is operand. ptr_cast, - /// Implements the `@qualCast` builtin. - /// Uses `pl_node` with payload `Bin`. `lhs` is dest type, `rhs` is operand. - qual_cast, /// Implements the `@truncate` builtin. /// Uses `pl_node` with payload `Bin`. `lhs` is dest type, `rhs` is operand. truncate, @@ -1198,7 +1195,6 @@ pub const Inst = struct { .float_cast, .int_cast, .ptr_cast, - .qual_cast, .truncate, .align_cast, .has_field, @@ -1488,7 +1484,6 @@ pub const Inst = struct { .float_cast, .int_cast, .ptr_cast, - .qual_cast, .truncate, .align_cast, .has_field, @@ -1760,7 +1755,6 @@ pub const Inst = struct { .float_cast = .pl_node, .int_cast = .pl_node, .ptr_cast = .pl_node, - .qual_cast = .pl_node, .truncate = .pl_node, .align_cast = .pl_node, .typeof_builtin = .pl_node, @@ -2004,6 +1998,12 @@ pub const Inst = struct { /// Implement builtin `@cVaStart`. /// `operand` is `src_node: i32`. c_va_start, + /// Implements the `@constCast` builtin. + /// `operand` is payload index to `UnNode`. + const_cast, + /// Implements the `@volatileCast` builtin. + /// `operand` is payload index to `UnNode`. + volatile_cast, pub const InstData = struct { opcode: Extended, diff --git a/src/print_zir.zig b/src/print_zir.zig index e5fc8815ed..8d97000582 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -332,7 +332,6 @@ const Writer = struct { .float_cast, .int_cast, .ptr_cast, - .qual_cast, .truncate, .align_cast, .div_exact, @@ -507,6 +506,8 @@ const Writer = struct { .reify, .c_va_copy, .c_va_end, + .const_cast, + .volatile_cast, => { const inst_data = self.code.extraData(Zir.Inst.UnNode, extended.operand).data; const src = LazySrcLoc.nodeOffset(inst_data.node); diff --git a/stage1/zig1.wasm b/stage1/zig1.wasm index ef7d7891c9..9a3b25eb30 100644 Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig index 30889aef04..16f3c6e2dd 100644 --- a/test/behavior/cast.zig +++ b/test/behavior/cast.zig @@ -1553,3 +1553,18 @@ test "peer type resolution forms error union" { } else @intCast(u32, foo); try expect(try result == 123); } + +test "@constCast without a result location" { + const x: i32 = 1234; + const y = @constCast(&x); + try expect(@TypeOf(y) == *i32); + try expect(y.* == 1234); +} + +test "@volatileCast without a result location" { + var x: i32 = 1234; + var y: *volatile i32 = &x; + const z = @volatileCast(y); + try expect(@TypeOf(z) == *i32); + try expect(z.* == 1234); +} diff --git a/test/cases/compile_errors/invalid_qualcast.zig b/test/cases/compile_errors/invalid_qualcast.zig deleted file mode 100644 index 20b223b727..0000000000 --- a/test/cases/compile_errors/invalid_qualcast.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub export fn entry() void { - var a: [*:0]const volatile u16 = undefined; - _ = @qualCast([*]u16, a); -} - -// error -// backend=stage2 -// target=native -// -// :3:9: error: '@qualCast' can only modify 'const' and 'volatile' qualifiers -// :3:9: note: expected type '[*]const volatile u16' -// :3:9: note: got type '[*:0]const volatile u16' diff --git a/test/cases/compile_errors/ptrCast_discards_const_qualifier.zig b/test/cases/compile_errors/ptrCast_discards_const_qualifier.zig index eedef01234..f27f5f4f93 100644 --- a/test/cases/compile_errors/ptrCast_discards_const_qualifier.zig +++ b/test/cases/compile_errors/ptrCast_discards_const_qualifier.zig @@ -9,4 +9,4 @@ export fn entry() void { // target=native // // :3:15: error: cast discards const qualifier -// :3:15: note: consider using '@qualCast' +// :3:15: note: consider using '@constCast' -- cgit v1.2.3 From 4c7ca20da56d86636dfcc35a6982ad10b147cece Mon Sep 17 00:00:00 2001 From: Motiejus Jakštys Date: Thu, 23 Jun 2022 06:41:30 +0300 Subject: [linker] add --no-undefined, -z undefs Refs https://github.com/zigchroot/zig-chroot/issues/1 --- src/main.zig | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index cacdb83334..b4a88ddc66 100644 --- a/src/main.zig +++ b/src/main.zig @@ -488,10 +488,12 @@ const usage_build_generic = \\ -fno-build-id (default) Saves a bit of time linking \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker \\ --emit-relocs Enable output of relocation sections for post build tools + \\ --no-undefined Alias of -z defs \\ -z [arg] Set linker extension flags \\ nodelete Indicate that the object cannot be deleted from a process \\ notext Permit read-only relocations in read-only segments \\ defs Force a fatal error if any undefined symbols remain + \\ undefs Reverse of -z defs \\ origin Indicate that the object must have its origin processed \\ nocopyreloc Disable the creation of copy relocations \\ now (default) Force all relocations to be processed on load @@ -1333,6 +1335,8 @@ fn buildOutputType( linker_z_notext = true; } else if (mem.eql(u8, z_arg, "defs")) { linker_z_defs = true; + } else if (mem.eql(u8, z_arg, "undefs")) { + linker_z_defs = false; } else if (mem.eql(u8, z_arg, "origin")) { linker_z_origin = true; } else if (mem.eql(u8, z_arg, "nocopyreloc")) { @@ -1879,6 +1883,8 @@ fn buildOutputType( linker_gc_sections = true; } else if (mem.eql(u8, arg, "-dead_strip_dylibs")) { dead_strip_dylibs = true; + } else if (mem.eql(u8, arg, "--no-undefined")) { + linker_z_defs = true; } else if (mem.eql(u8, arg, "--gc-sections")) { linker_gc_sections = true; } else if (mem.eql(u8, arg, "--no-gc-sections")) { @@ -1944,6 +1950,8 @@ fn buildOutputType( linker_z_notext = true; } else if (mem.eql(u8, z_arg, "defs")) { linker_z_defs = true; + } else if (mem.eql(u8, z_arg, "undefs")) { + linker_z_defs = false; } else if (mem.eql(u8, z_arg, "origin")) { linker_z_origin = true; } else if (mem.eql(u8, z_arg, "nocopyreloc")) { -- cgit v1.2.3 From 61da9a25b97289f0b1820ae2eae60066a7a871d8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 16 Feb 2023 16:35:06 -0700 Subject: CLI: remove option from usage text since it doesn't work --- src/main.zig | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index b4a88ddc66..2e83e98fff 100644 --- a/src/main.zig +++ b/src/main.zig @@ -488,7 +488,6 @@ const usage_build_generic = \\ -fno-build-id (default) Saves a bit of time linking \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker \\ --emit-relocs Enable output of relocation sections for post build tools - \\ --no-undefined Alias of -z defs \\ -z [arg] Set linker extension flags \\ nodelete Indicate that the object cannot be deleted from a process \\ notext Permit read-only relocations in read-only segments -- cgit v1.2.3 From b1b944a683a2143584055468ca7958d298a9742b Mon Sep 17 00:00:00 2001 From: Motiejus Jakštys Date: Thu, 23 Jun 2022 08:28:35 +0300 Subject: [elf linker] add --sort-section Tested by: created a "hello world" C file and compiled with: zig cc -v main.c -Wl,--sort-section=name -o main ... and verified the `--sort-section=name` is passed to ld.lld. Refs https://github.com/zigchroot/zig-chroot/issues/1 --- src/Compilation.zig | 3 +++ src/link.zig | 3 +++ src/link/Elf.zig | 6 ++++++ src/main.zig | 12 ++++++++++++ 4 files changed, 24 insertions(+) (limited to 'src') diff --git a/src/Compilation.zig b/src/Compilation.zig index c6737ed3eb..ce0bfb9908 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -989,6 +989,7 @@ pub const InitOptions = struct { linker_optimization: ?u8 = null, linker_compress_debug_sections: ?link.CompressDebugSections = null, linker_module_definition_file: ?[]const u8 = null, + linker_sort_section: ?link.SortSection = null, major_subsystem_version: ?u32 = null, minor_subsystem_version: ?u32 = null, clang_passthrough_mode: bool = false, @@ -1853,6 +1854,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .bind_global_refs_locally = options.linker_bind_global_refs_locally orelse false, .compress_debug_sections = options.linker_compress_debug_sections orelse .none, .module_definition_file = options.linker_module_definition_file, + .sort_section = options.linker_sort_section, .import_memory = options.linker_import_memory orelse false, .import_symbols = options.linker_import_symbols, .import_table = options.linker_import_table, @@ -2684,6 +2686,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes man.hash.add(comp.bin_file.options.hash_style); man.hash.add(comp.bin_file.options.compress_debug_sections); man.hash.add(comp.bin_file.options.include_compiler_rt); + man.hash.addOptional(comp.bin_file.options.sort_section); if (comp.bin_file.options.link_libc) { man.hash.add(comp.bin_file.options.libc_installation != null); if (comp.bin_file.options.libc_installation) |libc_installation| { diff --git a/src/link.zig b/src/link.zig index 5650e0679a..4c4915441d 100644 --- a/src/link.zig +++ b/src/link.zig @@ -25,6 +25,8 @@ pub const SystemLib = struct { weak: bool = false, }; +pub const SortSection = enum { name, alignment }; + pub const CacheMode = enum { incremental, whole }; pub fn hashAddSystemLibs( @@ -159,6 +161,7 @@ pub const Options = struct { disable_lld_caching: bool, is_test: bool, hash_style: HashStyle, + sort_section: ?SortSection, major_subsystem_version: ?u32, minor_subsystem_version: ?u32, gc_sections: ?bool = null, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 37ebfdc0dc..d936a347cf 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1324,6 +1324,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v man.hash.addOptionalBytes(self.base.options.entry); man.hash.addOptional(self.base.options.image_base_override); man.hash.add(gc_sections); + man.hash.addOptional(self.base.options.sort_section); man.hash.add(self.base.options.eh_frame_hdr); man.hash.add(self.base.options.emit_relocs); man.hash.add(self.base.options.rdynamic); @@ -1488,6 +1489,11 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v try argv.append(linker_script); } + if (self.base.options.sort_section) |how| { + const arg = try std.fmt.allocPrint(arena, "--sort-section={s}", .{@tagName(how)}); + try argv.append(arg); + } + if (gc_sections) { try argv.append("--gc-sections"); } diff --git a/src/main.zig b/src/main.zig index 2e83e98fff..07a8d8b9e7 100644 --- a/src/main.zig +++ b/src/main.zig @@ -509,6 +509,7 @@ const usage_build_generic = \\ zlib Compression with deflate/inflate \\ --gc-sections Force removal of functions and data that are unreachable by the entry point or exported symbols \\ --no-gc-sections Don't force removal of unreachable functions and data + \\ --sort-section=[value] Sort wildcard section patterns by 'name' or 'alignment' \\ --subsystem [subsystem] (Windows) /SUBSYSTEM: to the linker \\ --stack [size] Override default stack size \\ --image-base [addr] Set base address for executable image @@ -731,6 +732,7 @@ fn buildOutputType( var linker_script: ?[]const u8 = null; var version_script: ?[]const u8 = null; var disable_c_depfile = false; + var linker_sort_section: ?link.SortSection = null; var linker_gc_sections: ?bool = null; var linker_compress_debug_sections: ?link.CompressDebugSections = null; var linker_allow_shlib_undefined: ?bool = null; @@ -1894,6 +1896,15 @@ fn buildOutputType( linker_print_icf_sections = true; } else if (mem.eql(u8, arg, "--print-map")) { linker_print_map = true; + } else if (mem.eql(u8, arg, "--sort-section")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{s}'", .{arg}); + } + const arg1 = linker_args.items[i]; + linker_sort_section = std.meta.stringToEnum(link.SortSection, arg1) orelse { + fatal("expected [name|alignment] after --sort-section, found '{s}'", .{arg1}); + }; } else if (mem.eql(u8, arg, "--allow-shlib-undefined") or mem.eql(u8, arg, "-allow-shlib-undefined")) { @@ -3070,6 +3081,7 @@ fn buildOutputType( .version_script = version_script, .disable_c_depfile = disable_c_depfile, .soname = resolved_soname, + .linker_sort_section = linker_sort_section, .linker_gc_sections = linker_gc_sections, .linker_allow_shlib_undefined = linker_allow_shlib_undefined, .linker_bind_global_refs_locally = linker_bind_global_refs_locally, -- cgit v1.2.3 From 705e9cb3caaa049336104662a833716757ffcb4a Mon Sep 17 00:00:00 2001 From: Motiejus Jakštys Date: Thu, 23 Jun 2022 08:52:35 +0300 Subject: [linker] ignore --sort-common From ld.lld(1): --sort-common This option is ignored for GNU compatibility. Refs https://github.com/zigchroot/zig-chroot/issues/1 --- src/main.zig | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index 07a8d8b9e7..fcb52e93e9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1619,6 +1619,10 @@ fn buildOutputType( build_id = true; warn("ignoring build-id style argument: '{s}'", .{value}); continue; + } else if (mem.eql(u8, key, "--sort-common")) { + // this ignores --sort=common=; ignoring plain --sort-common + // is done below. + continue; } try linker_args.append(key); try linker_args.append(value); @@ -1631,6 +1635,9 @@ fn buildOutputType( needed = true; } else if (mem.eql(u8, linker_arg, "-no-pie")) { want_pie = false; + } else if (mem.eql(u8, linker_arg, "--sort-common")) { + // from ld.lld(1): --sort-common is ignored for GNU compatibility, + // this ignores plain --sort-common } else if (mem.eql(u8, linker_arg, "--whole-archive") or mem.eql(u8, linker_arg, "-whole-archive")) { -- cgit v1.2.3 From d2650eb570c5494c728a0a780feb310abe2f1222 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 16 Feb 2023 17:19:26 -0700 Subject: CLI: detect linker color diagnostics flags --- src/main.zig | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index fcb52e93e9..9d5eb45c7d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2055,6 +2055,14 @@ fn buildOutputType( // This option does not do anything. } else if (mem.eql(u8, arg, "--export-all-symbols")) { rdynamic = true; + } else if (mem.eql(u8, arg, "--color-diagnostics") or + mem.eql(u8, arg, "--color-diagnostics=always")) + { + color = .on; + } else if (mem.eql(u8, arg, "--no-color-diagnostics") or + mem.eql(u8, arg, "--color-diagnostics=never")) + { + color = .off; } else if (mem.eql(u8, arg, "-s") or mem.eql(u8, arg, "--strip-all") or mem.eql(u8, arg, "-S") or mem.eql(u8, arg, "--strip-debug")) { -- cgit v1.2.3 From 3f7e9ff597a3514bb1c4f1900027c40682ac9f13 Mon Sep 17 00:00:00 2001 From: Motiejus Jakštys Date: Wed, 22 Jun 2022 12:12:32 +0300 Subject: [all linkers] fail hard on unsupported flags Currently `zig cc`, when confronted with a linker argument it does not understand, skips the flag and emits a warning. This has been causing headaches for people that build third-party software (including me). Zig seemingly builds and links the final executable, only to segfault when running it. If there are linker warnings when compiling software, the first thing we have to do is add support for ones linker is complaining, and only then go file issues. If zig "successfully" (i.e. status code = 0) compiles a binary, there is instead a tendency to blaim "zig doing something weird". (I am guilty of this.) In my experience, adding the unsupported arguments has been quite easy; see #11679, #11875, #11874 for recent examples. With the current ones (+ prerequisites below) I was able to build all of the CGo programs that I am encountering at $dayjob. CGo is a reasonable example, because it is exercising the unusual linker args quite a bit. Prerequisites: #11614 and #11863. --- src/main.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/main.zig b/src/main.zig index 9d5eb45c7d..a0cdfb36b6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1355,7 +1355,7 @@ fn buildOutputType( } else if (mem.startsWith(u8, z_arg, "max-page-size=")) { linker_z_max_page_size = parseIntSuffix(z_arg, "max-page-size=".len); } else { - warn("unsupported linker extension flag: -z {s}", .{z_arg}); + fatal("unsupported linker extension flag: -z {s}", .{z_arg}); } } else if (mem.eql(u8, arg, "--import-memory")) { linker_import_memory = true; @@ -1993,7 +1993,7 @@ fn buildOutputType( } else if (mem.startsWith(u8, z_arg, "max-page-size=")) { linker_z_max_page_size = parseIntSuffix(z_arg, "max-page-size=".len); } else { - warn("unsupported linker extension flag: -z {s}", .{z_arg}); + fatal("unsupported linker extension flag: -z {s}", .{z_arg}); } } else if (mem.eql(u8, arg, "--major-image-version")) { i += 1; @@ -2226,7 +2226,7 @@ fn buildOutputType( have_version = true; } else { - warn("unsupported linker arg: {s}", .{arg}); + fatal("unsupported linker arg: {s}", .{arg}); } } -- cgit v1.2.3