diff options
Diffstat (limited to 'src')
36 files changed, 7302 insertions, 2091 deletions
diff --git a/src/Compilation.zig b/src/Compilation.zig index cc922998c3..5bfabfdcfc 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -358,7 +358,10 @@ pub const CObject = struct { pub const Win32Resource = struct { /// Relative to cwd. Owned by arena. - src: RcSourceFile, + src: union(enum) { + rc: RcSourceFile, + manifest: []const u8, + }, status: union(enum) { new, success: struct { @@ -582,6 +585,7 @@ pub const InitOptions = struct { symbol_wrap_set: std.StringArrayHashMapUnmanaged(void) = .{}, c_source_files: []const CSourceFile = &[0]CSourceFile{}, rc_source_files: []const RcSourceFile = &[0]RcSourceFile{}, + manifest_file: ?[]const u8 = null, rc_includes: RcIncludes = .any, link_objects: []LinkObject = &[0]LinkObject{}, framework_dirs: []const []const u8 = &[0][]const u8{}, @@ -735,6 +739,7 @@ pub const InitOptions = struct { pdb_source_path: ?[]const u8 = null, /// (Windows) PDB output path pdb_out_path: ?[]const u8 = null, + error_limit: ?Module.ErrorInt = null, }; fn addModuleTableToCacheHash( @@ -1414,6 +1419,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .local_zir_cache = local_zir_cache, .emit_h = emit_h, .tmp_hack_arena = std.heap.ArenaAllocator.init(gpa), + .error_limit = options.error_limit orelse (std.math.maxInt(u16) - 1), }; try module.init(); @@ -1749,16 +1755,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { comp.c_object_table.putAssumeCapacityNoClobber(c_object, {}); } - // Add a `Win32Resource` for each `rc_source_files`. + // Add a `Win32Resource` for each `rc_source_files` and one for `manifest_file`. if (!build_options.only_core_functionality) { - try comp.win32_resource_table.ensureTotalCapacity(gpa, options.rc_source_files.len); + try comp.win32_resource_table.ensureTotalCapacity(gpa, options.rc_source_files.len + @intFromBool(options.manifest_file != null)); for (options.rc_source_files) |rc_source_file| { const win32_resource = try gpa.create(Win32Resource); errdefer gpa.destroy(win32_resource); win32_resource.* = .{ .status = .{ .new = {} }, - .src = rc_source_file, + .src = .{ .rc = rc_source_file }, + }; + comp.win32_resource_table.putAssumeCapacityNoClobber(win32_resource, {}); + } + if (options.manifest_file) |manifest_path| { + const win32_resource = try gpa.create(Win32Resource); + errdefer gpa.destroy(win32_resource); + + win32_resource.* = .{ + .status = .{ .new = {} }, + .src = .{ .manifest = manifest_path }, }; comp.win32_resource_table.putAssumeCapacityNoClobber(win32_resource, {}); } @@ -2458,6 +2474,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes man.hash.add(comp.bin_file.options.skip_linker_dependencies); man.hash.add(comp.bin_file.options.parent_compilation_link_libc); man.hash.add(mod.emit_h != null); + man.hash.add(mod.error_limit); } try man.addOptionalFile(comp.bin_file.options.linker_script); @@ -2477,8 +2494,15 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes if (!build_options.only_core_functionality) { for (comp.win32_resource_table.keys()) |key| { - _ = try man.addFile(key.src.src_path, null); - man.hash.addListOfBytes(key.src.extra_flags); + switch (key.src) { + .rc => |rc_src| { + _ = try man.addFile(rc_src.src_path, null); + man.hash.addListOfBytes(rc_src.extra_flags); + }, + .manifest => |manifest_path| { + _ = try man.addFile(manifest_path, null); + }, + } } } @@ -2831,6 +2855,10 @@ pub fn totalErrorCount(self: *Compilation) u32 { } } } + + if (module.global_error_set.entries.len - 1 > module.error_limit) { + total += 1; + } } // The "no entry point found" error only counts if there are no semantic analysis errors. @@ -2981,6 +3009,22 @@ pub fn getAllErrorsAlloc(self: *Compilation) !ErrorBundle { for (module.failed_exports.values()) |value| { try addModuleErrorMsg(module, &bundle, value.*); } + + const actual_error_count = module.global_error_set.entries.len - 1; + if (actual_error_count > module.error_limit) { + try bundle.addRootErrorMessage(.{ + .msg = try bundle.printString("module used more errors than possible: used {d}, max {d}", .{ + actual_error_count, module.error_limit, + }), + .notes_len = 1, + }); + const notes_start = try bundle.reserveNotes(1); + bundle.extra.items[notes_start] = @intFromEnum(try bundle.addErrorMessage(.{ + .msg = try bundle.printString("use '--error-limit {d}' to increase limit", .{ + actual_error_count, + }), + })); + } } if (bundle.root_list.items.len == 0) { @@ -4173,7 +4217,10 @@ fn reportRetryableWin32ResourceError( try bundle.addRootErrorMessage(.{ .msg = try bundle.printString("{s}", .{@errorName(err)}), .src_loc = try bundle.addSourceLocation(.{ - .src_path = try bundle.addString(win32_resource.src.src_path), + .src_path = try bundle.addString(switch (win32_resource.src) { + .rc => |rc_src| rc_src.src_path, + .manifest => |manifest_src| manifest_src, + }), .line = 0, .column = 0, .span_start = 0, @@ -4543,7 +4590,17 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const tracy_trace = trace(@src()); defer tracy_trace.end(); - log.debug("updating win32 resource: {s}", .{win32_resource.src.src_path}); + const src_path = switch (win32_resource.src) { + .rc => |rc_src| rc_src.src_path, + .manifest => |src_path| src_path, + }; + const src_basename = std.fs.path.basename(src_path); + + log.debug("updating win32 resource: {s}", .{src_path}); + + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); if (win32_resource.clearStatus(comp.gpa)) { // There was previous failure. @@ -4554,24 +4611,113 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 _ = comp.failed_win32_resources.swapRemove(win32_resource); } + win32_resource_prog_node.activate(); + var child_progress_node = win32_resource_prog_node.start(src_basename, 0); + child_progress_node.activate(); + defer child_progress_node.end(); + var man = comp.obtainWin32ResourceCacheManifest(); defer man.deinit(); - _ = try man.addFile(win32_resource.src.src_path, null); - man.hash.addListOfBytes(win32_resource.src.extra_flags); + // For .manifest files, we ultimately just want to generate a .res with + // the XML data as a RT_MANIFEST resource. This means we can skip preprocessing, + // include paths, CLI options, etc. + if (win32_resource.src == .manifest) { + _ = try man.addFile(src_path, null); - var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); - defer arena_allocator.deinit(); - const arena = arena_allocator.allocator(); + const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); - const rc_basename = std.fs.path.basename(win32_resource.src.src_path); + const digest = if (try man.hit()) man.final() else blk: { + // The digest only depends on the .manifest file, so we can + // get the digest now and write the .res directly to the cache + const digest = man.final(); - win32_resource_prog_node.activate(); - var child_progress_node = win32_resource_prog_node.start(rc_basename, 0); - child_progress_node.activate(); - defer child_progress_node.end(); + const o_sub_path = try std.fs.path.join(arena, &.{ "o", &digest }); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + + var output_file = o_dir.createFile(res_basename, .{}) catch |err| { + const output_file_path = try comp.local_cache_directory.join(arena, &.{ o_sub_path, res_basename }); + return comp.failWin32Resource(win32_resource, "failed to create output file '{s}': {s}", .{ output_file_path, @errorName(err) }); + }; + var output_file_closed = false; + defer if (!output_file_closed) output_file.close(); + + var diagnostics = resinator.errors.Diagnostics.init(arena); + defer diagnostics.deinit(); + + var output_buffered_stream = std.io.bufferedWriter(output_file.writer()); + + // In .rc files, a " within a quoted string is escaped as "" + const fmtRcEscape = struct { + fn formatRcEscape(bytes: []const u8, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { + _ = fmt; + _ = options; + for (bytes) |byte| switch (byte) { + '"' => try writer.writeAll("\"\""), + '\\' => try writer.writeAll("\\\\"), + else => try writer.writeByte(byte), + }; + } + + pub fn fmtRcEscape(bytes: []const u8) std.fmt.Formatter(formatRcEscape) { + return .{ .data = bytes }; + } + }.fmtRcEscape; + + // 1 is CREATEPROCESS_MANIFEST_RESOURCE_ID which is the default ID used for RT_MANIFEST resources + // 24 is RT_MANIFEST + const input = try std.fmt.allocPrint(arena, "1 24 \"{s}\"", .{fmtRcEscape(src_path)}); + + resinator.compile.compile(arena, input, output_buffered_stream.writer(), .{ + .cwd = std.fs.cwd(), + .diagnostics = &diagnostics, + .ignore_include_env_var = true, + .default_code_page = .utf8, + }) catch |err| switch (err) { + error.ParseError, error.CompileError => { + // Delete the output file on error + output_file.close(); + output_file_closed = true; + // Failing to delete is not really a big deal, so swallow any errors + o_dir.deleteFile(res_basename) catch { + const output_file_path = try comp.local_cache_directory.join(arena, &.{ o_sub_path, res_basename }); + log.warn("failed to delete '{s}': {s}", .{ output_file_path, @errorName(err) }); + }; + return comp.failWin32ResourceCompile(win32_resource, input, &diagnostics, null); + }, + else => |e| return e, + }; + + try output_buffered_stream.flush(); + + break :blk digest; + }; - const rc_basename_noext = rc_basename[0 .. rc_basename.len - std.fs.path.extension(rc_basename).len]; + if (man.have_exclusive_lock) { + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ src_path, @errorName(err) }); + }; + } + + win32_resource.status = .{ + .success = .{ + .res_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{ + "o", &digest, res_basename, + }), + .lock = man.toOwnedLock(), + }, + }; + return; + } + + // We now know that we're compiling an .rc file + const rc_src = win32_resource.src.rc; + + _ = try man.addFile(rc_src.src_path, null); + man.hash.addListOfBytes(rc_src.extra_flags); + + const rc_basename_noext = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const digest = if (try man.hit()) man.final() else blk: { const rcpp_filename = try std.fmt.allocPrint(arena, "{s}.rcpp", .{rc_basename_noext}); @@ -4587,11 +4733,11 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const out_res_path = try comp.tmpFilePath(arena, res_filename); var options = options: { - var resinator_args = try std.ArrayListUnmanaged([]const u8).initCapacity(comp.gpa, win32_resource.src.extra_flags.len + 4); + var resinator_args = try std.ArrayListUnmanaged([]const u8).initCapacity(comp.gpa, rc_src.extra_flags.len + 4); defer resinator_args.deinit(comp.gpa); resinator_args.appendAssumeCapacity(""); // dummy 'process name' arg - resinator_args.appendSliceAssumeCapacity(win32_resource.src.extra_flags); + resinator_args.appendSliceAssumeCapacity(rc_src.extra_flags); resinator_args.appendSliceAssumeCapacity(&.{ "--", out_rcpp_path, out_res_path }); var cli_diagnostics = resinator.cli.Diagnostics.init(comp.gpa); @@ -4620,7 +4766,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 .nostdinc = false, // handled by addCCArgs }); - try argv.append(win32_resource.src.src_path); + try argv.append(rc_src.src_path); try argv.appendSlice(&[_][]const u8{ "-o", out_rcpp_path, @@ -4694,7 +4840,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 }, }; - var mapping_results = try resinator.source_mapping.parseAndRemoveLineCommands(arena, full_input, full_input, .{ .initial_filename = win32_resource.src.src_path }); + var mapping_results = try resinator.source_mapping.parseAndRemoveLineCommands(arena, full_input, full_input, .{ .initial_filename = rc_src.src_path }); defer mapping_results.mappings.deinit(arena); var final_input = resinator.comments.removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings); @@ -4777,7 +4923,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // the contents were the same, we hit the cache but the manifest is dirty and we need to update // it to prevent doing a full file content comparison the next time around. man.writeManifest() catch |err| { - log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ win32_resource.src.src_path, @errorName(err) }); + log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ rc_src.src_path, @errorName(err) }); }; } @@ -5115,7 +5261,7 @@ pub fn addCCArgs( try argv.append("-fno-unwind-tables"); } }, - .shared_library, .ll, .bc, .unknown, .static_library, .object, .def, .zig, .res => {}, + .shared_library, .ll, .bc, .unknown, .static_library, .object, .def, .zig, .res, .manifest => {}, .assembly, .assembly_with_cpp => { if (ext == .assembly_with_cpp) { const c_headers_dir = try std.fs.path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, "include" }); @@ -5341,7 +5487,10 @@ fn failWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, comptim try bundle.addRootErrorMessage(.{ .msg = try bundle.printString(format, args), .src_loc = try bundle.addSourceLocation(.{ - .src_path = try bundle.addString(win32_resource.src.src_path), + .src_path = try bundle.addString(switch (win32_resource.src) { + .rc => |rc_src| rc_src.src_path, + .manifest => |manifest_src| manifest_src, + }), .line = 0, .column = 0, .span_start = 0, @@ -5382,7 +5531,10 @@ fn failWin32ResourceCli( try bundle.addRootErrorMessage(.{ .msg = try bundle.addString("invalid command line option(s)"), .src_loc = try bundle.addSourceLocation(.{ - .src_path = try bundle.addString(win32_resource.src.src_path), + .src_path = try bundle.addString(switch (win32_resource.src) { + .rc => |rc_src| rc_src.src_path, + .manifest => |manifest_src| manifest_src, + }), .line = 0, .column = 0, .span_start = 0, @@ -5428,7 +5580,7 @@ fn failWin32ResourceCompile( win32_resource: *Win32Resource, source: []const u8, diagnostics: *resinator.errors.Diagnostics, - mappings: resinator.source_mapping.SourceMappings, + opt_mappings: ?resinator.source_mapping.SourceMappings, ) SemaError { @setCold(true); @@ -5452,19 +5604,26 @@ fn failWin32ResourceCompile( .note => if (cur_err == null) continue, .err => {}, } - const corresponding_span = mappings.get(err_details.token.line_number); - const corresponding_file = mappings.files.get(corresponding_span.filename_offset); + const err_line, const err_filename = blk: { + if (opt_mappings) |mappings| { + const corresponding_span = mappings.get(err_details.token.line_number); + const corresponding_file = mappings.files.get(corresponding_span.filename_offset); + const err_line = corresponding_span.start_line; + break :blk .{ err_line, corresponding_file }; + } else { + break :blk .{ err_details.token.line_number, "<generated rc>" }; + } + }; const source_line_start = err_details.token.getLineStart(source); const column = err_details.token.calculateColumn(source, 1, source_line_start); - const err_line = corresponding_span.start_line; msg_buf.clearRetainingCapacity(); try err_details.render(msg_buf.writer(comp.gpa), source, diagnostics.strings.items); const src_loc = src_loc: { var src_loc: ErrorBundle.SourceLocation = .{ - .src_path = try bundle.addString(corresponding_file), + .src_path = try bundle.addString(err_filename), .line = @intCast(err_line - 1), // 1-based -> 0-based .column = @intCast(column), .span_start = 0, @@ -5537,6 +5696,7 @@ pub const FileExt = enum { def, rc, res, + manifest, unknown, pub fn clangSupportsDepFile(ext: FileExt) bool { @@ -5554,6 +5714,7 @@ pub const FileExt = enum { .def, .rc, .res, + .manifest, .unknown, => false, }; @@ -5578,6 +5739,7 @@ pub const FileExt = enum { .def => ".def", .rc => ".rc", .res => ".res", + .manifest => ".manifest", .unknown => "", }; } @@ -5673,6 +5835,8 @@ pub fn classifyFileExt(filename: []const u8) FileExt { return .rc; } else if (std.ascii.endsWithIgnoreCase(filename, ".res")) { return .res; + } else if (std.ascii.endsWithIgnoreCase(filename, ".manifest")) { + return .manifest; } else { return .unknown; } diff --git a/src/InternPool.zig b/src/InternPool.zig index 14490c21db..36a5c65deb 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -1556,10 +1556,10 @@ pub const Key = union(enum) { // These are strange: we'll sometimes represent them as f128, even if the // underlying type is smaller. f80 is an exception: see float_c_longdouble_f80. const a_val = switch (a_info.storage) { - inline else => |val| @as(f128, @floatCast(val)), + inline else => |val| @as(u128, @bitCast(@as(f128, @floatCast(val)))), }; const b_val = switch (b_info.storage) { - inline else => |val| @as(f128, @floatCast(val)), + inline else => |val| @as(u128, @bitCast(@as(f128, @floatCast(val)))), }; return a_val == b_val; } @@ -1567,9 +1567,14 @@ pub const Key = union(enum) { const StorageTag = @typeInfo(Key.Float.Storage).Union.tag_type.?; assert(@as(StorageTag, a_info.storage) == @as(StorageTag, b_info.storage)); - return switch (a_info.storage) { - inline else => |val, tag| val == @field(b_info.storage, @tagName(tag)), - }; + switch (a_info.storage) { + inline else => |val, tag| { + const Bits = std.meta.Int(.unsigned, @bitSizeOf(@TypeOf(val))); + const a_bits: Bits = @bitCast(val); + const b_bits: Bits = @bitCast(@field(b_info.storage, @tagName(tag))); + return a_bits == b_bits; + }, + } }, .opaque_type => |a_info| { diff --git a/src/Module.zig b/src/Module.zig index 41f4ec2b41..36d9a4284c 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -137,6 +137,9 @@ deletion_set: std.AutoArrayHashMapUnmanaged(Decl.Index, void) = .{}, /// Key is the error name, index is the error tag value. Index 0 has a length-0 string. global_error_set: GlobalErrorSet = .{}, +/// Maximum amount of distinct error values, set by --error-limit +error_limit: ErrorInt, + /// Incrementing integer used to compare against the corresponding Decl /// field to determine whether a Decl's status applies to an ongoing update, or a /// previous analysis. @@ -5020,6 +5023,11 @@ pub fn getErrorValueFromSlice( return getErrorValue(mod, interned_name); } +pub fn errorSetBits(mod: *Module) u16 { + if (mod.error_limit == 0) return 0; + return std.math.log2_int_ceil(ErrorInt, mod.error_limit + 1); // +1 for no error +} + pub fn createAnonymousDecl(mod: *Module, block: *Sema.Block, typed_value: TypedValue) !Decl.Index { const src_decl = mod.declPtr(block.src_decl); return mod.createAnonymousDeclFromDecl(src_decl, block.namespace, block.wip_capture_scope, typed_value); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 5d315ffed4..50468f4c2c 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -81,6 +81,10 @@ pub const JobQueue = struct { wait_group: WaitGroup = .{}, global_cache: Cache.Directory, recursive: bool, + /// Dumps hash information to stdout which can be used to troubleshoot why + /// two hashes of the same package do not match. + /// If this is true, `recursive` must be false. + debug_hash: bool, work_around_btrfs_bug: bool, pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); @@ -752,12 +756,14 @@ const FileType = enum { tar, @"tar.gz", @"tar.xz", + @"tar.zst", git_pack, fn fromPath(file_path: []const u8) ?FileType { if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; return null; } @@ -975,6 +981,9 @@ fn unpackResource( if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) break :ft .@"tar.xz"; + if (ascii.eqlIgnoreCase(content_type, "application/zstd")) + break :ft .@"tar.zst"; + if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { return f.fail(f.location_tok, try eb.printString( "unrecognized 'Content-Type' header: '{s}'", @@ -1015,6 +1024,7 @@ fn unpackResource( .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), + .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper), .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { error.FetchFailed => return error.FetchFailed, error.OutOfMemory => return error.OutOfMemory, @@ -1026,6 +1036,18 @@ fn unpackResource( } } +// due to slight differences in the API of std.compress.(gzip|xz) and std.compress.zstd, zstd is +// wrapped for generic use in unpackTarballCompressed: see github.com/ziglang/zig/issues/14739 +const ZstdWrapper = struct { + fn DecompressType(comptime T: type) type { + return error{}!std.compress.zstd.DecompressStream(T, .{}); + } + + fn decompress(allocator: Allocator, reader: anytype) DecompressType(@TypeOf(reader)) { + return std.compress.zstd.decompressStream(allocator, reader); + } +}; + fn unpackTarballCompressed( f: *Fetch, out_dir: fs.Dir, @@ -1315,7 +1337,7 @@ fn computeHash( const kind: HashedFile.Kind = switch (entry.kind) { .directory => unreachable, .file => .file, - .sym_link => .sym_link, + .sym_link => .link, else => return f.fail(f.location_tok, try eb.printString( "package contains '{s}' which has illegal file type '{s}'", .{ entry.path, @tagName(entry.kind) }, @@ -1329,7 +1351,7 @@ fn computeHash( const hashed_file = try arena.create(HashedFile); hashed_file.* = .{ .fs_path = fs_path, - .normalized_path = try normalizePath(arena, fs_path), + .normalized_path = try normalizePathAlloc(arena, fs_path), .kind = kind, .hash = undefined, // to be populated by the worker .failure = undefined, // to be populated by the worker @@ -1399,9 +1421,36 @@ fn computeHash( } if (any_failures) return error.FetchFailed; + + if (f.job_queue.debug_hash) { + assert(!f.job_queue.recursive); + // Print something to stdout that can be text diffed to figure out why + // the package hash is different. + dumpHashInfo(all_files.items) catch |err| { + std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)}); + std.process.exit(1); + }; + } + return hasher.finalResult(); } +fn dumpHashInfo(all_files: []const *const HashedFile) !void { + const stdout = std.io.getStdOut(); + var bw = std.io.bufferedWriter(stdout.writer()); + const w = bw.writer(); + + for (all_files) |hashed_file| { + try w.print("{s}: {s}: {s}\n", .{ + @tagName(hashed_file.kind), + std.fmt.fmtSliceHexLower(&hashed_file.hash), + hashed_file.normalized_path, + }); + } + + try bw.flush(); +} + fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { defer wg.finish(); hashed_file.failure = hashFileFallible(dir, hashed_file); @@ -1427,8 +1476,14 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void hasher.update(buf[0..bytes_read]); } }, - .sym_link => { + .link => { const link_name = try dir.readLink(hashed_file.fs_path, &buf); + if (fs.path.sep != canonical_sep) { + // Package hashes are intended to be consistent across + // platforms which means we must normalize path separators + // inside symlinks. + normalizePath(link_name); + } hasher.update(link_name); }, } @@ -1474,7 +1529,7 @@ const HashedFile = struct { fs.File.StatError || fs.Dir.ReadLinkError; - const Kind = enum { file, sym_link }; + const Kind = enum { file, link }; fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { _ = context; @@ -1484,22 +1539,20 @@ const HashedFile = struct { /// Make a file system path identical independently of operating system path inconsistencies. /// This converts backslashes into forward slashes. -fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { - const canonical_sep = '/'; - - if (fs.path.sep == canonical_sep) - return fs_path; - +fn normalizePathAlloc(arena: Allocator, fs_path: []const u8) ![]const u8 { + if (fs.path.sep == canonical_sep) return fs_path; const normalized = try arena.dupe(u8, fs_path); - for (normalized) |*byte| { - switch (byte.*) { - fs.path.sep => byte.* = canonical_sep, - else => continue, - } - } + normalizePath(normalized); return normalized; } +const canonical_sep = fs.path.sep_posix; + +fn normalizePath(bytes: []u8) void { + assert(fs.path.sep != canonical_sep); + std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); +} + const Filter = struct { include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, diff --git a/src/Sema.zig b/src/Sema.zig index f25457a21d..f084020f95 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -9891,6 +9891,17 @@ fn zirIntFromPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError! if (!ptr_ty.isPtrAtRuntime(mod)) { return sema.fail(block, ptr_src, "expected pointer, found '{}'", .{ptr_ty.fmt(mod)}); } + const pointee_ty = ptr_ty.childType(mod); + if (try sema.typeRequiresComptime(ptr_ty)) { + const msg = msg: { + const msg = try sema.errMsg(block, ptr_src, "comptime-only type '{}' has no pointer address", .{pointee_ty.fmt(mod)}); + errdefer msg.destroy(sema.gpa); + const src_decl = mod.declPtr(block.src_decl); + try sema.explainWhyTypeIsComptime(msg, ptr_src.toSrcLoc(src_decl, mod), pointee_ty); + break :msg msg; + }; + return sema.failWithOwnedErrorMsg(block, msg); + } if (try sema.resolveMaybeUndefValIntable(operand)) |operand_val| ct: { if (!is_vector) { return Air.internedToRef((try mod.intValue( @@ -19765,15 +19776,39 @@ fn zirArrayInit( const is_tuple = array_ty.zigTypeTag(mod) == .Struct; const sentinel_val = array_ty.sentinel(mod); - const resolved_args = try gpa.alloc(Air.Inst.Ref, args.len - 1 + @intFromBool(sentinel_val != null)); + var root_msg: ?*Module.ErrorMsg = null; + errdefer if (root_msg) |msg| msg.destroy(sema.gpa); + + const final_len = try sema.usizeCast(block, src, array_ty.arrayLenIncludingSentinel(mod)); + const resolved_args = try gpa.alloc(Air.Inst.Ref, final_len); defer gpa.free(resolved_args); - for (args[1..], 0..) |arg, i| { + for (resolved_args, 0..) |*dest, i| { + // Less inits than needed. + if (i + 2 > args.len) if (is_tuple) { + const default_val = array_ty.structFieldDefaultValue(i, mod).toIntern(); + if (default_val == .unreachable_value) { + const template = "missing tuple field with index {d}"; + if (root_msg) |msg| { + try sema.errNote(block, src, msg, template, .{i}); + } else { + root_msg = try sema.errMsg(block, src, template, .{i}); + } + } else { + dest.* = Air.internedToRef(default_val); + } + continue; + } else { + dest.* = Air.internedToRef(sentinel_val.?.toIntern()); + break; + }; + + const arg = args[i + 1]; const resolved_arg = try sema.resolveInst(arg); const elem_ty = if (is_tuple) array_ty.structFieldType(i, mod) else array_ty.elemType2(mod); - resolved_args[i] = sema.coerce(block, elem_ty, resolved_arg, .unneeded) catch |err| switch (err) { + dest.* = sema.coerce(block, elem_ty, resolved_arg, .unneeded) catch |err| switch (err) { error.NeededSourceLocation => { const decl = mod.declPtr(block.src_decl); const elem_src = mod.initSrc(src.node_offset.x, decl, i); @@ -19783,7 +19818,7 @@ fn zirArrayInit( else => return err, }; if (is_tuple) if (try array_ty.structFieldValueComptime(mod, i)) |field_val| { - const init_val = try sema.resolveMaybeUndefVal(resolved_args[i]) orelse { + const init_val = try sema.resolveMaybeUndefVal(dest.*) orelse { const decl = mod.declPtr(block.src_decl); const elem_src = mod.initSrc(src.node_offset.x, decl, i); return sema.failWithNeededComptime(block, elem_src, .{ @@ -19798,8 +19833,10 @@ fn zirArrayInit( }; } - if (sentinel_val) |some| { - resolved_args[resolved_args.len - 1] = Air.internedToRef(some.toIntern()); + if (root_msg) |msg| { + try sema.addDeclaredHereNote(msg, array_ty); + root_msg = null; + return sema.failWithOwnedErrorMsg(block, msg); } const opt_runtime_index: ?u32 = for (resolved_args, 0..) |arg, i| { @@ -19810,7 +19847,7 @@ fn zirArrayInit( const runtime_index = opt_runtime_index orelse { const elem_vals = try sema.arena.alloc(InternPool.Index, resolved_args.len); for (elem_vals, resolved_args, 0..) |*val, arg, i| { - const elem_ty = if (array_ty.zigTypeTag(mod) == .Struct) + const elem_ty = if (is_tuple) array_ty.structFieldType(i, mod) else array_ty.elemType2(mod); @@ -19845,7 +19882,7 @@ fn zirArrayInit( const alloc = try block.addTy(.alloc, alloc_ty); const base_ptr = try sema.optEuBasePtrInit(block, alloc, src); - if (array_ty.isTuple(mod)) { + if (is_tuple) { for (resolved_args, 0..) |arg, i| { const elem_ptr_ty = try sema.ptrType(.{ .child = array_ty.structFieldType(i, mod).toIntern(), @@ -25277,7 +25314,7 @@ fn zirBuiltinExtern( if (!ty.isPtrAtRuntime(mod)) { return sema.fail(block, ty_src, "expected (optional) pointer", .{}); } - if (!try sema.validateExternType(ty.childType(mod), .other)) { + if (!try sema.validateExternType(ty, .other)) { const msg = msg: { const msg = try sema.errMsg(block, ty_src, "extern symbol cannot have type '{}'", .{ty.fmt(mod)}); errdefer msg.destroy(sema.gpa); @@ -25618,7 +25655,12 @@ fn validateExternType( .Float, .AnyFrame, => return true, - .Pointer => return !(ty.isSlice(mod) or try sema.typeRequiresComptime(ty)), + .Pointer => { + if (ty.childType(mod).zigTypeTag(mod) == .Fn) { + return ty.isConstPtr(mod) and try sema.validateExternType(ty.childType(mod), .other); + } + return !(ty.isSlice(mod) or try sema.typeRequiresComptime(ty)); + }, .Int => switch (ty.intInfo(mod).bits) { 0, 8, 16, 32, 64, 128 => return true, else => return false, @@ -25687,8 +25729,13 @@ fn explainWhyTypeIsNotExtern( try mod.errNoteNonLazy(src_loc, msg, "slices have no guaranteed in-memory representation", .{}); } else { const pointee_ty = ty.childType(mod); - try mod.errNoteNonLazy(src_loc, msg, "pointer to comptime-only type '{}'", .{pointee_ty.fmt(sema.mod)}); - try sema.explainWhyTypeIsComptime(msg, src_loc, pointee_ty); + if (!ty.isConstPtr(mod) and pointee_ty.zigTypeTag(mod) == .Fn) { + try mod.errNoteNonLazy(src_loc, msg, "pointer to extern function must be 'const'", .{}); + } else if (try sema.typeRequiresComptime(ty)) { + try mod.errNoteNonLazy(src_loc, msg, "pointer to comptime-only type '{}'", .{pointee_ty.fmt(sema.mod)}); + try sema.explainWhyTypeIsComptime(msg, src_loc, ty); + } + try sema.explainWhyTypeIsNotExtern(msg, src_loc, pointee_ty, position); } }, .Void => try mod.errNoteNonLazy(src_loc, msg, "'void' is a zero bit type; for C 'void' use 'anyopaque'", .{}), diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index cdd683390b..d0e068da73 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -4012,6 +4012,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .extern_got => unreachable, }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -4318,8 +4319,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier if (self.bin_file.cast(link.File.Elf)) |elf_file| { const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl); const sym = elf_file.symbol(sym_index); - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file))); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file))); try self.genSetReg(Type.usize, .x30, .{ .memory = got_addr }); } else if (self.bin_file.cast(link.File.MachO)) |macho_file| { const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl); @@ -5531,6 +5532,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .extern_got => unreachable, }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5652,6 +5654,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .got => .load_memory_got, .direct => .load_memory_direct, .import => .load_memory_import, + .extern_got => unreachable, }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5849,6 +5852,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .extern_got => unreachable, }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -6176,7 +6180,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .memory => |addr| .{ .memory = addr }, .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } }, .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, - .load_tlv => unreachable, // TODO + .load_extern_got, .load_tlv => unreachable, // TODO }, .fail => |msg| { self.err_msg = msg; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index 5afb944474..a7c437bb59 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -4304,8 +4304,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier if (self.bin_file.cast(link.File.Elf)) |elf_file| { const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl); const sym = elf_file.symbol(sym_index); - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file))); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file))); try self.genSetReg(Type.usize, .lr, .{ .memory = got_addr }); } else if (self.bin_file.cast(link.File.MachO)) |_| { unreachable; // unsupported architecture for MachO @@ -6135,7 +6135,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_extern_got, .load_direct, .load_tlv => unreachable, // TODO .immediate => |imm| .{ .immediate = @as(u32, @truncate(imm)) }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 6c2748e8f3..e4c4266531 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -1754,8 +1754,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier .func => |func| { const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl); const sym = elf_file.symbol(sym_index); - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file))); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file))); try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr }); _ = try self.addInst(.{ .tag = .jalr, @@ -2591,7 +2591,7 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_extern_got, .load_direct, .load_tlv => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index be02614327..ae4fc29e8a 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -1349,8 +1349,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: { const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl); const sym = elf_file.symbol(sym_index); - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - break :blk @as(u32, @intCast(sym.gotAddress(elf_file))); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + break :blk @as(u32, @intCast(sym.zigGotAddress(elf_file))); } else unreachable; try self.genSetReg(Type.usize, .o7, .{ .memory = got_addr }); @@ -4137,7 +4137,7 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_extern_got, .load_direct, .load_tlv => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index ebcccd781d..3eac273164 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -6253,8 +6253,10 @@ fn airMulWithOverflow(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { func.finishAir(inst, result_ptr, &.{ extra.lhs, extra.rhs }); } -fn airMaxMin(func: *CodeGen, inst: Air.Inst.Index, op: enum { max, min }) InnerError!void { +fn airMaxMin(func: *CodeGen, inst: Air.Inst.Index, op: Op) InnerError!void { + assert(op == .max or op == .min); const mod = func.bin_file.base.options.module.?; + const target = mod.getTarget(); const bin_op = func.air.instructions.items(.data)[inst].bin_op; const ty = func.typeOfIndex(inst); @@ -6269,13 +6271,25 @@ fn airMaxMin(func: *CodeGen, inst: Air.Inst.Index, op: enum { max, min }) InnerE const lhs = try func.resolveInst(bin_op.lhs); const rhs = try func.resolveInst(bin_op.rhs); - // operands to select from - try func.lowerToStack(lhs); - try func.lowerToStack(rhs); - _ = try func.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt); + if (ty.zigTypeTag(mod) == .Float) { + var fn_name_buf: [64]u8 = undefined; + const float_bits = ty.floatBits(target); + const fn_name = std.fmt.bufPrint(&fn_name_buf, "{s}f{s}{s}", .{ + target_util.libcFloatPrefix(float_bits), + @tagName(op), + target_util.libcFloatSuffix(float_bits), + }) catch unreachable; + const result = try func.callIntrinsic(fn_name, &.{ ty.ip_index, ty.ip_index }, ty, &.{ lhs, rhs }); + try func.lowerToStack(result); + } else { + // operands to select from + try func.lowerToStack(lhs); + try func.lowerToStack(rhs); + _ = try func.cmp(lhs, rhs, ty, if (op == .max) .gt else .lt); - // based on the result from comparison, return operand 0 or 1. - try func.addTag(.select); + // based on the result from comparison, return operand 0 or 1. + try func.addTag(.select); + } // store result in local const result_ty = if (isByRef(ty, mod)) Type.u32 else ty; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index a7809559e7..59d54bd04f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -207,6 +207,12 @@ pub const MCValue = union(enum) { /// The value is a pointer to a value referenced indirectly via GOT. /// Payload is a symbol index. lea_got: u32, + /// The value is an extern variable referenced via GOT. + /// Payload is a symbol index. + load_extern_got: u32, + /// The value is a pointer to an extern variable referenced via GOT. + /// Payload is a symbol index. + lea_extern_got: u32, /// The value is a threadlocal variable. /// Payload is a symbol index. load_tlv: u32, @@ -295,6 +301,7 @@ pub const MCValue = union(enum) { .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -308,6 +315,7 @@ pub const MCValue = union(enum) { .load_direct => |sym_index| .{ .lea_direct = sym_index }, .load_got => |sym_index| .{ .lea_got = sym_index }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, + .load_extern_got => |sym_index| .{ .lea_extern_got = sym_index }, .load_frame => |frame_addr| .{ .lea_frame = frame_addr }, }; } @@ -325,6 +333,7 @@ pub const MCValue = union(enum) { .indirect, .load_direct, .load_got, + .load_extern_got, .load_tlv, .load_frame, .reserved_frame, @@ -335,6 +344,7 @@ pub const MCValue = union(enum) { .register_offset => |reg_off| .{ .indirect = reg_off }, .lea_direct => |sym_index| .{ .load_direct = sym_index }, .lea_got => |sym_index| .{ .load_got = sym_index }, + .lea_extern_got => |sym_index| .{ .load_extern_got = sym_index }, .lea_tlv => |sym_index| .{ .load_tlv = sym_index }, .lea_frame => |frame_addr| .{ .load_frame = frame_addr }, }; @@ -358,6 +368,8 @@ pub const MCValue = union(enum) { .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -392,6 +404,8 @@ pub const MCValue = union(enum) { .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .lea_frame, @@ -434,6 +448,8 @@ pub const MCValue = union(enum) { .lea_direct => |pl| try writer.print("direct:{d}", .{pl}), .load_got => |pl| try writer.print("[got:{d}]", .{pl}), .lea_got => |pl| try writer.print("got:{d}", .{pl}), + .load_extern_got => |pl| try writer.print("[extern_got:{d}]", .{pl}), + .lea_extern_got => |pl| try writer.print("extern_got:{d}", .{pl}), .load_tlv => |pl| try writer.print("[tlv:{d}]", .{pl}), .lea_tlv => |pl| try writer.print("tlv:{d}", .{pl}), .load_frame => |pl| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }), @@ -461,6 +477,8 @@ const InstTracking = struct { .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -520,6 +538,8 @@ const InstTracking = struct { .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -555,6 +575,8 @@ const InstTracking = struct { .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .lea_frame, @@ -4371,6 +4393,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { .memory, .load_direct, .load_got, + .load_extern_got, .load_tlv, => try self.genSetReg(addr_reg, Type.usize, array.address()), .lea_direct, .lea_tlv => unreachable, @@ -5851,6 +5874,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro .register_offset, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()), @@ -5858,6 +5882,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro .indirect, .load_direct, .load_got, + .load_extern_got, .load_tlv, .load_frame, => { @@ -5996,6 +6021,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr .register_offset, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv), @@ -6003,6 +6029,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr .indirect, .load_direct, .load_got, + .load_extern_got, .load_tlv, .load_frame, => { @@ -6424,6 +6451,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -6431,7 +6459,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC => unreachable, // unmodifiable destination .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), .register_pair => unreachable, // unimplemented - .memory, .load_got, .load_direct, .load_tlv => { + .memory, .load_got, .load_extern_got, .load_direct, .load_tlv => { const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); @@ -7389,6 +7417,8 @@ fn genBinOp( .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .lea_frame, @@ -7445,6 +7475,8 @@ fn genBinOp( .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .lea_frame, @@ -8397,6 +8429,7 @@ fn genBinOpMir( .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -8485,6 +8518,8 @@ fn genBinOpMir( .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -8517,6 +8552,7 @@ fn genBinOpMir( .register_offset, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, => { @@ -8532,6 +8568,7 @@ fn genBinOpMir( .memory, .load_direct, .load_got, + .load_extern_got, .load_tlv, => { const ptr_ty = try mod.singleConstPtrType(ty); @@ -8552,13 +8589,13 @@ fn genBinOpMir( } } }, - .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => { + .memory, .indirect, .load_got, .load_extern_got, .load_direct, .load_tlv, .load_frame => { const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock }; const limb_abi_size: u32 = @min(abi_size, 8); const dst_info: OpInfo = switch (dst_mcv) { else => unreachable, - .memory, .load_got, .load_direct, .load_tlv => dst: { + .memory, .load_got, .load_extern_got, .load_direct, .load_tlv => dst: { const dst_addr_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64(); const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); @@ -8592,16 +8629,17 @@ fn genBinOpMir( .indirect, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .load_frame, .lea_frame, => null, - .memory, .load_got, .load_direct, .load_tlv => src: { + .memory, .load_got, .load_extern_got, .load_direct, .load_tlv => src: { switch (resolved_src_mcv) { .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) break :src null, - .load_got, .load_direct, .load_tlv => {}, + .load_got, .load_extern_got, .load_direct, .load_tlv => {}, else => unreachable, } @@ -8644,6 +8682,7 @@ fn genBinOpMir( switch (dst_mcv) { .memory, .load_got, + .load_extern_got, .load_direct, .load_tlv, => .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off }, @@ -8728,6 +8767,8 @@ fn genBinOpMir( .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -8743,6 +8784,7 @@ fn genBinOpMir( .register_offset, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, => switch (limb_i) { @@ -8792,6 +8834,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -8840,6 +8883,8 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .lea_frame, @@ -8878,7 +8923,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M } }, .register_pair => unreachable, // unimplemented - .memory, .indirect, .load_direct, .load_got, .load_tlv, .load_frame => { + .memory, .indirect, .load_direct, .load_got, .load_extern_got, .load_tlv, .load_frame => { const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); const tmp_mcv = MCValue{ .register = tmp_reg }; const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); @@ -8971,6 +9016,7 @@ fn genVarDbgInfo( //} }, .memory => |address| .{ .memory = address }, .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } }, + .load_extern_got => |sym_index| .{ .linker_load = .{ .type = .extern_got, .sym_index = sym_index } }, .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, .immediate => |x| .{ .immediate = x }, .undef => .undef, @@ -9189,16 +9235,20 @@ fn genCall(self: *Self, info: union(enum) { if (self.bin_file.cast(link.File.Elf)) |elf_file| { const sym_index = try elf_file.getOrCreateMetadataForDecl(owner_decl); const sym = elf_file.symbol(sym_index); - sym.flags.needs_got = true; - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - _ = try self.addInst(.{ - .tag = .call, - .ops = .direct_got_reloc, - .data = .{ .reloc = .{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym.esym_index, - } }, - }); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + if (self.bin_file.options.pic) { + try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym.esym_index }); + try self.asmRegister(.{ ._, .call }, .rax); + } else { + _ = try self.addInst(.{ + .tag = .call, + .ops = .direct_got_reloc, + .data = .{ .reloc = .{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym.esym_index, + } }, + }); + } } else if (self.bin_file.cast(link.File.Coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForDecl(owner_decl); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; @@ -9406,13 +9456,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { .indirect, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, .air_ref, => unreachable, .register_pair, .load_frame => null, - .memory, .load_got, .load_direct, .load_tlv => dst: { + .memory, .load_got, .load_extern_got, .load_direct, .load_tlv => dst: { switch (resolved_dst_mcv) { .memory => |addr| if (math.cast( i32, @@ -9421,7 +9472,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { i32, @as(i64, @bitCast(addr)) + abi_size - 8, ) != null) break :dst null, - .load_got, .load_direct, .load_tlv => {}, + .load_got, .load_extern_got, .load_direct, .load_tlv => {}, else => unreachable, } @@ -9464,13 +9515,14 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { .indirect, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, .air_ref, => unreachable, .register_pair, .load_frame => null, - .memory, .load_got, .load_direct, .load_tlv => src: { + .memory, .load_got, .load_extern_got, .load_direct, .load_tlv => src: { switch (resolved_src_mcv) { .memory => |addr| if (math.cast( i32, @@ -9479,7 +9531,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { i32, @as(i64, @bitCast(addr)) + abi_size - 8, ) != null) break :src null, - .load_got, .load_direct, .load_tlv => {}, + .load_got, .load_extern_got, .load_direct, .load_tlv => {}, else => unreachable, } @@ -9898,6 +9950,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -9924,6 +9977,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .memory, .load_got, + .load_extern_got, .load_direct, .load_tlv, => { @@ -10481,7 +10535,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| break :arg input_mcv, .indirect, .load_frame => break :arg input_mcv, - .load_direct, .load_got, .load_tlv => {}, + .load_direct, .load_got, .load_extern_got, .load_tlv => {}, else => { const temp_mcv = try self.allocTempRegOrMem(ty, false); try self.genCopy(ty, temp_mcv, input_mcv); @@ -11142,6 +11196,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError .register_overflow, .lea_direct, .lea_got, + .lea_extern_got, .lea_tlv, .lea_frame, .reserved_frame, @@ -11193,11 +11248,11 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError } }, .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv), - .memory, .load_direct, .load_got, .load_tlv => { + .memory, .load_direct, .load_got, .load_extern_got, .load_tlv => { switch (dst_mcv) { .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv), - .load_direct, .load_got, .load_tlv => {}, + .load_direct, .load_got, .load_extern_got, .load_tlv => {}, else => unreachable, } @@ -11359,7 +11414,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr else => unreachable, }, )), - .memory, .load_direct, .load_got, .load_tlv => { + .memory, .load_direct, .load_got, .load_extern_got, .load_tlv => { switch (src_mcv) { .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| return (try self.moveStrategy( @@ -11387,7 +11442,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr }, .Float, .Vector => {}, }, - .load_got, .load_tlv => {}, + .load_got, .load_extern_got, .load_tlv => {}, else => unreachable, } @@ -11401,17 +11456,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }), ); }, - .lea_direct, .lea_got => |sym_index| { + .lea_direct, .lea_got, .lea_extern_got => |sym_index| { const atom_index = try self.owner.getSymbolIndex(self); _ = try self.addInst(.{ .tag = switch (src_mcv) { .lea_direct => .lea, - .lea_got => .mov, + .lea_got, .lea_extern_got => .mov, else => unreachable, }, .ops = switch (src_mcv) { .lea_direct => .direct_reloc, .lea_got => .got_reloc, + .lea_extern_got => .extern_got_reloc, else => unreachable, }, .data = .{ .rx = .{ @@ -11547,6 +11603,8 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal .lea_direct, .load_got, .lea_got, + .load_extern_got, + .lea_extern_got, .load_tlv, .lea_tlv, .load_frame, @@ -11637,36 +11695,49 @@ fn genLazySymbolRef( const sym_index = elf_file.getOrCreateMetadataForLazySymbol(lazy_sym) catch |err| return self.fail("{s} creating lazy symbol", .{@errorName(err)}); const sym = elf_file.symbol(sym_index); - sym.flags.needs_got = true; - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - const reloc = Mir.Reloc{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym.esym_index, - }; - switch (tag) { - .lea, .mov => _ = try self.addInst(.{ - .tag = .mov, - .ops = .direct_got_reloc, - .data = .{ .rx = .{ - .r1 = reg.to64(), - .payload = try self.addExtra(reloc), - } }, - }), - .call => _ = try self.addInst(.{ - .tag = .call, - .ops = .direct_got_reloc, - .data = .{ .reloc = reloc }, - }), - else => unreachable, - } - switch (tag) { - .lea, .call => {}, - .mov => try self.asmRegisterMemory( - .{ ._, tag }, - reg.to64(), - Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }), - ), - else => unreachable, + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + + if (self.bin_file.options.pic) { + switch (tag) { + .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym.esym_index }), + .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym.esym_index }), + else => unreachable, + } + switch (tag) { + .lea, .mov => {}, + .call => try self.asmRegister(.{ ._, .call }, reg), + else => unreachable, + } + } else { + const reloc = Mir.Reloc{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym.esym_index, + }; + switch (tag) { + .lea, .mov => _ = try self.addInst(.{ + .tag = .mov, + .ops = .direct_got_reloc, + .data = .{ .rx = .{ + .r1 = reg.to64(), + .payload = try self.addExtra(reloc), + } }, + }), + .call => _ = try self.addInst(.{ + .tag = .call, + .ops = .direct_got_reloc, + .data = .{ .reloc = reloc }, + }), + else => unreachable, + } + switch (tag) { + .lea, .call => {}, + .mov => try self.asmRegisterMemory( + .{ ._, tag }, + reg.to64(), + Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }), + ), + else => unreachable, + } } } else if (self.bin_file.cast(link.File.Plan9)) |p9_file| { const atom_index = p9_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err| @@ -13539,6 +13610,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .memory => |addr| .{ .memory = addr }, .load_direct => |sym_index| .{ .load_direct = sym_index }, .load_got => |sym_index| .{ .lea_got = sym_index }, + .load_extern_got => |sym_index| .{ .lea_extern_got = sym_index }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, }, .fail => |msg| { @@ -13755,7 +13827,8 @@ fn resolveCallingConventionValues( } const param_size: u31 = @intCast(ty.abiSize(mod)); - const param_align: u31 = @intCast(ty.abiAlignment(mod).toByteUnitsOptional().?); + const param_align: u31 = + @intCast(@max(ty.abiAlignment(mod).toByteUnitsOptional().?, 8)); result.stack_byte_count = mem.alignForward(u31, result.stack_byte_count, param_align); arg.* = .{ .load_frame = .{ diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index d2a199da42..e03b0f01b5 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -79,20 +79,29 @@ pub fn emitMir(emit: *Emit) Error!void { @tagName(emit.bin_file.tag), }), .linker_got, + .linker_extern_got, .linker_direct, .linker_direct_got, .linker_import, .linker_tlv, => |symbol| if (emit.bin_file.cast(link.File.Elf)) |elf_file| { const r_type: u32 = switch (lowered_relocs[0].target) { - .linker_direct_got => std.elf.R_X86_64_GOT32, + .linker_direct_got => link.File.Elf.R_X86_64_ZIG_GOT32, + .linker_got => link.File.Elf.R_X86_64_ZIG_GOTPCREL, + .linker_extern_got => std.elf.R_X86_64_GOTPCREL, + .linker_direct => std.elf.R_X86_64_PC32, + else => unreachable, + }; + const r_addend: i64 = switch (lowered_relocs[0].target) { + .linker_direct_got => 0, + .linker_got, .linker_extern_got, .linker_direct => -4, else => unreachable, }; const atom_ptr = elf_file.symbol(symbol.atom_index).atom(elf_file).?; try atom_ptr.addReloc(elf_file, .{ .r_offset = end_offset - 4, .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | r_type, - .r_addend = 0, + .r_addend = r_addend, }); } else if (emit.bin_file.cast(link.File.MachO)) |macho_file| { const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = symbol.atom_index }).?; diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 71f411bdaf..ae5f86d6b0 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -51,6 +51,7 @@ pub const Reloc = struct { inst: Mir.Inst.Index, linker_extern_fn: Mir.Reloc, linker_got: Mir.Reloc, + linker_extern_got: Mir.Reloc, linker_direct: Mir.Reloc, linker_direct_got: Mir.Reloc, linker_import: Mir.Reloc, @@ -388,7 +389,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes, .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes, .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes, - .extern_fn_reloc, .got_reloc, .direct_reloc, .direct_got_reloc, .import_reloc, .tlv_reloc => ._, + .extern_fn_reloc, .got_reloc, .extern_got_reloc, .direct_reloc, .direct_got_reloc, .import_reloc, .tlv_reloc => ._, else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}), }; try lower.emit(switch (fixes) { @@ -532,11 +533,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { else => unreachable, } }, - .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: { + .got_reloc, .extern_got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: { const reg = inst.data.rx.r1; const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data; _ = lower.reloc(switch (inst.ops) { .got_reloc => .{ .linker_got = extra }, + .extern_got_reloc => .{ .linker_extern_got = extra }, .direct_reloc => .{ .linker_direct = extra }, .import_reloc => .{ .linker_import = extra }, .tlv_reloc => .{ .linker_tlv = extra }, diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index b17da20862..0e2a4cca6d 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -783,6 +783,9 @@ pub const Inst = struct { /// Linker relocation - GOT indirection. /// Uses `rx` payload with extra data of type `Reloc`. got_reloc, + /// Linker relocation - reference to an extern variable via GOT. + /// Uses `rx` payload with extra data of type `Reloc`. + extern_got_reloc, /// Linker relocation - direct reference. /// Uses `rx` payload with extra data of type `Reloc`. direct_reloc, diff --git a/src/codegen.zig b/src/codegen.zig index 2c5fd7b630..24269f38ba 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -793,11 +793,13 @@ fn lowerDeclRef( /// Helper struct to denote that the value is in memory but requires a linker relocation fixup: /// * got - the value is referenced indirectly via GOT entry index (the linker emits a got-type reloc) +/// * extern_got - pointer to extern variable referenced via GOT /// * direct - the value is referenced directly via symbol index index (the linker emits a displacement reloc) /// * import - the value is referenced indirectly via import entry index (the linker emits an import-type reloc) pub const LinkerLoad = struct { type: enum { got, + extern_got, direct, import, }, @@ -827,6 +829,8 @@ pub const GenResult = union(enum) { load_got: u32, /// Direct by-address reference to memory location. memory: u64, + /// Pointer to extern variable via GOT. + load_extern_got: u32, }; fn mcv(val: MCValue) GenResult { @@ -885,13 +889,26 @@ fn genDeclRef( try mod.markDeclAlive(decl); const is_threadlocal = tv.val.isPtrToThreadLocal(mod) and !bin_file.options.single_threaded; + const is_extern = decl.isExtern(mod); if (bin_file.cast(link.File.Elf)) |elf_file| { + if (is_extern) { + const name = mod.intern_pool.stringToSlice(decl.name); + // TODO audit this + const lib_name = if (decl.getOwnedVariable(mod)) |ov| + mod.intern_pool.stringToSliceUnwrap(ov.lib_name) + else + null; + return GenResult.mcv(.{ .load_extern_got = try elf_file.getGlobalSymbol(name, lib_name) }); + } const sym_index = try elf_file.getOrCreateMetadataForDecl(decl_index); const sym = elf_file.symbol(sym_index); - sym.flags.needs_got = true; - _ = try sym.getOrCreateGotEntry(sym_index, elf_file); - return GenResult.mcv(.{ .memory = sym.gotAddress(elf_file) }); + _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + if (bin_file.options.pic) { + return GenResult.mcv(.{ .load_got = sym.esym_index }); + } else { + return GenResult.mcv(.{ .memory = sym.zigGotAddress(elf_file) }); + } } else if (bin_file.cast(link.File.MachO)) |macho_file| { const atom_index = try macho_file.getOrCreateAtomForDecl(decl_index); const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?; @@ -926,7 +943,12 @@ fn genUnnamedConst( return GenResult.fail(bin_file.allocator, src_loc, "lowering unnamed constant failed: {s}", .{@errorName(err)}); }; if (bin_file.cast(link.File.Elf)) |elf_file| { - return GenResult.mcv(.{ .memory = elf_file.symbol(local_sym_index).value }); + const local = elf_file.symbol(local_sym_index); + if (bin_file.options.pic) { + return GenResult.mcv(.{ .load_direct = local.esym_index }); + } else { + return GenResult.mcv(.{ .memory = local.value }); + } } else if (bin_file.cast(link.File.MachO)) |_| { return GenResult.mcv(.{ .load_direct = local_sym_index }); } else if (bin_file.cast(link.File.Coff)) |_| { diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index e18ffece3d..cddd4340d7 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -3050,6 +3050,7 @@ pub const Object = struct { decl_val: InternPool.Index, llvm_addr_space: Builder.AddrSpace, ) Error!Builder.Variable.Index { + // TODO: Add address space to the anon_decl_map const gop = try o.anon_decl_map.getOrPut(o.gpa, decl_val); if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; errdefer assert(o.anon_decl_map.remove(decl_val)); diff --git a/src/codegen/spirv.zig b/src/codegen/spirv.zig index a4d3ec7f46..2c9018463d 100644 --- a/src/codegen/spirv.zig +++ b/src/codegen/spirv.zig @@ -52,18 +52,136 @@ const Block = struct { const BlockMap = std.AutoHashMapUnmanaged(Air.Inst.Index, *Block); -/// Maps Zig decl indices to linking SPIR-V linking information. -pub const DeclLinkMap = std.AutoHashMap(Module.Decl.Index, SpvModule.Decl.Index); +/// This structure holds information that is relevant to the entire compilation, +/// in contrast to `DeclGen`, which only holds relevant information about a +/// single decl. +pub const Object = struct { + /// A general-purpose allocator that can be used for any allocation for this Object. + gpa: Allocator, + + /// the SPIR-V module that represents the final binary. + spv: SpvModule, + + /// The Zig module that this object file is generated for. + /// A map of Zig decl indices to SPIR-V decl indices. + decl_link: std.AutoHashMapUnmanaged(Decl.Index, SpvModule.Decl.Index) = .{}, + + /// A map of Zig InternPool indices for anonymous decls to SPIR-V decl indices. + anon_decl_link: std.AutoHashMapUnmanaged(struct { InternPool.Index, StorageClass }, SpvModule.Decl.Index) = .{}, + + /// A map that maps AIR intern pool indices to SPIR-V cache references (which + /// is basically the same thing except for SPIR-V). + /// This map is typically only used for structures that are deemed heavy enough + /// that it is worth to store them here. The SPIR-V module also interns types, + /// and so the main purpose of this map is to avoid recomputation and to + /// cache extra information about the type rather than to aid in validity + /// of the SPIR-V module. + type_map: TypeMap = .{}, + + pub fn init(gpa: Allocator) Object { + return .{ + .gpa = gpa, + .spv = SpvModule.init(gpa), + }; + } + + pub fn deinit(self: *Object) void { + self.spv.deinit(); + self.decl_link.deinit(self.gpa); + self.anon_decl_link.deinit(self.gpa); + self.type_map.deinit(self.gpa); + } + + fn genDecl( + self: *Object, + mod: *Module, + decl_index: Decl.Index, + air: Air, + liveness: Liveness, + ) !void { + var decl_gen = DeclGen{ + .gpa = self.gpa, + .object = self, + .module = mod, + .spv = &self.spv, + .decl_index = decl_index, + .air = air, + .liveness = liveness, + .type_map = &self.type_map, + .current_block_label_id = undefined, + }; + defer decl_gen.deinit(); + + decl_gen.genDecl() catch |err| switch (err) { + error.CodegenFail => { + try mod.failed_decls.put(mod.gpa, decl_index, decl_gen.error_msg.?); + }, + else => |other| { + // There might be an error that happened *after* self.error_msg + // was already allocated, so be sure to free it. + if (decl_gen.error_msg) |error_msg| { + error_msg.deinit(mod.gpa); + } + + return other; + }, + }; + } + + pub fn updateFunc( + self: *Object, + mod: *Module, + func_index: InternPool.Index, + air: Air, + liveness: Liveness, + ) !void { + const decl_index = mod.funcInfo(func_index).owner_decl; + // TODO: Separate types for generating decls and functions? + try self.genDecl(mod, decl_index, air, liveness); + } + + pub fn updateDecl( + self: *Object, + mod: *Module, + decl_index: Decl.Index, + ) !void { + try self.genDecl(mod, decl_index, undefined, undefined); + } + + /// Fetch or allocate a result id for decl index. This function also marks the decl as alive. + /// Note: Function does not actually generate the decl, it just allocates an index. + pub fn resolveDecl(self: *Object, mod: *Module, decl_index: Decl.Index) !SpvModule.Decl.Index { + const decl = mod.declPtr(decl_index); + try mod.markDeclAlive(decl); + + const entry = try self.decl_link.getOrPut(self.gpa, decl_index); + if (!entry.found_existing) { + // TODO: Extern fn? + const kind: SpvModule.DeclKind = if (decl.val.isFuncBody(mod)) + .func + else + .global; + + entry.value_ptr.* = try self.spv.allocDecl(kind); + } + + return entry.value_ptr.*; + } +}; /// This structure is used to compile a declaration, and contains all relevant meta-information to deal with that. -pub const DeclGen = struct { +const DeclGen = struct { /// A general-purpose allocator that can be used for any allocations for this DeclGen. gpa: Allocator, + /// The object that this decl is generated into. + object: *Object, + /// The Zig module that we are generating decls for. module: *Module, /// The SPIR-V module that instructions should be emitted into. + /// This is the same as `self.object.spv`, repeated here for brevity. spv: *SpvModule, /// The decl we are currently generating code for. @@ -77,27 +195,19 @@ pub const DeclGen = struct { /// Note: If the declaration is not a function, this value will be undefined! liveness: Liveness, - /// Maps Zig Decl indices to SPIR-V globals. - decl_link: *DeclLinkMap, - /// An array of function argument result-ids. Each index corresponds with the /// function argument of the same index. args: std.ArrayListUnmanaged(IdRef) = .{}, /// A counter to keep track of how many `arg` instructions we've seen yet. - next_arg_index: u32, + next_arg_index: u32 = 0, /// A map keeping track of which instruction generated which result-id. inst_results: InstMap = .{}, - /// A map that maps AIR intern pool indices to SPIR-V cache references (which - /// is basically the same thing except for SPIR-V). - /// This map is typically only used for structures that are deemed heavy enough - /// that it is worth to store them here. The SPIR-V module also interns types, - /// and so the main purpose of this map is to avoid recomputation and to - /// cache extra information about the type rather than to aid in validity - /// of the SPIR-V module. - type_map: TypeMap = .{}, + /// A map that maps AIR intern pool indices to SPIR-V cache references. + /// See Object.type_map + type_map: *TypeMap, /// We need to keep track of result ids for block labels, as well as the 'incoming' /// blocks for a block. @@ -115,7 +225,7 @@ pub const DeclGen = struct { /// If `gen` returned `Error.CodegenFail`, this contains an explanatory message. /// Memory is owned by `module.gpa`. - error_msg: ?*Module.ErrorMsg, + error_msg: ?*Module.ErrorMsg = null, /// Possible errors the `genDecl` function may return. const Error = error{ CodegenFail, OutOfMemory }; @@ -154,6 +264,12 @@ pub const DeclGen = struct { /// This is the actual number of bits of the type, not the size of the backing integer. bits: u16, + /// The number of bits required to store the type. + /// For `integer` and `float`, this is equal to `bits`. + /// For `strange_integer` and `bool` this is the size of the backing integer. + /// For `composite_integer` this is 0 (TODO) + backing_bits: u16, + /// Whether the type is a vector. is_vector: bool, @@ -175,65 +291,10 @@ pub const DeclGen = struct { indirect, }; - /// Initialize the common resources of a DeclGen. Some fields are left uninitialized, - /// only set when `gen` is called. - pub fn init( - allocator: Allocator, - module: *Module, - spv: *SpvModule, - decl_link: *DeclLinkMap, - ) DeclGen { - return .{ - .gpa = allocator, - .module = module, - .spv = spv, - .decl_index = undefined, - .air = undefined, - .liveness = undefined, - .decl_link = decl_link, - .next_arg_index = undefined, - .current_block_label_id = undefined, - .error_msg = undefined, - }; - } - - /// Generate the code for `decl`. If a reportable error occurred during code generation, - /// a message is returned by this function. Callee owns the memory. If this function - /// returns such a reportable error, it is valid to be called again for a different decl. - pub fn gen(self: *DeclGen, decl_index: Decl.Index, air: Air, liveness: Liveness) !?*Module.ErrorMsg { - // Reset internal resources, we don't want to re-allocate these. - self.decl_index = decl_index; - self.air = air; - self.liveness = liveness; - self.args.items.len = 0; - self.next_arg_index = 0; - self.inst_results.clearRetainingCapacity(); - self.blocks.clearRetainingCapacity(); - self.current_block_label_id = undefined; - self.func.reset(); - self.base_line_stack.items.len = 0; - self.error_msg = null; - - self.genDecl() catch |err| switch (err) { - error.CodegenFail => return self.error_msg, - else => |others| { - // There might be an error that happened *after* self.error_msg - // was already allocated, so be sure to free it. - if (self.error_msg) |error_msg| { - error_msg.deinit(self.module.gpa); - } - return others; - }, - }; - - return null; - } - /// Free resources owned by the DeclGen. pub fn deinit(self: *DeclGen) void { self.args.deinit(self.gpa); self.inst_results.deinit(self.gpa); - self.type_map.deinit(self.gpa); self.blocks.deinit(self.gpa); self.func.deinit(self.gpa); self.base_line_stack.deinit(self.gpa); @@ -269,7 +330,7 @@ pub const DeclGen = struct { .func => |func| func.owner_decl, else => unreachable, }; - const spv_decl_index = try self.resolveDecl(fn_decl_index); + const spv_decl_index = try self.object.resolveDecl(mod, fn_decl_index); try self.func.decl_deps.put(self.spv.gpa, spv_decl_index, {}); return self.spv.declPtr(spv_decl_index).result_id; } @@ -280,25 +341,87 @@ pub const DeclGen = struct { return self.inst_results.get(index).?; // Assertion means instruction does not dominate usage. } - /// Fetch or allocate a result id for decl index. This function also marks the decl as alive. - /// Note: Function does not actually generate the decl. - fn resolveDecl(self: *DeclGen, decl_index: Module.Decl.Index) !SpvModule.Decl.Index { + fn resolveAnonDecl(self: *DeclGen, val: InternPool.Index, storage_class: StorageClass) !IdRef { + // TODO: This cannot be a function at this point, but it should probably be handled anyway. + const spv_decl_index = blk: { + const entry = try self.object.anon_decl_link.getOrPut(self.object.gpa, .{ val, storage_class }); + if (entry.found_existing) { + try self.func.decl_deps.put(self.spv.gpa, entry.value_ptr.*, {}); + return self.spv.declPtr(entry.value_ptr.*).result_id; + } + + const spv_decl_index = try self.spv.allocDecl(.global); + try self.func.decl_deps.put(self.spv.gpa, spv_decl_index, {}); + entry.value_ptr.* = spv_decl_index; + break :blk spv_decl_index; + }; + const mod = self.module; - const decl = mod.declPtr(decl_index); - try mod.markDeclAlive(decl); + const ty = mod.intern_pool.typeOf(val).toType(); + const ty_ref = try self.resolveType(ty, .indirect); + const ptr_ty_ref = try self.spv.ptrType(ty_ref, storage_class); - const entry = try self.decl_link.getOrPut(decl_index); - if (!entry.found_existing) { - // TODO: Extern fn? - const kind: SpvModule.DeclKind = if (decl.val.isFuncBody(mod)) - .func - else - .global; + const var_id = self.spv.declPtr(spv_decl_index).result_id; - entry.value_ptr.* = try self.spv.allocDecl(kind); - } + const section = &self.spv.sections.types_globals_constants; + try section.emit(self.spv.gpa, .OpVariable, .{ + .id_result_type = self.typeId(ptr_ty_ref), + .id_result = var_id, + .storage_class = storage_class, + }); - return entry.value_ptr.*; + // TODO: At some point we will be able to generate this all constant here, but then all of + // constant() will need to be implemented such that it doesn't generate any at-runtime code. + // NOTE: Because this is a global, we really only want to initialize it once. Therefore the + // constant lowering of this value will need to be deferred to some other function, which + // is then added to the list of initializers using endGlobal(). + + // Save the current state so that we can temporarily generate into a different function. + // TODO: This should probably be made a little more robust. + const func = self.func; + defer self.func = func; + const block_label_id = self.current_block_label_id; + defer self.current_block_label_id = block_label_id; + + self.func = .{}; + + // TODO: Merge this with genDecl? + const begin = self.spv.beginGlobal(); + + const void_ty_ref = try self.resolveType(Type.void, .direct); + const initializer_proto_ty_ref = try self.spv.resolve(.{ .function_type = .{ + .return_type = void_ty_ref, + .parameters = &.{}, + } }); + + const initializer_id = self.spv.allocId(); + try self.func.prologue.emit(self.spv.gpa, .OpFunction, .{ + .id_result_type = self.typeId(void_ty_ref), + .id_result = initializer_id, + .function_control = .{}, + .function_type = self.typeId(initializer_proto_ty_ref), + }); + const root_block_id = self.spv.allocId(); + try self.func.prologue.emit(self.spv.gpa, .OpLabel, .{ + .id_result = root_block_id, + }); + self.current_block_label_id = root_block_id; + + const val_id = try self.constant(ty, val.toValue(), .indirect); + try self.func.body.emit(self.spv.gpa, .OpStore, .{ + .pointer = var_id, + .object = val_id, + }); + + self.spv.endGlobal(spv_decl_index, begin, var_id, initializer_id); + try self.func.body.emit(self.spv.gpa, .OpReturn, {}); + try self.func.body.emit(self.spv.gpa, .OpFunctionEnd, {}); + try self.spv.addFunction(spv_decl_index, self.func); + + try self.spv.debugNameFmt(var_id, "__anon_{d}", .{@intFromEnum(val)}); + try self.spv.debugNameFmt(initializer_id, "initializer of __anon_{d}", .{@intFromEnum(val)}); + + return var_id; } /// Start a new SPIR-V block, Emits the label of the new block, and stores which @@ -376,12 +499,14 @@ pub const DeclGen = struct { return switch (ty.zigTypeTag(mod)) { .Bool => ArithmeticTypeInfo{ .bits = 1, // Doesn't matter for this class. + .backing_bits = self.backingIntBits(1).?, .is_vector = false, .signedness = .unsigned, // Technically, but doesn't matter for this class. .class = .bool, }, .Float => ArithmeticTypeInfo{ .bits = ty.floatBits(target), + .backing_bits = ty.floatBits(target), // TODO: F80? .is_vector = false, .signedness = .signed, // Technically, but doesn't matter for this class. .class = .float, @@ -392,6 +517,7 @@ pub const DeclGen = struct { const maybe_backing_bits = self.backingIntBits(int_info.bits); break :blk ArithmeticTypeInfo{ .bits = int_info.bits, + .backing_bits = maybe_backing_bits orelse 0, .is_vector = false, .signedness = int_info.signedness, .class = if (maybe_backing_bits) |backing_bits| @@ -403,8 +529,19 @@ pub const DeclGen = struct { .composite_integer, }; }, + .Enum => return self.arithmeticTypeInfo(ty.intTagType(mod)), // As of yet, there is no vector support in the self-hosted compiler. - .Vector => self.todo("implement arithmeticTypeInfo for Vector", .{}), + .Vector => blk: { + const child_type = ty.childType(mod); + const child_ty_info = try self.arithmeticTypeInfo(child_type); + break :blk ArithmeticTypeInfo{ + .bits = child_ty_info.bits, + .backing_bits = child_ty_info.backing_bits, + .is_vector = true, + .signedness = child_ty_info.signedness, + .class = child_ty_info.class, + }; + }, // TODO: For which types is this the case? // else => self.todo("implement arithmeticTypeInfo for {}", .{ty.fmt(self.module)}), else => unreachable, @@ -482,7 +619,7 @@ pub const DeclGen = struct { return result_id; } - /// Construct a struct at runtime. + /// Construct an array at runtime. /// result_ty_ref must be an array type. /// Constituents should be in `indirect` representation (as the elements of an array should be). /// Result is in `direct` representation. @@ -521,59 +658,6 @@ pub const DeclGen = struct { return result_id; } - fn constructDeclRef(self: *DeclGen, ty: Type, decl_index: Decl.Index) !IdRef { - const mod = self.module; - const ty_ref = try self.resolveType(ty, .direct); - const ty_id = self.typeId(ty_ref); - const decl = mod.declPtr(decl_index); - const spv_decl_index = try self.resolveDecl(decl_index); - switch (mod.intern_pool.indexToKey(decl.val.ip_index)) { - .func => { - // TODO: Properly lower function pointers. For now we are going to hack around it and - // just generate an empty pointer. Function pointers are represented by a pointer to usize. - // TODO: Add dependency - return try self.spv.constNull(ty_ref); - }, - .extern_func => unreachable, // TODO - else => { - const decl_id = self.spv.declPtr(spv_decl_index).result_id; - try self.func.decl_deps.put(self.spv.gpa, spv_decl_index, {}); - - const final_storage_class = spvStorageClass(decl.@"addrspace"); - - const decl_ty_ref = try self.resolveType(decl.ty, .indirect); - const decl_ptr_ty_ref = try self.spv.ptrType(decl_ty_ref, final_storage_class); - - const ptr_id = switch (final_storage_class) { - .Generic => blk: { - // Pointer should be Generic, but is actually placed in CrossWorkgroup. - const result_id = self.spv.allocId(); - try self.func.body.emit(self.spv.gpa, .OpPtrCastToGeneric, .{ - .id_result_type = self.typeId(decl_ptr_ty_ref), - .id_result = result_id, - .pointer = decl_id, - }); - break :blk result_id; - }, - else => decl_id, - }; - - if (decl_ptr_ty_ref != ty_ref) { - // Differing pointer types, insert a cast. - const casted_ptr_id = self.spv.allocId(); - try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ - .id_result_type = ty_id, - .id_result = casted_ptr_id, - .operand = ptr_id, - }); - return casted_ptr_id; - } else { - return ptr_id; - } - }, - } - } - /// This function generates a load for a constant in direct (ie, non-memory) representation. /// When the constant is simple, it can be generated directly using OpConstant instructions. /// When the constant is more complicated however, it needs to be constructed using multiple values. This @@ -738,7 +822,7 @@ pub const DeclGen = struct { return try self.constructStruct(result_ty_ref, &.{ payload_id, has_pl_id }); }, .aggregate => |aggregate| switch (ip.indexToKey(ty.ip_index)) { - .array_type => |array_type| { + inline .array_type, .vector_type => |array_type, tag| { const elem_ty = array_type.child.toType(); const elem_ty_ref = try self.resolveType(elem_ty, .indirect); @@ -765,9 +849,14 @@ pub const DeclGen = struct { } }, } - if (array_type.sentinel != .none) { - constituents[constituents.len - 1] = try self.constant(elem_ty, array_type.sentinel.toValue(), .indirect); + + switch (tag) { + inline .array_type => if (array_type.sentinel != .none) { + constituents[constituents.len - 1] = try self.constant(elem_ty, array_type.sentinel.toValue(), .indirect); + }, + else => {}, } + return try self.constructArray(result_ty_ref, constituents); }, .struct_type => { @@ -796,7 +885,7 @@ pub const DeclGen = struct { return try self.constructStruct(result_ty_ref, constituents.items); }, - .vector_type, .anon_struct_type => unreachable, // TODO + .anon_struct_type => unreachable, // TODO else => unreachable, }, .un => |un| { @@ -817,9 +906,9 @@ pub const DeclGen = struct { const result_ty_ref = try self.resolveType(ptr_ty, .direct); const mod = self.module; switch (mod.intern_pool.indexToKey(ptr_val.toIntern()).ptr.addr) { - .decl => |decl| return try self.constructDeclRef(ptr_ty, decl), - .anon_decl => @panic("TODO"), - .mut_decl => |decl_mut| return try self.constructDeclRef(ptr_ty, decl_mut.decl), + .decl => |decl| return try self.constantDeclRef(ptr_ty, decl), + .mut_decl => |decl_mut| return try self.constantDeclRef(ptr_ty, decl_mut.decl), + .anon_decl => |anon_decl| return try self.constantAnonDeclRef(ptr_ty, anon_decl), .int => |int| { const ptr_id = self.spv.allocId(); // TODO: This can probably be an OpSpecConstantOp Bitcast, but @@ -836,13 +925,158 @@ pub const DeclGen = struct { .opt_payload => unreachable, // TODO .comptime_field => unreachable, .elem => |elem_ptr| { - const elem_ptr_ty = mod.intern_pool.typeOf(elem_ptr.base).toType(); - const parent_ptr_id = try self.constantPtr(elem_ptr_ty, elem_ptr.base.toValue()); + const parent_ptr_ty = mod.intern_pool.typeOf(elem_ptr.base).toType(); + const parent_ptr_id = try self.constantPtr(parent_ptr_ty, elem_ptr.base.toValue()); const size_ty_ref = try self.sizeType(); const index_id = try self.constInt(size_ty_ref, elem_ptr.index); - return self.ptrAccessChain(result_ty_ref, parent_ptr_id, index_id, &.{}); + + const elem_ptr_id = try self.ptrElemPtr(parent_ptr_ty, parent_ptr_id, index_id); + + // TODO: Can we consolidate this in ptrElemPtr? + const elem_ty = parent_ptr_ty.elemType2(mod); // use elemType() so that we get T for *[N]T. + const elem_ty_ref = try self.resolveType(elem_ty, .direct); + const elem_ptr_ty_ref = try self.spv.ptrType(elem_ty_ref, spvStorageClass(parent_ptr_ty.ptrAddressSpace(mod))); + + if (elem_ptr_ty_ref == result_ty_ref) { + return elem_ptr_id; + } + // This may happen when we have pointer-to-array and the result is + // another pointer-to-array instead of a pointer-to-element. + const result_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ + .id_result_type = self.typeId(result_ty_ref), + .id_result = result_id, + .operand = elem_ptr_id, + }); + return result_id; + }, + .field => |field| { + const base_ptr_ty = mod.intern_pool.typeOf(field.base).toType(); + const base_ptr = try self.constantPtr(base_ptr_ty, field.base.toValue()); + const field_index: u32 = @intCast(field.index); + return try self.structFieldPtr(ptr_ty, base_ptr_ty, base_ptr, field_index); }, - .field => unreachable, // TODO + } + } + + fn constantAnonDeclRef(self: *DeclGen, ty: Type, decl_val: InternPool.Index) !IdRef { + // TODO: Merge this function with constantDeclRef. + + const mod = self.module; + const ip = &mod.intern_pool; + const ty_ref = try self.resolveType(ty, .direct); + const decl_ty = ip.typeOf(decl_val).toType(); + + if (decl_val.toValue().getFunction(mod)) |func| { + _ = func; + unreachable; // TODO + } else if (decl_val.toValue().getExternFunc(mod)) |func| { + _ = func; + unreachable; + } + + // const is_fn_body = decl_ty.zigTypeTag(mod) == .Fn; + if (!decl_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod)) { + // Pointer to nothing - return undefoined + return self.spv.constUndef(ty_ref); + } + + if (decl_ty.zigTypeTag(mod) == .Fn) { + unreachable; // TODO + } + + const final_storage_class = spvStorageClass(ty.ptrAddressSpace(mod)); + const actual_storage_class = switch (final_storage_class) { + .Generic => .CrossWorkgroup, + else => |other| other, + }; + + const decl_id = try self.resolveAnonDecl(decl_val, actual_storage_class); + const decl_ty_ref = try self.resolveType(decl_ty, .indirect); + const decl_ptr_ty_ref = try self.spv.ptrType(decl_ty_ref, final_storage_class); + + const ptr_id = switch (final_storage_class) { + .Generic => blk: { + const result_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpPtrCastToGeneric, .{ + .id_result_type = self.typeId(decl_ptr_ty_ref), + .id_result = result_id, + .pointer = decl_id, + }); + break :blk result_id; + }, + else => decl_id, + }; + + if (decl_ptr_ty_ref != ty_ref) { + // Differing pointer types, insert a cast. + const casted_ptr_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ + .id_result_type = self.typeId(ty_ref), + .id_result = casted_ptr_id, + .operand = ptr_id, + }); + return casted_ptr_id; + } else { + return ptr_id; + } + } + + fn constantDeclRef(self: *DeclGen, ty: Type, decl_index: Decl.Index) !IdRef { + const mod = self.module; + const ty_ref = try self.resolveType(ty, .direct); + const ty_id = self.typeId(ty_ref); + const decl = mod.declPtr(decl_index); + switch (mod.intern_pool.indexToKey(decl.val.ip_index)) { + .func => { + // TODO: Properly lower function pointers. For now we are going to hack around it and + // just generate an empty pointer. Function pointers are represented by a pointer to usize. + return try self.spv.constUndef(ty_ref); + }, + .extern_func => unreachable, // TODO + else => {}, + } + + if (!decl.ty.isFnOrHasRuntimeBitsIgnoreComptime(mod)) { + // Pointer to nothing - return undefined. + return self.spv.constUndef(ty_ref); + } + + const spv_decl_index = try self.object.resolveDecl(mod, decl_index); + + const decl_id = self.spv.declPtr(spv_decl_index).result_id; + try self.func.decl_deps.put(self.spv.gpa, spv_decl_index, {}); + + const final_storage_class = spvStorageClass(decl.@"addrspace"); + + const decl_ty_ref = try self.resolveType(decl.ty, .indirect); + const decl_ptr_ty_ref = try self.spv.ptrType(decl_ty_ref, final_storage_class); + + const ptr_id = switch (final_storage_class) { + .Generic => blk: { + // Pointer should be Generic, but is actually placed in CrossWorkgroup. + const result_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpPtrCastToGeneric, .{ + .id_result_type = self.typeId(decl_ptr_ty_ref), + .id_result = result_id, + .pointer = decl_id, + }); + break :blk result_id; + }, + else => decl_id, + }; + + if (decl_ptr_ty_ref != ty_ref) { + // Differing pointer types, insert a cast. + const casted_ptr_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ + .id_result_type = ty_id, + .id_result = casted_ptr_id, + .operand = ptr_id, + }); + return casted_ptr_id; + } else { + return ptr_id; } } @@ -968,6 +1202,22 @@ pub const DeclGen = struct { return ty_ref; } + fn resolveFnReturnType(self: *DeclGen, ret_ty: Type) !CacheRef { + const mod = self.module; + if (!ret_ty.hasRuntimeBitsIgnoreComptime(mod)) { + // If the return type is an error set or an error union, then we make this + // anyerror return type instead, so that it can be coerced into a function + // pointer type which has anyerror as the return type. + if (ret_ty.isError(mod)) { + return self.resolveType(Type.anyerror, .direct); + } else { + return self.resolveType(Type.void, .direct); + } + } + + return try self.resolveType(ret_ty, .direct); + } + /// Turn a Zig type into a SPIR-V Type, and return a reference to it. fn resolveType(self: *DeclGen, ty: Type, repr: Repr) Error!CacheRef { const mod = self.module; @@ -975,14 +1225,31 @@ pub const DeclGen = struct { log.debug("resolveType: ty = {}", .{ty.fmt(mod)}); const target = self.getTarget(); switch (ty.zigTypeTag(mod)) { - .Void, .NoReturn => return try self.spv.resolve(.void_type), + .NoReturn => { + assert(repr == .direct); + return try self.spv.resolve(.void_type); + }, + .Void => switch (repr) { + .direct => return try self.spv.resolve(.void_type), + // Pointers to void + .indirect => return try self.spv.resolve(.{ .opaque_type = .{ + .name = try self.spv.resolveString("void"), + } }), + }, .Bool => switch (repr) { .direct => return try self.spv.resolve(.bool_type), .indirect => return try self.intType(.unsigned, 1), }, .Int => { const int_info = ty.intInfo(mod); - // TODO: Integers in OpenCL kernels are always unsigned. + if (int_info.bits == 0) { + // Some times, the backend will be asked to generate a pointer to i0. OpTypeInt + // with 0 bits is invalid, so return an opaque type in this case. + assert(repr == .indirect); + return try self.spv.resolve(.{ .opaque_type = .{ + .name = try self.spv.resolveString("u0"), + } }); + } return try self.intType(int_info.signedness, int_info.bits); }, .Enum => { @@ -1012,10 +1279,32 @@ pub const DeclGen = struct { const elem_ty = ty.childType(mod); const elem_ty_ref = try self.resolveType(elem_ty, .indirect); - const total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(mod)) orelse { + var total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(mod)) orelse { return self.fail("array type of {} elements is too large", .{ty.arrayLenIncludingSentinel(mod)}); }; - const ty_ref = try self.spv.arrayType(total_len, elem_ty_ref); + const ty_ref = if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) blk: { + // The size of the array would be 0, but that is not allowed in SPIR-V. + // This path can be reached when the backend is asked to generate a pointer to + // an array of some zero-bit type. This should always be an indirect path. + assert(repr == .indirect); + + // We cannot use the child type here, so just use an opaque type. + break :blk try self.spv.resolve(.{ .opaque_type = .{ + .name = try self.spv.resolveString("zero-sized array"), + } }); + } else if (total_len == 0) blk: { + // The size of the array would be 0, but that is not allowed in SPIR-V. + // This path can be reached for example when there is a slicing of a pointer + // that produces a zero-length array. In all cases where this type can be generated, + // this should be an indirect path. + assert(repr == .indirect); + + // In this case, we have an array of a non-zero sized type. In this case, + // generate an array of 1 element instead, so that ptr_elem_ptr instructions + // can be lowered to ptrAccessChain instead of manually performing the math. + break :blk try self.spv.arrayType(1, elem_ty_ref); + } else try self.spv.arrayType(total_len, elem_ty_ref); + try self.type_map.put(self.gpa, ty.toIntern(), .{ .ty_ref = ty_ref }); return ty_ref; }, @@ -1030,14 +1319,19 @@ pub const DeclGen = struct { const param_ty_refs = try self.gpa.alloc(CacheRef, fn_info.param_types.len); defer self.gpa.free(param_ty_refs); - for (param_ty_refs, fn_info.param_types.get(ip)) |*param_type, fn_param_type| { - param_type.* = try self.resolveType(fn_param_type.toType(), .direct); + var param_index: usize = 0; + for (fn_info.param_types.get(ip)) |param_ty_index| { + const param_ty = param_ty_index.toType(); + if (!param_ty.hasRuntimeBitsIgnoreComptime(mod)) continue; + + param_ty_refs[param_index] = try self.resolveType(param_ty, .direct); + param_index += 1; } - const return_ty_ref = try self.resolveType(fn_info.return_type.toType(), .direct); + const return_ty_ref = try self.resolveFnReturnType(fn_info.return_type.toType()); const ty_ref = try self.spv.resolve(.{ .function_type = .{ .return_type = return_ty_ref, - .parameters = param_ty_refs, + .parameters = param_ty_refs[0..param_index], } }); try self.type_map.put(self.gpa, ty.toIntern(), .{ .ty_ref = ty_ref }); @@ -1072,19 +1366,14 @@ pub const DeclGen = struct { } }); }, .Vector => { - // Although not 100% the same, Zig vectors map quite neatly to SPIR-V vectors (including many integer and float operations - // which work on them), so simply use those. - // Note: SPIR-V vectors only support bools, ints and floats, so pointer vectors need to be supported another way. - // "composite integers" (larger than the largest supported native type) can probably be represented by an array of vectors. - // TODO: The SPIR-V spec mentions that vector sizes may be quite restricted! look into which we can use, and whether OpTypeVector - // is adequate at all for this. - - // TODO: Properly verify sizes and child type. - - return try self.spv.resolve(.{ .vector_type = .{ - .component_type = try self.resolveType(ty.childType(mod), repr), - .component_count = @as(u32, @intCast(ty.vectorLen(mod))), - } }); + if (self.type_map.get(ty.toIntern())) |info| return info.ty_ref; + + const elem_ty = ty.childType(mod); + const elem_ty_ref = try self.resolveType(elem_ty, .indirect); + + const ty_ref = try self.spv.arrayType(ty.vectorLen(mod), elem_ty_ref); + try self.type_map.put(self.gpa, ty.toIntern(), .{ .ty_ref = ty_ref }); + return ty_ref; }, .Struct => { if (self.type_map.get(ty.toIntern())) |info| return info.ty_ref; @@ -1126,10 +1415,16 @@ pub const DeclGen = struct { var it = struct_type.iterateRuntimeOrder(ip); while (it.next()) |field_index| { - const field_ty = struct_type.field_types.get(ip)[field_index]; - const field_name = ip.stringToSlice(struct_type.field_names.get(ip)[field_index]); - try member_types.append(try self.resolveType(field_ty.toType(), .indirect)); - try member_names.append(try self.spv.resolveString(field_name)); + const field_ty = struct_type.field_types.get(ip)[field_index].toType(); + if (!field_ty.hasRuntimeBitsIgnoreComptime(mod)) { + // This is a zero-bit field - we only needed it for the alignment. + continue; + } + + const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse + try ip.getOrPutStringFmt(mod.gpa, "{d}", .{field_index}); + try member_types.append(try self.resolveType(field_ty, .indirect)); + try member_names.append(try self.spv.resolveString(ip.stringToSlice(field_name))); } const ty_ref = try self.spv.resolve(.{ .struct_type = .{ @@ -1215,6 +1510,13 @@ pub const DeclGen = struct { try self.type_map.put(self.gpa, ty.toIntern(), .{ .ty_ref = ty_ref }); return ty_ref; }, + .Opaque => { + return try self.spv.resolve(.{ + .opaque_type = .{ + .name = .none, // TODO + }, + }); + }, .Null, .Undefined, @@ -1431,7 +1733,7 @@ pub const DeclGen = struct { const mod = self.module; const ip = &mod.intern_pool; const decl = mod.declPtr(self.decl_index); - const spv_decl_index = try self.resolveDecl(self.decl_index); + const spv_decl_index = try self.object.resolveDecl(mod, self.decl_index); const decl_id = self.spv.declPtr(spv_decl_index).result_id; @@ -1439,20 +1741,23 @@ pub const DeclGen = struct { if (decl.val.getFunction(mod)) |_| { assert(decl.ty.zigTypeTag(mod) == .Fn); + const fn_info = mod.typeToFunc(decl.ty).?; + const return_ty_ref = try self.resolveFnReturnType(fn_info.return_type.toType()); + const prototype_id = try self.resolveTypeId(decl.ty); try self.func.prologue.emit(self.spv.gpa, .OpFunction, .{ - .id_result_type = try self.resolveTypeId(decl.ty.fnReturnType(mod)), + .id_result_type = self.typeId(return_ty_ref), .id_result = decl_id, .function_control = .{}, // TODO: We can set inline here if the type requires it. .function_type = prototype_id, }); - const fn_info = mod.typeToFunc(decl.ty).?; - try self.args.ensureUnusedCapacity(self.gpa, fn_info.param_types.len); - for (0..fn_info.param_types.len) |i| { - const param_type = fn_info.param_types.get(ip)[i]; - const param_type_id = try self.resolveTypeId(param_type.toType()); + for (fn_info.param_types.get(ip)) |param_ty_index| { + const param_ty = param_ty_index.toType(); + if (!param_ty.hasRuntimeBitsIgnoreComptime(mod)) continue; + + const param_type_id = try self.resolveTypeId(param_ty); const arg_result_id = self.spv.allocId(); try self.func.prologue.emit(self.spv.gpa, .OpFunctionParameter, .{ .id_result_type = param_type_id, @@ -1693,11 +1998,15 @@ pub const DeclGen = struct { .shl => try self.airShift(inst, .OpShiftLeftLogical), + .min => try self.airMinMax(inst, .lt), + .max => try self.airMinMax(inst, .gt), + .bitcast => try self.airBitCast(inst), .intcast, .trunc => try self.airIntCast(inst), .int_from_ptr => try self.airIntFromPtr(inst), .float_from_int => try self.airFloatFromInt(inst), .int_from_float => try self.airIntFromFloat(inst), + .fpext, .fptrunc => try self.airFloatCast(inst), .not => try self.airNot(inst), .array_to_slice => try self.airArrayToSlice(inst), @@ -1717,18 +2026,20 @@ pub const DeclGen = struct { .union_init => try self.airUnionInit(inst), .struct_field_val => try self.airStructFieldVal(inst), + .field_parent_ptr => try self.airFieldParentPtr(inst), .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), - .cmp_eq => try self.airCmp(inst, .eq), - .cmp_neq => try self.airCmp(inst, .neq), - .cmp_gt => try self.airCmp(inst, .gt), - .cmp_gte => try self.airCmp(inst, .gte), - .cmp_lt => try self.airCmp(inst, .lt), - .cmp_lte => try self.airCmp(inst, .lte), + .cmp_eq => try self.airCmp(inst, .eq), + .cmp_neq => try self.airCmp(inst, .neq), + .cmp_gt => try self.airCmp(inst, .gt), + .cmp_gte => try self.airCmp(inst, .gte), + .cmp_lt => try self.airCmp(inst, .lt), + .cmp_lte => try self.airCmp(inst, .lte), + .cmp_vector => try self.airVectorCmp(inst), .arg => self.airArg(), .alloc => try self.airAlloc(inst), @@ -1784,13 +2095,30 @@ pub const DeclGen = struct { try self.inst_results.putNoClobber(self.gpa, inst, result_id); } - fn airBinOpSimple(self: *DeclGen, inst: Air.Inst.Index, comptime opcode: Opcode) !?IdRef { - if (self.liveness.isUnused(inst)) return null; - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const lhs_id = try self.resolve(bin_op.lhs); - const rhs_id = try self.resolve(bin_op.rhs); + fn binOpSimple(self: *DeclGen, ty: Type, lhs_id: IdRef, rhs_id: IdRef, comptime opcode: Opcode) !IdRef { + const mod = self.module; + + if (ty.isVector(mod)) { + const child_ty = ty.childType(mod); + const vector_len = ty.vectorLen(mod); + + var constituents = try self.gpa.alloc(IdRef, vector_len); + defer self.gpa.free(constituents); + + for (constituents, 0..) |*constituent, i| { + const lhs_index_id = try self.extractField(child_ty, lhs_id, @intCast(i)); + const rhs_index_id = try self.extractField(child_ty, rhs_id, @intCast(i)); + const result_id = try self.binOpSimple(child_ty, lhs_index_id, rhs_index_id, opcode); + constituent.* = try self.convertToIndirect(child_ty, result_id); + } + + const result_ty = try self.resolveType(child_ty, .indirect); + const result_ty_ref = try self.spv.arrayType(vector_len, result_ty); + return try self.constructArray(result_ty_ref, constituents); + } + const result_id = self.spv.allocId(); - const result_type_id = try self.resolveTypeId(self.typeOfIndex(inst)); + const result_type_id = try self.resolveTypeId(ty); try self.func.body.emit(self.spv.gpa, opcode, .{ .id_result_type = result_type_id, .id_result = result_id, @@ -1800,6 +2128,17 @@ pub const DeclGen = struct { return result_id; } + fn airBinOpSimple(self: *DeclGen, inst: Air.Inst.Index, comptime opcode: Opcode) !?IdRef { + if (self.liveness.isUnused(inst)) return null; + + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + const lhs_id = try self.resolve(bin_op.lhs); + const rhs_id = try self.resolve(bin_op.rhs); + const ty = self.typeOf(bin_op.lhs); + + return try self.binOpSimple(ty, lhs_id, rhs_id, opcode); + } + fn airShift(self: *DeclGen, inst: Air.Inst.Index, comptime opcode: Opcode) !?IdRef { if (self.liveness.isUnused(inst)) return null; const bin_op = self.air.instructions.items(.data)[inst].bin_op; @@ -1825,19 +2164,99 @@ pub const DeclGen = struct { return result_id; } - fn maskStrangeInt(self: *DeclGen, ty_ref: CacheRef, value_id: IdRef, bits: u16) !IdRef { - const mask_value = if (bits == 64) 0xFFFF_FFFF_FFFF_FFFF else (@as(u64, 1) << @as(u6, @intCast(bits))) - 1; + fn airMinMax(self: *DeclGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !?IdRef { + if (self.liveness.isUnused(inst)) return null; + + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + const lhs_id = try self.resolve(bin_op.lhs); + const rhs_id = try self.resolve(bin_op.rhs); + const result_ty = self.typeOfIndex(inst); + const result_ty_ref = try self.resolveType(result_ty, .direct); + + const info = try self.arithmeticTypeInfo(result_ty); + // TODO: Use fmin for OpenCL + const cmp_id = try self.cmp(op, result_ty, lhs_id, rhs_id); + const selection_id = switch (info.class) { + .float => blk: { + // cmp uses OpFOrd. When we have 0 [<>] nan this returns false, + // but we want it to pick lhs. Therefore we also have to check if + // rhs is nan. We don't need to care about the result when both + // are nan. + const rhs_is_nan_id = self.spv.allocId(); + const bool_ty_ref = try self.resolveType(Type.bool, .direct); + try self.func.body.emit(self.spv.gpa, .OpIsNan, .{ + .id_result_type = self.typeId(bool_ty_ref), + .id_result = rhs_is_nan_id, + .x = rhs_id, + }); + const float_cmp_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpLogicalOr, .{ + .id_result_type = self.typeId(bool_ty_ref), + .id_result = float_cmp_id, + .operand_1 = cmp_id, + .operand_2 = rhs_is_nan_id, + }); + break :blk float_cmp_id; + }, + else => cmp_id, + }; + const result_id = self.spv.allocId(); - const mask_id = try self.constInt(ty_ref, mask_value); - try self.func.body.emit(self.spv.gpa, .OpBitwiseAnd, .{ - .id_result_type = self.typeId(ty_ref), + try self.func.body.emit(self.spv.gpa, .OpSelect, .{ + .id_result_type = self.typeId(result_ty_ref), .id_result = result_id, - .operand_1 = value_id, - .operand_2 = mask_id, + .condition = selection_id, + .object_1 = lhs_id, + .object_2 = rhs_id, }); return result_id; } + /// This function canonicalizes a "strange" integer value: + /// For unsigned integers, the value is masked so that only the relevant bits can contain + /// non-zeros. + /// For signed integers, the value is also sign extended. + fn normalizeInt(self: *DeclGen, ty_ref: CacheRef, value_id: IdRef, info: ArithmeticTypeInfo) !IdRef { + assert(info.class != .composite_integer); // TODO + if (info.bits == info.backing_bits) { + return value_id; + } + + switch (info.signedness) { + .unsigned => { + const mask_value = if (info.bits == 64) 0xFFFF_FFFF_FFFF_FFFF else (@as(u64, 1) << @as(u6, @intCast(info.bits))) - 1; + const result_id = self.spv.allocId(); + const mask_id = try self.constInt(ty_ref, mask_value); + try self.func.body.emit(self.spv.gpa, .OpBitwiseAnd, .{ + .id_result_type = self.typeId(ty_ref), + .id_result = result_id, + .operand_1 = value_id, + .operand_2 = mask_id, + }); + return result_id; + }, + .signed => { + // Shift left and right so that we can copy the sight bit that way. + const shift_amt_id = try self.constInt(ty_ref, info.backing_bits - info.bits); + const left_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpShiftLeftLogical, .{ + .id_result_type = self.typeId(ty_ref), + .id_result = left_id, + .base = value_id, + .shift = shift_amt_id, + }); + const right_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpShiftRightArithmetic, .{ + .id_result_type = self.typeId(ty_ref), + .id_result = right_id, + .base = left_id, + .shift = shift_amt_id, + }); + return right_id; + }, + } + } + fn airArithOp( self: *DeclGen, inst: Air.Inst.Index, @@ -1848,18 +2267,52 @@ pub const DeclGen = struct { comptime modular: bool, ) !?IdRef { if (self.liveness.isUnused(inst)) return null; + // LHS and RHS are guaranteed to have the same type, and AIR guarantees // the result to be the same as the LHS and RHS, which matches SPIR-V. const ty = self.typeOfIndex(inst); const bin_op = self.air.instructions.items(.data)[inst].bin_op; - var lhs_id = try self.resolve(bin_op.lhs); - var rhs_id = try self.resolve(bin_op.rhs); - - const result_ty_ref = try self.resolveType(ty, .direct); + const lhs_id = try self.resolve(bin_op.lhs); + const rhs_id = try self.resolve(bin_op.rhs); assert(self.typeOf(bin_op.lhs).eql(ty, self.module)); assert(self.typeOf(bin_op.rhs).eql(ty, self.module)); + return try self.arithOp(ty, lhs_id, rhs_id, fop, sop, uop, modular); + } + + fn arithOp( + self: *DeclGen, + ty: Type, + lhs_id_: IdRef, + rhs_id_: IdRef, + comptime fop: Opcode, + comptime sop: Opcode, + comptime uop: Opcode, + /// true if this operation holds under modular arithmetic. + comptime modular: bool, + ) !IdRef { + var rhs_id = rhs_id_; + var lhs_id = lhs_id_; + + const mod = self.module; + const result_ty_ref = try self.resolveType(ty, .direct); + + if (ty.isVector(mod)) { + const child_ty = ty.childType(mod); + const vector_len = ty.vectorLen(mod); + var constituents = try self.gpa.alloc(IdRef, vector_len); + defer self.gpa.free(constituents); + + for (constituents, 0..) |*constituent, i| { + const lhs_index_id = try self.extractField(child_ty, lhs_id, @intCast(i)); + const rhs_index_id = try self.extractField(child_ty, rhs_id, @intCast(i)); + constituent.* = try self.arithOp(child_ty, lhs_index_id, rhs_index_id, fop, sop, uop, modular); + } + + return self.constructArray(result_ty_ref, constituents); + } + // Binary operations are generally applicable to both scalar and vector operations // in SPIR-V, but int and float versions of operations require different opcodes. const info = try self.arithmeticTypeInfo(ty); @@ -1870,8 +2323,8 @@ pub const DeclGen = struct { }, .strange_integer => blk: { if (!modular) { - lhs_id = try self.maskStrangeInt(result_ty_ref, lhs_id, info.bits); - rhs_id = try self.maskStrangeInt(result_ty_ref, rhs_id, info.bits); + lhs_id = try self.normalizeInt(result_ty_ref, lhs_id, info); + rhs_id = try self.normalizeInt(result_ty_ref, rhs_id, info); } break :blk switch (info.signedness) { .signed => @as(usize, 1), @@ -2174,8 +2627,7 @@ pub const DeclGen = struct { fn cmp( self: *DeclGen, - comptime op: std.math.CompareOperator, - bool_ty_id: IdRef, + op: std.math.CompareOperator, ty: Type, lhs_id: IdRef, rhs_id: IdRef, @@ -2183,38 +2635,104 @@ pub const DeclGen = struct { const mod = self.module; var cmp_lhs_id = lhs_id; var cmp_rhs_id = rhs_id; - const opcode: Opcode = opcode: { - const op_ty = switch (ty.zigTypeTag(mod)) { - .Int, .Bool, .Float => ty, - .Enum => ty.intTagType(mod), - .ErrorSet => Type.u16, - .Pointer => blk: { - // Note that while SPIR-V offers OpPtrEqual and OpPtrNotEqual, they are - // currently not implemented in the SPIR-V LLVM translator. Thus, we emit these using - // OpConvertPtrToU... - cmp_lhs_id = self.spv.allocId(); - cmp_rhs_id = self.spv.allocId(); - - const usize_ty_id = self.typeId(try self.sizeType()); - - try self.func.body.emit(self.spv.gpa, .OpConvertPtrToU, .{ - .id_result_type = usize_ty_id, - .id_result = cmp_lhs_id, - .pointer = lhs_id, - }); + const bool_ty_ref = try self.resolveType(Type.bool, .direct); + const op_ty = switch (ty.zigTypeTag(mod)) { + .Int, .Bool, .Float => ty, + .Enum => ty.intTagType(mod), + .ErrorSet => Type.u16, + .Pointer => blk: { + // Note that while SPIR-V offers OpPtrEqual and OpPtrNotEqual, they are + // currently not implemented in the SPIR-V LLVM translator. Thus, we emit these using + // OpConvertPtrToU... + cmp_lhs_id = self.spv.allocId(); + cmp_rhs_id = self.spv.allocId(); + + const usize_ty_id = self.typeId(try self.sizeType()); + + try self.func.body.emit(self.spv.gpa, .OpConvertPtrToU, .{ + .id_result_type = usize_ty_id, + .id_result = cmp_lhs_id, + .pointer = lhs_id, + }); - try self.func.body.emit(self.spv.gpa, .OpConvertPtrToU, .{ - .id_result_type = usize_ty_id, - .id_result = cmp_rhs_id, - .pointer = rhs_id, - }); + try self.func.body.emit(self.spv.gpa, .OpConvertPtrToU, .{ + .id_result_type = usize_ty_id, + .id_result = cmp_rhs_id, + .pointer = rhs_id, + }); - break :blk Type.usize; - }, - .Optional => unreachable, // TODO - else => unreachable, - }; + break :blk Type.usize; + }, + .Optional => { + const payload_ty = ty.optionalChild(mod); + if (ty.optionalReprIsPayload(mod)) { + assert(payload_ty.hasRuntimeBitsIgnoreComptime(mod)); + assert(!payload_ty.isSlice(mod)); + return self.cmp(op, payload_ty, lhs_id, rhs_id); + } + + const lhs_valid_id = if (payload_ty.hasRuntimeBitsIgnoreComptime(mod)) + try self.extractField(Type.bool, lhs_id, 1) + else + try self.convertToDirect(Type.bool, lhs_id); + + const rhs_valid_id = if (payload_ty.hasRuntimeBitsIgnoreComptime(mod)) + try self.extractField(Type.bool, rhs_id, 1) + else + try self.convertToDirect(Type.bool, rhs_id); + + const valid_cmp_id = try self.cmp(op, Type.bool, lhs_valid_id, rhs_valid_id); + if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) { + return valid_cmp_id; + } + // TODO: Should we short circuit here? It shouldn't affect correctness, but + // perhaps it will generate more efficient code. + + const lhs_pl_id = try self.extractField(payload_ty, lhs_id, 0); + const rhs_pl_id = try self.extractField(payload_ty, rhs_id, 0); + + const pl_cmp_id = try self.cmp(op, payload_ty, lhs_pl_id, rhs_pl_id); + + // op == .eq => lhs_valid == rhs_valid && lhs_pl == rhs_pl + // op == .neq => lhs_valid != rhs_valid || lhs_pl != rhs_pl + + const result_id = self.spv.allocId(); + const args = .{ + .id_result_type = self.typeId(bool_ty_ref), + .id_result = result_id, + .operand_1 = valid_cmp_id, + .operand_2 = pl_cmp_id, + }; + switch (op) { + .eq => try self.func.body.emit(self.spv.gpa, .OpLogicalAnd, args), + .neq => try self.func.body.emit(self.spv.gpa, .OpLogicalOr, args), + else => unreachable, + } + return result_id; + }, + .Vector => { + const child_ty = ty.childType(mod); + const vector_len = ty.vectorLen(mod); + const bool_ty_ref_indirect = try self.resolveType(Type.bool, .indirect); + + var constituents = try self.gpa.alloc(IdRef, vector_len); + defer self.gpa.free(constituents); + + for (constituents, 0..) |*constituent, i| { + const lhs_index_id = try self.extractField(child_ty, cmp_lhs_id, @intCast(i)); + const rhs_index_id = try self.extractField(child_ty, cmp_rhs_id, @intCast(i)); + const result_id = try self.cmp(op, child_ty, lhs_index_id, rhs_index_id); + constituent.* = try self.convertToIndirect(Type.bool, result_id); + } + + const result_ty_ref = try self.spv.arrayType(vector_len, bool_ty_ref_indirect); + return try self.constructArray(result_ty_ref, constituents); + }, + else => unreachable, + }; + + const opcode: Opcode = opcode: { const info = try self.arithmeticTypeInfo(op_ty); const signedness = switch (info.class) { .composite_integer => { @@ -2222,7 +2740,7 @@ pub const DeclGen = struct { }, .float => break :opcode switch (op) { .eq => .OpFOrdEqual, - .neq => .OpFOrdNotEqual, + .neq => .OpFUnordNotEqual, .lt => .OpFOrdLessThan, .lte => .OpFOrdLessThanEqual, .gt => .OpFOrdGreaterThan, @@ -2236,8 +2754,8 @@ pub const DeclGen = struct { .strange_integer => sign: { const op_ty_ref = try self.resolveType(op_ty, .direct); // Mask operands before performing comparison. - cmp_lhs_id = try self.maskStrangeInt(op_ty_ref, cmp_lhs_id, info.bits); - cmp_rhs_id = try self.maskStrangeInt(op_ty_ref, cmp_rhs_id, info.bits); + cmp_lhs_id = try self.normalizeInt(op_ty_ref, cmp_lhs_id, info); + cmp_rhs_id = try self.normalizeInt(op_ty_ref, cmp_rhs_id, info); break :sign info.signedness; }, .integer => info.signedness, @@ -2265,7 +2783,7 @@ pub const DeclGen = struct { const result_id = self.spv.allocId(); try self.func.body.emitRaw(self.spv.gpa, opcode, 4); - self.func.body.writeOperand(spec.IdResultType, bool_ty_id); + self.func.body.writeOperand(spec.IdResultType, self.typeId(bool_ty_ref)); self.func.body.writeOperand(spec.IdResult, result_id); self.func.body.writeOperand(spec.IdResultType, cmp_lhs_id); self.func.body.writeOperand(spec.IdResultType, cmp_rhs_id); @@ -2281,11 +2799,22 @@ pub const DeclGen = struct { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const lhs_id = try self.resolve(bin_op.lhs); const rhs_id = try self.resolve(bin_op.rhs); - const bool_ty_id = try self.resolveTypeId(Type.bool); const ty = self.typeOf(bin_op.lhs); - assert(ty.eql(self.typeOf(bin_op.rhs), self.module)); - return try self.cmp(op, bool_ty_id, ty, lhs_id, rhs_id); + return try self.cmp(op, ty, lhs_id, rhs_id); + } + + fn airVectorCmp(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { + if (self.liveness.isUnused(inst)) return null; + + const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; + const vec_cmp = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; + const lhs_id = try self.resolve(vec_cmp.lhs); + const rhs_id = try self.resolve(vec_cmp.rhs); + const op = vec_cmp.compareOperator(); + const ty = self.typeOf(vec_cmp.lhs); + + return try self.cmp(op, ty, lhs_id, rhs_id); } fn bitCast( @@ -2295,26 +2824,58 @@ pub const DeclGen = struct { src_id: IdRef, ) !IdRef { const mod = self.module; + const src_ty_ref = try self.resolveType(src_ty, .direct); const dst_ty_ref = try self.resolveType(dst_ty, .direct); - const result_id = self.spv.allocId(); + if (src_ty_ref == dst_ty_ref) { + return src_id; + } // TODO: Some more cases are missing here // See fn bitCast in llvm.zig if (src_ty.zigTypeTag(mod) == .Int and dst_ty.isPtrAtRuntime(mod)) { + const result_id = self.spv.allocId(); try self.func.body.emit(self.spv.gpa, .OpConvertUToPtr, .{ .id_result_type = self.typeId(dst_ty_ref), .id_result = result_id, .integer_value = src_id, }); - } else { + return result_id; + } + + // We can only use OpBitcast for specific conversions: between numerical types, and + // between pointers. If the resolved spir-v types fall into this category then emit OpBitcast, + // otherwise use a temporary and perform a pointer cast. + const src_key = self.spv.cache.lookup(src_ty_ref); + const dst_key = self.spv.cache.lookup(dst_ty_ref); + + if ((src_key.isNumericalType() and dst_key.isNumericalType()) or (src_key == .ptr_type and dst_key == .ptr_type)) { + const result_id = self.spv.allocId(); try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ .id_result_type = self.typeId(dst_ty_ref), .id_result = result_id, .operand = src_id, }); + return result_id; } - return result_id; + + const src_ptr_ty_ref = try self.spv.ptrType(src_ty_ref, .Function); + const dst_ptr_ty_ref = try self.spv.ptrType(dst_ty_ref, .Function); + + const tmp_id = self.spv.allocId(); + try self.func.prologue.emit(self.spv.gpa, .OpVariable, .{ + .id_result_type = self.typeId(src_ptr_ty_ref), + .id_result = tmp_id, + .storage_class = .Function, + }); + try self.store(src_ty, tmp_id, src_id, false); + const casted_ptr_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpBitcast, .{ + .id_result_type = self.typeId(dst_ptr_ty_ref), + .id_result = casted_ptr_id, + .operand = tmp_id, + }); + return try self.load(dst_ty, casted_ptr_id, false); } fn airBitCast(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { @@ -2331,37 +2892,40 @@ pub const DeclGen = struct { const ty_op = self.air.instructions.items(.data)[inst].ty_op; const operand_id = try self.resolve(ty_op.operand); - const dest_ty = self.typeOfIndex(inst); - const dest_ty_id = try self.resolveTypeId(dest_ty); + const src_ty = self.typeOf(ty_op.operand); + const dst_ty = self.typeOfIndex(inst); + const src_ty_ref = try self.resolveType(src_ty, .direct); + const dst_ty_ref = try self.resolveType(dst_ty, .direct); - const mod = self.module; - const dest_info = dest_ty.intInfo(mod); + const src_info = try self.arithmeticTypeInfo(src_ty); + const dst_info = try self.arithmeticTypeInfo(dst_ty); + + // While intcast promises that the value already fits, the upper bits of a + // strange integer may contain garbage. Therefore, mask/sign extend it before. + const src_id = try self.normalizeInt(src_ty_ref, operand_id, src_info); - // TODO: Masking? + if (src_info.backing_bits == dst_info.backing_bits) { + return src_id; + } const result_id = self.spv.allocId(); - switch (dest_info.signedness) { + switch (dst_info.signedness) { .signed => try self.func.body.emit(self.spv.gpa, .OpSConvert, .{ - .id_result_type = dest_ty_id, + .id_result_type = self.typeId(dst_ty_ref), .id_result = result_id, - .signed_value = operand_id, + .signed_value = src_id, }), .unsigned => try self.func.body.emit(self.spv.gpa, .OpUConvert, .{ - .id_result_type = dest_ty_id, + .id_result_type = self.typeId(dst_ty_ref), .id_result = result_id, - .unsigned_value = operand_id, + .unsigned_value = src_id, }), } return result_id; } - fn airIntFromPtr(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { - if (self.liveness.isUnused(inst)) return null; - - const un_op = self.air.instructions.items(.data)[inst].un_op; - const operand_id = try self.resolve(un_op); + fn intFromPtr(self: *DeclGen, operand_id: IdRef) !IdRef { const result_type_id = try self.resolveTypeId(Type.usize); - const result_id = self.spv.allocId(); try self.func.body.emit(self.spv.gpa, .OpConvertPtrToU, .{ .id_result_type = result_type_id, @@ -2371,6 +2935,14 @@ pub const DeclGen = struct { return result_id; } + fn airIntFromPtr(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { + if (self.liveness.isUnused(inst)) return null; + + const un_op = self.air.instructions.items(.data)[inst].un_op; + const operand_id = try self.resolve(un_op); + return try self.intFromPtr(operand_id); + } + fn airFloatFromInt(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { if (self.liveness.isUnused(inst)) return null; @@ -2422,6 +2994,23 @@ pub const DeclGen = struct { return result_id; } + fn airFloatCast(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { + if (self.liveness.isUnused(inst)) return null; + + const ty_op = self.air.instructions.items(.data)[inst].ty_op; + const operand_id = try self.resolve(ty_op.operand); + const dest_ty = self.typeOfIndex(inst); + const dest_ty_id = try self.resolveTypeId(dest_ty); + + const result_id = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpFConvert, .{ + .id_result_type = dest_ty_id, + .id_result = result_id, + .float_value = operand_id, + }); + return result_id; + } + fn airNot(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { if (self.liveness.isUnused(inst)) return null; const ty_op = self.air.instructions.items(.data)[inst].ty_op; @@ -2461,22 +3050,23 @@ pub const DeclGen = struct { const ty_op = self.air.instructions.items(.data)[inst].ty_op; const array_ptr_ty = self.typeOf(ty_op.operand); const array_ty = array_ptr_ty.childType(mod); - const elem_ty = array_ptr_ty.elemType2(mod); // use elemType() so that we get T for *[N]T. - const elem_ty_ref = try self.resolveType(elem_ty, .indirect); - const elem_ptr_ty_ref = try self.spv.ptrType(elem_ty_ref, spvStorageClass(array_ptr_ty.ptrAddressSpace(mod))); const slice_ty = self.typeOfIndex(inst); + const elem_ptr_ty = slice_ty.slicePtrFieldType(mod); + + const elem_ptr_ty_ref = try self.resolveType(elem_ptr_ty, .direct); const slice_ty_ref = try self.resolveType(slice_ty, .direct); const size_ty_ref = try self.sizeType(); const array_ptr_id = try self.resolve(ty_op.operand); const len_id = try self.constInt(size_ty_ref, array_ty.arrayLen(mod)); - if (!array_ty.hasRuntimeBitsIgnoreComptime(mod)) { - unreachable; // TODO - } + const elem_ptr_id = if (!array_ty.hasRuntimeBitsIgnoreComptime(mod)) + // Note: The pointer is something like *opaque{}, so we need to bitcast it to the element type. + try self.bitCast(elem_ptr_ty, array_ptr_ty, array_ptr_id) + else + // Convert the pointer-to-array to a pointer to the first element. + try self.accessChain(elem_ptr_ty_ref, array_ptr_id, &.{0}); - // Convert the pointer-to-array to a pointer to the first element. - const elem_ptr_id = try self.accessChain(elem_ptr_ty_ref, array_ptr_id, &.{0}); return try self.constructStruct(slice_ty_ref, &.{ elem_ptr_id, len_id }); } @@ -2500,6 +3090,7 @@ pub const DeclGen = struct { if (self.liveness.isUnused(inst)) return null; const mod = self.module; + const ip = &mod.intern_pool; const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const result_ty = self.typeOfIndex(inst); const result_ty_ref = try self.resolveType(result_ty, .direct); @@ -2508,15 +3099,53 @@ pub const DeclGen = struct { switch (result_ty.zigTypeTag(mod)) { .Vector => unreachable, // TODO - .Struct => unreachable, // TODO + .Struct => { + if (mod.typeToPackedStruct(result_ty)) |struct_type| { + _ = struct_type; + unreachable; // TODO + } + + const constituents = try self.gpa.alloc(IdRef, elements.len); + defer self.gpa.free(constituents); + var index: usize = 0; + + switch (ip.indexToKey(result_ty.toIntern())) { + .anon_struct_type => |tuple| { + for (tuple.types.get(ip), elements, 0..) |field_ty, element, i| { + if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue; + assert(field_ty.toType().hasRuntimeBits(mod)); + + const id = try self.resolve(element); + constituents[index] = try self.convertToIndirect(field_ty.toType(), id); + index += 1; + } + }, + .struct_type => |struct_type| { + var it = struct_type.iterateRuntimeOrder(ip); + for (elements, 0..) |element, i| { + const field_index = it.next().?; + if ((try result_ty.structFieldValueComptime(mod, i)) != null) continue; + const field_ty = struct_type.field_types.get(ip)[field_index].toType(); + assert(field_ty.hasRuntimeBitsIgnoreComptime(mod)); + + const id = try self.resolve(element); + constituents[index] = try self.convertToIndirect(field_ty, id); + index += 1; + } + }, + else => unreachable, + } + + return try self.constructStruct(result_ty_ref, constituents[0..index]); + }, .Array => { const array_info = result_ty.arrayInfo(mod); const n_elems: usize = @intCast(result_ty.arrayLenIncludingSentinel(mod)); const elem_ids = try self.gpa.alloc(IdRef, n_elems); defer self.gpa.free(elem_ids); - for (elements, 0..) |elem_inst, i| { - const id = try self.resolve(elem_inst); + for (elements, 0..) |element, i| { + const id = try self.resolve(element); elem_ids[i] = try self.convertToIndirect(array_info.elem_type, id); } @@ -2540,7 +3169,8 @@ pub const DeclGen = struct { fn airSliceElemPtr(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { const mod = self.module; - const bin_op = self.air.instructions.items(.data)[inst].bin_op; + const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const slice_ty = self.typeOf(bin_op.lhs); if (!slice_ty.isVolatilePtr(mod) and self.liveness.isUnused(inst)) return null; @@ -2593,14 +3223,17 @@ pub const DeclGen = struct { const mod = self.module; const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr_ty = self.typeOf(bin_op.lhs); - const elem_ty = ptr_ty.childType(mod); - // TODO: Make this return a null ptr or something - if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) return null; - + const src_ptr_ty = self.typeOf(bin_op.lhs); + const elem_ty = src_ptr_ty.childType(mod); const ptr_id = try self.resolve(bin_op.lhs); + + if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) { + const dst_ptr_ty = self.typeOfIndex(inst); + return try self.bitCast(dst_ptr_ty, src_ptr_ty, ptr_id); + } + const index_id = try self.resolve(bin_op.rhs); - return try self.ptrElemPtr(ptr_ty, ptr_id, index_id); + return try self.ptrElemPtr(src_ptr_ty, ptr_id, index_id); } fn airArrayElemVal(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { @@ -2850,13 +3483,44 @@ pub const DeclGen = struct { } } + fn airFieldParentPtr(self: *DeclGen, inst: Air.Inst.Index) !?IdRef { + const mod = self.module; + const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; + const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + + const parent_ty = self.air.getRefType(ty_pl.ty).childType(mod); + const res_ty = try self.resolveType(self.air.getRefType(ty_pl.ty), .indirect); + const usize_ty = Type.usize; + const usize_ty_ref = try self.resolveType(usize_ty, .direct); + + const field_ptr = try self.resolve(extra.field_ptr); + const field_ptr_int = try self.intFromPtr(field_ptr); + const field_offset = parent_ty.structFieldOffset(extra.field_index, mod); + + const base_ptr_int = base_ptr_int: { + if (field_offset == 0) break :base_ptr_int field_ptr_int; + + const field_offset_id = try self.constInt(usize_ty_ref, field_offset); + break :base_ptr_int try self.binOpSimple(usize_ty, field_ptr_int, field_offset_id, .OpISub); + }; + + const base_ptr = self.spv.allocId(); + try self.func.body.emit(self.spv.gpa, .OpConvertUToPtr, .{ + .id_result_type = self.spv.resultId(res_ty), + .id_result = base_ptr, + .integer_value = base_ptr_int, + }); + + return base_ptr; + } + fn structFieldPtr( self: *DeclGen, result_ptr_ty: Type, object_ptr_ty: Type, object_ptr: IdRef, field_index: u32, - ) !?IdRef { + ) !IdRef { const result_ty_ref = try self.resolveType(result_ptr_ty, .direct); const mod = self.module; @@ -3101,15 +3765,25 @@ pub const DeclGen = struct { fn airRet(self: *DeclGen, inst: Air.Inst.Index) !void { const operand = self.air.instructions.items(.data)[inst].un_op; - const operand_ty = self.typeOf(operand); + const ret_ty = self.typeOf(operand); const mod = self.module; - if (operand_ty.hasRuntimeBits(mod)) { - // TODO: If we return an empty struct, this branch is also hit incorrectly. - const operand_id = try self.resolve(operand); - try self.func.body.emit(self.spv.gpa, .OpReturnValue, .{ .value = operand_id }); - } else { - try self.func.body.emit(self.spv.gpa, .OpReturn, {}); + if (!ret_ty.hasRuntimeBitsIgnoreComptime(mod)) { + const decl = mod.declPtr(self.decl_index); + const fn_info = mod.typeToFunc(decl.ty).?; + if (fn_info.return_type.toType().isError(mod)) { + // Functions with an empty error set are emitted with an error code + // return type and return zero so they can be function pointers coerced + // to functions that return anyerror. + const err_ty_ref = try self.resolveType(Type.anyerror, .direct); + const no_err_id = try self.constInt(err_ty_ref, 0); + return try self.func.body.emit(self.spv.gpa, .OpReturnValue, .{ .value = no_err_id }); + } else { + return try self.func.body.emit(self.spv.gpa, .OpReturn, {}); + } } + + const operand_id = try self.resolve(operand); + try self.func.body.emit(self.spv.gpa, .OpReturnValue, .{ .value = operand_id }); } fn airRetLoad(self: *DeclGen, inst: Air.Inst.Index) !void { @@ -3119,8 +3793,18 @@ pub const DeclGen = struct { const ret_ty = ptr_ty.childType(mod); if (!ret_ty.hasRuntimeBitsIgnoreComptime(mod)) { - try self.func.body.emit(self.spv.gpa, .OpReturn, {}); - return; + const decl = mod.declPtr(self.decl_index); + const fn_info = mod.typeToFunc(decl.ty).?; + if (fn_info.return_type.toType().isError(mod)) { + // Functions with an empty error set are emitted with an error code + // return type and return zero so they can be function pointers coerced + // to functions that return anyerror. + const err_ty_ref = try self.resolveType(Type.anyerror, .direct); + const no_err_id = try self.constInt(err_ty_ref, 0); + return try self.func.body.emit(self.spv.gpa, .OpReturnValue, .{ .value = no_err_id }); + } else { + return try self.func.body.emit(self.spv.gpa, .OpReturn, {}); + } } const ptr = try self.resolve(un_op); @@ -3297,31 +3981,26 @@ pub const DeclGen = struct { payload_ty; const ptr_id = if (payload_ty.isSlice(mod)) - try self.extractField(Type.bool, operand_id, 0) + try self.extractField(ptr_ty, operand_id, 0) else operand_id; const payload_ty_ref = try self.resolveType(ptr_ty, .direct); const null_id = try self.spv.constNull(payload_ty_ref); - const result_id = self.spv.allocId(); - const operands = .{ - .id_result_type = self.typeId(bool_ty_ref), - .id_result = result_id, - .operand_1 = ptr_id, - .operand_2 = null_id, + const op: std.math.CompareOperator = switch (pred) { + .is_null => .eq, + .is_non_null => .neq, }; - switch (pred) { - .is_null => try self.func.body.emit(self.spv.gpa, .OpPtrEqual, operands), - .is_non_null => try self.func.body.emit(self.spv.gpa, .OpPtrNotEqual, operands), - } - return result_id; + return try self.cmp(op, ptr_ty, ptr_id, null_id); } - const is_non_null_id = if (optional_ty.hasRuntimeBitsIgnoreComptime(mod)) + const is_non_null_id = if (payload_ty.hasRuntimeBitsIgnoreComptime(mod)) try self.extractField(Type.bool, operand_id, 1) else // Optional representation is bool indicating whether the optional is set - operand_id; + // Optionals with no payload are represented as an (indirect) bool, so convert + // it back to the direct bool here. + try self.convertToDirect(Type.bool, operand_id); return switch (pred) { .is_null => blk: { @@ -3400,17 +4079,19 @@ pub const DeclGen = struct { const payload_ty = self.typeOf(ty_op.operand); if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) { - return try self.constBool(true, .direct); + return try self.constBool(true, .indirect); } const operand_id = try self.resolve(ty_op.operand); + const optional_ty = self.typeOfIndex(inst); if (optional_ty.optionalReprIsPayload(mod)) { return operand_id; } const optional_ty_ref = try self.resolveType(optional_ty, .direct); - const members = [_]IdRef{ operand_id, try self.constBool(true, .indirect) }; + const payload_id = try self.convertToIndirect(payload_ty, operand_id); + const members = [_]IdRef{ payload_id, try self.constBool(true, .indirect) }; return try self.constructStruct(optional_ty_ref, &members); } @@ -3437,6 +4118,7 @@ pub const DeclGen = struct { }; break :blk if (backing_bits <= 32) @as(u32, 1) else 2; }, + .ErrorSet => 1, else => return self.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(mod))}), // TODO: Figure out which types apply here, and work around them as we can only do integers. }; @@ -3490,6 +4172,7 @@ pub const DeclGen = struct { // TODO: figure out of cond_ty is correct (something with enum literals) break :blk (try value.intFromEnum(cond_ty, mod)).toUnsignedInt(mod); // TODO: composite integer constants }, + .ErrorSet => value.getErrorInt(mod), else => unreachable, }; const int_lit: spec.LiteralContextDependentNumber = switch (cond_words) { @@ -3533,10 +4216,10 @@ pub const DeclGen = struct { fn airDbgStmt(self: *DeclGen, inst: Air.Inst.Index) !void { const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt; - const src_fname_id = try self.spv.resolveSourceFileName( - self.module, - self.module.declPtr(self.decl_index), - ); + const mod = self.module; + const decl = mod.declPtr(self.decl_index); + const path = decl.getFileScope(mod).sub_file_path; + const src_fname_id = try self.spv.resolveSourceFileName(path); const base_line = self.base_line_stack.getLast(); try self.func.body.emit(self.spv.gpa, .OpLine, .{ .file = src_fname_id, @@ -3710,7 +4393,7 @@ pub const DeclGen = struct { const fn_info = mod.typeToFunc(zig_fn_ty).?; const return_type = fn_info.return_type; - const result_type_id = try self.resolveTypeId(return_type.toType()); + const result_type_ref = try self.resolveFnReturnType(return_type.toType()); const result_id = self.spv.allocId(); const callee_id = try self.resolve(pl_op.operand); @@ -3722,16 +4405,16 @@ pub const DeclGen = struct { // Note: resolve() might emit instructions, so we need to call it // before starting to emit OpFunctionCall instructions. Hence the // temporary params buffer. - const arg_id = try self.resolve(arg); const arg_ty = self.typeOf(arg); if (!arg_ty.hasRuntimeBitsIgnoreComptime(mod)) continue; + const arg_id = try self.resolve(arg); params[n_params] = arg_id; n_params += 1; } try self.func.body.emit(self.spv.gpa, .OpFunctionCall, .{ - .id_result_type = result_type_id, + .id_result_type = self.typeId(result_type_ref), .id_result = result_id, .function = callee_id, .id_ref_3 = params[0..n_params], diff --git a/src/codegen/spirv/Cache.zig b/src/codegen/spirv/Cache.zig index 68fea5c47a..ad2d3442e5 100644 --- a/src/codegen/spirv/Cache.zig +++ b/src/codegen/spirv/Cache.zig @@ -81,6 +81,9 @@ const Tag = enum { /// have member names trailing. /// data is payload to SimpleStructType type_struct_simple_with_member_names, + /// Opaque type. + /// data is name string. + type_opaque, // -- Values /// Value of type u8 @@ -235,6 +238,7 @@ pub const Key = union(enum) { function_type: FunctionType, ptr_type: PointerType, struct_type: StructType, + opaque_type: OpaqueType, // -- values int: Int, @@ -289,6 +293,10 @@ pub const Key = union(enum) { } }; + pub const OpaqueType = struct { + name: String = .none, + }; + pub const Int = struct { /// The type: any bitness integer. ty: Ref, @@ -427,6 +435,13 @@ pub const Key = union(enum) { else => unreachable, }; } + + pub fn isNumericalType(self: Key) bool { + return switch (self) { + .int_type, .float_type => true, + else => false, + }; + } }; pub fn deinit(self: *Self, spv: *const Module) void { @@ -539,6 +554,13 @@ fn emit( } // TODO: Decorations? }, + .opaque_type => |opaque_type| { + const name = if (self.getString(opaque_type.name)) |name| name else ""; + try section.emit(spv.gpa, .OpTypeOpaque, .{ + .id_result = result_id, + .literal_string = name, + }); + }, .int => |int| { const int_type = self.lookup(int.ty).int_type; const ty_id = self.resultId(int.ty); @@ -697,6 +719,11 @@ pub fn resolve(self: *Self, spv: *Module, key: Key) !Ref { }; } }, + .opaque_type => |opaque_type| Item{ + .tag = .type_opaque, + .result_id = result_id, + .data = @intFromEnum(opaque_type.name), + }, .int => |int| blk: { const int_type = self.lookup(int.ty).int_type; if (int_type.signedness == .unsigned and int_type.bits == 8) { @@ -874,6 +901,11 @@ pub fn lookup(self: *const Self, ref: Ref) Key { }, }; }, + .type_opaque => .{ + .opaque_type = .{ + .name = @as(String, @enumFromInt(data)), + }, + }, .float16 => .{ .float = .{ .ty = self.get(.{ .float_type = .{ .bits = 16 } }), .value = .{ .float16 = @as(f16, @bitCast(@as(u16, @intCast(data)))) }, diff --git a/src/codegen/spirv/Module.zig b/src/codegen/spirv/Module.zig index 81b97ebae5..1936b78826 100644 --- a/src/codegen/spirv/Module.zig +++ b/src/codegen/spirv/Module.zig @@ -11,9 +11,6 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const ZigModule = @import("../../Module.zig"); -const ZigDecl = ZigModule.Decl; - const spec = @import("spec.zig"); const Word = spec.Word; const IdRef = spec.IdRef; @@ -103,15 +100,12 @@ pub const EntryPoint = struct { /// The declaration that should be exported. decl_index: Decl.Index, /// The name of the kernel to be exported. - name: []const u8, + name: CacheString, }; /// A general-purpose allocator which may be used to allocate resources for this module gpa: Allocator, -/// An arena allocator used to store things that have the same lifetime as this module. -arena: Allocator, - /// Module layout, according to SPIR-V Spec section 2.4, "Logical Layout of a Module". sections: struct { /// Capability instructions @@ -150,7 +144,7 @@ next_result_id: Word, /// Cache for results of OpString instructions for module file names fed to OpSource. /// Since OpString is pretty much only used for those, we don't need to keep track of all strings, /// just the ones for OpLine. Note that OpLine needs the result of OpString, and not that of OpSource. -source_file_names: std.StringHashMapUnmanaged(IdRef) = .{}, +source_file_names: std.AutoArrayHashMapUnmanaged(CacheString, IdRef) = .{}, /// SPIR-V type- and constant cache. This structure is used to store information about these in a more /// efficient manner. @@ -176,10 +170,9 @@ globals: struct { section: Section = .{}, } = .{}, -pub fn init(gpa: Allocator, arena: Allocator) Module { +pub fn init(gpa: Allocator) Module { return .{ .gpa = gpa, - .arena = arena, .next_result_id = 1, // 0 is an invalid SPIR-V result id, so start counting at 1. }; } @@ -321,7 +314,7 @@ fn entryPoints(self: *Module) !Section { try entry_points.emit(self.gpa, .OpEntryPoint, .{ .execution_model = .Kernel, .entry_point = entry_point_id, - .name = entry_point.name, + .name = self.cache.getString(entry_point.name).?, .interface = interface.items, }); } @@ -422,6 +415,18 @@ pub fn flush(self: *Module, file: std.fs.File) !void { 0, // Schema (currently reserved for future use) }; + var source = Section{}; + defer source.deinit(self.gpa); + try self.sections.debug_strings.emit(self.gpa, .OpSource, .{ + .source_language = .Unknown, + .version = 0, + // We cannot emit these because the Khronos translator does not parse this instruction + // correctly. + // See https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/2188 + .file = null, + .source = null, + }); + // Note: needs to be kept in order according to section 2.3! const buffers = &[_][]const Word{ &header, @@ -429,6 +434,7 @@ pub fn flush(self: *Module, file: std.fs.File) !void { self.sections.extensions.toWords(), entry_points.toWords(), self.sections.execution_modes.toWords(), + source.toWords(), self.sections.debug_strings.toWords(), self.sections.debug_names.toWords(), self.sections.annotations.toWords(), @@ -464,9 +470,9 @@ pub fn addFunction(self: *Module, decl_index: Decl.Index, func: Fn) !void { /// Fetch the result-id of an OpString instruction that encodes the path of the source /// file of the decl. This function may also emit an OpSource with source-level information regarding /// the decl. -pub fn resolveSourceFileName(self: *Module, zig_module: *ZigModule, zig_decl: *ZigDecl) !IdRef { - const path = zig_decl.getFileScope(zig_module).sub_file_path; - const result = try self.source_file_names.getOrPut(self.gpa, path); +pub fn resolveSourceFileName(self: *Module, path: []const u8) !IdRef { + const path_ref = try self.resolveString(path); + const result = try self.source_file_names.getOrPut(self.gpa, path_ref); if (!result.found_existing) { const file_result_id = self.allocId(); result.value_ptr.* = file_result_id; @@ -474,13 +480,6 @@ pub fn resolveSourceFileName(self: *Module, zig_module: *ZigModule, zig_decl: *Z .id_result = file_result_id, .string = path, }); - - try self.sections.debug_strings.emit(self.gpa, .OpSource, .{ - .source_language = .Unknown, // TODO: Register Zig source language. - .version = 0, // TODO: Zig version as u32? - .file = file_result_id, - .source = null, // TODO: Store actual source also? - }); } return result.value_ptr.*; @@ -641,7 +640,7 @@ pub fn endGlobal(self: *Module, global_index: Decl.Index, begin_inst: u32, resul pub fn declareEntryPoint(self: *Module, decl_index: Decl.Index, name: []const u8) !void { try self.entry_points.append(self.gpa, .{ .decl_index = decl_index, - .name = try self.arena.dupe(u8, name), + .name = try self.resolveString(name), }); } diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 931dbacdb9..a8f4125644 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -1389,6 +1389,7 @@ pub fn commitDeclState( .prev_vaddr = 0, }); }, + .elf => {}, // TODO else => unreachable, } } @@ -1850,8 +1851,7 @@ pub fn writeDbgInfoHeader(self: *Dwarf, module: *Module, low_pc: u64, high_pc: u // not including the initial length itself. // We have to come back and write it later after we know the size. const after_init_len = di_buf.items.len + init_len_size; - // +1 for the final 0 that ends the compilation unit children. - const dbg_info_end = self.getDebugInfoEnd().? + 1; + const dbg_info_end = self.getDebugInfoEnd().?; const init_len = dbg_info_end - after_init_len; if (self.bin_file.tag == .macho) { mem.writeIntLittle(u32, di_buf.addManyAsArrayAssumeCapacity(4), @as(u32, @intCast(init_len))); @@ -2500,7 +2500,7 @@ fn getDebugInfoOff(self: Dwarf) ?u32 { fn getDebugInfoEnd(self: Dwarf) ?u32 { const last_index = self.di_atom_last_index orelse return null; const last = self.getAtom(.di_atom, last_index); - return last.off + last.len; + return last.off + last.len + 1; } fn getDebugLineProgramOff(self: Dwarf) ?u32 { @@ -2642,7 +2642,7 @@ fn addDIFile(self: *Dwarf, mod: *Module, decl_index: Module.Decl.Index) !u28 { switch (self.bin_file.tag) { .elf => { const elf_file = self.bin_file.cast(File.Elf).?; - elf_file.markDirty(elf_file.debug_line_section_index.?, null); + elf_file.markDirty(elf_file.debug_line_section_index.?); }, .macho => { const d_sym = self.bin_file.cast(File.MachO).?.getDebugSymbols().?; diff --git a/src/link/Elf.zig b/src/link/Elf.zig index f75047424a..49b95da7ea 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -14,6 +14,7 @@ files: std.MultiArrayList(File.Entry) = .{}, zig_module_index: ?File.Index = null, linker_defined_index: ?File.Index = null, objects: std.ArrayListUnmanaged(File.Index) = .{}, +shared_objects: std.ArrayListUnmanaged(File.Index) = .{}, /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. /// Same order as in the file. @@ -22,35 +23,47 @@ shdrs: std.ArrayListUnmanaged(elf.Elf64_Shdr) = .{}, phdr_to_shdr_table: std.AutoHashMapUnmanaged(u16, u16) = .{}, /// File offset into the shdr table. shdr_table_offset: ?u64 = null, +/// Table of lists of atoms per output section. +/// This table is not used to track incrementally generated atoms. +output_sections: std.AutoArrayHashMapUnmanaged(u16, std.ArrayListUnmanaged(Atom.Index)) = .{}, /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write. /// Same order as in the file. phdrs: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{}, -/// The index into the program headers of the PT_PHDR program header -phdr_table_index: ?u16 = null, -/// The index into the program headers of the PT_LOAD program header containing the phdr -/// Most linkers would merge this with phdr_load_ro_index, -/// but incremental linking means we can't ensure they are consecutive. -phdr_table_load_index: ?u16 = null, + +/// Tracked loadable segments during incremental linking. /// The index into the program headers of a PT_LOAD program header with Read and Execute flags -phdr_load_re_index: ?u16 = null, +phdr_zig_load_re_index: ?u16 = null, /// The index into the program headers of the global offset table. /// It needs PT_LOAD and Read flags. -phdr_got_index: ?u16 = null, +phdr_zig_got_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Read flag -phdr_load_ro_index: ?u16 = null, +phdr_zig_load_ro_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Write flag -phdr_load_rw_index: ?u16 = null, +phdr_zig_load_rw_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with zerofill data. -phdr_load_zerofill_index: ?u16 = null, -/// The index into the program headers of the PT_TLS program header. +phdr_zig_load_zerofill_index: ?u16 = null, + +/// Special program headers +/// PT_PHDR +phdr_table_index: ?u16 = null, +/// PT_LOAD for PHDR table +/// We add this special load segment to ensure the PHDR table is always +/// loaded into memory. +phdr_table_load_index: ?u16 = null, +/// PT_INTERP +phdr_interp_index: ?u16 = null, +/// PT_DYNAMIC +phdr_dynamic_index: ?u16 = null, +/// PT_GNU_EH_FRAME +phdr_gnu_eh_frame_index: ?u16 = null, +/// PT_GNU_STACK +phdr_gnu_stack_index: ?u16 = null, +/// PT_TLS +/// TODO I think ELF permits multiple TLS segments but for now, assume one per file. phdr_tls_index: ?u16 = null, -/// The index into the program headers of a PT_LOAD program header with TLS data. -phdr_load_tls_data_index: ?u16 = null, -/// The index into the program headers of a PT_LOAD program header with TLS zerofill data. -phdr_load_tls_zerofill_index: ?u16 = null, -entry_addr: ?u64 = null, +entry_index: ?Symbol.Index = null, page_size: u32, default_sym_version: elf.Elf64_Versym, @@ -58,29 +71,76 @@ default_sym_version: elf.Elf64_Versym, shstrtab: StringTable(.strtab) = .{}, /// .strtab buffer strtab: StringTable(.strtab) = .{}, - -/// Representation of the GOT table as committed to the file. +/// Dynamic symbol table. Only populated and emitted when linking dynamically. +dynsym: DynsymSection = .{}, +/// .dynstrtab buffer +dynstrtab: StringTable(.dynstrtab) = .{}, +/// Version symbol table. Only populated and emitted when linking dynamically. +versym: std.ArrayListUnmanaged(elf.Elf64_Versym) = .{}, +/// .verneed section +verneed: VerneedSection = .{}, +/// .got section got: GotSection = .{}, +/// .rela.dyn section +rela_dyn: std.ArrayListUnmanaged(elf.Elf64_Rela) = .{}, +/// .dynamic section +dynamic: DynamicSection = .{}, +/// .hash section +hash: HashSection = .{}, +/// .gnu.hash section +gnu_hash: GnuHashSection = .{}, +/// .plt section +plt: PltSection = .{}, +/// .got.plt section +got_plt: GotPltSection = .{}, +/// .plt.got section +plt_got: PltGotSection = .{}, +/// .copyrel section +copy_rel: CopyRelSection = .{}, +/// .rela.plt section +rela_plt: std.ArrayListUnmanaged(elf.Elf64_Rela) = .{}, +/// .zig.got section +zig_got: ZigGotSection = .{}, + +/// Tracked section headers with incremental updates to Zig module +zig_text_section_index: ?u16 = null, +zig_rodata_section_index: ?u16 = null, +zig_data_section_index: ?u16 = null, +zig_bss_section_index: ?u16 = null, +zig_got_section_index: ?u16 = null, + +debug_info_section_index: ?u16 = null, +debug_abbrev_section_index: ?u16 = null, +debug_str_section_index: ?u16 = null, +debug_aranges_section_index: ?u16 = null, +debug_line_section_index: ?u16 = null, -/// Tracked section headers -text_section_index: ?u16 = null, -rodata_section_index: ?u16 = null, -data_section_index: ?u16 = null, -bss_section_index: ?u16 = null, -tdata_section_index: ?u16 = null, -tbss_section_index: ?u16 = null, +/// Size contribution of Zig's metadata to each debug section. +/// Used to track start of metadata from input object files. +debug_info_section_zig_size: u64 = 0, +debug_abbrev_section_zig_size: u64 = 0, +debug_str_section_zig_size: u64 = 0, +debug_aranges_section_zig_size: u64 = 0, +debug_line_section_zig_size: u64 = 0, + +copy_rel_section_index: ?u16 = null, +dynamic_section_index: ?u16 = null, +dynstrtab_section_index: ?u16 = null, +dynsymtab_section_index: ?u16 = null, eh_frame_section_index: ?u16 = null, eh_frame_hdr_section_index: ?u16 = null, -dynamic_section_index: ?u16 = null, +hash_section_index: ?u16 = null, +gnu_hash_section_index: ?u16 = null, got_section_index: ?u16 = null, got_plt_section_index: ?u16 = null, +interp_section_index: ?u16 = null, plt_section_index: ?u16 = null, +plt_got_section_index: ?u16 = null, rela_dyn_section_index: ?u16 = null, -debug_info_section_index: ?u16 = null, -debug_abbrev_section_index: ?u16 = null, -debug_str_section_index: ?u16 = null, -debug_aranges_section_index: ?u16 = null, -debug_line_section_index: ?u16 = null, +rela_plt_section_index: ?u16 = null, +versym_section_index: ?u16 = null, +verneed_section_index: ?u16 = null, + shstrtab_section_index: ?u16 = null, strtab_section_index: ?u16 = null, symtab_section_index: ?u16 = null, @@ -109,11 +169,8 @@ symbols_extra: std.ArrayListUnmanaged(u32) = .{}, resolver: std.AutoArrayHashMapUnmanaged(u32, Symbol.Index) = .{}, symbols_free_list: std.ArrayListUnmanaged(Symbol.Index) = .{}, -phdr_table_dirty: bool = false, -shdr_table_dirty: bool = false, -shstrtab_dirty: bool = false, -strtab_dirty: bool = false, -got_dirty: bool = false, +has_text_reloc: bool = false, +num_ifunc_dynrelocs: usize = 0, debug_strtab_dirty: bool = false, debug_abbrev_section_dirty: bool = false, @@ -161,6 +218,7 @@ comdat_groups: std.ArrayListUnmanaged(ComdatGroup) = .{}, comdat_groups_owners: std.ArrayListUnmanaged(ComdatGroupOwner) = .{}, comdat_groups_table: std.AutoHashMapUnmanaged(u32, ComdatGroupOwner.Index) = .{}, +const AtomList = std.ArrayListUnmanaged(Atom.Index); const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Symbol.Index)); const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, Symbol.Index); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata); @@ -183,6 +241,9 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option const self = try createEmpty(allocator, options); errdefer self.base.destroy(); + const is_obj = options.output_mode == .Obj; + const is_obj_or_ar = is_obj or (options.output_mode == .Lib and options.link_mode == .Static); + if (options.use_llvm) { const use_lld = build_options.have_llvm and self.base.options.use_lld; if (use_lld) return self; @@ -192,6 +253,10 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option sub_path, options.target.ofmt.fileExt(options.target.cpu.arch), }); } + if (is_obj) { + // TODO until we implement -r option, we don't want to open a file at this stage. + return self; + } } errdefer if (self.base.intermediary_basename) |path| allocator.free(path); @@ -201,8 +266,6 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option .mode = link.determineMode(options), }); - self.shdr_table_dirty = true; - // Index 0 is always a null symbol. try self.symbols.append(allocator, .{}); // Index 0 is always a null symbol. @@ -211,21 +274,71 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option try self.atoms.append(allocator, .{}); // Append null file at index 0 try self.files.append(allocator, .null); + // Append null byte to string tables + try self.shstrtab.buffer.append(allocator, 0); + try self.strtab.buffer.append(allocator, 0); // There must always be a null shdr in index 0 - try self.shdrs.append(allocator, .{ - .sh_name = 0, - .sh_type = elf.SHT_NULL, - .sh_flags = 0, - .sh_addr = 0, - .sh_offset = 0, - .sh_size = 0, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = 0, - .sh_entsize = 0, - }); + _ = try self.addSection(.{ .name = "" }); - try self.populateMissingMetadata(); + if (!is_obj_or_ar) { + try self.dynstrtab.buffer.append(allocator, 0); + + // Initialize PT_PHDR program header + const p_align: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Phdr), + .p64 => @alignOf(elf.Elf64_Phdr), + }; + const image_base = self.calcImageBase(); + const offset: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Ehdr), + .p64 => @sizeOf(elf.Elf64_Ehdr), + }; + self.phdr_table_index = try self.addPhdr(.{ + .type = elf.PT_PHDR, + .flags = elf.PF_R, + .@"align" = p_align, + .addr = image_base + offset, + .offset = offset, + }); + self.phdr_table_load_index = try self.addPhdr(.{ + .type = elf.PT_LOAD, + .flags = elf.PF_R, + .@"align" = self.page_size, + .addr = image_base, + }); + } + + if (options.module != null and !options.use_llvm) { + if (!options.strip) { + self.dwarf = Dwarf.init(allocator, &self.base, options.target); + } + + const index = @as(File.Index, @intCast(try self.files.addOne(allocator))); + self.files.set(index, .{ .zig_module = .{ + .index = index, + .path = options.module.?.main_mod.root_src_path, + } }); + self.zig_module_index = index; + const zig_module = self.file(index).?.zig_module; + + try zig_module.atoms.append(allocator, 0); // null input section + + const name_off = try self.strtab.insert(allocator, std.fs.path.stem(options.module.?.main_mod.root_src_path)); + const symbol_index = try self.addSymbol(); + try zig_module.local_symbols.append(allocator, symbol_index); + const symbol_ptr = self.symbol(symbol_index); + symbol_ptr.file_index = zig_module.index; + symbol_ptr.name_offset = name_off; + + const esym_index = try zig_module.addLocalEsym(allocator); + const esym = &zig_module.local_esyms.items[esym_index]; + esym.st_name = name_off; + esym.st_info |= elf.STT_FILE; + esym.st_shndx = elf.SHN_ABS; + symbol_ptr.esym_index = esym_index; + + try self.initMetadata(); + } return self; } @@ -244,17 +357,12 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf { .sparc64 => 0x2000, else => 0x1000, }; - const default_sym_version: elf.Elf64_Versym = if (options.output_mode == .Lib and options.link_mode == .Dynamic) + const is_dyn_lib = options.output_mode == .Lib and options.link_mode == .Dynamic; + const default_sym_version: elf.Elf64_Versym = if (is_dyn_lib or options.rdynamic) elf.VER_NDX_GLOBAL else elf.VER_NDX_LOCAL; - const use_llvm = options.use_llvm; - var dwarf: ?Dwarf = if (!options.strip and options.module != null and !use_llvm) - Dwarf.init(gpa, &self.base, options.target) - else - null; - self.* = .{ .base = .{ .tag = .elf, @@ -262,12 +370,11 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf { .allocator = gpa, .file = null, }, - .dwarf = dwarf, .ptr_width = ptr_width, .page_size = page_size, .default_sym_version = default_sym_version, }; - if (use_llvm and options.module != null) { + if (options.use_llvm and options.module != null) { self.llvm_object = try LlvmObject.create(gpa, options); } @@ -284,20 +391,24 @@ pub fn deinit(self: *Elf) void { .zig_module => data.zig_module.deinit(gpa), .linker_defined => data.linker_defined.deinit(gpa), .object => data.object.deinit(gpa), - // .shared_object => data.shared_object.deinit(gpa), + .shared_object => data.shared_object.deinit(gpa), }; self.files.deinit(gpa); self.objects.deinit(gpa); + self.shared_objects.deinit(gpa); self.shdrs.deinit(gpa); self.phdr_to_shdr_table.deinit(gpa); self.phdrs.deinit(gpa); + for (self.output_sections.values()) |*list| { + list.deinit(gpa); + } + self.output_sections.deinit(gpa); self.shstrtab.deinit(gpa); self.strtab.deinit(gpa); self.symbols.deinit(gpa); self.symbols_extra.deinit(gpa); self.symbols_free_list.deinit(gpa); - self.got.deinit(gpa); self.resolver.deinit(gpa); self.start_stop_indexes.deinit(gpa); @@ -333,6 +444,19 @@ pub fn deinit(self: *Elf) void { self.comdat_groups.deinit(gpa); self.comdat_groups_owners.deinit(gpa); self.comdat_groups_table.deinit(gpa); + + self.got.deinit(gpa); + self.plt.deinit(gpa); + self.plt_got.deinit(gpa); + self.dynsym.deinit(gpa); + self.dynstrtab.deinit(gpa); + self.dynamic.deinit(gpa); + self.hash.deinit(gpa); + self.versym.deinit(gpa); + self.verneed.deinit(gpa); + self.copy_rel.deinit(gpa); + self.rela_dyn.deinit(gpa); + self.rela_plt.deinit(gpa); } pub fn getDeclVAddr(self: *Elf, decl_index: Module.Decl.Index, reloc_info: link.File.RelocInfo) !u64 { @@ -368,7 +492,7 @@ pub fn lowerAnonDecl(self: *Elf, decl_val: InternPool.Index, src_loc: Module.Src const tv = TypedValue{ .ty = ty, .val = val }; const name = try std.fmt.allocPrint(gpa, "__anon_{d}", .{@intFromEnum(decl_val)}); defer gpa.free(name); - const res = self.lowerConst(name, tv, self.rodata_section_index.?, src_loc) catch |err| switch (err) { + const res = self.lowerConst(name, tv, self.zig_rodata_section_index.?, src_loc) catch |err| switch (err) { else => { // TODO improve error message const em = try Module.ErrorMsg.create(gpa, src_loc, "lowerAnonDecl failed with error: {s}", .{ @@ -420,21 +544,23 @@ fn detectAllocCollision(self: *Elf, start: u64, size: u64) ?u64 { } for (self.shdrs.items) |shdr| { - // SHT_NOBITS takes no physical space in the output file so set its size to 0. - const sh_size = if (shdr.sh_type == elf.SHT_NOBITS) 0 else shdr.sh_size; - const increased_size = padToIdeal(sh_size); + if (shdr.sh_type == elf.SHT_NOBITS) continue; + const increased_size = padToIdeal(shdr.sh_size); const test_end = shdr.sh_offset + increased_size; if (end > shdr.sh_offset and start < test_end) { return test_end; } } + for (self.phdrs.items) |phdr| { + if (phdr.p_type != elf.PT_LOAD) continue; const increased_size = padToIdeal(phdr.p_filesz); const test_end = phdr.p_offset + increased_size; if (end > phdr.p_offset and start < test_end) { return test_end; } } + return null; } @@ -474,55 +600,34 @@ fn findFreeSpace(self: *Elf, object_size: u64, min_alignment: u64) u64 { } const AllocateSegmentOpts = struct { - size: u64, + addr: u64, + memsz: u64, + filesz: u64, alignment: u64, - addr: ?u64 = null, flags: u32 = elf.PF_R, }; pub fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16 { - const gpa = self.base.allocator; - const index = @as(u16, @intCast(self.phdrs.items.len)); - try self.phdrs.ensureUnusedCapacity(gpa, 1); - const off = self.findFreeSpace(opts.size, opts.alignment); - // Currently, we automatically allocate memory in sequence by finding the largest - // allocated virtual address and going from there. - // TODO we want to keep machine code segment in the furthest memory range among all - // segments as it is most likely to grow. - const addr = opts.addr orelse blk: { - const reserved_capacity = self.calcImageBase() * 4; - // Calculate largest VM address - var addresses = std.ArrayList(u64).init(gpa); - defer addresses.deinit(); - try addresses.ensureTotalCapacityPrecise(self.phdrs.items.len); - for (self.phdrs.items) |phdr| { - if (phdr.p_type != elf.PT_LOAD) continue; - addresses.appendAssumeCapacity(phdr.p_vaddr + reserved_capacity); - } - mem.sort(u64, addresses.items, {}, std.sort.asc(u64)); - break :blk mem.alignForward(u64, addresses.pop(), opts.alignment); - }; + const off = self.findFreeSpace(opts.filesz, opts.alignment); + const index = try self.addPhdr(.{ + .type = elf.PT_LOAD, + .offset = off, + .filesz = opts.filesz, + .addr = opts.addr, + .memsz = opts.memsz, + .@"align" = opts.alignment, + .flags = opts.flags, + }); log.debug("allocating phdr({d})({c}{c}{c}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{ index, if (opts.flags & elf.PF_R != 0) @as(u8, 'R') else '_', if (opts.flags & elf.PF_W != 0) @as(u8, 'W') else '_', if (opts.flags & elf.PF_X != 0) @as(u8, 'X') else '_', off, - off + opts.size, - addr, - addr + opts.size, - }); - self.phdrs.appendAssumeCapacity(.{ - .p_type = elf.PT_LOAD, - .p_offset = off, - .p_filesz = opts.size, - .p_vaddr = addr, - .p_paddr = addr, - .p_memsz = opts.size, - .p_align = opts.alignment, - .p_flags = opts.flags, + off + opts.filesz, + opts.addr, + opts.addr + opts.memsz, }); - self.phdr_table_dirty = true; return index; } @@ -537,9 +642,13 @@ const AllocateAllocSectionOpts = struct { pub fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{OutOfMemory}!u16 { const gpa = self.base.allocator; const phdr = &self.phdrs.items[opts.phdr_index]; - const index = @as(u16, @intCast(self.shdrs.items.len)); - try self.shdrs.ensureUnusedCapacity(gpa, 1); - const sh_name = try self.shstrtab.insert(gpa, opts.name); + const index = try self.addSection(.{ + .name = opts.name, + .type = opts.type, + .flags = opts.flags, + .addralign = opts.alignment, + }); + const shdr = &self.shdrs.items[index]; try self.phdr_to_shdr_table.putNoClobber(gpa, index, opts.phdr_index); log.debug("allocating '{s}' in phdr({d}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{ opts.name, @@ -549,19 +658,9 @@ pub fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{Ou phdr.p_vaddr, phdr.p_vaddr + phdr.p_memsz, }); - self.shdrs.appendAssumeCapacity(.{ - .sh_name = sh_name, - .sh_type = opts.type, - .sh_flags = opts.flags, - .sh_addr = phdr.p_vaddr, - .sh_offset = phdr.p_offset, - .sh_size = phdr.p_memsz, - .sh_link = 0, - .sh_info = 0, - .sh_addralign = opts.alignment, - .sh_entsize = 0, - }); - self.shdr_table_dirty = true; + shdr.sh_addr = phdr.p_vaddr; + shdr.sh_offset = phdr.p_offset; + shdr.sh_size = phdr.p_memsz; return index; } @@ -577,268 +676,132 @@ const AllocateNonAllocSectionOpts = struct { }; fn allocateNonAllocSection(self: *Elf, opts: AllocateNonAllocSectionOpts) error{OutOfMemory}!u16 { - const index = @as(u16, @intCast(self.shdrs.items.len)); - try self.shdrs.ensureUnusedCapacity(self.base.allocator, 1); - const sh_name = try self.shstrtab.insert(self.base.allocator, opts.name); + const index = try self.addSection(.{ + .name = opts.name, + .type = opts.type, + .flags = opts.flags, + .link = opts.link, + .info = opts.info, + .addralign = opts.alignment, + .entsize = opts.entsize, + }); + const shdr = &self.shdrs.items[index]; const off = self.findFreeSpace(opts.size, opts.alignment); log.debug("allocating '{s}' from 0x{x} to 0x{x} ", .{ opts.name, off, off + opts.size }); - self.shdrs.appendAssumeCapacity(.{ - .sh_name = sh_name, - .sh_type = opts.type, - .sh_flags = opts.flags, - .sh_addr = 0, - .sh_offset = off, - .sh_size = opts.size, - .sh_link = opts.link, - .sh_info = opts.info, - .sh_addralign = opts.alignment, - .sh_entsize = opts.entsize, - }); - self.shdr_table_dirty = true; + shdr.sh_offset = off; + shdr.sh_size = opts.size; return index; } -pub fn populateMissingMetadata(self: *Elf) !void { +/// TODO move to ZigModule +pub fn initMetadata(self: *Elf) !void { const gpa = self.base.allocator; - const small_ptr = switch (self.ptr_width) { - .p32 => true, - .p64 => false, - }; - const ptr_size: u8 = self.ptrWidthBytes(); + const ptr_size = self.ptrWidthBytes(); + const ptr_bit_width = self.base.options.target.ptrBitWidth(); const is_linux = self.base.options.target.os.tag == .linux; - const image_base = self.calcImageBase(); - - if (self.phdr_table_index == null) { - self.phdr_table_index = @intCast(self.phdrs.items.len); - const p_align: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Phdr), - .p64 => @alignOf(elf.Elf64_Phdr), - }; - try self.phdrs.append(gpa, .{ - .p_type = elf.PT_PHDR, - .p_offset = 0, - .p_filesz = 0, - .p_vaddr = image_base, - .p_paddr = image_base, - .p_memsz = 0, - .p_align = p_align, - .p_flags = elf.PF_R, - }); - self.phdr_table_dirty = true; - } - - if (self.phdr_table_load_index == null) { - self.phdr_table_load_index = try self.allocateSegment(.{ - .addr = image_base, - .size = 0, - .alignment = self.page_size, - }); - self.phdr_table_dirty = true; - } - if (self.phdr_load_re_index == null) { - self.phdr_load_re_index = try self.allocateSegment(.{ - .size = self.base.options.program_code_size_hint, + if (self.phdr_zig_load_re_index == null) { + self.phdr_zig_load_re_index = try self.allocateSegment(.{ + .addr = if (ptr_bit_width >= 32) 0x8000000 else 0x8000, + .memsz = self.base.options.program_code_size_hint, + .filesz = self.base.options.program_code_size_hint, .alignment = self.page_size, .flags = elf.PF_X | elf.PF_R | elf.PF_W, }); - self.entry_addr = null; } - if (self.phdr_got_index == null) { + if (self.phdr_zig_got_index == null) { // We really only need ptr alignment but since we are using PROGBITS, linux requires // page align. const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_got_index = try self.allocateSegment(.{ - .size = @as(u64, ptr_size) * self.base.options.symbol_count_hint, + self.phdr_zig_got_index = try self.allocateSegment(.{ + .addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000, + .memsz = @as(u64, ptr_size) * self.base.options.symbol_count_hint, + .filesz = @as(u64, ptr_size) * self.base.options.symbol_count_hint, .alignment = alignment, .flags = elf.PF_R | elf.PF_W, }); } - if (self.phdr_load_ro_index == null) { + if (self.phdr_zig_load_ro_index == null) { const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_load_ro_index = try self.allocateSegment(.{ - .size = 1024, + self.phdr_zig_load_ro_index = try self.allocateSegment(.{ + .addr = if (ptr_bit_width >= 32) 0xc000000 else 0xa000, + .memsz = 1024, + .filesz = 1024, .alignment = alignment, .flags = elf.PF_R | elf.PF_W, }); } - if (self.phdr_load_rw_index == null) { + if (self.phdr_zig_load_rw_index == null) { const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_load_rw_index = try self.allocateSegment(.{ - .size = 1024, + self.phdr_zig_load_rw_index = try self.allocateSegment(.{ + .addr = if (ptr_bit_width >= 32) 0x10000000 else 0xc000, + .memsz = 1024, + .filesz = 1024, .alignment = alignment, .flags = elf.PF_R | elf.PF_W, }); } - if (self.phdr_load_zerofill_index == null) { + if (self.phdr_zig_load_zerofill_index == null) { const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_load_zerofill_index = try self.allocateSegment(.{ - .size = 0, - .alignment = alignment, + self.phdr_zig_load_zerofill_index = try self.addPhdr(.{ + .type = elf.PT_LOAD, + .addr = if (ptr_bit_width >= 32) 0x14000000 else 0xf000, + .memsz = 1024, + .@"align" = alignment, .flags = elf.PF_R | elf.PF_W, }); - const phdr = &self.phdrs.items[self.phdr_load_zerofill_index.?]; - phdr.p_offset = self.phdrs.items[self.phdr_load_rw_index.?].p_offset; // .bss overlaps .data - phdr.p_memsz = 1024; - } - - if (!self.base.options.single_threaded) { - if (self.phdr_load_tls_data_index == null) { - const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_load_tls_data_index = try self.allocateSegment(.{ - .size = 1024, - .alignment = alignment, - .flags = elf.PF_R | elf.PF_W, - }); - } - - if (self.phdr_load_tls_zerofill_index == null) { - // TODO .tbss doesn't need any physical or memory representation (aka a loadable segment) - // since the loader only cares about the PT_TLS to work out TLS size. However, when - // relocating we need to have .tdata and .tbss contiguously laid out so that we can - // work out correct offsets to the start/end of the TLS segment. I am thinking that - // perhaps it's possible to completely spoof it by having an abstracted mechanism - // for this that wouldn't require us to explicitly track .tbss. Anyhow, for now, - // we go the savage route of treating .tbss like .bss. - const alignment = if (is_linux) self.page_size else @as(u16, ptr_size); - self.phdr_load_tls_zerofill_index = try self.allocateSegment(.{ - .size = 0, - .alignment = alignment, - .flags = elf.PF_R | elf.PF_W, - }); - const phdr = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?]; - phdr.p_offset = self.phdrs.items[self.phdr_load_tls_data_index.?].p_offset; // .tbss overlaps .tdata - phdr.p_memsz = 1024; - } - - if (self.phdr_tls_index == null) { - self.phdr_tls_index = @intCast(self.phdrs.items.len); - const phdr_tdata = &self.phdrs.items[self.phdr_load_tls_data_index.?]; - const phdr_tbss = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?]; - try self.phdrs.append(gpa, .{ - .p_type = elf.PT_TLS, - .p_offset = phdr_tdata.p_offset, - .p_vaddr = phdr_tdata.p_vaddr, - .p_paddr = phdr_tdata.p_paddr, - .p_filesz = phdr_tdata.p_filesz, - .p_memsz = phdr_tbss.p_vaddr + phdr_tbss.p_memsz - phdr_tdata.p_vaddr, - .p_align = ptr_size, - .p_flags = elf.PF_R, - }); - self.phdr_table_dirty = true; - } } - if (self.shstrtab_section_index == null) { - assert(self.shstrtab.buffer.items.len == 0); - try self.shstrtab.buffer.append(gpa, 0); // need a 0 at position 0 - self.shstrtab_section_index = try self.allocateNonAllocSection(.{ - .name = ".shstrtab", - .size = @intCast(self.shstrtab.buffer.items.len), - .type = elf.SHT_STRTAB, - }); - self.shstrtab_dirty = true; - } - - if (self.strtab_section_index == null) { - assert(self.strtab.buffer.items.len == 0); - try self.strtab.buffer.append(gpa, 0); // need a 0 at position 0 - self.strtab_section_index = try self.allocateNonAllocSection(.{ - .name = ".strtab", - .size = @intCast(self.strtab.buffer.items.len), - .type = elf.SHT_STRTAB, - }); - self.strtab_dirty = true; - } - - if (self.text_section_index == null) { - self.text_section_index = try self.allocateAllocSection(.{ - .name = ".text", - .phdr_index = self.phdr_load_re_index.?, + if (self.zig_text_section_index == null) { + self.zig_text_section_index = try self.allocateAllocSection(.{ + .name = ".zig.text", + .phdr_index = self.phdr_zig_load_re_index.?, .flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.text_section_index.?, .{}); + try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_text_section_index.?, .{}); } - if (self.got_section_index == null) { - self.got_section_index = try self.allocateAllocSection(.{ - .name = ".got", - .phdr_index = self.phdr_got_index.?, + if (self.zig_got_section_index == null) { + self.zig_got_section_index = try self.allocateAllocSection(.{ + .name = ".zig.got", + .phdr_index = self.phdr_zig_got_index.?, .alignment = ptr_size, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, }); } - if (self.rodata_section_index == null) { - self.rodata_section_index = try self.allocateAllocSection(.{ - .name = ".rodata", - .phdr_index = self.phdr_load_ro_index.?, + if (self.zig_rodata_section_index == null) { + self.zig_rodata_section_index = try self.allocateAllocSection(.{ + .name = ".zig.rodata", + .phdr_index = self.phdr_zig_load_ro_index.?, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, // TODO rename this section to .data.rel.ro }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.rodata_section_index.?, .{}); + try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_rodata_section_index.?, .{}); } - if (self.data_section_index == null) { - self.data_section_index = try self.allocateAllocSection(.{ - .name = ".data", - .phdr_index = self.phdr_load_rw_index.?, + if (self.zig_data_section_index == null) { + self.zig_data_section_index = try self.allocateAllocSection(.{ + .name = ".zig.data", + .phdr_index = self.phdr_zig_load_rw_index.?, .alignment = ptr_size, .flags = elf.SHF_ALLOC | elf.SHF_WRITE, }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.data_section_index.?, .{}); + try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_data_section_index.?, .{}); } - if (self.bss_section_index == null) { - self.bss_section_index = try self.allocateAllocSection(.{ - .name = ".bss", - .phdr_index = self.phdr_load_zerofill_index.?, + if (self.zig_bss_section_index == null) { + self.zig_bss_section_index = try self.allocateAllocSection(.{ + .name = ".zig.bss", + .phdr_index = self.phdr_zig_load_zerofill_index.?, .alignment = ptr_size, .flags = elf.SHF_ALLOC | elf.SHF_WRITE, .type = elf.SHT_NOBITS, }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.bss_section_index.?, .{}); - } - - if (self.phdr_load_tls_data_index) |phdr_index| { - if (self.tdata_section_index == null) { - self.tdata_section_index = try self.allocateAllocSection(.{ - .name = ".tdata", - .phdr_index = phdr_index, - .alignment = ptr_size, - .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS, - }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.tdata_section_index.?, .{}); - } - } - - if (self.phdr_load_tls_zerofill_index) |phdr_index| { - if (self.tbss_section_index == null) { - self.tbss_section_index = try self.allocateAllocSection(.{ - .name = ".tbss", - .phdr_index = phdr_index, - .alignment = ptr_size, - .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS, - .type = elf.SHT_NOBITS, - }); - try self.last_atom_and_free_list_table.putNoClobber(gpa, self.tbss_section_index.?, .{}); - } - } - - if (self.symtab_section_index == null) { - const min_align: u16 = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym); - const each_size: u64 = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym); - self.symtab_section_index = try self.allocateNonAllocSection(.{ - .name = ".symtab", - .size = self.base.options.symbol_count_hint * each_size, - .alignment = min_align, - .type = elf.SHT_SYMTAB, - .link = self.strtab_section_index.?, // Index of associated string table - .info = @intCast(self.symbols.items.len), - .entsize = each_size, - }); - self.shdr_table_dirty = true; + try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_bss_section_index.?, .{}); } if (self.dwarf) |*dw| { @@ -890,68 +853,6 @@ pub fn populateMissingMetadata(self: *Elf) !void { self.debug_line_header_dirty = true; } } - - const shsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Shdr), - .p64 => @sizeOf(elf.Elf64_Shdr), - }; - const shalign: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Shdr), - .p64 => @alignOf(elf.Elf64_Shdr), - }; - if (self.shdr_table_offset == null) { - self.shdr_table_offset = self.findFreeSpace(self.shdrs.items.len * shsize, shalign); - self.shdr_table_dirty = true; - } - - { - // Iterate over symbols, populating free_list and last_text_block. - if (self.symbols.items.len != 1) { - @panic("TODO implement setting up free_list and last_text_block from existing ELF file"); - } - // We are starting with an empty file. The default values are correct, null and empty list. - } - - if (self.shdr_table_dirty) { - // We need to find out what the max file offset is according to section headers. - // Otherwise, we may end up with an ELF binary with file size not matching the final section's - // offset + it's filesize. - var max_file_offset: u64 = 0; - - for (self.shdrs.items) |shdr| { - if (shdr.sh_offset + shdr.sh_size > max_file_offset) { - max_file_offset = shdr.sh_offset + shdr.sh_size; - } - } - - try self.base.file.?.pwriteAll(&[_]u8{0}, max_file_offset); - } - - if (self.base.options.module) |module| { - if (self.zig_module_index == null and !self.base.options.use_llvm) { - const index: File.Index = @intCast(try self.files.addOne(gpa)); - self.files.set(index, .{ .zig_module = .{ - .index = index, - .path = module.main_mod.root_src_path, - } }); - self.zig_module_index = index; - const zig_module = self.file(index).?.zig_module; - - const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_mod.root_src_path)); - const symbol_index = try self.addSymbol(); - try zig_module.local_symbols.append(gpa, symbol_index); - const symbol_ptr = self.symbol(symbol_index); - symbol_ptr.file_index = zig_module.index; - symbol_ptr.name_offset = name_off; - - const esym_index = try zig_module.addLocalEsym(gpa); - const esym = &zig_module.local_esyms.items[esym_index]; - esym.st_name = name_off; - esym.st_info |= elf.STT_FILE; - esym.st_shndx = elf.SHN_ABS; - symbol_ptr.esym_index = esym_index; - } - } } pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void { @@ -987,89 +888,17 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void { const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr); if (needed_size > mem_capacity) { - // We are exceeding our allocated VM capacity so we need to shift everything in memory - // and grow. - { - const dirty_addr = phdr.p_vaddr + phdr.p_memsz; - self.got_dirty = for (self.got.entries.items) |entry| { - if (self.symbol(entry.symbol_index).value >= dirty_addr) break true; - } else false; - - // TODO mark relocs dirty - } - try self.growSegment(shdr_index, needed_size); - - if (self.zig_module_index != null) { - // TODO self-hosted backends cannot yet handle this condition correctly as the linker - // cannot update emitted virtual addresses of symbols already committed to the final file. - var err = try self.addErrorWithNotes(2); - try err.addMsg(self, "fatal linker error: cannot expand load segment phdr({d}) in virtual memory", .{ - phdr_index, - }); - try err.addNote(self, "TODO: emit relocations to memory locations in self-hosted backends", .{}); - try err.addNote(self, "as a workaround, try increasing pre-allocated virtual memory of each segment", .{}); - } + var err = try self.addErrorWithNotes(2); + try err.addMsg(self, "fatal linker error: cannot expand load segment phdr({d}) in virtual memory", .{ + phdr_index, + }); + try err.addNote(self, "TODO: emit relocations to memory locations in self-hosted backends", .{}); + try err.addNote(self, "as a workaround, try increasing pre-allocated virtual memory of each segment", .{}); } phdr.p_memsz = needed_size; - self.markDirty(shdr_index, phdr_index); -} - -fn growSegment(self: *Elf, shndx: u16, needed_size: u64) !void { - const phdr_index = self.phdr_to_shdr_table.get(shndx).?; - const phdr = &self.phdrs.items[phdr_index]; - const increased_size = padToIdeal(needed_size); - const end_addr = phdr.p_vaddr + phdr.p_memsz; - const old_aligned_end = phdr.p_vaddr + mem.alignForward(u64, phdr.p_memsz, phdr.p_align); - const new_aligned_end = phdr.p_vaddr + mem.alignForward(u64, increased_size, phdr.p_align); - const diff = new_aligned_end - old_aligned_end; - log.debug("growing phdr({d}) in memory by {x}", .{ phdr_index, diff }); - - // Update symbols and atoms. - var files = std.ArrayList(File.Index).init(self.base.allocator); - defer files.deinit(); - try files.ensureTotalCapacityPrecise(self.objects.items.len + 1); - - if (self.zig_module_index) |index| files.appendAssumeCapacity(index); - files.appendSliceAssumeCapacity(self.objects.items); - - for (files.items) |index| { - const file_ptr = self.file(index).?; - - for (file_ptr.locals()) |sym_index| { - const sym = self.symbol(sym_index); - const atom_ptr = sym.atom(self) orelse continue; - if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue; - if (sym.value >= end_addr) sym.value += diff; - } - - for (file_ptr.globals()) |sym_index| { - const sym = self.symbol(sym_index); - if (sym.file_index != index) continue; - const atom_ptr = sym.atom(self) orelse continue; - if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue; - if (sym.value >= end_addr) sym.value += diff; - } - - for (file_ptr.atoms()) |atom_index| { - const atom_ptr = self.atom(atom_index) orelse continue; - if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue; - if (atom_ptr.value >= end_addr) atom_ptr.value += diff; - } - } - - // Finally, update section headers. - for (self.shdrs.items, 0..) |*other_shdr, other_shndx| { - if (other_shdr.sh_flags & elf.SHF_ALLOC == 0) continue; - if (other_shndx == shndx) continue; - const other_phdr_index = self.phdr_to_shdr_table.get(@intCast(other_shndx)) orelse continue; - const other_phdr = &self.phdrs.items[other_phdr_index]; - if (other_phdr.p_vaddr < end_addr) continue; - other_shdr.sh_addr += diff; - other_phdr.p_vaddr += diff; - other_phdr.p_paddr += diff; - } + self.markDirty(shdr_index); } pub fn growNonAllocSection( @@ -1106,18 +935,12 @@ pub fn growNonAllocSection( shdr.sh_offset = new_offset; } - shdr.sh_size = needed_size; // anticipating adding the global symbols later + shdr.sh_size = needed_size; - self.markDirty(shdr_index, null); + self.markDirty(shdr_index); } -pub fn markDirty(self: *Elf, shdr_index: u16, phdr_index: ?u16) void { - self.shdr_table_dirty = true; // TODO look into only writing one section - - if (phdr_index) |_| { - self.phdr_table_dirty = true; // TODO look into making only the one program header dirty - } - +pub fn markDirty(self: *Elf, shdr_index: u16) void { if (self.dwarf) |_| { if (self.debug_info_section_index.? == shdr_index) { self.debug_info_header_dirty = true; @@ -1144,10 +967,11 @@ pub fn flush(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) link if (use_lld) { return self.linkWithLLD(comp, prog_node); } - switch (self.base.options.output_mode) { - .Exe, .Obj => return self.flushModule(comp, prog_node), - .Lib => return error.TODOImplementWritingLibFiles, + if (self.base.options.output_mode == .Lib and self.isStatic()) { + // TODO writing static library files + return error.TODOImplementWritingLibFiles; } + try self.flushModule(comp, prog_node); } pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) link.File.FlushError!void { @@ -1173,21 +997,296 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node const target = self.base.options.target; const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type. const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path}); + const module_obj_path: ?[]const u8 = if (self.base.intermediary_basename) |path| blk: { + if (fs.path.dirname(full_out_path)) |dirname| { + break :blk try fs.path.join(arena, &.{ dirname, path }); + } else { + break :blk path; + } + } else null; + const gc_sections = self.base.options.gc_sections orelse false; + + if (self.base.options.output_mode == .Obj and self.zig_module_index == null) { + // TODO this will become -r route I guess. For now, just copy the object file. + assert(self.base.file == null); // TODO uncomment once we implement -r + const the_object_path = blk: { + if (self.base.options.objects.len != 0) { + break :blk self.base.options.objects[0].path; + } + + if (comp.c_object_table.count() != 0) + break :blk comp.c_object_table.keys()[0].status.success.object_path; + + if (module_obj_path) |p| + break :blk p; + + // TODO I think this is unreachable. Audit this situation when solving the above TODO + // regarding eliding redundant object -> object transformations. + return error.NoObjectsToLink; + }; + // This can happen when using --enable-cache and using the stage1 backend. In this case + // we can skip the file copy. + if (!mem.eql(u8, the_object_path, full_out_path)) { + try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{}); + } + return; + } + + var csu = try CsuObjects.init(arena, self.base.options, comp); + const compiler_rt_path: ?[]const u8 = blk: { + if (comp.compiler_rt_lib) |x| break :blk x.full_object_path; + if (comp.compiler_rt_obj) |x| break :blk x.full_object_path; + break :blk null; + }; + + // --verbose-link + if (self.base.options.verbose_link) { + var argv = std.ArrayList([]const u8).init(arena); + + try argv.append("zig"); + try argv.append("ld"); + + try argv.append("-o"); + try argv.append(full_out_path); + + if (self.base.options.entry) |entry| { + try argv.append("--entry"); + try argv.append(entry); + } + + if (self.base.options.dynamic_linker) |path| { + try argv.append("-dynamic-linker"); + try argv.append(path); + } + + if (self.base.options.soname) |name| { + try argv.append("-soname"); + try argv.append(name); + } + + for (self.base.options.rpath_list) |rpath| { + try argv.append("-rpath"); + try argv.append(rpath); + } + + if (self.base.options.each_lib_rpath) { + for (self.base.options.lib_dirs) |lib_dir_path| { + try argv.append("-rpath"); + try argv.append(lib_dir_path); + } + for (self.base.options.objects) |obj| { + if (Compilation.classifyFileExt(obj.path) == .shared_library) { + const lib_dir_path = std.fs.path.dirname(obj.path) orelse continue; + if (obj.loption) continue; + + try argv.append("-rpath"); + try argv.append(lib_dir_path); + } + } + } + + if (self.base.options.stack_size_override) |ss| { + try argv.append("-z"); + try argv.append(try std.fmt.allocPrint(arena, "stack-size={d}", .{ss})); + } + + if (self.base.options.image_base_override) |image_base| { + try argv.append(try std.fmt.allocPrint(arena, "--image-base={d}", .{image_base})); + } + + if (gc_sections) { + try argv.append("--gc-sections"); + } + + if (self.base.options.print_gc_sections) { + try argv.append("--print-gc-sections"); + } + + if (self.base.options.eh_frame_hdr) { + try argv.append("--eh-frame-hdr"); + } + + if (self.base.options.rdynamic) { + try argv.append("--export-dynamic"); + } + + if (self.base.options.strip) { + try argv.append("-s"); + } + + if (self.base.options.z_notext) { + try argv.append("-z"); + try argv.append("notext"); + } + + if (self.base.options.z_nocopyreloc) { + try argv.append("-z"); + try argv.append("nocopyreloc"); + } + + if (self.base.options.z_now) { + try argv.append("-z"); + try argv.append("now"); + } + + if (self.isStatic()) { + try argv.append("-static"); + } else if (self.isDynLib()) { + try argv.append("-shared"); + } + + if (self.base.options.pie and self.isExe()) { + try argv.append("-pie"); + } + + // csu prelude + if (csu.crt0) |v| try argv.append(v); + if (csu.crti) |v| try argv.append(v); + if (csu.crtbegin) |v| try argv.append(v); + + for (self.base.options.lib_dirs) |lib_dir| { + try argv.append("-L"); + try argv.append(lib_dir); + } + + if (self.base.options.link_libc) { + if (self.base.options.libc_installation) |libc_installation| { + try argv.append("-L"); + try argv.append(libc_installation.crt_dir.?); + } + } + + var whole_archive = false; + for (self.base.options.objects) |obj| { + if (obj.must_link and !whole_archive) { + try argv.append("-whole-archive"); + whole_archive = true; + } else if (!obj.must_link and whole_archive) { + try argv.append("-no-whole-archive"); + whole_archive = false; + } + + if (obj.loption) { + assert(obj.path[0] == ':'); + try argv.append("-l"); + } + try argv.append(obj.path); + } + if (whole_archive) { + try argv.append("-no-whole-archive"); + whole_archive = false; + } + + for (comp.c_object_table.keys()) |key| { + try argv.append(key.status.success.object_path); + } + + if (module_obj_path) |p| { + try argv.append(p); + } + + // TSAN + if (self.base.options.tsan) { + try argv.append(comp.tsan_static_lib.?.full_object_path); + } + + // libc + if (!self.base.options.skip_linker_dependencies and + !self.base.options.link_libc) + { + if (comp.libc_static_lib) |lib| { + try argv.append(lib.full_object_path); + } + } + + // stack-protector. + // Related: https://github.com/ziglang/zig/issues/7265 + if (comp.libssp_static_lib) |ssp| { + try argv.append(ssp.full_object_path); + } + + // Shared libraries. + // Worst-case, we need an --as-needed argument for every lib, as well + // as one before and one after. + try argv.ensureUnusedCapacity(self.base.options.system_libs.keys().len * 2 + 2); + argv.appendAssumeCapacity("--as-needed"); + var as_needed = true; + + for (self.base.options.system_libs.values()) |lib_info| { + const lib_as_needed = !lib_info.needed; + switch ((@as(u2, @intFromBool(lib_as_needed)) << 1) | @intFromBool(as_needed)) { + 0b00, 0b11 => {}, + 0b01 => { + argv.appendAssumeCapacity("--no-as-needed"); + as_needed = false; + }, + 0b10 => { + argv.appendAssumeCapacity("--as-needed"); + as_needed = true; + }, + } + argv.appendAssumeCapacity(lib_info.path.?); + } + + if (!as_needed) { + argv.appendAssumeCapacity("--as-needed"); + as_needed = true; + } + + // libc++ dep + if (self.base.options.link_libcpp) { + try argv.append(comp.libcxxabi_static_lib.?.full_object_path); + try argv.append(comp.libcxx_static_lib.?.full_object_path); + } + + // libunwind dep + if (self.base.options.link_libunwind) { + try argv.append(comp.libunwind_static_lib.?.full_object_path); + } + + // libc dep + if (self.base.options.link_libc) { + if (self.base.options.libc_installation != null) { + const needs_grouping = self.base.options.link_mode == .Static; + if (needs_grouping) try argv.append("--start-group"); + try argv.appendSlice(target_util.libcFullLinkFlags(target)); + if (needs_grouping) try argv.append("--end-group"); + } else if (target.isGnuLibC()) { + for (glibc.libs) |lib| { + const lib_path = try std.fmt.allocPrint(arena, "{s}{c}lib{s}.so.{d}", .{ + comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover, + }); + try argv.append(lib_path); + } + try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a")); + } else if (target.isMusl()) { + try argv.append(try comp.get_libc_crt_file(arena, switch (self.base.options.link_mode) { + .Static => "libc.a", + .Dynamic => "libc.so", + })); + } + } + + // compiler-rt + if (compiler_rt_path) |p| { + try argv.append(p); + } + + // crt postlude + if (csu.crtend) |v| try argv.append(v); + if (csu.crtn) |v| try argv.append(v); + + Compilation.dump_argv(argv.items); + } // Here we will parse input positional and library files (if referenced). // This will roughly match in any linker backend we support. var positionals = std.ArrayList(Compilation.LinkObject).init(arena); - if (self.base.intermediary_basename) |path| { - const full_path = blk: { - if (fs.path.dirname(full_out_path)) |dirname| { - break :blk try fs.path.join(arena, &.{ dirname, path }); - } else { - break :blk path; - } - }; - try positionals.append(.{ .path = full_path }); - } + // csu prelude + if (csu.crt0) |v| try positionals.append(.{ .path = v }); + if (csu.crti) |v| try positionals.append(.{ .path = v }); + if (csu.crtbegin) |v| try positionals.append(.{ .path = v }); try positionals.ensureUnusedCapacity(self.base.options.objects.len); positionals.appendSliceAssumeCapacity(self.base.options.objects); @@ -1199,11 +1298,60 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node try positionals.append(.{ .path = key.status.success.object_path }); } - // csu prelude - var csu = try CsuObjects.init(arena, self.base.options, comp); - if (csu.crt0) |v| try positionals.append(.{ .path = v }); - if (csu.crti) |v| try positionals.append(.{ .path = v }); - if (csu.crtbegin) |v| try positionals.append(.{ .path = v }); + if (module_obj_path) |path| try positionals.append(.{ .path = path }); + + // rpaths + var rpath_table = std.StringArrayHashMap(void).init(self.base.allocator); + defer rpath_table.deinit(); + for (self.base.options.rpath_list) |rpath| { + _ = try rpath_table.put(rpath, {}); + } + + if (self.base.options.each_lib_rpath) { + var test_path = std.ArrayList(u8).init(self.base.allocator); + defer test_path.deinit(); + for (self.base.options.lib_dirs) |lib_dir_path| { + for (self.base.options.system_libs.keys()) |link_lib| { + test_path.clearRetainingCapacity(); + const sep = fs.path.sep_str; + try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ + lib_dir_path, link_lib, + }); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => continue, + else => |e| return e, + }; + _ = try rpath_table.put(lib_dir_path, {}); + } + } + for (self.base.options.objects) |obj| { + if (Compilation.classifyFileExt(obj.path) == .shared_library) { + const lib_dir_path = std.fs.path.dirname(obj.path) orelse continue; + if (obj.loption) continue; + _ = try rpath_table.put(lib_dir_path, {}); + } + } + } + + // TSAN + if (self.base.options.tsan) { + try positionals.append(.{ .path = comp.tsan_static_lib.?.full_object_path }); + } + + // libc + if (!self.base.options.skip_linker_dependencies and + !self.base.options.link_libc) + { + if (comp.libc_static_lib) |lib| { + try positionals.append(.{ .path = lib.full_object_path }); + } + } + + // stack-protector. + // Related: https://github.com/ziglang/zig/issues/7265 + if (comp.libssp_static_lib) |ssp| { + try positionals.append(.{ .path = ssp.full_object_path }); + } for (positionals.items) |obj| { const in_file = try std.fs.cwd().openFile(obj.path, .{}); @@ -1215,6 +1363,23 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node var system_libs = std.ArrayList(SystemLib).init(arena); + try system_libs.ensureUnusedCapacity(self.base.options.system_libs.values().len); + for (self.base.options.system_libs.values()) |lib_info| { + system_libs.appendAssumeCapacity(.{ .needed = lib_info.needed, .path = lib_info.path.? }); + } + + // libc++ dep + if (self.base.options.link_libcpp) { + try system_libs.ensureUnusedCapacity(2); + system_libs.appendAssumeCapacity(.{ .path = comp.libcxxabi_static_lib.?.full_object_path }); + system_libs.appendAssumeCapacity(.{ .path = comp.libcxx_static_lib.?.full_object_path }); + } + + // libunwind dep + if (self.base.options.link_libunwind) { + try system_libs.append(.{ .path = comp.libunwind_static_lib.?.full_object_path }); + } + // libc dep self.error_flags.missing_libc = false; if (self.base.options.link_libc) { @@ -1256,11 +1421,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node // compiler-rt. Since compiler_rt exports symbols like `memset`, it needs // to be after the shared libraries, so they are picked up from the shared // libraries, not libcompiler_rt. - const compiler_rt_path: ?[]const u8 = blk: { - if (comp.compiler_rt_lib) |x| break :blk x.full_object_path; - if (comp.compiler_rt_obj) |x| break :blk x.full_object_path; - break :blk null; - }; if (compiler_rt_path) |path| try positionals.append(.{ .path = path }); // csu postlude @@ -1301,13 +1461,28 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node if (metadata.rodata_state != .unused) metadata.rodata_state = .flushed; } - const target_endian = self.base.options.target.cpu.arch.endian(); - const foreign_endian = target_endian != builtin.cpu.arch.endian(); - if (self.dwarf) |*dw| { try dw.flushModule(self.base.options.module.?); } + // Dedup shared objects + { + var seen_dsos = std.StringHashMap(void).init(gpa); + defer seen_dsos.deinit(); + try seen_dsos.ensureTotalCapacity(@as(u32, @intCast(self.shared_objects.items.len))); + + var i: usize = 0; + while (i < self.shared_objects.items.len) { + const index = self.shared_objects.items[i]; + const shared_object = self.file(index).?.shared_object; + const soname = shared_object.soname(); + const gop = seen_dsos.getOrPutAssumeCapacity(soname); + if (gop.found_existing) { + _ = self.shared_objects.orderedRemove(i); + } else i += 1; + } + } + // If we haven't already, create a linker-generated input file comprising of // linker-defined synthetic symbols only such as `_DYNAMIC`, etc. if (self.linker_defined_index == null) { @@ -1315,113 +1490,51 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node self.files.set(index, .{ .linker_defined = .{ .index = index } }); self.linker_defined_index = index; } - try self.addLinkerDefinedSymbols(); // Now, we are ready to resolve the symbols across all input files. // We will first resolve the files in the ZigModule, next in the parsed // input Object files. // Any qualifing unresolved symbol will be upgraded to an absolute, weak // symbol for potential resolution at load-time. - try self.resolveSymbols(); + self.resolveSymbols(); + self.markEhFrameAtomsDead(); + try self.convertCommonSymbols(); self.markImportsExports(); - self.claimUnresolved(); - - // Scan and create missing synthetic entries such as GOT indirection. - try self.scanRelocs(); - - // Allocate atoms parsed from input object files, followed by allocating - // linker-defined synthetic symbols. - try self.allocateObjects(); - self.allocateLinkerDefinedSymbols(); - - // .bss always overlaps .data in file offset, but is zero-sized in file so it doesn't - // get mapped by the loader - if (self.data_section_index) |data_shndx| blk: { - const bss_shndx = self.bss_section_index orelse break :blk; - const data_phndx = self.phdr_to_shdr_table.get(data_shndx).?; - const bss_phndx = self.phdr_to_shdr_table.get(bss_shndx).?; - self.shdrs.items[bss_shndx].sh_offset = self.shdrs.items[data_shndx].sh_offset; - self.phdrs.items[bss_phndx].p_offset = self.phdrs.items[data_phndx].p_offset; - } - - // Same treatment for .tbss section. - if (self.tdata_section_index) |tdata_shndx| blk: { - const tbss_shndx = self.tbss_section_index orelse break :blk; - const tdata_phndx = self.phdr_to_shdr_table.get(tdata_shndx).?; - const tbss_phndx = self.phdr_to_shdr_table.get(tbss_shndx).?; - self.shdrs.items[tbss_shndx].sh_offset = self.shdrs.items[tdata_shndx].sh_offset; - self.phdrs.items[tbss_phndx].p_offset = self.phdrs.items[tdata_phndx].p_offset; - } - - if (self.phdr_tls_index) |tls_index| { - const tdata_phdr = &self.phdrs.items[self.phdr_load_tls_data_index.?]; - const tbss_phdr = &self.phdrs.items[self.phdr_load_tls_zerofill_index.?]; - const phdr = &self.phdrs.items[tls_index]; - phdr.p_offset = tdata_phdr.p_offset; - phdr.p_filesz = tdata_phdr.p_filesz; - phdr.p_vaddr = tdata_phdr.p_vaddr; - phdr.p_paddr = tdata_phdr.p_vaddr; - phdr.p_memsz = tbss_phdr.p_vaddr + tbss_phdr.p_memsz - tdata_phdr.p_vaddr; - } - - // Beyond this point, everything has been allocated a virtual address and we can resolve - // the relocations, and commit objects to file. - if (self.zig_module_index) |index| { - const zig_module = self.file(index).?.zig_module; - for (zig_module.atoms.keys()) |atom_index| { - const atom_ptr = self.atom(atom_index).?; - if (!atom_ptr.flags.alive) continue; - const shdr = &self.shdrs.items[atom_ptr.outputShndx().?]; - if (shdr.sh_type == elf.SHT_NOBITS) continue; - const code = try zig_module.codeAlloc(self, atom_index); - defer gpa.free(code); - const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr; - try atom_ptr.resolveRelocs(self, code); - try self.base.file.?.pwriteAll(code, file_offset); - } - } - try self.writeObjects(); - - if (self.got_dirty) { - const shdr = &self.shdrs.items[self.got_section_index.?]; - var buffer = try std.ArrayList(u8).initCapacity(gpa, self.got.size(self)); - defer buffer.deinit(); - try self.got.writeAllEntries(self, buffer.writer()); - try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); - self.got_dirty = false; - } // Look for entry address in objects if not set by the incremental compiler. - if (self.entry_addr == null) { + if (self.entry_index == null) { const entry: ?[]const u8 = entry: { if (self.base.options.entry) |entry| break :entry entry; if (!self.isDynLib()) break :entry "_start"; break :entry null; }; - self.entry_addr = if (entry) |name| entry_addr: { - const global_index = self.globalByName(name) orelse break :entry_addr null; - break :entry_addr self.symbol(global_index).value; - } else null; + self.entry_index = if (entry) |name| self.globalByName(name) else null; + } + + if (gc_sections) { + try gc.gcAtoms(self); + + if (self.base.options.print_gc_sections) { + try gc.dumpPrunedAtoms(self); + } } - // Generate and emit the symbol table. - try self.updateSymtabSize(); - try self.writeSymtab(); + try self.addLinkerDefinedSymbols(); + self.claimUnresolved(); + + // Scan and create missing synthetic entries such as GOT indirection. + try self.scanRelocs(); + // TODO I need to re-think how to handle ZigModule's debug sections AND debug sections + // extracted from input object files correctly. if (self.dwarf) |*dw| { if (self.debug_abbrev_section_dirty) { try dw.writeDbgAbbrev(); - if (!self.shdr_table_dirty) { - // Then it won't get written with the others and we need to do it. - try self.writeShdr(self.debug_abbrev_section_index.?); - } self.debug_abbrev_section_dirty = false; } if (self.debug_info_header_dirty) { - // Currently only one compilation unit is supported, so the address range is simply - // identical to the main program header virtual address and memory size. - const text_phdr = &self.phdrs.items[self.phdr_load_re_index.?]; + const text_phdr = &self.phdrs.items[self.phdr_zig_load_re_index.?]; const low_pc = text_phdr.p_vaddr; const high_pc = text_phdr.p_vaddr + text_phdr.p_memsz; try dw.writeDbgInfoHeader(self.base.options.module.?, low_pc, high_pc); @@ -1429,14 +1542,8 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node } if (self.debug_aranges_section_dirty) { - // Currently only one compilation unit is supported, so the address range is simply - // identical to the main program header virtual address and memory size. - const text_phdr = &self.phdrs.items[self.phdr_load_re_index.?]; + const text_phdr = &self.phdrs.items[self.phdr_zig_load_re_index.?]; try dw.writeDbgAranges(text_phdr.p_vaddr, text_phdr.p_memsz); - if (!self.shdr_table_dirty) { - // Then it won't get written with the others and we need to do it. - try self.writeShdr(self.debug_aranges_section_index.?); - } self.debug_aranges_section_dirty = false; } @@ -1444,160 +1551,83 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node try dw.writeDbgLineHeader(); self.debug_line_header_dirty = false; } - } - if (self.phdr_table_dirty) { - const phsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Phdr), - .p64 => @sizeOf(elf.Elf64_Phdr), - }; - - const phdr_table_index = self.phdr_table_index.?; - const phdr_table = &self.phdrs.items[phdr_table_index]; - const phdr_table_load = &self.phdrs.items[self.phdr_table_load_index.?]; - - const allocated_size = self.allocatedSize(phdr_table.p_offset); - const needed_size = self.phdrs.items.len * phsize; - - if (needed_size > allocated_size) { - phdr_table.p_offset = 0; // free the space - phdr_table.p_offset = self.findFreeSpace(needed_size, @as(u32, @intCast(phdr_table.p_align))); - } - - phdr_table_load.p_offset = mem.alignBackward(u64, phdr_table.p_offset, phdr_table_load.p_align); - const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset; - phdr_table_load.p_filesz = load_align_offset + needed_size; - phdr_table_load.p_memsz = load_align_offset + needed_size; - - phdr_table.p_filesz = needed_size; - phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset; - phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset; - phdr_table.p_memsz = needed_size; - - switch (self.ptr_width) { - .p32 => { - const buf = try gpa.alloc(elf.Elf32_Phdr, self.phdrs.items.len); - defer gpa.free(buf); - - for (buf, 0..) |*phdr, i| { - phdr.* = phdrTo32(self.phdrs.items[i]); - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf32_Phdr, phdr); - } - } - try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset); - }, - .p64 => { - const buf = try gpa.alloc(elf.Elf64_Phdr, self.phdrs.items.len); - defer gpa.free(buf); - - for (buf, 0..) |*phdr, i| { - phdr.* = self.phdrs.items[i]; - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf64_Phdr, phdr); - } - } - try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset); - }, + if (self.debug_str_section_index) |shndx| { + if (self.debug_strtab_dirty or dw.strtab.buffer.items.len != self.shdrs.items[shndx].sh_size) { + try self.growNonAllocSection(shndx, dw.strtab.buffer.items.len, 1, false); + const shdr = self.shdrs.items[shndx]; + try self.base.file.?.pwriteAll(dw.strtab.buffer.items, shdr.sh_offset); + self.debug_strtab_dirty = false; + } } - // We don't actually care if the phdr load section overlaps, only the phdr section matters. - phdr_table_load.p_offset = 0; - phdr_table_load.p_filesz = 0; - - self.phdr_table_dirty = false; + self.saveDebugSectionsSizes(); } - { - const shdr_index = self.shstrtab_section_index.?; - if (self.shstrtab_dirty or self.shstrtab.buffer.items.len != self.shdrs.items[shdr_index].sh_size) { - try self.growNonAllocSection(shdr_index, self.shstrtab.buffer.items.len, 1, false); - const shstrtab_sect = &self.shdrs.items[shdr_index]; - try self.base.file.?.pwriteAll(self.shstrtab.buffer.items, shstrtab_sect.sh_offset); - self.shstrtab_dirty = false; - } - } + // Generate and emit non-incremental sections. + try self.initSections(); + try self.initSpecialPhdrs(); + try self.sortShdrs(); + for (self.objects.items) |index| { + try self.file(index).?.object.addAtomsToOutputSections(self); + } + try self.sortInitFini(); + try self.setDynamicSection(rpath_table.keys()); + self.sortDynamicSymtab(); + try self.setHashSections(); + try self.setVersionSymtab(); + try self.updateSectionSizes(); + + self.allocatePhdrTable(); + try self.allocateAllocSections(); + try self.sortPhdrs(); + try self.allocateNonAllocSections(); + self.allocateSpecialPhdrs(); + self.allocateAtoms(); + self.allocateLinkerDefinedSymbols(); - { - const shdr_index = self.strtab_section_index.?; - if (self.strtab_dirty or self.strtab.buffer.items.len != self.shdrs.items[shdr_index].sh_size) { - try self.growNonAllocSection(shdr_index, self.strtab.buffer.items.len, 1, false); - const strtab_sect = self.shdrs.items[shdr_index]; - try self.base.file.?.pwriteAll(self.strtab.buffer.items, strtab_sect.sh_offset); - self.strtab_dirty = false; - } + // Dump the state for easy debugging. + // State can be dumped via `--debug-log link_state`. + if (build_options.enable_logging) { + state_log.debug("{}", .{self.dumpState()}); } - if (self.dwarf) |dwarf| { - const shdr_index = self.debug_str_section_index.?; - if (self.debug_strtab_dirty or dwarf.strtab.buffer.items.len != self.shdrs.items[shdr_index].sh_size) { - try self.growNonAllocSection(shdr_index, dwarf.strtab.buffer.items.len, 1, false); - const debug_strtab_sect = self.shdrs.items[shdr_index]; - try self.base.file.?.pwriteAll(dwarf.strtab.buffer.items, debug_strtab_sect.sh_offset); - self.debug_strtab_dirty = false; + // Beyond this point, everything has been allocated a virtual address and we can resolve + // the relocations, and commit objects to file. + if (self.zig_module_index) |index| { + const zig_module = self.file(index).?.zig_module; + for (zig_module.atoms.items) |atom_index| { + const atom_ptr = self.atom(atom_index) orelse continue; + if (!atom_ptr.flags.alive) continue; + const out_shndx = atom_ptr.outputShndx() orelse continue; + const shdr = &self.shdrs.items[out_shndx]; + if (shdr.sh_type == elf.SHT_NOBITS) continue; + const code = try zig_module.codeAlloc(self, atom_index); + defer gpa.free(code); + const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr; + atom_ptr.resolveRelocsAlloc(self, code) catch |err| switch (err) { + // TODO + error.RelaxFail, error.InvalidInstruction, error.CannotEncode => { + log.err("relaxing intructions failed; TODO this should be a fatal linker error", .{}); + }, + else => |e| return e, + }; + try self.base.file.?.pwriteAll(code, file_offset); } } - if (self.shdr_table_dirty) { - const shsize: u64 = switch (self.ptr_width) { - .p32 => @sizeOf(elf.Elf32_Shdr), - .p64 => @sizeOf(elf.Elf64_Shdr), - }; - const shalign: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Shdr), - .p64 => @alignOf(elf.Elf64_Shdr), - }; - const allocated_size = self.allocatedSize(self.shdr_table_offset.?); - const needed_size = self.shdrs.items.len * shsize; - - if (needed_size > allocated_size) { - self.shdr_table_offset = null; // free the space - self.shdr_table_offset = self.findFreeSpace(needed_size, shalign); - } - - switch (self.ptr_width) { - .p32 => { - const buf = try gpa.alloc(elf.Elf32_Shdr, self.shdrs.items.len); - defer gpa.free(buf); + try self.writePhdrTable(); + try self.writeShdrTable(); + try self.writeAtoms(); + try self.writeSyntheticSections(); - for (buf, 0..) |*shdr, i| { - shdr.* = shdrTo32(self.shdrs.items[i]); - log.debug("writing section {?s}: {}", .{ self.shstrtab.get(shdr.sh_name), shdr.* }); - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf32_Shdr, shdr); - } - } - try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); - }, - .p64 => { - const buf = try gpa.alloc(elf.Elf64_Shdr, self.shdrs.items.len); - defer gpa.free(buf); - - for (buf, 0..) |*shdr, i| { - shdr.* = self.shdrs.items[i]; - log.debug("writing section {?s}: {}", .{ self.shstrtab.get(shdr.sh_name), shdr.* }); - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf64_Shdr, shdr); - } - } - try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); - }, - } - self.shdr_table_dirty = false; - } - if (self.entry_addr == null and self.base.options.effectiveOutputMode() == .Exe) { + if (self.entry_index == null and self.base.options.effectiveOutputMode() == .Exe) { log.debug("flushing. no_entry_point_found = true", .{}); self.error_flags.no_entry_point_found = true; } else { log.debug("flushing. no_entry_point_found = false", .{}); self.error_flags.no_entry_point_found = false; - try self.writeElfHeader(); - } - - // Dump the state for easy debugging. - // State can be dumped via `--debug-log link_state`. - if (build_options.enable_logging) { - state_log.debug("{}", .{self.dumpState()}); + try self.writeHeader(); } // The point of flush() is to commit changes, so in theory, nothing should @@ -1606,12 +1636,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node // such as debug_line_header_dirty and debug_info_header_dirty. assert(!self.debug_abbrev_section_dirty); assert(!self.debug_aranges_section_dirty); - assert(!self.phdr_table_dirty); - assert(!self.shdr_table_dirty); - assert(!self.shstrtab_dirty); - assert(!self.strtab_dirty); assert(!self.debug_strtab_dirty); - assert(!self.got.dirty); } const ParseError = error{ @@ -1655,6 +1680,8 @@ fn parseLibrary( if (Archive.isArchive(in_file)) { try self.parseArchive(in_file, lib.path, must_link, ctx); + } else if (SharedObject.isSharedObject(in_file)) { + try self.parseSharedObject(in_file, lib, ctx); } else return error.UnknownFileType; } @@ -1709,6 +1736,34 @@ fn parseArchive( } } +fn parseSharedObject( + self: *Elf, + in_file: std.fs.File, + lib: SystemLib, + ctx: *ParseErrorCtx, +) ParseError!void { + const tracy = trace(@src()); + defer tracy.end(); + + const gpa = self.base.allocator; + const data = try in_file.readToEndAlloc(gpa, std.math.maxInt(u32)); + const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); + self.files.set(index, .{ .shared_object = .{ + .path = lib.path, + .data = data, + .index = index, + .needed = lib.needed, + .alive = lib.needed, + } }); + try self.shared_objects.append(gpa, index); + + const shared_object = self.file(index).?.shared_object; + try shared_object.parse(self); + + ctx.detected_cpu_arch = shared_object.header.?.e_machine.toTargetCpuArch().?; + if (ctx.detected_cpu_arch != self.base.options.target.cpu.arch) return error.InvalidCpuArch; +} + /// When resolving symbols, we approach the problem similarly to `mold`. /// 1. Resolve symbols across all objects (including those preemptively extracted archives). /// 2. Resolve symbols across all shared objects. @@ -1716,11 +1771,12 @@ fn parseArchive( /// 4. Reset state of all resolved globals since we will redo this bit on the pruned set. /// 5. Remove references to dead objects/shared objects /// 6. Re-run symbol resolution on pruned objects and shared objects sets. -fn resolveSymbols(self: *Elf) error{Overflow}!void { +fn resolveSymbols(self: *Elf) void { // Resolve symbols in the ZigModule. For now, we assume that it's always live. if (self.zig_module_index) |index| self.file(index).?.resolveSymbols(self); // Resolve symbols on the set of all objects and shared objects (even if some are unneeded). for (self.objects.items) |index| self.file(index).?.resolveSymbols(self); + for (self.shared_objects.items) |index| self.file(index).?.resolveSymbols(self); // Mark live objects. self.markLive(); @@ -1728,6 +1784,7 @@ fn resolveSymbols(self: *Elf) error{Overflow}!void { // Reset state of all globals after marking live objects. if (self.zig_module_index) |index| self.file(index).?.resetGlobals(self); for (self.objects.items) |index| self.file(index).?.resetGlobals(self); + for (self.shared_objects.items) |index| self.file(index).?.resetGlobals(self); // Prune dead objects and shared objects. var i: usize = 0; @@ -1737,6 +1794,13 @@ fn resolveSymbols(self: *Elf) error{Overflow}!void { _ = self.objects.orderedRemove(i); } else i += 1; } + i = 0; + while (i < self.shared_objects.items.len) { + const index = self.shared_objects.items[i]; + if (!self.file(index).?.isAlive()) { + _ = self.shared_objects.orderedRemove(i); + } else i += 1; + } // Dedup comdat groups. for (self.objects.items) |index| { @@ -1758,11 +1822,11 @@ fn resolveSymbols(self: *Elf) error{Overflow}!void { const cg = self.comdatGroup(cg_index); const cg_owner = self.comdatGroupOwner(cg.owner); if (cg_owner.file != index) { - for (try object.comdatGroupMembers(cg.shndx)) |shndx| { + for (object.comdatGroupMembers(cg.shndx)) |shndx| { const atom_index = object.atoms.items[shndx]; if (self.atom(atom_index)) |atom_ptr| { atom_ptr.flags.alive = false; - // atom_ptr.markFdesDead(self); + atom_ptr.markFdesDead(self); } } } @@ -1772,6 +1836,7 @@ fn resolveSymbols(self: *Elf) error{Overflow}!void { // Re-resolve the symbols. if (self.zig_module_index) |index| self.file(index).?.resolveSymbols(self); for (self.objects.items) |index| self.file(index).?.resolveSymbols(self); + for (self.shared_objects.items) |index| self.file(index).?.resolveSymbols(self); } /// Traverses all objects and shared objects marking any object referenced by @@ -1784,6 +1849,24 @@ fn markLive(self: *Elf) void { const file_ptr = self.file(index).?; if (file_ptr.isAlive()) file_ptr.markLive(self); } + for (self.shared_objects.items) |index| { + const file_ptr = self.file(index).?; + if (file_ptr.isAlive()) file_ptr.markLive(self); + } +} + +fn markEhFrameAtomsDead(self: *Elf) void { + for (self.objects.items) |index| { + const file_ptr = self.file(index).?; + if (!file_ptr.isAlive()) continue; + file_ptr.object.markEhFrameAtomsDead(self); + } +} + +fn convertCommonSymbols(self: *Elf) !void { + for (self.objects.items) |index| { + try self.file(index).?.object.convertCommonSymbols(self); + } } fn markImportsExports(self: *Elf) void { @@ -1795,10 +1878,10 @@ fn markImportsExports(self: *Elf) void { const file_ptr = global.file(elf_file) orelse continue; const vis = @as(elf.STV, @enumFromInt(global.elfSym(elf_file).st_other)); if (vis == .HIDDEN) continue; - // if (file == .shared and !global.isAbs(self)) { - // global.flags.import = true; - // continue; - // } + if (file_ptr == .shared_object and !global.isAbs(elf_file)) { + global.flags.import = true; + continue; + } if (file_ptr.index() == file_index) { global.flags.@"export" = true; if (elf_file.isDynLib() and vis != .PROTECTED) { @@ -1809,6 +1892,17 @@ fn markImportsExports(self: *Elf) void { } }.mark; + if (!self.isDynLib()) { + for (self.shared_objects.items) |index| { + for (self.file(index).?.globals()) |global_index| { + const global = self.symbol(global_index); + const file_ptr = global.file(self) orelse continue; + const vis = @as(elf.STV, @enumFromInt(global.elfSym(self).st_other)); + if (file_ptr != .shared_object and vis != .HIDDEN) global.flags.@"export" = true; + } + } + } + if (self.zig_module_index) |index| { mark(self, index); } @@ -1856,65 +1950,51 @@ fn scanRelocs(self: *Elf) !void { try self.reportUndefined(&undefs); - for (self.symbols.items, 0..) |*sym, sym_index| { + for (self.symbols.items, 0..) |*sym, i| { + const index = @as(u32, @intCast(i)); + if (!sym.isLocal() and !sym.flags.has_dynamic) { + log.debug("'{s}' is non-local", .{sym.name(self)}); + try self.dynsym.addSymbol(index, self); + } if (sym.flags.needs_got) { log.debug("'{s}' needs GOT", .{sym.name(self)}); - // TODO how can we tell we need to write it again, aka the entry is dirty? - const gop = try sym.getOrCreateGotEntry(@intCast(sym_index), self); - try self.got.writeEntry(self, gop.index); + _ = try self.got.addGotSymbol(index, self); } - } -} - -fn allocateObjects(self: *Elf) !void { - for (self.objects.items) |index| { - const object = self.file(index).?.object; - - for (object.atoms.items) |atom_index| { - const atom_ptr = self.atom(atom_index) orelse continue; - if (!atom_ptr.flags.alive or atom_ptr.flags.allocated) continue; - try atom_ptr.allocate(self); + if (sym.flags.needs_plt) { + if (sym.flags.is_canonical) { + log.debug("'{s}' needs CPLT", .{sym.name(self)}); + sym.flags.@"export" = true; + try self.plt.addSymbol(index, self); + } else if (sym.flags.needs_got) { + log.debug("'{s}' needs PLTGOT", .{sym.name(self)}); + try self.plt_got.addSymbol(index, self); + } else { + log.debug("'{s}' needs PLT", .{sym.name(self)}); + try self.plt.addSymbol(index, self); + } } - - for (object.locals()) |local_index| { - const local = self.symbol(local_index); - const atom_ptr = local.atom(self) orelse continue; - if (!atom_ptr.flags.alive) continue; - local.value = local.elfSym(self).st_value + atom_ptr.value; + if (sym.flags.needs_copy_rel and !sym.flags.has_copy_rel) { + log.debug("'{s}' needs COPYREL", .{sym.name(self)}); + try self.copy_rel.addSymbol(index, self); } - - for (object.globals()) |global_index| { - const global = self.symbol(global_index); - const atom_ptr = global.atom(self) orelse continue; - if (!atom_ptr.flags.alive) continue; - if (global.file_index == index) { - global.value = global.elfSym(self).st_value + atom_ptr.value; - } + if (sym.flags.needs_tlsgd) { + log.debug("'{s}' needs TLSGD", .{sym.name(self)}); + try self.got.addTlsGdSymbol(index, self); + } + if (sym.flags.needs_gottp) { + log.debug("'{s}' needs GOTTP", .{sym.name(self)}); + try self.got.addGotTpSymbol(index, self); + } + if (sym.flags.needs_tlsdesc) { + log.debug("'{s}' needs TLSDESC", .{sym.name(self)}); + try self.dynsym.addSymbol(index, self); + try self.got.addTlsDescSymbol(index, self); } } -} - -fn writeObjects(self: *Elf) !void { - const gpa = self.base.allocator; - - for (self.objects.items) |index| { - const object = self.file(index).?.object; - for (object.atoms.items) |atom_index| { - const atom_ptr = self.atom(atom_index) orelse continue; - if (!atom_ptr.flags.alive) continue; - - const shdr = &self.shdrs.items[atom_ptr.outputShndx().?]; - if (shdr.sh_type == elf.SHT_NOBITS) continue; - if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; // TODO we don't yet know how to handle non-alloc sections - - const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr; - log.debug("writing atom({d}) at 0x{x}", .{ atom_ptr.atom_index, file_offset }); - const code = try object.codeDecompressAlloc(self, atom_ptr.atom_index); - defer gpa.free(code); - try atom_ptr.resolveRelocs(self, code); - try self.base.file.?.pwriteAll(code, file_offset); - } + if (self.got.flags.needs_tlsld) { + log.debug("program needs TLSLD", .{}); + try self.got.addTlsLdSymbol(self); } } @@ -2618,7 +2698,100 @@ fn writeDwarfAddrAssumeCapacity(self: *Elf, buf: *std.ArrayList(u8), addr: u64) } } -fn writeElfHeader(self: *Elf) !void { +fn writeShdrTable(self: *Elf) !void { + const gpa = self.base.allocator; + const target_endian = self.base.options.target.cpu.arch.endian(); + const foreign_endian = target_endian != builtin.cpu.arch.endian(); + const shsize: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Shdr), + .p64 => @sizeOf(elf.Elf64_Shdr), + }; + const shalign: u16 = switch (self.ptr_width) { + .p32 => @alignOf(elf.Elf32_Shdr), + .p64 => @alignOf(elf.Elf64_Shdr), + }; + + const shoff = self.shdr_table_offset orelse 0; + const needed_size = self.shdrs.items.len * shsize; + + if (needed_size > self.allocatedSize(shoff)) { + self.shdr_table_offset = null; + self.shdr_table_offset = self.findFreeSpace(needed_size, shalign); + } + + log.debug("writing section headers from 0x{x} to 0x{x}", .{ + self.shdr_table_offset.?, + self.shdr_table_offset.? + needed_size, + }); + + switch (self.ptr_width) { + .p32 => { + const buf = try gpa.alloc(elf.Elf32_Shdr, self.shdrs.items.len); + defer gpa.free(buf); + + for (buf, 0..) |*shdr, i| { + shdr.* = shdrTo32(self.shdrs.items[i]); + if (foreign_endian) { + mem.byteSwapAllFields(elf.Elf32_Shdr, shdr); + } + } + try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + }, + .p64 => { + const buf = try gpa.alloc(elf.Elf64_Shdr, self.shdrs.items.len); + defer gpa.free(buf); + + for (buf, 0..) |*shdr, i| { + shdr.* = self.shdrs.items[i]; + if (foreign_endian) { + mem.byteSwapAllFields(elf.Elf64_Shdr, shdr); + } + } + try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), self.shdr_table_offset.?); + }, + } +} + +fn writePhdrTable(self: *Elf) !void { + const gpa = self.base.allocator; + const target_endian = self.base.options.target.cpu.arch.endian(); + const foreign_endian = target_endian != builtin.cpu.arch.endian(); + const phdr_table = &self.phdrs.items[self.phdr_table_index.?]; + + log.debug("writing program headers from 0x{x} to 0x{x}", .{ + phdr_table.p_offset, + phdr_table.p_offset + phdr_table.p_filesz, + }); + + switch (self.ptr_width) { + .p32 => { + const buf = try gpa.alloc(elf.Elf32_Phdr, self.phdrs.items.len); + defer gpa.free(buf); + + for (buf, 0..) |*phdr, i| { + phdr.* = phdrTo32(self.phdrs.items[i]); + if (foreign_endian) { + mem.byteSwapAllFields(elf.Elf32_Phdr, phdr); + } + } + try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset); + }, + .p64 => { + const buf = try gpa.alloc(elf.Elf64_Phdr, self.phdrs.items.len); + defer gpa.free(buf); + + for (buf, 0..) |*phdr, i| { + phdr.* = self.phdrs.items[i]; + if (foreign_endian) { + mem.byteSwapAllFields(elf.Elf64_Phdr, phdr); + } + } + try self.base.file.?.pwriteAll(mem.sliceAsBytes(buf), phdr_table.p_offset); + }, + } +} + +fn writeHeader(self: *Elf) !void { var hdr_buf: [@sizeOf(elf.Elf64_Ehdr)]u8 = undefined; var index: usize = 0; @@ -2649,12 +2822,12 @@ fn writeElfHeader(self: *Elf) !void { assert(index == 16); - const elf_type = switch (self.base.options.effectiveOutputMode()) { - .Exe => elf.ET.EXEC, - .Obj => elf.ET.REL, + const elf_type: elf.ET = switch (self.base.options.effectiveOutputMode()) { + .Exe => if (self.base.options.pic) .DYN else .EXEC, + .Obj => .REL, .Lib => switch (self.base.options.link_mode) { - .Static => elf.ET.REL, - .Dynamic => elf.ET.DYN, + .Static => @as(elf.ET, .REL), + .Dynamic => .DYN, }, }; mem.writeInt(u16, hdr_buf[index..][0..2], @intFromEnum(elf_type), endian); @@ -2668,8 +2841,7 @@ fn writeElfHeader(self: *Elf) !void { mem.writeInt(u32, hdr_buf[index..][0..4], 1, endian); index += 4; - const e_entry = if (elf_type == .REL) 0 else self.entry_addr.?; - + const e_entry = if (self.entry_index) |entry_index| self.symbol(entry_index).value else 0; const phdr_table_offset = self.phdrs.items[self.phdr_table_index.?].p_offset; switch (self.ptr_width) { .p32 => { @@ -2826,24 +2998,24 @@ fn getDeclShdrIndex(self: *Elf, decl_index: Module.Decl.Index, code: []const u8) const decl = mod.declPtr(decl_index); const shdr_index = switch (decl.ty.zigTypeTag(mod)) { // TODO: what if this is a function pointer? - .Fn => self.text_section_index.?, + .Fn => self.zig_text_section_index.?, else => blk: { if (decl.getOwnedVariable(mod)) |variable| { - if (variable.is_const) break :blk self.rodata_section_index.?; + if (variable.is_const) break :blk self.zig_rodata_section_index.?; if (variable.init.toValue().isUndefDeep(mod)) { const mode = self.base.options.optimize_mode; - if (mode == .Debug or mode == .ReleaseSafe) break :blk self.data_section_index.?; - break :blk self.bss_section_index.?; + if (mode == .Debug or mode == .ReleaseSafe) break :blk self.zig_data_section_index.?; + break :blk self.zig_bss_section_index.?; } // TODO I blatantly copied the logic from the Wasm linker, but is there a less // intrusive check for all zeroes than this? const is_all_zeroes = for (code) |byte| { if (byte != 0) break false; } else true; - if (is_all_zeroes) break :blk self.bss_section_index.?; - break :blk self.data_section_index.?; + if (is_all_zeroes) break :blk self.zig_bss_section_index.?; + break :blk self.zig_data_section_index.?; } - break :blk self.rodata_section_index.?; + break :blk self.zig_rodata_section_index.?; }, }; return shdr_index; @@ -2897,8 +3069,9 @@ fn updateDeclCode( esym.st_value = atom_ptr.value; log.debug(" (writing new offset table entry)", .{}); + assert(sym.flags.has_zig_got); const extra = sym.extra(self).?; - try self.got.writeEntry(self, extra.got); + try self.zig_got.writeOne(self, extra.zig_got); } } else if (code.len < old_size) { atom_ptr.shrink(self); @@ -2910,9 +3083,8 @@ fn updateDeclCode( sym.value = atom_ptr.value; esym.st_value = atom_ptr.value; - sym.flags.needs_got = true; - const gop = try sym.getOrCreateGotEntry(sym_index, self); - try self.got.writeEntry(self, gop.index); + const gop = try sym.getOrCreateZigGotEntry(sym_index, self); + try self.zig_got.writeOne(self, gop.index); } if (self.base.child_pid) |pid| { @@ -3016,12 +3188,16 @@ pub fn updateDecl( const decl = mod.declPtr(decl_index); if (decl.val.getExternFunc(mod)) |_| { - return; // TODO Should we do more when front-end analyzed extern decl? + return; } - if (decl.val.getVariable(mod)) |variable| { - if (variable.is_extern) { - return; // TODO Should we do more when front-end analyzed extern decl? - } + + if (decl.isExtern(mod)) { + // Extern variable gets a .got entry only. + const variable = decl.getOwnedVariable(mod).?; + const name = mod.intern_pool.stringToSlice(decl.name); + const lib_name = mod.intern_pool.stringToSliceUnwrap(variable.lib_name); + _ = try self.getGlobalSymbol(name, lib_name); + return; } const sym_index = try self.getOrCreateMetadataForDecl(decl_index); @@ -3122,8 +3298,8 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol. }; const output_section_index = switch (sym.kind) { - .code => self.text_section_index.?, - .const_data => self.rodata_section_index.?, + .code => self.zig_text_section_index.?, + .const_data => self.zig_rodata_section_index.?, }; const local_sym = self.symbol(symbol_index); const phdr_index = self.phdr_to_shdr_table.get(output_section_index).?; @@ -3146,9 +3322,8 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol. local_sym.value = atom_ptr.value; local_esym.st_value = atom_ptr.value; - local_sym.flags.needs_got = true; - const gop = try local_sym.getOrCreateGotEntry(symbol_index, self); - try self.got.writeEntry(self, gop.index); + const gop = try local_sym.getOrCreateZigGotEntry(symbol_index, self); + try self.zig_got.writeOne(self, gop.index); const section_offset = atom_ptr.value - self.phdrs.items[phdr_index].p_vaddr; const file_offset = self.shdrs.items[output_section_index].sh_offset + section_offset; @@ -3168,7 +3343,7 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module const index = unnamed_consts.items.len; const name = try std.fmt.allocPrint(gpa, "__unnamed_{s}_{d}", .{ decl_name, index }); defer gpa.free(name); - const sym_index = switch (try self.lowerConst(name, typed_value, self.rodata_section_index.?, decl.srcLoc(mod))) { + const sym_index = switch (try self.lowerConst(name, typed_value, self.zig_rodata_section_index.?, decl.srcLoc(mod))) { .ok => |sym_index| sym_index, .fail => |em| { decl.analysis = .codegen_failure; @@ -3265,8 +3440,7 @@ pub fn updateDeclExports( const zig_module = self.file(self.zig_module_index.?).?.zig_module; const decl = mod.declPtr(decl_index); const decl_sym_index = try self.getOrCreateMetadataForDecl(decl_index); - const decl_sym = self.symbol(decl_sym_index); - const decl_esym = zig_module.local_esyms.items[decl_sym.esym_index]; + const decl_esym = zig_module.local_esyms.items[self.symbol(decl_sym_index).esym_index]; const decl_metadata = self.decls.getPtr(decl_index).?; for (exports) |exp| { @@ -3283,13 +3457,7 @@ pub fn updateDeclExports( } const stb_bits: u8 = switch (exp.opts.linkage) { .Internal => elf.STB_LOCAL, - .Strong => blk: { - const entry_name = self.base.options.entry orelse "_start"; - if (mem.eql(u8, exp_name, entry_name)) { - self.entry_addr = decl_sym.value; - } - break :blk elf.STB_GLOBAL; - }, + .Strong => elf.STB_GLOBAL, .Weak => elf.STB_WEAK, .LinkOnce => { try mod.failed_exports.ensureUnusedCapacity(mod.gpa, 1); @@ -3315,8 +3483,8 @@ pub fn updateDeclExports( break :blk sym_index; }; const esym = &zig_module.global_esyms.items[sym_index & 0x0fffffff]; - esym.st_value = decl_sym.value; - esym.st_shndx = decl_sym.atom_index; + esym.st_value = self.symbol(decl_sym_index).value; + esym.st_shndx = decl_esym.st_shndx; esym.st_info = (stb_bits << 4) | stt_bits; esym.st_name = name_off; } @@ -3388,23 +3556,23 @@ fn addLinkerDefinedSymbols(self: *Elf) !void { self.rela_iplt_start_index = try linker_defined.addGlobal("__rela_iplt_start", self); self.rela_iplt_end_index = try linker_defined.addGlobal("__rela_iplt_end", self); - // for (self.objects.items) |index| { - // const object = self.getFile(index).?.object; - // for (object.atoms.items) |atom_index| { - // if (self.getStartStopBasename(atom_index)) |name| { - // const gpa = self.base.allocator; - // try self.start_stop_indexes.ensureUnusedCapacity(gpa, 2); + for (self.objects.items) |index| { + const object = self.file(index).?.object; + for (object.atoms.items) |atom_index| { + if (self.getStartStopBasename(atom_index)) |name| { + const gpa = self.base.allocator; + try self.start_stop_indexes.ensureUnusedCapacity(gpa, 2); - // const start = try std.fmt.allocPrintZ(gpa, "__start_{s}", .{name}); - // defer gpa.free(start); - // const stop = try std.fmt.allocPrintZ(gpa, "__stop_{s}", .{name}); - // defer gpa.free(stop); + const start = try std.fmt.allocPrintZ(gpa, "__start_{s}", .{name}); + defer gpa.free(start); + const stop = try std.fmt.allocPrintZ(gpa, "__stop_{s}", .{name}); + defer gpa.free(stop); - // self.start_stop_indexes.appendAssumeCapacity(try internal.addSyntheticGlobal(start, self)); - // self.start_stop_indexes.appendAssumeCapacity(try internal.addSyntheticGlobal(stop, self)); - // } - // } - // } + self.start_stop_indexes.appendAssumeCapacity(try linker_defined.addGlobal(start, self)); + self.start_stop_indexes.appendAssumeCapacity(try linker_defined.addGlobal(stop, self)); + } + } + } linker_defined.resolveSymbols(self); } @@ -3507,14 +3675,9 @@ fn allocateLinkerDefinedSymbols(self: *Elf) void { // _end { const end_symbol = self.symbol(self.end_index.?); - end_symbol.value = 0; - for (self.shdrs.items, 0..) |*shdr, shndx| { - if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; - const phdr_index = self.phdr_to_shdr_table.get(@intCast(shndx)).?; - const phdr = self.phdrs.items[phdr_index]; - const value = phdr.p_vaddr + phdr.p_memsz; - if (end_symbol.value < value) { - end_symbol.value = value; + for (self.shdrs.items, 0..) |shdr, shndx| { + if (shdr.sh_flags & elf.SHF_ALLOC != 0) { + end_symbol.value = shdr.sh_addr + shdr.sh_size; end_symbol.output_section_index = @intCast(shndx); } } @@ -3537,6 +3700,1168 @@ fn allocateLinkerDefinedSymbols(self: *Elf) void { } } +fn initSections(self: *Elf) !void { + const small_ptr = switch (self.ptr_width) { + .p32 => true, + .p64 => false, + }; + const ptr_size = self.ptrWidthBytes(); + + for (self.objects.items) |index| { + try self.file(index).?.object.initOutputSections(self); + } + + const needs_eh_frame = for (self.objects.items) |index| { + if (self.file(index).?.object.cies.items.len > 0) break true; + } else false; + if (needs_eh_frame) { + self.eh_frame_section_index = try self.addSection(.{ + .name = ".eh_frame", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC, + .addralign = ptr_size, + }); + + if (self.base.options.eh_frame_hdr) { + self.eh_frame_hdr_section_index = try self.addSection(.{ + .name = ".eh_frame_hdr", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC, + .addralign = 4, + }); + } + } + + if (self.got.entries.items.len > 0) { + self.got_section_index = try self.addSection(.{ + .name = ".got", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + .addralign = ptr_size, + }); + } + + const needs_rela_dyn = blk: { + if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or + self.zig_got.flags.needs_rela or self.copy_rel.symbols.items.len > 0) break :blk true; + if (self.zig_module_index) |index| { + if (self.file(index).?.zig_module.num_dynrelocs > 0) break :blk true; + } + for (self.objects.items) |index| { + if (self.file(index).?.object.num_dynrelocs > 0) break :blk true; + } + break :blk false; + }; + if (needs_rela_dyn) { + self.rela_dyn_section_index = try self.addSection(.{ + .name = ".rela.dyn", + .type = elf.SHT_RELA, + .flags = elf.SHF_ALLOC, + .addralign = @alignOf(elf.Elf64_Rela), + .entsize = @sizeOf(elf.Elf64_Rela), + }); + } + + if (self.plt.symbols.items.len > 0) { + self.plt_section_index = try self.addSection(.{ + .name = ".plt", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, + .addralign = 16, + }); + self.got_plt_section_index = try self.addSection(.{ + .name = ".got.plt", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + .addralign = @alignOf(u64), + }); + self.rela_plt_section_index = try self.addSection(.{ + .name = ".rela.plt", + .type = elf.SHT_RELA, + .flags = elf.SHF_ALLOC, + .addralign = @alignOf(elf.Elf64_Rela), + .entsize = @sizeOf(elf.Elf64_Rela), + }); + } + + if (self.plt_got.symbols.items.len > 0) { + self.plt_got_section_index = try self.addSection(.{ + .name = ".plt.got", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC | elf.SHF_EXECINSTR, + .addralign = 16, + }); + } + + if (self.copy_rel.symbols.items.len > 0) { + self.copy_rel_section_index = try self.addSection(.{ + .name = ".copyrel", + .type = elf.SHT_NOBITS, + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + }); + } + + const needs_interp = blk: { + // On Ubuntu with musl-gcc, we get a weird combo of options looking like this: + // -dynamic-linker=<path> -static + // In this case, if we do generate .interp section and segment, we will get + // a segfault in the dynamic linker trying to load a binary that is static + // and doesn't contain .dynamic section. + if (self.isStatic() and !self.base.options.pie) break :blk false; + break :blk self.base.options.dynamic_linker != null; + }; + if (needs_interp) { + self.interp_section_index = try self.addSection(.{ + .name = ".interp", + .type = elf.SHT_PROGBITS, + .flags = elf.SHF_ALLOC, + .addralign = 1, + }); + } + + if (self.isDynLib() or self.shared_objects.items.len > 0 or self.base.options.pie) { + self.dynstrtab_section_index = try self.addSection(.{ + .name = ".dynstr", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_STRTAB, + .entsize = 1, + .addralign = 1, + }); + self.dynamic_section_index = try self.addSection(.{ + .name = ".dynamic", + .flags = elf.SHF_ALLOC | elf.SHF_WRITE, + .type = elf.SHT_DYNAMIC, + .entsize = @sizeOf(elf.Elf64_Dyn), + .addralign = @alignOf(elf.Elf64_Dyn), + }); + self.dynsymtab_section_index = try self.addSection(.{ + .name = ".dynsym", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_DYNSYM, + .addralign = @alignOf(elf.Elf64_Sym), + .entsize = @sizeOf(elf.Elf64_Sym), + .info = 1, + }); + self.hash_section_index = try self.addSection(.{ + .name = ".hash", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_HASH, + .addralign = 4, + .entsize = 4, + }); + self.gnu_hash_section_index = try self.addSection(.{ + .name = ".gnu.hash", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_HASH, + .addralign = 8, + }); + + const needs_versions = for (self.dynsym.entries.items) |entry| { + const sym = self.symbol(entry.symbol_index); + if (sym.flags.import and sym.version_index & elf.VERSYM_VERSION > elf.VER_NDX_GLOBAL) break true; + } else false; + if (needs_versions) { + self.versym_section_index = try self.addSection(.{ + .name = ".gnu.version", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_VERSYM, + .addralign = @alignOf(elf.Elf64_Versym), + .entsize = @sizeOf(elf.Elf64_Versym), + }); + self.verneed_section_index = try self.addSection(.{ + .name = ".gnu.version_r", + .flags = elf.SHF_ALLOC, + .type = elf.SHT_GNU_VERNEED, + .addralign = @alignOf(elf.Elf64_Verneed), + }); + } + } + + if (self.symtab_section_index == null) { + self.symtab_section_index = try self.addSection(.{ + .name = ".symtab", + .type = elf.SHT_SYMTAB, + .addralign = if (small_ptr) @alignOf(elf.Elf32_Sym) else @alignOf(elf.Elf64_Sym), + .entsize = if (small_ptr) @sizeOf(elf.Elf32_Sym) else @sizeOf(elf.Elf64_Sym), + }); + } + if (self.strtab_section_index == null) { + self.strtab_section_index = try self.addSection(.{ + .name = ".strtab", + .type = elf.SHT_STRTAB, + .entsize = 1, + .addralign = 1, + }); + } + if (self.shstrtab_section_index == null) { + self.shstrtab_section_index = try self.addSection(.{ + .name = ".shstrtab", + .type = elf.SHT_STRTAB, + .entsize = 1, + .addralign = 1, + }); + } +} + +fn initSpecialPhdrs(self: *Elf) !void { + if (self.interp_section_index != null) { + self.phdr_interp_index = try self.addPhdr(.{ + .type = elf.PT_INTERP, + .flags = elf.PF_R, + .@"align" = 1, + }); + } + if (self.dynamic_section_index != null) { + self.phdr_dynamic_index = try self.addPhdr(.{ + .type = elf.PT_DYNAMIC, + .flags = elf.PF_R | elf.PF_W, + }); + } + if (self.eh_frame_hdr_section_index != null) { + self.phdr_gnu_eh_frame_index = try self.addPhdr(.{ + .type = elf.PT_GNU_EH_FRAME, + .flags = elf.PF_R, + }); + } + self.phdr_gnu_stack_index = try self.addPhdr(.{ + .type = elf.PT_GNU_STACK, + .flags = elf.PF_W | elf.PF_R, + .memsz = self.base.options.stack_size_override orelse 0, + .@"align" = 1, + }); + + const has_tls = for (self.shdrs.items) |shdr| { + if (shdr.sh_flags & elf.SHF_TLS != 0) break true; + } else false; + if (has_tls) { + self.phdr_tls_index = try self.addPhdr(.{ + .type = elf.PT_TLS, + .flags = elf.PF_R, + .@"align" = 1, + }); + } +} + +/// We need to sort constructors/destuctors in the following sections: +/// * .init_array +/// * .fini_array +/// * .preinit_array +/// * .ctors +/// * .dtors +/// The prority of inclusion is defined as part of the input section's name. For example, .init_array.10000. +/// If no priority value has been specified, +/// * for .init_array, .fini_array and .preinit_array, we automatically assign that section max value of maxInt(i32) +/// and push it to the back of the queue, +/// * for .ctors and .dtors, we automatically assign that section min value of -1 +/// and push it to the front of the queue, +/// crtbegin and ctrend are assigned minInt(i32) and maxInt(i32) respectively. +/// Ties are broken by the file prority which corresponds to the inclusion of input sections in this output section +/// we are about to sort. +fn sortInitFini(self: *Elf) !void { + const gpa = self.base.allocator; + + const Entry = struct { + priority: i32, + atom_index: Atom.Index, + + pub fn lessThan(ctx: *Elf, lhs: @This(), rhs: @This()) bool { + if (lhs.priority == rhs.priority) { + return ctx.atom(lhs.atom_index).?.priority(ctx) < ctx.atom(rhs.atom_index).?.priority(ctx); + } + return lhs.priority < rhs.priority; + } + }; + + for (self.shdrs.items, 0..) |*shdr, shndx| { + if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; + + var is_init_fini = false; + var is_ctor_dtor = false; + switch (shdr.sh_type) { + elf.SHT_PREINIT_ARRAY, + elf.SHT_INIT_ARRAY, + elf.SHT_FINI_ARRAY, + => is_init_fini = true, + else => { + const name = self.shstrtab.getAssumeExists(shdr.sh_name); + is_ctor_dtor = mem.indexOf(u8, name, ".ctors") != null or mem.indexOf(u8, name, ".dtors") != null; + }, + } + + if (!is_init_fini and !is_ctor_dtor) continue; + + const atom_list = self.output_sections.getPtr(@intCast(shndx)) orelse continue; + + var entries = std.ArrayList(Entry).init(gpa); + try entries.ensureTotalCapacityPrecise(atom_list.items.len); + defer entries.deinit(); + + for (atom_list.items) |atom_index| { + const atom_ptr = self.atom(atom_index).?; + const object = atom_ptr.file(self).?.object; + const priority = blk: { + if (is_ctor_dtor) { + if (mem.indexOf(u8, object.path, "crtbegin") != null) break :blk std.math.minInt(i32); + if (mem.indexOf(u8, object.path, "crtend") != null) break :blk std.math.maxInt(i32); + } + const default: i32 = if (is_ctor_dtor) -1 else std.math.maxInt(i32); + const name = atom_ptr.name(self); + var it = mem.splitBackwards(u8, name, "."); + const priority = std.fmt.parseUnsigned(u16, it.first(), 10) catch default; + break :blk priority; + }; + entries.appendAssumeCapacity(.{ .priority = priority, .atom_index = atom_index }); + } + + mem.sort(Entry, entries.items, self, Entry.lessThan); + + atom_list.clearRetainingCapacity(); + for (entries.items) |entry| { + atom_list.appendAssumeCapacity(entry.atom_index); + } + } +} + +fn setDynamicSection(self: *Elf, rpaths: []const []const u8) !void { + if (self.dynamic_section_index == null) return; + + for (self.shared_objects.items) |index| { + const shared_object = self.file(index).?.shared_object; + if (!shared_object.alive) continue; + try self.dynamic.addNeeded(shared_object, self); + } + + if (self.base.options.soname) |soname| { + try self.dynamic.setSoname(soname, self); + } + + try self.dynamic.setRpath(rpaths, self); +} + +fn sortDynamicSymtab(self: *Elf) void { + if (self.gnu_hash_section_index == null) return; + self.dynsym.sort(self); +} + +fn setVersionSymtab(self: *Elf) !void { + if (self.versym_section_index == null) return; + try self.versym.resize(self.base.allocator, self.dynsym.count()); + self.versym.items[0] = elf.VER_NDX_LOCAL; + for (self.dynsym.entries.items, 1..) |entry, i| { + const sym = self.symbol(entry.symbol_index); + self.versym.items[i] = sym.version_index; + } + + if (self.verneed_section_index) |shndx| { + try self.verneed.generate(self); + const shdr = &self.shdrs.items[shndx]; + shdr.sh_info = @as(u32, @intCast(self.verneed.verneed.items.len)); + } +} + +fn setHashSections(self: *Elf) !void { + if (self.hash_section_index != null) { + try self.hash.generate(self); + } + if (self.gnu_hash_section_index != null) { + try self.gnu_hash.calcSize(self); + } +} + +fn phdrRank(phdr: elf.Elf64_Phdr) u8 { + switch (phdr.p_type) { + elf.PT_NULL => return 0, + elf.PT_PHDR => return 1, + elf.PT_INTERP => return 2, + elf.PT_LOAD => return 3, + elf.PT_DYNAMIC, elf.PT_TLS => return 4, + elf.PT_GNU_EH_FRAME => return 5, + elf.PT_GNU_STACK => return 6, + else => return 7, + } +} + +fn sortPhdrs(self: *Elf) error{OutOfMemory}!void { + const Entry = struct { + phndx: u16, + + pub fn lessThan(elf_file: *Elf, lhs: @This(), rhs: @This()) bool { + const lhs_phdr = elf_file.phdrs.items[lhs.phndx]; + const rhs_phdr = elf_file.phdrs.items[rhs.phndx]; + const lhs_rank = phdrRank(lhs_phdr); + const rhs_rank = phdrRank(rhs_phdr); + if (lhs_rank == rhs_rank) return lhs_phdr.p_vaddr < rhs_phdr.p_vaddr; + return lhs_rank < rhs_rank; + } + }; + + const gpa = self.base.allocator; + var entries = try std.ArrayList(Entry).initCapacity(gpa, self.phdrs.items.len); + defer entries.deinit(); + for (0..self.phdrs.items.len) |phndx| { + entries.appendAssumeCapacity(.{ .phndx = @as(u16, @intCast(phndx)) }); + } + + mem.sort(Entry, entries.items, self, Entry.lessThan); + + const backlinks = try gpa.alloc(u16, entries.items.len); + defer gpa.free(backlinks); + for (entries.items, 0..) |entry, i| { + backlinks[entry.phndx] = @as(u16, @intCast(i)); + } + + var slice = try self.phdrs.toOwnedSlice(gpa); + defer gpa.free(slice); + + try self.phdrs.ensureTotalCapacityPrecise(gpa, slice.len); + for (entries.items) |sorted| { + self.phdrs.appendAssumeCapacity(slice[sorted.phndx]); + } + + for (&[_]*?u16{ + &self.phdr_zig_load_re_index, + &self.phdr_zig_got_index, + &self.phdr_zig_load_ro_index, + &self.phdr_zig_load_zerofill_index, + &self.phdr_table_index, + &self.phdr_table_load_index, + &self.phdr_interp_index, + &self.phdr_dynamic_index, + &self.phdr_gnu_eh_frame_index, + &self.phdr_tls_index, + }) |maybe_index| { + if (maybe_index.*) |*index| { + index.* = backlinks[index.*]; + } + } + + { + var it = self.phdr_to_shdr_table.iterator(); + while (it.next()) |entry| { + entry.value_ptr.* = backlinks[entry.value_ptr.*]; + } + } +} + +fn shdrRank(self: *Elf, shndx: u16) u8 { + const shdr = self.shdrs.items[shndx]; + const name = self.shstrtab.getAssumeExists(shdr.sh_name); + const flags = shdr.sh_flags; + + switch (shdr.sh_type) { + elf.SHT_NULL => return 0, + elf.SHT_DYNSYM => return 2, + elf.SHT_HASH => return 3, + elf.SHT_GNU_HASH => return 3, + elf.SHT_GNU_VERSYM => return 4, + elf.SHT_GNU_VERDEF => return 4, + elf.SHT_GNU_VERNEED => return 4, + + elf.SHT_PREINIT_ARRAY, + elf.SHT_INIT_ARRAY, + elf.SHT_FINI_ARRAY, + => return 0xf2, + + elf.SHT_DYNAMIC => return 0xf3, + + elf.SHT_RELA => return 0xf, + + elf.SHT_PROGBITS => if (flags & elf.SHF_ALLOC != 0) { + if (flags & elf.SHF_EXECINSTR != 0) { + return 0xf1; + } else if (flags & elf.SHF_WRITE != 0) { + return if (flags & elf.SHF_TLS != 0) 0xf4 else 0xf6; + } else if (mem.eql(u8, name, ".interp")) { + return 1; + } else { + return 0xf0; + } + } else { + if (mem.startsWith(u8, name, ".debug")) { + return 0xf8; + } else { + return 0xf9; + } + }, + + elf.SHT_NOBITS => return if (flags & elf.SHF_TLS != 0) 0xf5 else 0xf7, + elf.SHT_SYMTAB => return 0xfa, + elf.SHT_STRTAB => return if (mem.eql(u8, name, ".dynstr")) 0x4 else 0xfb, + else => return 0xff, + } +} + +fn sortShdrs(self: *Elf) !void { + const Entry = struct { + shndx: u16, + + pub fn lessThan(elf_file: *Elf, lhs: @This(), rhs: @This()) bool { + return elf_file.shdrRank(lhs.shndx) < elf_file.shdrRank(rhs.shndx); + } + }; + + const gpa = self.base.allocator; + var entries = try std.ArrayList(Entry).initCapacity(gpa, self.shdrs.items.len); + defer entries.deinit(); + for (0..self.shdrs.items.len) |shndx| { + entries.appendAssumeCapacity(.{ .shndx = @as(u16, @intCast(shndx)) }); + } + + mem.sort(Entry, entries.items, self, Entry.lessThan); + + const backlinks = try gpa.alloc(u16, entries.items.len); + defer gpa.free(backlinks); + for (entries.items, 0..) |entry, i| { + backlinks[entry.shndx] = @as(u16, @intCast(i)); + } + + var slice = try self.shdrs.toOwnedSlice(gpa); + defer gpa.free(slice); + + try self.shdrs.ensureTotalCapacityPrecise(gpa, slice.len); + for (entries.items) |sorted| { + self.shdrs.appendAssumeCapacity(slice[sorted.shndx]); + } + + for (&[_]*?u16{ + &self.eh_frame_section_index, + &self.eh_frame_hdr_section_index, + &self.got_section_index, + &self.symtab_section_index, + &self.strtab_section_index, + &self.shstrtab_section_index, + &self.interp_section_index, + &self.dynamic_section_index, + &self.dynsymtab_section_index, + &self.dynstrtab_section_index, + &self.hash_section_index, + &self.gnu_hash_section_index, + &self.plt_section_index, + &self.got_plt_section_index, + &self.plt_got_section_index, + &self.rela_dyn_section_index, + &self.rela_plt_section_index, + &self.copy_rel_section_index, + &self.versym_section_index, + &self.verneed_section_index, + &self.zig_text_section_index, + &self.zig_got_section_index, + &self.zig_rodata_section_index, + &self.zig_data_section_index, + &self.zig_bss_section_index, + &self.debug_str_section_index, + &self.debug_info_section_index, + &self.debug_abbrev_section_index, + &self.debug_aranges_section_index, + &self.debug_line_section_index, + }) |maybe_index| { + if (maybe_index.*) |*index| { + index.* = backlinks[index.*]; + } + } + + if (self.symtab_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.strtab_section_index.?; + } + + if (self.dynamic_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynstrtab_section_index.?; + } + + if (self.dynsymtab_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynstrtab_section_index.?; + } + + if (self.hash_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynsymtab_section_index.?; + } + + if (self.gnu_hash_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynsymtab_section_index.?; + } + + if (self.versym_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynsymtab_section_index.?; + } + + if (self.verneed_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynstrtab_section_index.?; + } + + if (self.rela_dyn_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynsymtab_section_index orelse 0; + } + + if (self.rela_plt_section_index) |index| { + const shdr = &self.shdrs.items[index]; + shdr.sh_link = self.dynsymtab_section_index.?; + shdr.sh_info = self.plt_section_index.?; + } + + { + var phdr_to_shdr_table = try self.phdr_to_shdr_table.clone(gpa); + defer phdr_to_shdr_table.deinit(gpa); + + self.phdr_to_shdr_table.clearRetainingCapacity(); + + var it = phdr_to_shdr_table.iterator(); + while (it.next()) |entry| { + const shndx = entry.key_ptr.*; + const phndx = entry.value_ptr.*; + self.phdr_to_shdr_table.putAssumeCapacityNoClobber(backlinks[shndx], phndx); + } + } + + if (self.zig_module_index) |index| { + const zig_module = self.file(index).?.zig_module; + for (zig_module.atoms.items) |atom_index| { + const atom_ptr = self.atom(atom_index) orelse continue; + if (!atom_ptr.flags.alive) continue; + const out_shndx = atom_ptr.outputShndx() orelse continue; + atom_ptr.output_section_index = backlinks[out_shndx]; + } + + for (zig_module.locals()) |local_index| { + const local = self.symbol(local_index); + const atom_ptr = local.atom(self) orelse continue; + if (!atom_ptr.flags.alive) continue; + const out_shndx = local.outputShndx() orelse continue; + local.output_section_index = backlinks[out_shndx]; + } + + for (zig_module.globals()) |global_index| { + const global = self.symbol(global_index); + const atom_ptr = global.atom(self) orelse continue; + if (!atom_ptr.flags.alive) continue; + if (global.file(self).?.index() != index) continue; + const out_shndx = global.outputShndx() orelse continue; + global.output_section_index = backlinks[out_shndx]; + } + } +} + +fn saveDebugSectionsSizes(self: *Elf) void { + if (self.debug_info_section_index) |shndx| { + self.debug_info_section_zig_size = self.shdrs.items[shndx].sh_size; + } + if (self.debug_abbrev_section_index) |shndx| { + self.debug_abbrev_section_zig_size = self.shdrs.items[shndx].sh_size; + } + if (self.debug_str_section_index) |shndx| { + self.debug_str_section_zig_size = self.shdrs.items[shndx].sh_size; + } + if (self.debug_aranges_section_index) |shndx| { + self.debug_aranges_section_zig_size = self.shdrs.items[shndx].sh_size; + } + if (self.debug_line_section_index) |shndx| { + self.debug_line_section_zig_size = self.shdrs.items[shndx].sh_size; + } +} + +fn updateSectionSizes(self: *Elf) !void { + for (self.output_sections.keys(), self.output_sections.values()) |shndx, atom_list| { + if (atom_list.items.len == 0) continue; + const shdr = &self.shdrs.items[shndx]; + for (atom_list.items) |atom_index| { + const atom_ptr = self.atom(atom_index) orelse continue; + if (!atom_ptr.flags.alive) continue; + const offset = atom_ptr.alignment.forward(shdr.sh_size); + const padding = offset - shdr.sh_size; + atom_ptr.value = offset; + shdr.sh_size += padding + atom_ptr.size; + shdr.sh_addralign = @max(shdr.sh_addralign, atom_ptr.alignment.toByteUnits(1)); + } + } + + if (self.eh_frame_section_index) |index| { + self.shdrs.items[index].sh_size = try eh_frame.calcEhFrameSize(self); + } + + if (self.eh_frame_hdr_section_index) |index| { + self.shdrs.items[index].sh_size = eh_frame.calcEhFrameHdrSize(self); + } + + if (self.got_section_index) |index| { + self.shdrs.items[index].sh_size = self.got.size(self); + } + + if (self.plt_section_index) |index| { + self.shdrs.items[index].sh_size = self.plt.size(); + } + + if (self.got_plt_section_index) |index| { + self.shdrs.items[index].sh_size = self.got_plt.size(self); + } + + if (self.plt_got_section_index) |index| { + self.shdrs.items[index].sh_size = self.plt_got.size(); + } + + if (self.rela_dyn_section_index) |shndx| { + var num = self.got.numRela(self) + self.copy_rel.numRela() + self.zig_got.numRela(); + if (self.zig_module_index) |index| { + num += self.file(index).?.zig_module.num_dynrelocs; + } + for (self.objects.items) |index| { + num += self.file(index).?.object.num_dynrelocs; + } + self.shdrs.items[shndx].sh_size = num * @sizeOf(elf.Elf64_Rela); + } + + if (self.rela_plt_section_index) |index| { + self.shdrs.items[index].sh_size = self.plt.numRela() * @sizeOf(elf.Elf64_Rela); + } + + if (self.copy_rel_section_index) |index| { + try self.copy_rel.updateSectionSize(index, self); + } + + if (self.interp_section_index) |index| { + self.shdrs.items[index].sh_size = self.base.options.dynamic_linker.?.len + 1; + } + + if (self.hash_section_index) |index| { + self.shdrs.items[index].sh_size = self.hash.size(); + } + + if (self.gnu_hash_section_index) |index| { + self.shdrs.items[index].sh_size = self.gnu_hash.size(); + } + + if (self.dynamic_section_index) |index| { + self.shdrs.items[index].sh_size = self.dynamic.size(self); + } + + if (self.dynsymtab_section_index) |index| { + self.shdrs.items[index].sh_size = self.dynsym.size(); + } + + if (self.dynstrtab_section_index) |index| { + self.shdrs.items[index].sh_size = self.dynstrtab.buffer.items.len; + } + + if (self.versym_section_index) |index| { + self.shdrs.items[index].sh_size = self.versym.items.len * @sizeOf(elf.Elf64_Versym); + } + + if (self.verneed_section_index) |index| { + self.shdrs.items[index].sh_size = self.verneed.size(); + } + + if (self.symtab_section_index != null) { + try self.updateSymtabSize(); + } + + if (self.strtab_section_index) |index| { + // TODO I don't really this here but we need it to add symbol names from GOT and other synthetic + // sections into .strtab for easier debugging. + if (self.zig_got_section_index) |_| { + try self.zig_got.updateStrtab(self); + } + if (self.got_section_index) |_| { + try self.got.updateStrtab(self); + } + if (self.plt_section_index) |_| { + try self.plt.updateStrtab(self); + } + if (self.plt_got_section_index) |_| { + try self.plt_got.updateStrtab(self); + } + self.shdrs.items[index].sh_size = self.strtab.buffer.items.len; + } + + if (self.shstrtab_section_index) |index| { + self.shdrs.items[index].sh_size = self.shstrtab.buffer.items.len; + } +} + +fn shdrToPhdrFlags(sh_flags: u64) u32 { + const write = sh_flags & elf.SHF_WRITE != 0; + const exec = sh_flags & elf.SHF_EXECINSTR != 0; + var out_flags: u32 = elf.PF_R; + if (write) out_flags |= elf.PF_W; + if (exec) out_flags |= elf.PF_X; + return out_flags; +} + +/// Calculates how many segments (PT_LOAD progam headers) are required +/// to cover the set of sections. +/// We permit a maximum of 3**2 number of segments. +fn calcNumberOfSegments(self: *Elf) usize { + var covers: [9]bool = [_]bool{false} ** 9; + for (self.shdrs.items, 0..) |shdr, shndx| { + if (shdr.sh_type == elf.SHT_NULL) continue; + if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; + if (self.isZigSection(@intCast(shndx))) continue; + const flags = shdrToPhdrFlags(shdr.sh_flags); + covers[flags - 1] = true; + } + var count: usize = 0; + for (covers) |cover| { + if (cover) count += 1; + } + return count; +} + +/// Allocates PHDR table in virtual memory and in file. +fn allocatePhdrTable(self: *Elf) void { + const new_load_segments = self.calcNumberOfSegments(); + const phdr_table = &self.phdrs.items[self.phdr_table_index.?]; + const phdr_table_load = &self.phdrs.items[self.phdr_table_load_index.?]; + + const phsize: u64 = switch (self.ptr_width) { + .p32 => @sizeOf(elf.Elf32_Phdr), + .p64 => @sizeOf(elf.Elf64_Phdr), + }; + const needed_size = (self.phdrs.items.len + new_load_segments) * phsize; + + if (needed_size > self.allocatedSize(phdr_table.p_offset)) { + phdr_table.p_offset = 0; + phdr_table.p_offset = self.findFreeSpace(needed_size, phdr_table.p_align); + } + + phdr_table_load.p_offset = mem.alignBackward(u64, phdr_table.p_offset, phdr_table_load.p_align); + const load_align_offset = phdr_table.p_offset - phdr_table_load.p_offset; + phdr_table_load.p_filesz = load_align_offset + needed_size; + phdr_table_load.p_memsz = load_align_offset + needed_size; + + phdr_table.p_filesz = needed_size; + phdr_table.p_vaddr = phdr_table_load.p_vaddr + load_align_offset; + phdr_table.p_paddr = phdr_table_load.p_paddr + load_align_offset; + phdr_table.p_memsz = needed_size; +} + +/// Allocates alloc sections and creates load segments for sections +/// extracted from input object files. +fn allocateAllocSections(self: *Elf) error{OutOfMemory}!void { + // We use this struct to track maximum alignment of all TLS sections. + // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold, + // in-file offsets have to be aligned against the start of TLS program header. + // If that's not ensured, then in a multi-threaded context, TLS variables across a shared object + // boundary may not get correctly loaded at an aligned address. + const Align = struct { + tls_start_align: u64 = 1, + first_tls_index: ?usize = null, + + fn isFirstTlsShdr(this: @This(), other: usize) bool { + if (this.first_tls_index) |index| return index == other; + return false; + } + + fn @"align"(this: @This(), index: usize, sh_addralign: u64, addr: u64) u64 { + const alignment = if (this.isFirstTlsShdr(index)) this.tls_start_align else sh_addralign; + return mem.alignForward(u64, addr, alignment); + } + }; + + var alignment = Align{}; + for (self.shdrs.items, 0..) |shdr, i| { + if (shdr.sh_type == elf.SHT_NULL) continue; + if (shdr.sh_flags & elf.SHF_TLS == 0) continue; + if (alignment.first_tls_index == null) alignment.first_tls_index = i; + alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign); + } + + // Next, calculate segment covers by scanning all alloc sections. + // If a section matches segment flags with the preceeding section, + // we put it in the same segment. Otherwise, we create a new cover. + // This algorithm is simple but suboptimal in terms of space re-use: + // normally we would also take into account any gaps in allocated + // virtual and file offsets. However, the simple one will do for one + // as we are more interested in quick turnaround and compatibility + // with `findFreeSpace` mechanics than anything else. + const Cover = std.ArrayList(u16); + const gpa = self.base.allocator; + var covers: [9]Cover = undefined; + for (&covers) |*cover| { + cover.* = Cover.init(gpa); + } + defer for (&covers) |*cover| { + cover.deinit(); + }; + + for (self.shdrs.items, 0..) |shdr, shndx| { + if (shdr.sh_type == elf.SHT_NULL) continue; + if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; + if (self.isZigSection(@intCast(shndx))) continue; + const flags = shdrToPhdrFlags(shdr.sh_flags); + try covers[flags - 1].append(@intCast(shndx)); + } + + // Now we can proceed with allocating the sections in virtual memory. + // As the base address we take the end address of the PHDR table. + // When allocating we first find the largest required alignment + // of any section that is contained in a cover and use it to align + // the start address of the segement (and first section). + const phdr_table = &self.phdrs.items[self.phdr_table_load_index.?]; + var addr = phdr_table.p_vaddr + phdr_table.p_memsz; + + for (covers) |cover| { + if (cover.items.len == 0) continue; + + var @"align": u64 = self.page_size; + for (cover.items) |shndx| { + const shdr = self.shdrs.items[shndx]; + if (shdr.sh_type == elf.SHT_NOBITS and shdr.sh_flags & elf.SHF_TLS != 0) continue; + @"align" = @max(@"align", shdr.sh_addralign); + } + + addr = mem.alignForward(u64, addr, @"align"); + + var memsz: u64 = 0; + var filesz: u64 = 0; + var i: usize = 0; + while (i < cover.items.len) : (i += 1) { + const shndx = cover.items[i]; + const shdr = &self.shdrs.items[shndx]; + if (shdr.sh_type == elf.SHT_NOBITS and shdr.sh_flags & elf.SHF_TLS != 0) { + // .tbss is a little special as it's used only by the loader meaning it doesn't + // need to be actually mmap'ed at runtime. We still need to correctly increment + // the addresses of every TLS zerofill section tho. Thus, we hack it so that + // we increment the start address like normal, however, after we are done, + // the next ALLOC section will get its start address allocated within the same + // range as the .tbss sections. We will get something like this: + // + // ... + // .tbss 0x10 + // .tcommon 0x20 + // .data 0x10 + // ... + var tbss_addr = addr; + while (i < cover.items.len and + self.shdrs.items[cover.items[i]].sh_type == elf.SHT_NOBITS and + self.shdrs.items[cover.items[i]].sh_flags & elf.SHF_TLS != 0) : (i += 1) + { + const tbss_shndx = cover.items[i]; + const tbss_shdr = &self.shdrs.items[tbss_shndx]; + tbss_addr = alignment.@"align"(tbss_shndx, tbss_shdr.sh_addralign, tbss_addr); + tbss_shdr.sh_addr = tbss_addr; + tbss_addr += tbss_shdr.sh_size; + } + i -= 1; + continue; + } + const next = alignment.@"align"(shndx, shdr.sh_addralign, addr); + const padding = next - addr; + addr = next; + shdr.sh_addr = addr; + if (shdr.sh_type != elf.SHT_NOBITS) { + filesz += padding + shdr.sh_size; + } + memsz += padding + shdr.sh_size; + addr += shdr.sh_size; + } + + const first = self.shdrs.items[cover.items[0]]; + var off = self.findFreeSpace(filesz, @"align"); + const phndx = try self.addPhdr(.{ + .type = elf.PT_LOAD, + .offset = off, + .addr = first.sh_addr, + .memsz = memsz, + .filesz = filesz, + .@"align" = @"align", + .flags = shdrToPhdrFlags(first.sh_flags), + }); + + for (cover.items) |shndx| { + const shdr = &self.shdrs.items[shndx]; + if (shdr.sh_type == elf.SHT_NOBITS) continue; + off = alignment.@"align"(shndx, shdr.sh_addralign, off); + shdr.sh_offset = off; + off += shdr.sh_size; + try self.phdr_to_shdr_table.putNoClobber(gpa, shndx, phndx); + } + + addr = mem.alignForward(u64, addr, self.page_size); + } +} + +/// Allocates non-alloc sections (debug info, symtabs, etc.). +fn allocateNonAllocSections(self: *Elf) !void { + for (self.shdrs.items, 0..) |*shdr, shndx| { + if (shdr.sh_type == elf.SHT_NULL) continue; + if (shdr.sh_flags & elf.SHF_ALLOC != 0) continue; + const needed_size = shdr.sh_size; + if (needed_size > self.allocatedSize(shdr.sh_offset)) { + shdr.sh_size = 0; + const new_offset = self.findFreeSpace(needed_size, shdr.sh_addralign); + + if (self.isDebugSection(@intCast(shndx))) { + log.debug("moving {s} from 0x{x} to 0x{x}", .{ + self.shstrtab.getAssumeExists(shdr.sh_name), + shdr.sh_offset, + new_offset, + }); + const existing_size = blk: { + if (shndx == self.debug_info_section_index.?) break :blk self.debug_info_section_zig_size; + if (shndx == self.debug_abbrev_section_index.?) break :blk self.debug_abbrev_section_zig_size; + if (shndx == self.debug_str_section_index.?) break :blk self.debug_str_section_zig_size; + if (shndx == self.debug_aranges_section_index.?) break :blk self.debug_aranges_section_zig_size; + if (shndx == self.debug_line_section_index.?) break :blk self.debug_line_section_zig_size; + unreachable; + }; + const amt = try self.base.file.?.copyRangeAll( + shdr.sh_offset, + self.base.file.?, + new_offset, + existing_size, + ); + if (amt != existing_size) return error.InputOutput; + } + + shdr.sh_offset = new_offset; + shdr.sh_size = needed_size; + } + } +} + +fn allocateSpecialPhdrs(self: *Elf) void { + for (&[_]struct { ?u16, ?u16 }{ + .{ self.phdr_interp_index, self.interp_section_index }, + .{ self.phdr_dynamic_index, self.dynamic_section_index }, + .{ self.phdr_gnu_eh_frame_index, self.eh_frame_hdr_section_index }, + }) |pair| { + if (pair[0]) |index| { + const shdr = self.shdrs.items[pair[1].?]; + const phdr = &self.phdrs.items[index]; + phdr.p_align = shdr.sh_addralign; + phdr.p_offset = shdr.sh_offset; + phdr.p_vaddr = shdr.sh_addr; + phdr.p_filesz = shdr.sh_size; + phdr.p_memsz = shdr.sh_size; + } + } + + // Set the TLS segment boundaries. + // We assume TLS sections are laid out contiguously and that there is + // a single TLS segment. + if (self.phdr_tls_index) |index| { + const slice = self.shdrs.items; + const phdr = &self.phdrs.items[index]; + var shndx: u16 = 0; + while (shndx < slice.len) { + const shdr = slice[shndx]; + if (shdr.sh_flags & elf.SHF_TLS == 0) { + shndx += 1; + continue; + } + phdr.p_offset = shdr.sh_offset; + phdr.p_vaddr = shdr.sh_addr; + phdr.p_paddr = shdr.sh_addr; + phdr.p_align = shdr.sh_addralign; + shndx += 1; + phdr.p_align = @max(phdr.p_align, shdr.sh_addralign); + if (shdr.sh_type != elf.SHT_NOBITS) { + phdr.p_filesz = shdr.sh_offset + shdr.sh_size - phdr.p_offset; + } + phdr.p_memsz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr; + + while (shndx < slice.len) : (shndx += 1) { + const next = slice[shndx]; + if (next.sh_flags & elf.SHF_TLS == 0) break; + phdr.p_align = @max(phdr.p_align, next.sh_addralign); + if (next.sh_type != elf.SHT_NOBITS) { + phdr.p_filesz = next.sh_offset + next.sh_size - phdr.p_offset; + } + phdr.p_memsz = next.sh_addr + next.sh_size - phdr.p_vaddr; + } + } + } +} + +fn allocateAtoms(self: *Elf) void { + for (self.objects.items) |index| { + self.file(index).?.object.allocateAtoms(self); + } +} + +fn writeAtoms(self: *Elf) !void { + const gpa = self.base.allocator; + + var undefs = std.AutoHashMap(Symbol.Index, std.ArrayList(Atom.Index)).init(gpa); + defer { + var it = undefs.iterator(); + while (it.next()) |entry| { + entry.value_ptr.deinit(); + } + undefs.deinit(); + } + + // TODO iterate over `output_sections` directly + for (self.shdrs.items, 0..) |shdr, shndx| { + if (shdr.sh_type == elf.SHT_NULL) continue; + if (shdr.sh_type == elf.SHT_NOBITS) continue; + + const atom_list = self.output_sections.get(@intCast(shndx)) orelse continue; + + log.debug("writing atoms in '{s}' section", .{self.shstrtab.getAssumeExists(shdr.sh_name)}); + + // TODO really, really handle debug section separately + const base_offset = if (self.isDebugSection(@intCast(shndx))) blk: { + if (shndx == self.debug_info_section_index.?) break :blk self.debug_info_section_zig_size; + if (shndx == self.debug_abbrev_section_index.?) break :blk self.debug_abbrev_section_zig_size; + if (shndx == self.debug_str_section_index.?) break :blk self.debug_str_section_zig_size; + if (shndx == self.debug_aranges_section_index.?) break :blk self.debug_aranges_section_zig_size; + if (shndx == self.debug_line_section_index.?) break :blk self.debug_line_section_zig_size; + unreachable; + } else 0; + const sh_offset = shdr.sh_offset + base_offset; + const sh_size = math.cast(usize, shdr.sh_size - base_offset) orelse return error.Overflow; + + const buffer = try gpa.alloc(u8, sh_size); + defer gpa.free(buffer); + const padding_byte: u8 = if (shdr.sh_type == elf.SHT_PROGBITS and + shdr.sh_flags & elf.SHF_EXECINSTR != 0) + 0xcc // int3 + else + 0; + @memset(buffer, padding_byte); + + for (atom_list.items) |atom_index| { + const atom_ptr = self.atom(atom_index).?; + assert(atom_ptr.flags.alive); + + const object = atom_ptr.file(self).?.object; + const offset = math.cast(usize, atom_ptr.value - shdr.sh_addr - base_offset) orelse + return error.Overflow; + const size = math.cast(usize, atom_ptr.size) orelse return error.Overflow; + + log.debug("writing atom({d}) at 0x{x}", .{ atom_index, sh_offset + offset }); + + // TODO decompress directly into provided buffer + const out_code = buffer[offset..][0..size]; + const in_code = try object.codeDecompressAlloc(self, atom_index); + defer gpa.free(in_code); + @memcpy(out_code, in_code); + + if (shdr.sh_flags & elf.SHF_ALLOC == 0) { + try atom_ptr.resolveRelocsNonAlloc(self, out_code, &undefs); + } else { + atom_ptr.resolveRelocsAlloc(self, out_code) catch |err| switch (err) { + // TODO + error.RelaxFail, error.InvalidInstruction, error.CannotEncode => { + log.err("relaxing intructions failed; TODO this should be a fatal linker error", .{}); + }, + else => |e| return e, + }; + } + } + + try self.base.file.?.pwriteAll(buffer, sh_offset); + } + + try self.reportUndefined(&undefs); +} + fn updateSymtabSize(self: *Elf) !void { var sizes = SymtabSize{}; @@ -3554,11 +4879,32 @@ fn updateSymtabSize(self: *Elf) !void { sizes.nglobals += object.output_symtab_size.nglobals; } + for (self.shared_objects.items) |index| { + const shared_object = self.file(index).?.shared_object; + shared_object.updateSymtabSize(self); + sizes.nglobals += shared_object.output_symtab_size.nglobals; + } + + if (self.zig_got_section_index) |_| { + self.zig_got.updateSymtabSize(self); + sizes.nlocals += self.zig_got.output_symtab_size.nlocals; + } + if (self.got_section_index) |_| { self.got.updateSymtabSize(self); sizes.nlocals += self.got.output_symtab_size.nlocals; } + if (self.plt_section_index) |_| { + self.plt.updateSymtabSize(self); + sizes.nlocals += self.plt.output_symtab_size.nlocals; + } + + if (self.plt_got_section_index) |_| { + self.plt_got.updateSymtabSize(self); + sizes.nlocals += self.plt_got.output_symtab_size.nlocals; + } + if (self.linker_defined_index) |index| { const linker_defined = self.file(index).?.linker_defined; linker_defined.updateSymtabSize(self); @@ -3567,19 +4913,155 @@ fn updateSymtabSize(self: *Elf) !void { const shdr = &self.shdrs.items[self.symtab_section_index.?]; shdr.sh_info = sizes.nlocals + 1; - self.markDirty(self.symtab_section_index.?, null); + shdr.sh_link = self.strtab_section_index.?; const sym_size: u64 = switch (self.ptr_width) { .p32 => @sizeOf(elf.Elf32_Sym), .p64 => @sizeOf(elf.Elf64_Sym), }; - const sym_align: u16 = switch (self.ptr_width) { - .p32 => @alignOf(elf.Elf32_Sym), - .p64 => @alignOf(elf.Elf64_Sym), - }; const needed_size = (sizes.nlocals + sizes.nglobals + 1) * sym_size; shdr.sh_size = needed_size; - try self.growNonAllocSection(self.symtab_section_index.?, needed_size, sym_align, true); +} + +fn writeSyntheticSections(self: *Elf) !void { + const gpa = self.base.allocator; + + if (self.interp_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + const sh_size = math.cast(usize, shdr.sh_size) orelse return error.Overflow; + var buffer = try gpa.alloc(u8, sh_size); + defer gpa.free(buffer); + const dylinker = self.base.options.dynamic_linker.?; + @memcpy(buffer[0..dylinker.len], dylinker); + buffer[dylinker.len] = 0; + try self.base.file.?.pwriteAll(buffer, shdr.sh_offset); + } + + if (self.hash_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + try self.base.file.?.pwriteAll(self.hash.buffer.items, shdr.sh_offset); + } + + if (self.gnu_hash_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.gnu_hash.size()); + defer buffer.deinit(); + try self.gnu_hash.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.versym_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.versym.items), shdr.sh_offset); + } + + if (self.verneed_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.verneed.size()); + defer buffer.deinit(); + try self.verneed.write(buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.dynamic_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.dynamic.size(self)); + defer buffer.deinit(); + try self.dynamic.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.dynsymtab_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.dynsym.size()); + defer buffer.deinit(); + try self.dynsym.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.dynstrtab_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + try self.base.file.?.pwriteAll(self.dynstrtab.buffer.items, shdr.sh_offset); + } + + if (self.eh_frame_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + const sh_size = math.cast(usize, shdr.sh_size) orelse return error.Overflow; + var buffer = try std.ArrayList(u8).initCapacity(gpa, sh_size); + defer buffer.deinit(); + try eh_frame.writeEhFrame(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.eh_frame_hdr_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + const sh_size = math.cast(usize, shdr.sh_size) orelse return error.Overflow; + var buffer = try std.ArrayList(u8).initCapacity(gpa, sh_size); + defer buffer.deinit(); + try eh_frame.writeEhFrameHdr(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.got_section_index) |index| { + const shdr = self.shdrs.items[index]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.got.size(self)); + defer buffer.deinit(); + try self.got.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.rela_dyn_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + try self.got.addRela(self); + try self.copy_rel.addRela(self); + try self.zig_got.addRela(self); + self.sortRelaDyn(); + try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.rela_dyn.items), shdr.sh_offset); + } + + if (self.plt_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.plt.size()); + defer buffer.deinit(); + try self.plt.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.got_plt_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.got_plt.size(self)); + defer buffer.deinit(); + try self.got_plt.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.plt_got_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + var buffer = try std.ArrayList(u8).initCapacity(gpa, self.plt_got.size()); + defer buffer.deinit(); + try self.plt_got.write(self, buffer.writer()); + try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset); + } + + if (self.rela_plt_section_index) |shndx| { + const shdr = self.shdrs.items[shndx]; + try self.plt.addRela(self); + try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.rela_plt.items), shdr.sh_offset); + } + + if (self.shstrtab_section_index) |index| { + const shdr = self.shdrs.items[index]; + try self.base.file.?.pwriteAll(self.shstrtab.buffer.items, shdr.sh_offset); + } + + if (self.strtab_section_index) |index| { + const shdr = self.shdrs.items[index]; + try self.base.file.?.pwriteAll(self.strtab.buffer.items, shdr.sh_offset); + } + + if (self.symtab_section_index) |_| { + try self.writeSymtab(); + } } fn writeSymtab(self: *Elf) !void { @@ -3617,11 +5099,32 @@ fn writeSymtab(self: *Elf) !void { ctx.iglobal += object.output_symtab_size.nglobals; } + for (self.shared_objects.items) |index| { + const shared_object = self.file(index).?.shared_object; + shared_object.writeSymtab(self, ctx); + ctx.iglobal += shared_object.output_symtab_size.nglobals; + } + + if (self.zig_got_section_index) |_| { + try self.zig_got.writeSymtab(self, ctx); + ctx.ilocal += self.zig_got.output_symtab_size.nlocals; + } + if (self.got_section_index) |_| { try self.got.writeSymtab(self, ctx); ctx.ilocal += self.got.output_symtab_size.nlocals; } + if (self.plt_section_index) |_| { + try self.plt.writeSymtab(self, ctx); + ctx.ilocal += self.plt.output_symtab_size.nlocals; + } + + if (self.plt_got_section_index) |_| { + try self.plt_got.writeSymtab(self, ctx); + ctx.ilocal += self.plt_got.output_symtab_size.nlocals; + } + if (self.linker_defined_index) |index| { const linker_defined = self.file(index).?.linker_defined; linker_defined.writeSymtab(self, ctx); @@ -3683,29 +5186,6 @@ fn phdrTo32(phdr: elf.Elf64_Phdr) elf.Elf32_Phdr { }; } -fn writeShdr(self: *Elf, index: usize) !void { - const foreign_endian = self.base.options.target.cpu.arch.endian() != builtin.cpu.arch.endian(); - switch (self.ptr_width) { - .p32 => { - var shdr: [1]elf.Elf32_Shdr = undefined; - shdr[0] = shdrTo32(self.shdrs.items[index]); - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf32_Shdr, &shdr[0]); - } - const offset = self.shdr_table_offset.? + index * @sizeOf(elf.Elf32_Shdr); - return self.base.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); - }, - .p64 => { - var shdr = [1]elf.Elf64_Shdr{self.shdrs.items[index]}; - if (foreign_endian) { - mem.byteSwapAllFields(elf.Elf64_Shdr, &shdr[0]); - } - const offset = self.shdr_table_offset.? + index * @sizeOf(elf.Elf64_Shdr); - return self.base.file.?.pwriteAll(mem.sliceAsBytes(&shdr), offset); - }, - } -} - fn shdrTo32(shdr: elf.Elf64_Shdr) elf.Elf32_Shdr { return .{ .sh_name = shdr.sh_name, @@ -3979,8 +5459,94 @@ pub fn isStatic(self: Elf) bool { return self.base.options.link_mode == .Static; } +pub fn isExe(self: Elf) bool { + return self.base.options.effectiveOutputMode() == .Exe; +} + pub fn isDynLib(self: Elf) bool { - return self.base.options.output_mode == .Lib and self.base.options.link_mode == .Dynamic; + return self.base.options.effectiveOutputMode() == .Lib and self.base.options.link_mode == .Dynamic; +} + +pub fn isZigSection(self: Elf, shndx: u16) bool { + inline for (&[_]?u16{ + self.zig_text_section_index, + self.zig_rodata_section_index, + self.zig_data_section_index, + self.zig_bss_section_index, + self.zig_got_section_index, + }) |maybe_index| { + if (maybe_index) |index| { + if (index == shndx) return true; + } + } + return false; +} + +pub fn isDebugSection(self: Elf, shndx: u16) bool { + inline for (&[_]?u16{ + self.debug_info_section_index, + self.debug_abbrev_section_index, + self.debug_str_section_index, + self.debug_aranges_section_index, + self.debug_line_section_index, + }) |maybe_index| { + if (maybe_index) |index| { + if (index == shndx) return true; + } + } + return false; +} + +fn addPhdr(self: *Elf, opts: struct { + type: u32 = 0, + flags: u32 = 0, + @"align": u64 = 0, + offset: u64 = 0, + addr: u64 = 0, + filesz: u64 = 0, + memsz: u64 = 0, +}) error{OutOfMemory}!u16 { + const index = @as(u16, @intCast(self.phdrs.items.len)); + try self.phdrs.append(self.base.allocator, .{ + .p_type = opts.type, + .p_flags = opts.flags, + .p_offset = opts.offset, + .p_vaddr = opts.addr, + .p_paddr = opts.addr, + .p_filesz = opts.filesz, + .p_memsz = opts.memsz, + .p_align = opts.@"align", + }); + return index; +} + +pub const AddSectionOpts = struct { + name: [:0]const u8, + type: u32 = elf.SHT_NULL, + flags: u64 = 0, + link: u32 = 0, + info: u32 = 0, + addralign: u64 = 0, + entsize: u64 = 0, +}; + +pub fn addSection(self: *Elf, opts: AddSectionOpts) !u16 { + const gpa = self.base.allocator; + const index = @as(u16, @intCast(self.shdrs.items.len)); + const shdr = try self.shdrs.addOne(gpa); + shdr.* = .{ + .sh_name = try self.shstrtab.insert(gpa, opts.name), + .sh_type = opts.type, + .sh_flags = opts.flags, + .sh_addr = 0, + .sh_offset = 0, + .sh_size = 0, + .sh_link = opts.link, + .sh_info = opts.info, + .sh_addralign = opts.addralign, + .sh_entsize = opts.entsize, + }; + return index; } pub fn sectionByName(self: *Elf, name: [:0]const u8) ?u16 { @@ -3990,9 +5556,77 @@ pub fn sectionByName(self: *Elf, name: [:0]const u8) ?u16 { } else return null; } -pub fn calcNumIRelativeRelocs(self: *Elf) u64 { - _ = self; - unreachable; // TODO +const RelaDyn = struct { + offset: u64, + sym: u64 = 0, + type: u32, + addend: i64 = 0, +}; + +pub fn addRelaDyn(self: *Elf, opts: RelaDyn) !void { + try self.rela_dyn.ensureUnusedCapacity(self.base.alloctor, 1); + self.addRelaDynAssumeCapacity(opts); +} + +pub fn addRelaDynAssumeCapacity(self: *Elf, opts: RelaDyn) void { + self.rela_dyn.appendAssumeCapacity(.{ + .r_offset = opts.offset, + .r_info = (opts.sym << 32) | opts.type, + .r_addend = opts.addend, + }); +} + +fn sortRelaDyn(self: *Elf) void { + const Sort = struct { + fn rank(rel: elf.Elf64_Rela) u2 { + return switch (rel.r_type()) { + elf.R_X86_64_RELATIVE => 0, + elf.R_X86_64_IRELATIVE => 2, + else => 1, + }; + } + + pub fn lessThan(ctx: void, lhs: elf.Elf64_Rela, rhs: elf.Elf64_Rela) bool { + _ = ctx; + if (rank(lhs) == rank(rhs)) { + if (lhs.r_sym() == rhs.r_sym()) return lhs.r_offset < rhs.r_offset; + return lhs.r_sym() < rhs.r_sym(); + } + return rank(lhs) < rank(rhs); + } + }; + mem.sort(elf.Elf64_Rela, self.rela_dyn.items, {}, Sort.lessThan); +} + +fn calcNumIRelativeRelocs(self: *Elf) usize { + var count: usize = self.num_ifunc_dynrelocs; + + for (self.got.entries.items) |entry| { + if (entry.tag != .got) continue; + const sym = self.symbol(entry.symbol_index); + if (sym.isIFunc(self)) count += 1; + } + + return count; +} + +pub fn isCIdentifier(name: []const u8) bool { + if (name.len == 0) return false; + const first_c = name[0]; + if (!std.ascii.isAlphabetic(first_c) and first_c != '_') return false; + for (name[1..]) |c| { + if (!std.ascii.isAlphanumeric(c) and c != '_') return false; + } + return true; +} + +fn getStartStopBasename(self: *Elf, atom_index: Atom.Index) ?[]const u8 { + const atom_ptr = self.atom(atom_index) orelse return null; + const name = atom_ptr.name(self); + if (atom_ptr.inputShdr(self).sh_flags & elf.SHF_ALLOC != 0 and name.len > 0) { + if (isCIdentifier(name)) return name; + } + return null; } pub fn atom(self: *Elf, atom_index: Atom.Index) ?*Atom { @@ -4015,6 +5649,7 @@ pub fn file(self: *Elf, index: File.Index) ?File { .linker_defined => .{ .linker_defined = &self.files.items(.data)[index].linker_defined }, .zig_module => .{ .zig_module = &self.files.items(.data)[index].zig_module }, .object => .{ .object = &self.files.items(.data)[index].object }, + .shared_object => .{ .shared_object = &self.files.items(.data)[index].shared_object }, }; } @@ -4288,52 +5923,78 @@ fn reportParseError( try err.addNote(self, "while parsing {s}", .{path}); } -fn fmtShdrs(self: *Elf) std.fmt.Formatter(formatShdrs) { - return .{ .data = self }; +const FormatShdrCtx = struct { + elf_file: *Elf, + shdr: elf.Elf64_Shdr, +}; + +fn fmtShdr(self: *Elf, shdr: elf.Elf64_Shdr) std.fmt.Formatter(formatShdr) { + return .{ .data = .{ + .shdr = shdr, + .elf_file = self, + } }; } -fn formatShdrs( - self: *Elf, +fn formatShdr( + ctx: FormatShdrCtx, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = options; _ = unused_fmt_string; - for (self.shdrs.items, 0..) |shdr, i| { - try writer.print("shdr({d}) : phdr({?d}) : {s} : @{x} ({x}) : align({x}) : size({x})\n", .{ - i, self.phdr_to_shdr_table.get(@intCast(i)), - self.shstrtab.getAssumeExists(shdr.sh_name), shdr.sh_offset, - shdr.sh_addr, shdr.sh_addralign, - shdr.sh_size, - }); - } + const shdr = ctx.shdr; + try writer.print("{s} : @{x} ({x}) : align({x}) : size({x})", .{ + ctx.elf_file.shstrtab.getAssumeExists(shdr.sh_name), shdr.sh_offset, + shdr.sh_addr, shdr.sh_addralign, + shdr.sh_size, + }); } -fn fmtPhdrs(self: *Elf) std.fmt.Formatter(formatPhdrs) { - return .{ .data = self }; +const FormatPhdrCtx = struct { + elf_file: *Elf, + phdr: elf.Elf64_Phdr, +}; + +fn fmtPhdr(self: *Elf, phdr: elf.Elf64_Phdr) std.fmt.Formatter(formatPhdr) { + return .{ .data = .{ + .phdr = phdr, + .elf_file = self, + } }; } -fn formatPhdrs( - self: *Elf, +fn formatPhdr( + ctx: FormatPhdrCtx, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = options; _ = unused_fmt_string; - for (self.phdrs.items, 0..) |phdr, i| { - const write = phdr.p_flags & elf.PF_W != 0; - const read = phdr.p_flags & elf.PF_R != 0; - const exec = phdr.p_flags & elf.PF_X != 0; - var flags: [3]u8 = [_]u8{'_'} ** 3; - if (exec) flags[0] = 'X'; - if (write) flags[1] = 'W'; - if (read) flags[2] = 'R'; - try writer.print("phdr({d}) : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})\n", .{ - i, flags, phdr.p_offset, phdr.p_vaddr, phdr.p_align, phdr.p_filesz, phdr.p_memsz, - }); - } + const phdr = ctx.phdr; + const write = phdr.p_flags & elf.PF_W != 0; + const read = phdr.p_flags & elf.PF_R != 0; + const exec = phdr.p_flags & elf.PF_X != 0; + var flags: [3]u8 = [_]u8{'_'} ** 3; + if (exec) flags[0] = 'X'; + if (write) flags[1] = 'W'; + if (read) flags[2] = 'R'; + const p_type = switch (phdr.p_type) { + elf.PT_LOAD => "LOAD", + elf.PT_TLS => "TLS", + elf.PT_GNU_EH_FRAME => "GNU_EH_FRAME", + elf.PT_GNU_STACK => "GNU_STACK", + elf.PT_DYNAMIC => "DYNAMIC", + elf.PT_INTERP => "INTERP", + elf.PT_NULL => "NULL", + elf.PT_PHDR => "PHDR", + elf.PT_NOTE => "NOTE", + else => "UNKNOWN", + }; + try writer.print("{s} : {s} : @{x} ({x}) : align({x}) : filesz({x}) : memsz({x})", .{ + p_type, flags, phdr.p_offset, phdr.p_vaddr, + phdr.p_align, phdr.p_filesz, phdr.p_memsz, + }); } fn dumpState(self: *Elf) std.fmt.Formatter(fmtDumpState) { @@ -4352,7 +6013,10 @@ fn fmtDumpState( if (self.zig_module_index) |index| { const zig_module = self.file(index).?.zig_module; try writer.print("zig_module({d}) : {s}\n", .{ index, zig_module.path }); - try writer.print("{}\n", .{zig_module.fmtSymtab(self)}); + try writer.print("{}{}\n", .{ + zig_module.fmtAtoms(self), + zig_module.fmtSymtab(self), + }); } for (self.objects.items) |index| { @@ -4369,16 +6033,35 @@ fn fmtDumpState( }); } + for (self.shared_objects.items) |index| { + const shared_object = self.file(index).?.shared_object; + try writer.print("shared_object({d}) : ", .{index}); + try writer.print("{s}", .{shared_object.path}); + try writer.print(" : needed({})", .{shared_object.needed}); + if (!shared_object.alive) try writer.writeAll(" : [*]"); + try writer.writeByte('\n'); + try writer.print("{}\n", .{shared_object.fmtSymtab(self)}); + } + if (self.linker_defined_index) |index| { const linker_defined = self.file(index).?.linker_defined; try writer.print("linker_defined({d}) : (linker defined)\n", .{index}); try writer.print("{}\n", .{linker_defined.fmtSymtab(self)}); } try writer.print("{}\n", .{self.got.fmt(self)}); + try writer.print("{}\n", .{self.zig_got.fmt(self)}); try writer.writeAll("Output shdrs\n"); - try writer.print("{}\n", .{self.fmtShdrs()}); - try writer.writeAll("Output phdrs\n"); - try writer.print("{}\n", .{self.fmtPhdrs()}); + for (self.shdrs.items, 0..) |shdr, shndx| { + try writer.print("shdr({d}) : phdr({?d}) : {}\n", .{ + shndx, + self.phdr_to_shdr_table.get(@intCast(shndx)), + self.fmtShdr(shdr), + }); + } + try writer.writeAll("\nOutput phdrs\n"); + for (self.phdrs.items, 0..) |phdr, phndx| { + try writer.print("phdr{d} : {}\n", .{ phndx, self.fmtPhdr(phdr) }); + } } /// Binary search @@ -4486,11 +6169,27 @@ pub const null_sym = elf.Elf64_Sym{ .st_size = 0, }; +pub const null_shdr = elf.Elf64_Shdr{ + .sh_name = 0, + .sh_type = 0, + .sh_flags = 0, + .sh_addr = 0, + .sh_offset = 0, + .sh_size = 0, + .sh_link = 0, + .sh_info = 0, + .sh_addralign = 0, + .sh_entsize = 0, +}; + const SystemLib = struct { needed: bool = false, path: []const u8, }; +pub const R_X86_64_ZIG_GOT32 = elf.R_X86_64_NUM + 1; +pub const R_X86_64_ZIG_GOTPCREL = elf.R_X86_64_NUM + 2; + const std = @import("std"); const build_options = @import("build_options"); const builtin = @import("builtin"); @@ -4503,6 +6202,8 @@ const math = std.math; const mem = std.mem; const codegen = @import("../codegen.zig"); +const eh_frame = @import("Elf/eh_frame.zig"); +const gc = @import("Elf/gc.zig"); const glibc = @import("../glibc.zig"); const link = @import("../link.zig"); const lldMain = @import("../main.zig").lldMain; @@ -4517,10 +6218,16 @@ const Archive = @import("Elf/Archive.zig"); pub const Atom = @import("Elf/Atom.zig"); const Cache = std.Build.Cache; const Compilation = @import("../Compilation.zig"); +const CopyRelSection = synthetic_sections.CopyRelSection; +const DynamicSection = synthetic_sections.DynamicSection; +const DynsymSection = synthetic_sections.DynsymSection; const Dwarf = @import("Dwarf.zig"); const Elf = @This(); const File = @import("Elf/file.zig").File; +const GnuHashSection = synthetic_sections.GnuHashSection; const GotSection = synthetic_sections.GotSection; +const GotPltSection = synthetic_sections.GotPltSection; +const HashSection = synthetic_sections.HashSection; const LinkerDefined = @import("Elf/LinkerDefined.zig"); const Liveness = @import("../Liveness.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; @@ -4528,10 +6235,15 @@ const Module = @import("../Module.zig"); const Object = @import("Elf/Object.zig"); const InternPool = @import("../InternPool.zig"); const Package = @import("../Package.zig"); +const PltSection = synthetic_sections.PltSection; +const PltGotSection = synthetic_sections.PltGotSection; +const SharedObject = @import("Elf/SharedObject.zig"); const Symbol = @import("Elf/Symbol.zig"); const StringTable = @import("strtab.zig").StringTable; const TableSection = @import("table_section.zig").TableSection; const Type = @import("../type.zig").Type; const TypedValue = @import("../TypedValue.zig"); const Value = @import("../value.zig").Value; +const VerneedSection = synthetic_sections.VerneedSection; +const ZigGotSection = synthetic_sections.ZigGotSection; const ZigModule = @import("Elf/ZigModule.zig"); diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 14c10bb244..a876a4d16c 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -14,13 +14,13 @@ size: u64 = 0, alignment: Alignment = .@"1", /// Index of the input section. -input_section_index: Index = 0, +input_section_index: u16 = 0, /// Index of the output section. output_section_index: u16 = 0, /// Index of the input section containing this atom's relocs. -relocs_section_index: Index = 0, +relocs_section_index: u16 = 0, /// Index of this atom in the linker's atoms table. atom_index: Index = 0, @@ -49,9 +49,12 @@ pub fn file(self: Atom, elf_file: *Elf) ?File { return elf_file.file(self.file_index); } -pub fn inputShdr(self: Atom, elf_file: *Elf) elf.Elf64_Shdr { - const object = self.file(elf_file).?.object; - return object.shdrs.items[self.input_section_index]; +pub fn inputShdr(self: Atom, elf_file: *Elf) Object.ElfShdr { + return switch (self.file(elf_file).?) { + .object => |x| x.shdrs.items[self.input_section_index], + .zig_module => |x| x.inputShdr(self.atom_index, elf_file), + else => unreachable, + }; } pub fn outputShndx(self: Atom) ?u16 { @@ -199,7 +202,7 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !void { _ = free_list.swapRemove(i); } - self.flags.allocated = true; + self.flags.alive = true; } pub fn shrink(self: *Atom, elf_file: *Elf) void { @@ -216,7 +219,6 @@ pub fn free(self: *Atom, elf_file: *Elf) void { log.debug("freeAtom {d} ({s})", .{ self.atom_index, self.name(elf_file) }); const gpa = elf_file.base.allocator; - const zig_module = self.file(elf_file).?.zig_module; const shndx = self.outputShndx().?; const meta = elf_file.last_atom_and_free_list_table.getPtr(shndx).?; const free_list = &meta.free_list; @@ -267,11 +269,13 @@ pub fn free(self: *Atom, elf_file: *Elf) void { // TODO create relocs free list self.freeRelocs(elf_file); - assert(zig_module.atoms.swapRemove(self.atom_index)); + // TODO figure out how to free input section mappind in ZigModule + // const zig_module = self.file(elf_file).?.zig_module; + // assert(zig_module.atoms.swapRemove(self.atom_index)); self.* = .{}; } -pub fn relocs(self: Atom, elf_file: *Elf) error{Overflow}![]align(1) const elf.Elf64_Rela { +pub fn relocs(self: Atom, elf_file: *Elf) []align(1) const elf.Elf64_Rela { return switch (self.file(elf_file).?) { .zig_module => |x| x.relocs.items[self.relocs_section_index].items, .object => |x| x.getRelocs(self.relocs_section_index), @@ -279,6 +283,18 @@ pub fn relocs(self: Atom, elf_file: *Elf) error{Overflow}![]align(1) const elf.E }; } +pub fn fdes(self: Atom, elf_file: *Elf) []Fde { + if (self.fde_start == self.fde_end) return &[0]Fde{}; + const object = self.file(elf_file).?.object; + return object.fdes.items[self.fde_start..self.fde_end]; +} + +pub fn markFdesDead(self: Atom, elf_file: *Elf) void { + for (self.fdes(elf_file)) |*fde| { + fde.alive = false; + } +} + pub fn addReloc(self: Atom, elf_file: *Elf, reloc: elf.Elf64_Rela) !void { const gpa = elf_file.base.allocator; const file_ptr = self.file(elf_file).?; @@ -295,17 +311,18 @@ pub fn freeRelocs(self: Atom, elf_file: *Elf) void { zig_module.relocs.items[self.relocs_section_index].clearRetainingCapacity(); } -pub fn scanRelocsRequiresCode(self: Atom, elf_file: *Elf) error{Overflow}!bool { - for (try self.relocs(elf_file)) |rel| { +pub fn scanRelocsRequiresCode(self: Atom, elf_file: *Elf) bool { + for (self.relocs(elf_file)) |rel| { if (rel.r_type() == elf.R_X86_64_GOTTPOFF) return true; } return false; } pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype) !void { + const is_static = elf_file.isStatic(); const is_dyn_lib = elf_file.isDynLib(); const file_ptr = self.file(elf_file).?; - const rels = try self.relocs(elf_file); + const rels = self.relocs(elf_file); var i: usize = 0; while (i < rels.len) : (i += 1) { const rel = rels[i]; @@ -335,17 +352,24 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype // Report an undefined symbol. try self.reportUndefined(elf_file, symbol, symbol_index, rel, undefs); + if (symbol.isIFunc(elf_file)) { + symbol.flags.needs_got = true; + symbol.flags.needs_plt = true; + } + // While traversing relocations, mark symbols that require special handling such as // pointer indirection via GOT, or a stub trampoline via PLT. switch (rel.r_type()) { - elf.R_X86_64_64 => {}, + elf.R_X86_64_64 => { + try self.scanReloc(symbol, rel, dynAbsRelocAction(symbol, elf_file), elf_file); + }, elf.R_X86_64_32, elf.R_X86_64_32S, - => {}, + => { + try self.scanReloc(symbol, rel, dynAbsRelocAction(symbol, elf_file), elf_file); + }, - elf.R_X86_64_GOT32, - elf.R_X86_64_GOT64, elf.R_X86_64_GOTPC32, elf.R_X86_64_GOTPC64, elf.R_X86_64_GOTPCREL, @@ -364,23 +388,14 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype } }, - elf.R_X86_64_PC32 => {}, - - elf.R_X86_64_TPOFF32, - elf.R_X86_64_TPOFF64, - => { - if (is_dyn_lib) { - // TODO - // self.picError(symbol, rel, elf_file); - } + elf.R_X86_64_PC32 => { + try self.scanReloc(symbol, rel, pcRelocAction(symbol, elf_file), elf_file); }, elf.R_X86_64_TLSGD => { // TODO verify followed by appropriate relocation such as PLT32 __tls_get_addr - if (elf_file.isStatic() or - (!symbol.flags.import and !is_dyn_lib)) - { + if (is_static or (!symbol.flags.import and !is_dyn_lib)) { // Relax if building with -static flag as __tls_get_addr() will not be present in libc.a // We skip the next relocation. i += 1; @@ -392,9 +407,21 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype } }, + elf.R_X86_64_TLSLD => { + // TODO verify followed by appropriate relocation such as PLT32 __tls_get_addr + + if (is_static or !is_dyn_lib) { + // Relax if building with -static flag as __tls_get_addr() will not be present in libc.a + // We skip the next relocation. + i += 1; + } else { + elf_file.got.flags.needs_tlsld = true; + } + }, + elf.R_X86_64_GOTTPOFF => { const should_relax = blk: { - // if (!elf_file.options.relax or is_shared or symbol.flags.import) break :blk false; + if (is_dyn_lib or symbol.flags.import) break :blk false; if (!x86_64.canRelaxGotTpOff(code.?[r_offset - 3 ..])) break :blk false; break :blk true; }; @@ -403,21 +430,255 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype } }, - else => { - var err = try elf_file.addErrorWithNotes(1); - try err.addMsg(elf_file, "fatal linker error: unhandled relocation type {}", .{ - fmtRelocType(rel.r_type()), - }); - try err.addNote(elf_file, "in {}:{s} at offset 0x{x}", .{ - self.file(elf_file).?.fmtPath(), - self.name(elf_file), - r_offset, - }); + elf.R_X86_64_GOTPC32_TLSDESC => { + const should_relax = is_static or (!is_dyn_lib and !symbol.flags.import); + if (!should_relax) { + symbol.flags.needs_tlsdesc = true; + } + }, + + elf.R_X86_64_TPOFF32, + elf.R_X86_64_TPOFF64, + => { + if (is_dyn_lib) try self.reportPicError(symbol, rel, elf_file); }, + + elf.R_X86_64_GOTOFF64, + elf.R_X86_64_DTPOFF32, + elf.R_X86_64_DTPOFF64, + elf.R_X86_64_SIZE32, + elf.R_X86_64_SIZE64, + elf.R_X86_64_TLSDESC_CALL, + => {}, + + // Zig custom relocations + Elf.R_X86_64_ZIG_GOT32, + Elf.R_X86_64_ZIG_GOTPCREL, + => { + assert(symbol.flags.has_zig_got); + }, + + else => try self.reportUnhandledRelocError(rel, elf_file), } } } +fn scanReloc( + self: Atom, + symbol: *Symbol, + rel: elf.Elf64_Rela, + action: RelocAction, + elf_file: *Elf, +) error{OutOfMemory}!void { + const is_writeable = self.inputShdr(elf_file).sh_flags & elf.SHF_WRITE != 0; + const num_dynrelocs = switch (self.file(elf_file).?) { + .linker_defined => unreachable, + .shared_object => unreachable, + inline else => |x| &x.num_dynrelocs, + }; + + switch (action) { + .none => {}, + + .@"error" => if (symbol.isAbs(elf_file)) + try self.reportNoPicError(symbol, rel, elf_file) + else + try self.reportPicError(symbol, rel, elf_file), + + .copyrel => { + if (elf_file.base.options.z_nocopyreloc) { + if (symbol.isAbs(elf_file)) + try self.reportNoPicError(symbol, rel, elf_file) + else + try self.reportPicError(symbol, rel, elf_file); + } + symbol.flags.needs_copy_rel = true; + }, + + .dyn_copyrel => { + if (is_writeable or elf_file.base.options.z_nocopyreloc) { + if (!is_writeable) { + if (elf_file.base.options.z_notext) { + elf_file.has_text_reloc = true; + } else { + try self.reportTextRelocError(symbol, rel, elf_file); + } + } + num_dynrelocs.* += 1; + } else { + symbol.flags.needs_copy_rel = true; + } + }, + + .plt => { + symbol.flags.needs_plt = true; + }, + + .cplt => { + symbol.flags.needs_plt = true; + symbol.flags.is_canonical = true; + }, + + .dyn_cplt => { + if (is_writeable) { + num_dynrelocs.* += 1; + } else { + symbol.flags.needs_plt = true; + symbol.flags.is_canonical = true; + } + }, + + .dynrel, .baserel, .ifunc => { + if (!is_writeable) { + if (elf_file.base.options.z_notext) { + elf_file.has_text_reloc = true; + } else { + try self.reportTextRelocError(symbol, rel, elf_file); + } + } + num_dynrelocs.* += 1; + + if (action == .ifunc) elf_file.num_ifunc_dynrelocs += 1; + }, + } +} + +const RelocAction = enum { + none, + @"error", + copyrel, + dyn_copyrel, + plt, + dyn_cplt, + cplt, + dynrel, + baserel, + ifunc, +}; + +fn pcRelocAction(symbol: *const Symbol, elf_file: *Elf) RelocAction { + // zig fmt: off + const table: [3][4]RelocAction = .{ + // Abs Local Import data Import func + .{ .@"error", .none, .@"error", .plt }, // Shared object + .{ .@"error", .none, .copyrel, .plt }, // PIE + .{ .none, .none, .copyrel, .cplt }, // Non-PIE + }; + // zig fmt: on + const output = outputType(elf_file); + const data = dataType(symbol, elf_file); + return table[output][data]; +} + +fn absRelocAction(symbol: *const Symbol, elf_file: *Elf) RelocAction { + // zig fmt: off + const table: [3][4]RelocAction = .{ + // Abs Local Import data Import func + .{ .none, .@"error", .@"error", .@"error" }, // Shared object + .{ .none, .@"error", .@"error", .@"error" }, // PIE + .{ .none, .none, .copyrel, .cplt }, // Non-PIE + }; + // zig fmt: on + const output = outputType(elf_file); + const data = dataType(symbol, elf_file); + return table[output][data]; +} + +fn dynAbsRelocAction(symbol: *const Symbol, elf_file: *Elf) RelocAction { + if (symbol.isIFunc(elf_file)) return .ifunc; + // zig fmt: off + const table: [3][4]RelocAction = .{ + // Abs Local Import data Import func + .{ .none, .baserel, .dynrel, .dynrel }, // Shared object + .{ .none, .baserel, .dynrel, .dynrel }, // PIE + .{ .none, .none, .dyn_copyrel, .dyn_cplt }, // Non-PIE + }; + // zig fmt: on + const output = outputType(elf_file); + const data = dataType(symbol, elf_file); + return table[output][data]; +} + +fn outputType(elf_file: *Elf) u2 { + return switch (elf_file.base.options.output_mode) { + .Obj => unreachable, + .Lib => 0, + .Exe => if (elf_file.base.options.pie) 1 else 2, + }; +} + +fn dataType(symbol: *const Symbol, elf_file: *Elf) u2 { + if (symbol.isAbs(elf_file)) return 0; + if (!symbol.flags.import) return 1; + if (symbol.type(elf_file) != elf.STT_FUNC) return 2; + return 3; +} + +fn reportUnhandledRelocError(self: Atom, rel: elf.Elf64_Rela, elf_file: *Elf) error{OutOfMemory}!void { + var err = try elf_file.addErrorWithNotes(1); + try err.addMsg(elf_file, "fatal linker error: unhandled relocation type {} at offset 0x{x}", .{ + fmtRelocType(rel.r_type()), + rel.r_offset, + }); + try err.addNote(elf_file, "in {}:{s}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + }); +} + +fn reportTextRelocError( + self: Atom, + symbol: *const Symbol, + rel: elf.Elf64_Rela, + elf_file: *Elf, +) error{OutOfMemory}!void { + var err = try elf_file.addErrorWithNotes(1); + try err.addMsg(elf_file, "relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + rel.r_offset, + symbol.name(elf_file), + }); + try err.addNote(elf_file, "in {}:{s}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + }); +} + +fn reportPicError( + self: Atom, + symbol: *const Symbol, + rel: elf.Elf64_Rela, + elf_file: *Elf, +) error{OutOfMemory}!void { + var err = try elf_file.addErrorWithNotes(2); + try err.addMsg(elf_file, "relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + rel.r_offset, + symbol.name(elf_file), + }); + try err.addNote(elf_file, "in {}:{s}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + }); + try err.addNote(elf_file, "recompile with -fPIC", .{}); +} + +fn reportNoPicError( + self: Atom, + symbol: *const Symbol, + rel: elf.Elf64_Rela, + elf_file: *Elf, +) error{OutOfMemory}!void { + var err = try elf_file.addErrorWithNotes(2); + try err.addMsg(elf_file, "relocation at offset 0x{x} against symbol '{s}' cannot be used", .{ + rel.r_offset, + symbol.name(elf_file), + }); + try err.addNote(elf_file, "in {}:{s}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + }); + try err.addNote(elf_file, "recompile with -fno-PIC", .{}); +} + // This function will report any undefined non-weak symbols that are not imports. fn reportUndefined( self: Atom, @@ -447,15 +708,14 @@ fn reportUndefined( } } -/// TODO mark relocs dirty -pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void { +pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) !void { relocs_log.debug("0x{x}: {s}", .{ self.value, self.name(elf_file) }); const file_ptr = self.file(elf_file).?; var stream = std.io.fixedBufferStream(code); const cwriter = stream.writer(); - const rels = try self.relocs(elf_file); + const rels = self.relocs(elf_file); var i: usize = 0; while (i < rels.len) : (i += 1) { const rel = rels[i]; @@ -488,19 +748,22 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void { null; break :blk if (shndx) |index| @as(i64, @intCast(elf_file.shdrs.items[index].sh_addr)) else 0; }; + // Address of the .zig.got table entry if any. + const ZIG_GOT = @as(i64, @intCast(target.zigGotAddress(elf_file))); // Relative offset to the start of the global offset table. const G = @as(i64, @intCast(target.gotAddress(elf_file))) - GOT; // // Address of the thread pointer. const TP = @as(i64, @intCast(elf_file.tpAddress())); - // // Address of the dynamic thread pointer. - // const DTP = @as(i64, @intCast(elf_file.dtpAddress())); + // Address of the dynamic thread pointer. + const DTP = @as(i64, @intCast(elf_file.dtpAddress())); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ fmtRelocType(r_type), r_offset, P, S + A, G + GOT + A, + ZIG_GOT + A, target.name(elf_file), }); @@ -509,18 +772,20 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void { switch (rel.r_type()) { elf.R_X86_64_NONE => unreachable, - elf.R_X86_64_64 => try cwriter.writeIntLittle(i64, S + A), - - elf.R_X86_64_32 => try cwriter.writeIntLittle(u32, @as(u32, @truncate(@as(u64, @intCast(S + A))))), - elf.R_X86_64_32S => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A))), + elf.R_X86_64_64 => { + try self.resolveDynAbsReloc( + target, + rel, + dynAbsRelocAction(target, elf_file), + elf_file, + cwriter, + ); + }, elf.R_X86_64_PLT32, elf.R_X86_64_PC32, => try cwriter.writeIntLittle(i32, @as(i32, @intCast(S + A - P))), - elf.R_X86_64_GOT32 => try cwriter.writeIntLittle(u32, @as(u32, @intCast(G + GOT + A))), - elf.R_X86_64_GOT64 => try cwriter.writeIntLittle(u64, @as(u64, @intCast(G + GOT + A))), - elf.R_X86_64_GOTPCREL => try cwriter.writeIntLittle(i32, @as(i32, @intCast(G + GOT + A - P))), elf.R_X86_64_GOTPC32 => try cwriter.writeIntLittle(i32, @as(i32, @intCast(GOT + A - P))), elf.R_X86_64_GOTPC64 => try cwriter.writeIntLittle(i64, GOT + A - P), @@ -543,18 +808,22 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void { try cwriter.writeIntLittle(i32, @as(i32, @intCast(G + GOT + A - P))); }, + elf.R_X86_64_32 => try cwriter.writeIntLittle(u32, @as(u32, @truncate(@as(u64, @intCast(S + A))))), + elf.R_X86_64_32S => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A))), + elf.R_X86_64_TPOFF32 => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A - TP))), elf.R_X86_64_TPOFF64 => try cwriter.writeIntLittle(i64, S + A - TP), + elf.R_X86_64_DTPOFF32 => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A - DTP))), + elf.R_X86_64_DTPOFF64 => try cwriter.writeIntLittle(i64, S + A - DTP), + elf.R_X86_64_TLSGD => { if (target.flags.has_tlsgd) { - // TODO - // const S_ = @as(i64, @intCast(target.tlsGdAddress(elf_file))); - // try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); + const S_ = @as(i64, @intCast(target.tlsGdAddress(elf_file))); + try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); } else if (target.flags.has_gottp) { - // TODO - // const S_ = @as(i64, @intCast(target.getGotTpAddress(elf_file))); - // try relaxTlsGdToIe(relocs[i .. i + 2], @intCast(S_ - P), elf_file, &stream); + const S_ = @as(i64, @intCast(target.gotTpAddress(elf_file))); + try x86_64.relaxTlsGdToIe(self, rels[i .. i + 2], @intCast(S_ - P), elf_file, &stream); i += 1; } else { try x86_64.relaxTlsGdToLe( @@ -568,22 +837,245 @@ pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void { } }, + elf.R_X86_64_TLSLD => { + if (elf_file.got.tlsld_index) |entry_index| { + const tlsld_entry = elf_file.got.entries.items[entry_index]; + const S_ = @as(i64, @intCast(tlsld_entry.address(elf_file))); + try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); + } else { + try x86_64.relaxTlsLdToLe( + self, + rels[i .. i + 2], + @as(i32, @intCast(TP - @as(i64, @intCast(elf_file.tlsAddress())))), + elf_file, + &stream, + ); + i += 1; + } + }, + + elf.R_X86_64_GOTPC32_TLSDESC => { + if (target.flags.has_tlsdesc) { + const S_ = @as(i64, @intCast(target.tlsDescAddress(elf_file))); + try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); + } else { + try x86_64.relaxGotPcTlsDesc(code[r_offset - 3 ..]); + try cwriter.writeIntLittle(i32, @as(i32, @intCast(S - TP))); + } + }, + + elf.R_X86_64_TLSDESC_CALL => if (!target.flags.has_tlsdesc) { + // call -> nop + try cwriter.writeAll(&.{ 0x66, 0x90 }); + }, + elf.R_X86_64_GOTTPOFF => { if (target.flags.has_gottp) { - // TODO - // const S_ = @as(i64, @intCast(target.gotTpAddress(elf_file))); - // try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); + const S_ = @as(i64, @intCast(target.gotTpAddress(elf_file))); + try cwriter.writeIntLittle(i32, @as(i32, @intCast(S_ + A - P))); } else { x86_64.relaxGotTpOff(code[r_offset - 3 ..]) catch unreachable; try cwriter.writeIntLittle(i32, @as(i32, @intCast(S - TP))); } }, + // Zig custom relocations + Elf.R_X86_64_ZIG_GOT32 => try cwriter.writeIntLittle(u32, @as(u32, @intCast(ZIG_GOT + A))), + Elf.R_X86_64_ZIG_GOTPCREL => try cwriter.writeIntLittle(i32, @as(i32, @intCast(ZIG_GOT + A - P))), + else => {}, } } } +fn resolveDynAbsReloc( + self: Atom, + target: *const Symbol, + rel: elf.Elf64_Rela, + action: RelocAction, + elf_file: *Elf, + writer: anytype, +) !void { + const P = self.value + rel.r_offset; + const A = rel.r_addend; + const S = @as(i64, @intCast(target.address(.{}, elf_file))); + const is_writeable = self.inputShdr(elf_file).sh_flags & elf.SHF_WRITE != 0; + + const num_dynrelocs = switch (self.file(elf_file).?) { + .linker_defined => unreachable, + .shared_object => unreachable, + inline else => |x| x.num_dynrelocs, + }; + try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, num_dynrelocs); + + switch (action) { + .@"error", + .plt, + => unreachable, + + .copyrel, + .cplt, + .none, + => try writer.writeIntLittle(i32, @as(i32, @truncate(S + A))), + + .dyn_copyrel => { + if (is_writeable or elf_file.base.options.z_nocopyreloc) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = P, + .sym = target.extra(elf_file).?.dynamic, + .type = elf.R_X86_64_64, + .addend = A, + }); + try applyDynamicReloc(A, elf_file, writer); + } else { + try writer.writeIntLittle(i32, @as(i32, @truncate(S + A))); + } + }, + + .dyn_cplt => { + if (is_writeable) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = P, + .sym = target.extra(elf_file).?.dynamic, + .type = elf.R_X86_64_64, + .addend = A, + }); + try applyDynamicReloc(A, elf_file, writer); + } else { + try writer.writeIntLittle(i32, @as(i32, @truncate(S + A))); + } + }, + + .dynrel => { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = P, + .sym = target.extra(elf_file).?.dynamic, + .type = elf.R_X86_64_64, + .addend = A, + }); + try applyDynamicReloc(A, elf_file, writer); + }, + + .baserel => { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = P, + .type = elf.R_X86_64_RELATIVE, + .addend = S + A, + }); + try applyDynamicReloc(S + A, elf_file, writer); + }, + + .ifunc => { + const S_ = @as(i64, @intCast(target.address(.{ .plt = false }, elf_file))); + elf_file.addRelaDynAssumeCapacity(.{ + .offset = P, + .type = elf.R_X86_64_IRELATIVE, + .addend = S_ + A, + }); + try applyDynamicReloc(S_ + A, elf_file, writer); + }, + } +} + +fn applyDynamicReloc(value: i64, elf_file: *Elf, writer: anytype) !void { + _ = elf_file; + // if (elf_file.options.apply_dynamic_relocs) { + try writer.writeIntLittle(i64, value); + // } +} + +pub fn resolveRelocsNonAlloc(self: Atom, elf_file: *Elf, code: []u8, undefs: anytype) !void { + relocs_log.debug("0x{x}: {s}", .{ self.value, self.name(elf_file) }); + + const file_ptr = self.file(elf_file).?; + var stream = std.io.fixedBufferStream(code); + const cwriter = stream.writer(); + + const rels = self.relocs(elf_file); + var i: usize = 0; + while (i < rels.len) : (i += 1) { + const rel = rels[i]; + const r_type = rel.r_type(); + if (r_type == elf.R_X86_64_NONE) continue; + + const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; + + const target_index = switch (file_ptr) { + .zig_module => |x| x.symbol(rel.r_sym()), + .object => |x| x.symbols.items[rel.r_sym()], + else => unreachable, + }; + const target = elf_file.symbol(target_index); + + // Check for violation of One Definition Rule for COMDATs. + if (target.file(elf_file) == null) { + // TODO convert into an error + log.debug("{}: {s}: {s} refers to a discarded COMDAT section", .{ + file_ptr.fmtPath(), + self.name(elf_file), + target.name(elf_file), + }); + continue; + } + + // Report an undefined symbol. + try self.reportUndefined(elf_file, target, target_index, rel, undefs); + + // We will use equation format to resolve relocations: + // https://intezer.com/blog/malware-analysis/executable-and-linkable-format-101-part-3-relocations/ + // + const P = @as(i64, @intCast(self.value + rel.r_offset)); + // Addend from the relocation. + const A = rel.r_addend; + // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub. + const S = @as(i64, @intCast(target.address(.{}, elf_file))); + // Address of the global offset table. + const GOT = blk: { + const shndx = if (elf_file.got_plt_section_index) |shndx| + shndx + else if (elf_file.got_section_index) |shndx| + shndx + else + null; + break :blk if (shndx) |index| @as(i64, @intCast(elf_file.shdrs.items[index].sh_addr)) else 0; + }; + // Address of the dynamic thread pointer. + const DTP = @as(i64, @intCast(elf_file.dtpAddress())); + + relocs_log.debug(" {s}: {x}: [{x} => {x}] ({s})", .{ + fmtRelocType(r_type), + rel.r_offset, + P, + S + A, + target.name(elf_file), + }); + + try stream.seekTo(r_offset); + + switch (r_type) { + elf.R_X86_64_NONE => unreachable, + elf.R_X86_64_8 => try cwriter.writeIntLittle(u8, @as(u8, @bitCast(@as(i8, @intCast(S + A))))), + elf.R_X86_64_16 => try cwriter.writeIntLittle(u16, @as(u16, @bitCast(@as(i16, @intCast(S + A))))), + elf.R_X86_64_32 => try cwriter.writeIntLittle(u32, @as(u32, @bitCast(@as(i32, @intCast(S + A))))), + elf.R_X86_64_32S => try cwriter.writeIntLittle(i32, @as(i32, @intCast(S + A))), + elf.R_X86_64_64 => try cwriter.writeIntLittle(i64, S + A), + elf.R_X86_64_DTPOFF32 => try cwriter.writeIntLittle(i32, @as(i32, @intCast(S + A - DTP))), + elf.R_X86_64_DTPOFF64 => try cwriter.writeIntLittle(i64, S + A - DTP), + elf.R_X86_64_GOTOFF64 => try cwriter.writeIntLittle(i64, S + A - GOT), + elf.R_X86_64_GOTPC64 => try cwriter.writeIntLittle(i64, GOT + A), + elf.R_X86_64_SIZE32 => { + const size = @as(i64, @intCast(target.elfSym(elf_file).st_size)); + try cwriter.writeIntLittle(u32, @as(u32, @bitCast(@as(i32, @intCast(size + A))))); + }, + elf.R_X86_64_SIZE64 => { + const size = @as(i64, @intCast(target.elfSym(elf_file).st_size)); + try cwriter.writeIntLittle(i64, @as(i64, @intCast(size + A))); + }, + else => try self.reportUnhandledRelocError(rel, elf_file), + } + } +} + pub fn fmtRelocType(r_type: u32) std.fmt.Formatter(formatRelocType) { return .{ .data = r_type }; } @@ -639,6 +1131,9 @@ fn formatRelocType( elf.R_X86_64_GOTPCRELX => "R_X86_64_GOTPCRELX", elf.R_X86_64_REX_GOTPCRELX => "R_X86_64_REX_GOTPCRELX", elf.R_X86_64_NUM => "R_X86_64_NUM", + // Zig custom relocations + Elf.R_X86_64_ZIG_GOT32 => "R_X86_64_ZIG_GOT32", + Elf.R_X86_64_ZIG_GOTPCREL => "R_X86_64_ZIG_GOTPCREL", else => "R_X86_64_UNKNOWN", }; try writer.print("{s}", .{str}); @@ -679,39 +1174,32 @@ fn format2( _ = unused_fmt_string; const atom = ctx.atom; const elf_file = ctx.elf_file; - try writer.print("atom({d}) : {s} : @{x} : sect({d}) : align({x}) : size({x})", .{ + try writer.print("atom({d}) : {s} : @{x} : shdr({d}) : align({x}) : size({x})", .{ atom.atom_index, atom.name(elf_file), atom.value, atom.output_section_index, atom.alignment, atom.size, }); - // if (atom.fde_start != atom.fde_end) { - // try writer.writeAll(" : fdes{ "); - // for (atom.getFdes(elf_file), atom.fde_start..) |fde, i| { - // try writer.print("{d}", .{i}); - // if (!fde.alive) try writer.writeAll("([*])"); - // if (i < atom.fde_end - 1) try writer.writeAll(", "); - // } - // try writer.writeAll(" }"); - // } - const gc_sections = if (elf_file.base.options.gc_sections) |gc_sections| gc_sections else false; - if (gc_sections and !atom.flags.alive) { + if (atom.fde_start != atom.fde_end) { + try writer.writeAll(" : fdes{ "); + for (atom.fdes(elf_file), atom.fde_start..) |fde, i| { + try writer.print("{d}", .{i}); + if (!fde.alive) try writer.writeAll("([*])"); + if (i < atom.fde_end - 1) try writer.writeAll(", "); + } + try writer.writeAll(" }"); + } + if (!atom.flags.alive) { try writer.writeAll(" : [*]"); } } -// TODO this has to be u32 but for now, to avoid redesigning elfSym machinery for -// ZigModule, keep it at u16 with the intention of bumping it to u32 in the near -// future. -pub const Index = u16; +pub const Index = u32; pub const Flags = packed struct { /// Specifies whether this atom is alive or has been garbage collected. - alive: bool = false, + alive: bool = true, /// Specifies if the atom has been visited during garbage collection. visited: bool = false, - - /// Specifies whether this atom has been allocated in the output section. - allocated: bool = false, }; const x86_64 = struct { @@ -745,6 +1233,95 @@ const x86_64 = struct { } } + pub fn relaxTlsGdToIe( + self: Atom, + rels: []align(1) const elf.Elf64_Rela, + value: i32, + elf_file: *Elf, + stream: anytype, + ) !void { + assert(rels.len == 2); + const writer = stream.writer(); + switch (rels[1].r_type()) { + elf.R_X86_64_PC32, + elf.R_X86_64_PLT32, + => { + var insts = [_]u8{ + 0x64, 0x48, 0x8b, 0x04, 0x25, 0, 0, 0, 0, // movq %fs:0,%rax + 0x48, 0x03, 0x05, 0, 0, 0, 0, // add foo@gottpoff(%rip), %rax + }; + std.mem.writeIntLittle(i32, insts[12..][0..4], value - 12); + try stream.seekBy(-4); + try writer.writeAll(&insts); + }, + + else => { + var err = try elf_file.addErrorWithNotes(1); + try err.addMsg(elf_file, "fatal linker error: rewrite {} when followed by {}", .{ + fmtRelocType(rels[0].r_type()), + fmtRelocType(rels[1].r_type()), + }); + try err.addNote(elf_file, "in {}:{s} at offset 0x{x}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + rels[0].r_offset, + }); + }, + } + } + + pub fn relaxTlsLdToLe( + self: Atom, + rels: []align(1) const elf.Elf64_Rela, + value: i32, + elf_file: *Elf, + stream: anytype, + ) !void { + assert(rels.len == 2); + const writer = stream.writer(); + switch (rels[1].r_type()) { + elf.R_X86_64_PC32, + elf.R_X86_64_PLT32, + => { + var insts = [_]u8{ + 0x31, 0xc0, // xor %eax, %eax + 0x64, 0x48, 0x8b, 0, // mov %fs:(%rax), %rax + 0x48, 0x2d, 0, 0, 0, 0, // sub $tls_size, %rax + }; + std.mem.writeIntLittle(i32, insts[8..][0..4], value); + try stream.seekBy(-3); + try writer.writeAll(&insts); + }, + + elf.R_X86_64_GOTPCREL, + elf.R_X86_64_GOTPCRELX, + => { + var insts = [_]u8{ + 0x31, 0xc0, // xor %eax, %eax + 0x64, 0x48, 0x8b, 0, // mov %fs:(%rax), %rax + 0x48, 0x2d, 0, 0, 0, 0, // sub $tls_size, %rax + 0x90, // nop + }; + std.mem.writeIntLittle(i32, insts[8..][0..4], value); + try stream.seekBy(-3); + try writer.writeAll(&insts); + }, + + else => { + var err = try elf_file.addErrorWithNotes(1); + try err.addMsg(elf_file, "fatal linker error: rewrite {} when followed by {}", .{ + fmtRelocType(rels[0].r_type()), + fmtRelocType(rels[1].r_type()), + }); + try err.addNote(elf_file, "in {}:{s} at offset 0x{x}", .{ + self.file(elf_file).?.fmtPath(), + self.name(elf_file), + rels[0].r_offset, + }); + }, + } + } + pub fn canRelaxGotTpOff(code: []const u8) bool { const old_inst = disassemble(code) orelse return false; switch (old_inst.encoding.mnemonic) { @@ -776,6 +1353,22 @@ const x86_64 = struct { } } + pub fn relaxGotPcTlsDesc(code: []u8) !void { + const old_inst = disassemble(code) orelse return error.RelaxFail; + switch (old_inst.encoding.mnemonic) { + .lea => { + const inst = try Instruction.new(old_inst.prefix, .mov, &.{ + old_inst.ops[0], + // TODO: hack to force imm32s in the assembler + .{ .imm = Immediate.s(-129) }, + }); + relocs_log.debug(" relaxing {} => {}", .{ old_inst.encoding, inst.encoding }); + encode(&.{inst}, code) catch return error.RelaxFail; + }, + else => return error.RelaxFail, + } + } + pub fn relaxTlsGdToLe( self: Atom, rels: []align(1) const elf.Elf64_Rela, @@ -843,11 +1436,14 @@ const x86_64 = struct { const std = @import("std"); const assert = std.debug.assert; const elf = std.elf; +const eh_frame = @import("eh_frame.zig"); const log = std.log.scoped(.link); const relocs_log = std.log.scoped(.link_relocs); const Allocator = std.mem.Allocator; const Atom = @This(); const Elf = @import("../Elf.zig"); +const Fde = eh_frame.Fde; const File = @import("file.zig").File; +const Object = @import("Object.zig"); const Symbol = @import("Symbol.zig"); diff --git a/src/link/Elf/Object.zig b/src/link/Elf/Object.zig index 191f6774a5..4edc5b62b1 100644 --- a/src/link/Elf/Object.zig +++ b/src/link/Elf/Object.zig @@ -4,7 +4,7 @@ data: []const u8, index: File.Index, header: ?elf.Elf64_Ehdr = null, -shdrs: std.ArrayListUnmanaged(elf.Elf64_Shdr) = .{}, +shdrs: std.ArrayListUnmanaged(ElfShdr) = .{}, strings: StringTable(.object_strings) = .{}, symtab: []align(1) const elf.Elf64_Sym = &[0]elf.Elf64_Sym{}, strtab: []const u8 = &[0]u8{}, @@ -59,8 +59,13 @@ pub fn parse(self: *Object, elf_file: *Elf) !void { [*]align(1) const elf.Elf64_Shdr, @ptrCast(self.data.ptr + shoff), )[0..self.header.?.e_shnum]; - try self.shdrs.appendUnalignedSlice(gpa, shdrs); - try self.strings.buffer.appendSlice(gpa, try self.shdrContents(self.header.?.e_shstrndx)); + try self.shdrs.ensureTotalCapacityPrecise(gpa, shdrs.len); + + for (shdrs) |shdr| { + self.shdrs.appendAssumeCapacity(try ElfShdr.fromElf64Shdr(shdr)); + } + + try self.strings.buffer.appendSlice(gpa, self.shdrContents(self.header.?.e_shstrndx)); const symtab_index = for (self.shdrs.items, 0..) |shdr, i| switch (shdr.sh_type) { elf.SHT_SYMTAB => break @as(u16, @intCast(i)), @@ -71,21 +76,21 @@ pub fn parse(self: *Object, elf_file: *Elf) !void { const shdr = shdrs[index]; self.first_global = shdr.sh_info; - const symtab = try self.shdrContents(index); + const symtab = self.shdrContents(index); const nsyms = @divExact(symtab.len, @sizeOf(elf.Elf64_Sym)); self.symtab = @as([*]align(1) const elf.Elf64_Sym, @ptrCast(symtab.ptr))[0..nsyms]; - self.strtab = try self.shdrContents(@as(u16, @intCast(shdr.sh_link))); + self.strtab = self.shdrContents(@as(u16, @intCast(shdr.sh_link))); } try self.initAtoms(elf_file); try self.initSymtab(elf_file); - // for (self.shdrs.items, 0..) |shdr, i| { - // const atom = elf_file.atom(self.atoms.items[i]) orelse continue; - // if (!atom.alive) continue; - // if (shdr.sh_type == elf.SHT_X86_64_UNWIND or mem.eql(u8, atom.name(elf_file), ".eh_frame")) - // try self.parseEhFrame(@as(u16, @intCast(i)), elf_file); - // } + for (self.shdrs.items, 0..) |shdr, i| { + const atom = elf_file.atom(self.atoms.items[i]) orelse continue; + if (!atom.flags.alive) continue; + if (shdr.sh_type == elf.SHT_X86_64_UNWIND or mem.eql(u8, atom.name(elf_file), ".eh_frame")) + try self.parseEhFrame(@as(u16, @intCast(i)), elf_file); + } } fn initAtoms(self: *Object, elf_file: *Elf) !void { @@ -115,7 +120,7 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void { }; const shndx = @as(u16, @intCast(i)); - const group_raw_data = try self.shdrContents(shndx); + const group_raw_data = self.shdrContents(shndx); const group_nmembers = @divExact(group_raw_data.len, @sizeOf(u32)); const group_members = @as([*]align(1) const u32, @ptrCast(group_raw_data.ptr))[0..group_nmembers]; @@ -125,7 +130,10 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void { continue; } - const group_signature_off = try self.strings.insert(elf_file.base.allocator, group_signature); + // Note the assumption about a global strtab used here to disambiguate common + // COMDAT owners. + const gpa = elf_file.base.allocator; + const group_signature_off = try elf_file.strtab.insert(gpa, group_signature); const gop = try elf_file.getOrCreateComdatGroupOwner(group_signature_off); const comdat_group_index = try elf_file.addComdatGroup(); const comdat_group = elf_file.comdatGroup(comdat_group_index); @@ -133,7 +141,7 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void { .owner = gop.index, .shndx = shndx, }; - try self.comdat_groups.append(elf_file.base.allocator, comdat_group_index); + try self.comdat_groups.append(gpa, comdat_group_index); }, elf.SHT_SYMTAB_SHNDX => @panic("TODO SHT_SYMTAB_SHNDX"), @@ -168,23 +176,21 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void { fn addAtom( self: *Object, - shdr: elf.Elf64_Shdr, + shdr: ElfShdr, shndx: u16, name: [:0]const u8, elf_file: *Elf, -) error{ OutOfMemory, Overflow }!void { +) error{OutOfMemory}!void { const atom_index = try elf_file.addAtom(); const atom = elf_file.atom(atom_index).?; atom.atom_index = atom_index; atom.name_offset = try elf_file.strtab.insert(elf_file.base.allocator, name); atom.file_index = self.index; atom.input_section_index = shndx; - atom.output_section_index = try self.getOutputSectionIndex(elf_file, shdr); - atom.flags.alive = true; self.atoms.items[shndx] = atom_index; if (shdr.sh_flags & elf.SHF_COMPRESSED != 0) { - const data = try self.shdrContents(shndx); + const data = self.shdrContents(shndx); const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*; atom.size = chdr.ch_size; atom.alignment = Alignment.fromNonzeroByteUnits(chdr.ch_addralign); @@ -194,10 +200,10 @@ fn addAtom( } } -fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) error{OutOfMemory}!u16 { +fn initOutputSection(self: Object, elf_file: *Elf, shdr: ElfShdr) error{OutOfMemory}!u16 { const name = blk: { const name = self.strings.getAssumeExists(shdr.sh_name); - // if (shdr.sh_flags & elf.SHF_MERGE != 0) break :blk name; + if (shdr.sh_flags & elf.SHF_MERGE != 0) break :blk name; const sh_name_prefixes: []const [:0]const u8 = &.{ ".text", ".data.rel.ro", ".data", ".rodata", ".bss.rel.ro", ".bss", ".init_array", ".fini_array", ".tbss", ".tdata", ".gcc_except_table", ".ctors", @@ -208,8 +214,6 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er break :blk prefix; } } - if (std.mem.eql(u8, name, ".tcommon")) break :blk ".tbss"; - if (std.mem.eql(u8, name, ".common")) break :blk ".bss"; break :blk name; }; const @"type" = switch (shdr.sh_type) { @@ -231,47 +235,23 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) er else => flags, }; }; - const out_shndx = elf_file.sectionByName(name) orelse blk: { - const is_alloc = flags & elf.SHF_ALLOC != 0; - const is_write = flags & elf.SHF_WRITE != 0; - const is_exec = flags & elf.SHF_EXECINSTR != 0; - if (!is_alloc) { - log.err("{}: output section {s} not found", .{ self.fmtPath(), name }); - @panic("TODO: missing output section!"); - } - var phdr_flags: u32 = elf.PF_R; - if (is_write) phdr_flags |= elf.PF_W; - if (is_exec) phdr_flags |= elf.PF_X; - const phdr_index = try elf_file.allocateSegment(.{ - .size = Elf.padToIdeal(shdr.sh_size), - .alignment = elf_file.page_size, - .flags = phdr_flags, - }); - const shndx = try elf_file.allocateAllocSection(.{ - .name = name, - .phdr_index = phdr_index, - .alignment = shdr.sh_addralign, - .flags = flags, - .type = @"type", - }); - try elf_file.last_atom_and_free_list_table.putNoClobber(elf_file.base.allocator, shndx, .{}); - break :blk shndx; - }; + const out_shndx = elf_file.sectionByName(name) orelse try elf_file.addSection(.{ + .type = @"type", + .flags = flags, + .name = name, + }); return out_shndx; } fn skipShdr(self: *Object, index: u16, elf_file: *Elf) bool { - _ = elf_file; const shdr = self.shdrs.items[index]; const name = self.strings.getAssumeExists(shdr.sh_name); const ignore = blk: { if (mem.startsWith(u8, name, ".note")) break :blk true; if (mem.startsWith(u8, name, ".comment")) break :blk true; if (mem.startsWith(u8, name, ".llvm_addrsig")) break :blk true; - if (mem.startsWith(u8, name, ".eh_frame")) break :blk true; - // if (elf_file.base.options.strip and shdr.sh_flags & elf.SHF_ALLOC == 0 and - // mem.startsWith(u8, name, ".debug")) break :blk true; - if (shdr.sh_flags & elf.SHF_ALLOC == 0 and mem.startsWith(u8, name, ".debug")) break :blk true; + if (elf_file.base.options.strip and shdr.sh_flags & elf.SHF_ALLOC == 0 and + mem.startsWith(u8, name, ".debug")) break :blk true; break :blk false; }; return ignore; @@ -300,10 +280,6 @@ fn initSymtab(self: *Object, elf_file: *Elf) !void { sym_ptr.esym_index = @as(u32, @intCast(i)); sym_ptr.atom_index = if (sym.st_shndx == elf.SHN_ABS) 0 else self.atoms.items[sym.st_shndx]; sym_ptr.file_index = self.index; - sym_ptr.output_section_index = if (sym_ptr.atom(elf_file)) |atom_ptr| - atom_ptr.outputShndx().? - else - elf.SHN_UNDEF; } for (self.symtab[first_global..]) |sym| { @@ -324,8 +300,8 @@ fn parseEhFrame(self: *Object, shndx: u16, elf_file: *Elf) !void { }; const gpa = elf_file.base.allocator; - const raw = try self.shdrContents(shndx); - const relocs = try self.getRelocs(relocs_shndx); + const raw = self.shdrContents(shndx); + const relocs = self.getRelocs(relocs_shndx); const fdes_start = self.fdes.items.len; const cies_start = self.cies.items.len; @@ -429,7 +405,7 @@ pub fn scanRelocs(self: *Object, elf_file: *Elf, undefs: anytype) !void { const shdr = atom.inputShdr(elf_file); if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue; if (shdr.sh_type == elf.SHT_NOBITS) continue; - if (try atom.scanRelocsRequiresCode(elf_file)) { + if (atom.scanRelocsRequiresCode(elf_file)) { // TODO ideally, we don't have to decompress at this stage (should already be done) // and we just fetch the code slice. const code = try self.codeDecompressAlloc(elf_file, atom_index); @@ -439,7 +415,7 @@ pub fn scanRelocs(self: *Object, elf_file: *Elf, undefs: anytype) !void { } for (self.cies.items) |cie| { - for (try cie.relocs(elf_file)) |rel| { + for (cie.relocs(elf_file)) |rel| { const sym = elf_file.symbol(self.symbols.items[rel.r_sym()]); if (sym.flags.import) { if (sym.type(elf_file) != elf.STT_FUNC) @@ -474,15 +450,10 @@ pub fn resolveSymbols(self: *Object, elf_file: *Elf) void { elf.SHN_ABS, elf.SHN_COMMON => 0, else => self.atoms.items[esym.st_shndx], }; - const output_section_index = if (elf_file.atom(atom_index)) |atom| - atom.outputShndx().? - else - elf.SHN_UNDEF; global.value = esym.st_value; global.atom_index = atom_index; global.esym_index = esym_index; global.file_index = self.index; - global.output_section_index = output_section_index; global.version_index = elf_file.default_sym_version; if (esym.st_bind() == elf.STB_WEAK) global.flags.weak = true; } @@ -544,6 +515,15 @@ pub fn markLive(self: *Object, elf_file: *Elf) void { } } +pub fn markEhFrameAtomsDead(self: Object, elf_file: *Elf) void { + for (self.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + const is_eh_frame = atom.inputShdr(elf_file).sh_type == elf.SHT_X86_64_UNWIND or + mem.eql(u8, atom.name(elf_file), ".eh_frame"); + if (atom.flags.alive and is_eh_frame) atom.flags.alive = false; + } +} + pub fn checkDuplicates(self: *Object, elf_file: *Elf) void { const first_global = self.first_global orelse return; for (self.globals(), 0..) |index, i| { @@ -581,14 +561,14 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void { if (this_sym.st_shndx != elf.SHN_COMMON) continue; const global = elf_file.symbol(index); - const global_file = global.getFile(elf_file).?; - if (global_file.getIndex() != self.index) { - if (elf_file.options.warn_common) { - elf_file.base.warn("{}: multiple common symbols: {s}", .{ - self.fmtPath(), - global.getName(elf_file), - }); - } + const global_file = global.file(elf_file).?; + if (global_file.index() != self.index) { + // if (elf_file.options.warn_common) { + // elf_file.base.warn("{}: multiple common symbols: {s}", .{ + // self.fmtPath(), + // global.getName(elf_file), + // }); + // } continue; } @@ -597,13 +577,13 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void { const atom_index = try elf_file.addAtom(); try self.atoms.append(gpa, atom_index); - const is_tls = global.getType(elf_file) == elf.STT_TLS; - const name = if (is_tls) ".tbss" else ".bss"; + const is_tls = global.type(elf_file) == elf.STT_TLS; + const name = if (is_tls) ".tls_common" else ".common"; const atom = elf_file.atom(atom_index).?; atom.atom_index = atom_index; - atom.name = try elf_file.strtab.insert(gpa, name); - atom.file = self.index; + atom.name_offset = try elf_file.strtab.insert(gpa, name); + atom.file_index = self.index; atom.size = this_sym.st_size; const alignment = this_sym.st_value; atom.alignment = Alignment.fromNonzeroByteUnits(alignment); @@ -612,26 +592,76 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void { if (is_tls) sh_flags |= elf.SHF_TLS; const shndx = @as(u16, @intCast(self.shdrs.items.len)); const shdr = try self.shdrs.addOne(gpa); + const sh_size = math.cast(usize, this_sym.st_size) orelse return error.Overflow; shdr.* = .{ .sh_name = try self.strings.insert(gpa, name), .sh_type = elf.SHT_NOBITS, .sh_flags = sh_flags, .sh_addr = 0, .sh_offset = 0, - .sh_size = this_sym.st_size, + .sh_size = sh_size, .sh_link = 0, .sh_info = 0, .sh_addralign = alignment, .sh_entsize = 0, }; - atom.shndx = shndx; + atom.input_section_index = shndx; global.value = 0; - global.atom = atom_index; + global.atom_index = atom_index; global.flags.weak = false; } } +pub fn initOutputSections(self: Object, elf_file: *Elf) !void { + for (self.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (!atom.flags.alive) continue; + const shdr = atom.inputShdr(elf_file); + _ = try self.initOutputSection(elf_file, shdr); + } +} + +pub fn addAtomsToOutputSections(self: *Object, elf_file: *Elf) !void { + for (self.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (!atom.flags.alive) continue; + const shdr = atom.inputShdr(elf_file); + atom.output_section_index = self.initOutputSection(elf_file, shdr) catch unreachable; + + const gpa = elf_file.base.allocator; + const gop = try elf_file.output_sections.getOrPut(gpa, atom.output_section_index); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.append(gpa, atom_index); + } +} + +pub fn allocateAtoms(self: Object, elf_file: *Elf) void { + for (self.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (!atom.flags.alive) continue; + const shdr = elf_file.shdrs.items[atom.output_section_index]; + atom.value += shdr.sh_addr; + } + + for (self.locals()) |local_index| { + const local = elf_file.symbol(local_index); + const atom = local.atom(elf_file) orelse continue; + if (!atom.flags.alive) continue; + local.value += atom.value; + local.output_section_index = atom.output_section_index; + } + + for (self.globals()) |global_index| { + const global = elf_file.symbol(global_index); + const atom = global.atom(elf_file) orelse continue; + if (!atom.flags.alive) continue; + if (global.file(elf_file).?.index() != self.index) continue; + global.value += atom.value; + global.output_section_index = atom.output_section_index; + } +} + pub fn updateSymtabSize(self: *Object, elf_file: *Elf) void { for (self.locals()) |local_index| { const local = elf_file.symbol(local_index); @@ -682,22 +712,20 @@ pub fn writeSymtab(self: *Object, elf_file: *Elf, ctx: anytype) void { } } -pub fn locals(self: *Object) []const Symbol.Index { +pub fn locals(self: Object) []const Symbol.Index { const end = self.first_global orelse self.symbols.items.len; return self.symbols.items[0..end]; } -pub fn globals(self: *Object) []const Symbol.Index { +pub fn globals(self: Object) []const Symbol.Index { const start = self.first_global orelse self.symbols.items.len; return self.symbols.items[start..]; } -fn shdrContents(self: Object, index: u32) error{Overflow}![]const u8 { +pub fn shdrContents(self: Object, index: u32) []const u8 { assert(index < self.shdrs.items.len); const shdr = self.shdrs.items[index]; - const offset = math.cast(usize, shdr.sh_offset) orelse return error.Overflow; - const size = math.cast(usize, shdr.sh_size) orelse return error.Overflow; - return self.data[offset..][0..size]; + return self.data[shdr.sh_offset..][0..shdr.sh_size]; } /// Returns atom's code and optionally uncompresses data if required (for compressed sections). @@ -706,7 +734,7 @@ pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index) const gpa = elf_file.base.allocator; const atom_ptr = elf_file.atom(atom_index).?; assert(atom_ptr.file_index == self.index); - const data = try self.shdrContents(atom_ptr.input_section_index); + const data = self.shdrContents(atom_ptr.input_section_index); const shdr = atom_ptr.inputShdr(elf_file); if (shdr.sh_flags & elf.SHF_COMPRESSED != 0) { const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*; @@ -734,8 +762,8 @@ fn getString(self: *Object, off: u32) [:0]const u8 { return mem.sliceTo(@as([*:0]const u8, @ptrCast(self.strtab.ptr + off)), 0); } -pub fn comdatGroupMembers(self: *Object, index: u16) error{Overflow}![]align(1) const u32 { - const raw = try self.shdrContents(index); +pub fn comdatGroupMembers(self: *Object, index: u16) []align(1) const u32 { + const raw = self.shdrContents(index); const nmembers = @divExact(raw.len, @sizeOf(u32)); const members = @as([*]align(1) const u32, @ptrCast(raw.ptr))[1..nmembers]; return members; @@ -745,8 +773,8 @@ pub fn asFile(self: *Object) File { return .{ .object = self }; } -pub fn getRelocs(self: *Object, shndx: u32) error{Overflow}![]align(1) const elf.Elf64_Rela { - const raw = try self.shdrContents(shndx); +pub fn getRelocs(self: *Object, shndx: u32) []align(1) const elf.Elf64_Rela { + const raw = self.shdrContents(shndx); const num = @divExact(raw.len, @sizeOf(elf.Elf64_Rela)); return @as([*]align(1) const elf.Elf64_Rela, @ptrCast(raw.ptr))[0..num]; } @@ -886,7 +914,7 @@ fn formatComdatGroups( const cg = elf_file.comdatGroup(cg_index); const cg_owner = elf_file.comdatGroupOwner(cg.owner); if (cg_owner.file != object.index) continue; - const cg_members = object.comdatGroupMembers(cg.shndx) catch continue; + const cg_members = object.comdatGroupMembers(cg.shndx); for (cg_members) |shndx| { const atom_index = object.atoms.items[shndx]; const atom = elf_file.atom(atom_index) orelse continue; @@ -915,6 +943,34 @@ fn formatPath( } else try writer.writeAll(object.path); } +pub const ElfShdr = struct { + sh_name: u32, + sh_type: u32, + sh_flags: u64, + sh_addr: u64, + sh_offset: usize, + sh_size: usize, + sh_link: u32, + sh_info: u32, + sh_addralign: u64, + sh_entsize: u64, + + pub fn fromElf64Shdr(shdr: elf.Elf64_Shdr) error{Overflow}!ElfShdr { + return .{ + .sh_name = shdr.sh_name, + .sh_type = shdr.sh_type, + .sh_flags = shdr.sh_flags, + .sh_addr = shdr.sh_addr, + .sh_offset = math.cast(usize, shdr.sh_offset) orelse return error.Overflow, + .sh_size = math.cast(usize, shdr.sh_size) orelse return error.Overflow, + .sh_link = shdr.sh_link, + .sh_info = shdr.sh_info, + .sh_addralign = shdr.sh_addralign, + .sh_entsize = shdr.sh_entsize, + }; + } +}; + const Object = @This(); const std = @import("std"); diff --git a/src/link/Elf/SharedObject.zig b/src/link/Elf/SharedObject.zig new file mode 100644 index 0000000000..19e688b58d --- /dev/null +++ b/src/link/Elf/SharedObject.zig @@ -0,0 +1,363 @@ +path: []const u8, +data: []const u8, +index: File.Index, + +header: ?elf.Elf64_Ehdr = null, +shdrs: std.ArrayListUnmanaged(ElfShdr) = .{}, +symtab: []align(1) const elf.Elf64_Sym = &[0]elf.Elf64_Sym{}, +strtab: []const u8 = &[0]u8{}, +/// Version symtab contains version strings of the symbols if present. +versyms: std.ArrayListUnmanaged(elf.Elf64_Versym) = .{}, +verstrings: std.ArrayListUnmanaged(u32) = .{}, + +dynamic_sect_index: ?u16 = null, +versym_sect_index: ?u16 = null, +verdef_sect_index: ?u16 = null, + +symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, +aliases: ?std.ArrayListUnmanaged(u32) = null, + +needed: bool, +alive: bool, + +output_symtab_size: Elf.SymtabSize = .{}, + +pub fn isSharedObject(file: std.fs.File) bool { + const reader = file.reader(); + const header = reader.readStruct(elf.Elf64_Ehdr) catch return false; + defer file.seekTo(0) catch {}; + if (!mem.eql(u8, header.e_ident[0..4], "\x7fELF")) return false; + if (header.e_ident[elf.EI_VERSION] != 1) return false; + if (header.e_type != elf.ET.DYN) return false; + return true; +} + +pub fn deinit(self: *SharedObject, allocator: Allocator) void { + allocator.free(self.data); + self.versyms.deinit(allocator); + self.verstrings.deinit(allocator); + self.symbols.deinit(allocator); + if (self.aliases) |*aliases| aliases.deinit(allocator); + self.shdrs.deinit(allocator); +} + +pub fn parse(self: *SharedObject, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + var stream = std.io.fixedBufferStream(self.data); + const reader = stream.reader(); + + self.header = try reader.readStruct(elf.Elf64_Ehdr); + const shoff = std.math.cast(usize, self.header.?.e_shoff) orelse return error.Overflow; + + var dynsym_index: ?u16 = null; + const shdrs = @as( + [*]align(1) const elf.Elf64_Shdr, + @ptrCast(self.data.ptr + shoff), + )[0..self.header.?.e_shnum]; + try self.shdrs.ensureTotalCapacityPrecise(gpa, shdrs.len); + + for (shdrs, 0..) |shdr, i| { + self.shdrs.appendAssumeCapacity(try ElfShdr.fromElf64Shdr(shdr)); + switch (shdr.sh_type) { + elf.SHT_DYNSYM => dynsym_index = @as(u16, @intCast(i)), + elf.SHT_DYNAMIC => self.dynamic_sect_index = @as(u16, @intCast(i)), + elf.SHT_GNU_VERSYM => self.versym_sect_index = @as(u16, @intCast(i)), + elf.SHT_GNU_VERDEF => self.verdef_sect_index = @as(u16, @intCast(i)), + else => {}, + } + } + + if (dynsym_index) |index| { + const shdr = self.shdrs.items[index]; + const symtab = self.shdrContents(index); + const nsyms = @divExact(symtab.len, @sizeOf(elf.Elf64_Sym)); + self.symtab = @as([*]align(1) const elf.Elf64_Sym, @ptrCast(symtab.ptr))[0..nsyms]; + self.strtab = self.shdrContents(@as(u16, @intCast(shdr.sh_link))); + } + + try self.parseVersions(elf_file); + try self.initSymtab(elf_file); +} + +fn parseVersions(self: *SharedObject, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + + try self.verstrings.resize(gpa, 2); + self.verstrings.items[elf.VER_NDX_LOCAL] = 0; + self.verstrings.items[elf.VER_NDX_GLOBAL] = 0; + + if (self.verdef_sect_index) |shndx| { + const verdefs = self.shdrContents(shndx); + const nverdefs = self.verdefNum(); + try self.verstrings.resize(gpa, self.verstrings.items.len + nverdefs); + + var i: u32 = 0; + var offset: u32 = 0; + while (i < nverdefs) : (i += 1) { + const verdef = @as(*align(1) const elf.Elf64_Verdef, @ptrCast(verdefs.ptr + offset)).*; + defer offset += verdef.vd_next; + if (verdef.vd_flags == elf.VER_FLG_BASE) continue; // Skip BASE entry + const vda_name = if (verdef.vd_cnt > 0) + @as(*align(1) const elf.Elf64_Verdaux, @ptrCast(verdefs.ptr + offset + verdef.vd_aux)).vda_name + else + 0; + self.verstrings.items[verdef.vd_ndx] = vda_name; + } + } + + try self.versyms.ensureTotalCapacityPrecise(gpa, self.symtab.len); + + if (self.versym_sect_index) |shndx| { + const versyms_raw = self.shdrContents(shndx); + const nversyms = @divExact(versyms_raw.len, @sizeOf(elf.Elf64_Versym)); + const versyms = @as([*]align(1) const elf.Elf64_Versym, @ptrCast(versyms_raw.ptr))[0..nversyms]; + for (versyms) |ver| { + const normalized_ver = if (ver & elf.VERSYM_VERSION >= self.verstrings.items.len - 1) + elf.VER_NDX_GLOBAL + else + ver; + self.versyms.appendAssumeCapacity(normalized_ver); + } + } else for (0..self.symtab.len) |_| { + self.versyms.appendAssumeCapacity(elf.VER_NDX_GLOBAL); + } +} + +fn initSymtab(self: *SharedObject, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + + try self.symbols.ensureTotalCapacityPrecise(gpa, self.symtab.len); + + for (self.symtab, 0..) |sym, i| { + const hidden = self.versyms.items[i] & elf.VERSYM_HIDDEN != 0; + const name = self.getString(sym.st_name); + // We need to garble up the name so that we don't pick this symbol + // during symbol resolution. Thank you GNU! + const off = if (hidden) blk: { + const full_name = try std.fmt.allocPrint(gpa, "{s}@{s}", .{ + name, + self.versionString(self.versyms.items[i]), + }); + defer gpa.free(full_name); + break :blk try elf_file.strtab.insert(gpa, full_name); + } else try elf_file.strtab.insert(gpa, name); + const gop = try elf_file.getOrPutGlobal(off); + self.symbols.addOneAssumeCapacity().* = gop.index; + } +} + +pub fn resolveSymbols(self: *SharedObject, elf_file: *Elf) void { + for (self.globals(), 0..) |index, i| { + const esym_index = @as(u32, @intCast(i)); + const this_sym = self.symtab[esym_index]; + + if (this_sym.st_shndx == elf.SHN_UNDEF) continue; + + const global = elf_file.symbol(index); + if (self.asFile().symbolRank(this_sym, false) < global.symbolRank(elf_file)) { + global.value = this_sym.st_value; + global.atom_index = 0; + global.esym_index = esym_index; + global.version_index = self.versyms.items[esym_index]; + global.file_index = self.index; + } + } +} + +pub fn resetGlobals(self: *SharedObject, elf_file: *Elf) void { + for (self.globals()) |index| { + const global = elf_file.symbol(index); + const off = global.name_offset; + global.* = .{}; + global.name_offset = off; + } +} + +pub fn markLive(self: *SharedObject, elf_file: *Elf) void { + for (self.globals(), 0..) |index, i| { + const sym = self.symtab[i]; + if (sym.st_shndx != elf.SHN_UNDEF) continue; + + const global = elf_file.symbol(index); + const file = global.file(elf_file) orelse continue; + const should_drop = switch (file) { + .shared_object => |sh| !sh.needed and sym.st_bind() == elf.STB_WEAK, + else => false, + }; + if (!should_drop and !file.isAlive()) { + file.setAlive(); + file.markLive(elf_file); + } + } +} + +pub fn updateSymtabSize(self: *SharedObject, elf_file: *Elf) void { + for (self.globals()) |global_index| { + const global = elf_file.symbol(global_index); + if (global.file(elf_file)) |file| if (file.index() != self.index) continue; + if (global.isLocal()) continue; + global.flags.output_symtab = true; + self.output_symtab_size.nglobals += 1; + } +} + +pub fn writeSymtab(self: *SharedObject, elf_file: *Elf, ctx: anytype) void { + var iglobal = ctx.iglobal; + for (self.globals()) |global_index| { + const global = elf_file.symbol(global_index); + if (global.file(elf_file)) |file| if (file.index() != self.index) continue; + if (!global.flags.output_symtab) continue; + global.setOutputSym(elf_file, &ctx.symtab[iglobal]); + iglobal += 1; + } +} + +pub fn globals(self: SharedObject) []const Symbol.Index { + return self.symbols.items; +} + +pub fn shdrContents(self: SharedObject, index: u16) []const u8 { + const shdr = self.shdrs.items[index]; + return self.data[shdr.sh_offset..][0..shdr.sh_size]; +} + +pub fn getString(self: SharedObject, off: u32) [:0]const u8 { + assert(off < self.strtab.len); + return mem.sliceTo(@as([*:0]const u8, @ptrCast(self.strtab.ptr + off)), 0); +} + +pub fn versionString(self: SharedObject, index: elf.Elf64_Versym) [:0]const u8 { + const off = self.verstrings.items[index & elf.VERSYM_VERSION]; + return self.getString(off); +} + +pub fn asFile(self: *SharedObject) File { + return .{ .shared_object = self }; +} + +fn dynamicTable(self: *SharedObject) []align(1) const elf.Elf64_Dyn { + const shndx = self.dynamic_sect_index orelse return &[0]elf.Elf64_Dyn{}; + const raw = self.shdrContents(shndx); + const num = @divExact(raw.len, @sizeOf(elf.Elf64_Dyn)); + return @as([*]align(1) const elf.Elf64_Dyn, @ptrCast(raw.ptr))[0..num]; +} + +fn verdefNum(self: *SharedObject) u32 { + const entries = self.dynamicTable(); + for (entries) |entry| switch (entry.d_tag) { + elf.DT_VERDEFNUM => return @as(u32, @intCast(entry.d_val)), + else => {}, + }; + return 0; +} + +pub fn soname(self: *SharedObject) []const u8 { + const entries = self.dynamicTable(); + for (entries) |entry| switch (entry.d_tag) { + elf.DT_SONAME => return self.getString(@as(u32, @intCast(entry.d_val))), + else => {}, + }; + return std.fs.path.basename(self.path); +} + +pub fn initSymbolAliases(self: *SharedObject, elf_file: *Elf) !void { + assert(self.aliases == null); + + const SortAlias = struct { + pub fn lessThan(ctx: *Elf, lhs: Symbol.Index, rhs: Symbol.Index) bool { + const lhs_sym = ctx.symbol(lhs).elfSym(ctx); + const rhs_sym = ctx.symbol(rhs).elfSym(ctx); + return lhs_sym.st_value < rhs_sym.st_value; + } + }; + + const gpa = elf_file.base.allocator; + var aliases = std.ArrayList(Symbol.Index).init(gpa); + defer aliases.deinit(); + try aliases.ensureTotalCapacityPrecise(self.globals().len); + + for (self.globals()) |index| { + const global = elf_file.symbol(index); + const global_file = global.file(elf_file) orelse continue; + if (global_file.index() != self.index) continue; + aliases.appendAssumeCapacity(index); + } + + std.mem.sort(u32, aliases.items, elf_file, SortAlias.lessThan); + + self.aliases = aliases.moveToUnmanaged(); +} + +pub fn symbolAliases(self: *SharedObject, index: u32, elf_file: *Elf) []const u32 { + assert(self.aliases != null); + + const symbol = elf_file.symbol(index).elfSym(elf_file); + const aliases = self.aliases.?; + + const start = for (aliases.items, 0..) |alias, i| { + const alias_sym = elf_file.symbol(alias).elfSym(elf_file); + if (symbol.st_value == alias_sym.st_value) break i; + } else aliases.items.len; + + const end = for (aliases.items[start..], 0..) |alias, i| { + const alias_sym = elf_file.symbol(alias).elfSym(elf_file); + if (symbol.st_value < alias_sym.st_value) break i + start; + } else aliases.items.len; + + return aliases.items[start..end]; +} + +pub fn format( + self: SharedObject, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, +) !void { + _ = self; + _ = unused_fmt_string; + _ = options; + _ = writer; + @compileError("do not format shared objects directly"); +} + +pub fn fmtSymtab(self: SharedObject, elf_file: *Elf) std.fmt.Formatter(formatSymtab) { + return .{ .data = .{ + .shared = self, + .elf_file = elf_file, + } }; +} + +const FormatContext = struct { + shared: SharedObject, + elf_file: *Elf, +}; + +fn formatSymtab( + ctx: FormatContext, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, +) !void { + _ = unused_fmt_string; + _ = options; + const shared = ctx.shared; + try writer.writeAll(" globals\n"); + for (shared.symbols.items) |index| { + const global = ctx.elf_file.symbol(index); + try writer.print(" {}\n", .{global.fmt(ctx.elf_file)}); + } +} + +const SharedObject = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const elf = std.elf; +const log = std.log.scoped(.elf); +const mem = std.mem; + +const Allocator = mem.Allocator; +const Elf = @import("../Elf.zig"); +const ElfShdr = @import("Object.zig").ElfShdr; +const File = @import("file.zig").File; +const Symbol = @import("Symbol.zig"); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index a0236f4c87..00e6e15a8a 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -32,7 +32,7 @@ extra_index: u32 = 0, pub fn isAbs(symbol: Symbol, elf_file: *Elf) bool { const file_ptr = symbol.file(elf_file).?; - // if (file_ptr == .shared) return symbol.sourceSymbol(elf_file).st_shndx == elf.SHN_ABS; + if (file_ptr == .shared_object) return symbol.elfSym(elf_file).st_shndx == elf.SHN_ABS; return !symbol.flags.import and symbol.atom(elf_file) == null and symbol.outputShndx() == null and file_ptr != .linker_defined; } @@ -51,10 +51,10 @@ pub fn isIFunc(symbol: Symbol, elf_file: *Elf) bool { } pub fn @"type"(symbol: Symbol, elf_file: *Elf) u4 { - const s_sym = symbol.elfSym(elf_file); - // const file_ptr = symbol.file(elf_file).?; - // if (s_sym.st_type() == elf.STT_GNU_IFUNC and file_ptr == .shared) return elf.STT_FUNC; - return s_sym.st_type(); + const esym = symbol.elfSym(elf_file); + const file_ptr = symbol.file(elf_file).?; + if (esym.st_type() == elf.STT_GNU_IFUNC and file_ptr == .shared_object) return elf.STT_FUNC; + return esym.st_type(); } pub fn name(symbol: Symbol, elf_file: *Elf) [:0]const u8 { @@ -74,7 +74,7 @@ pub fn elfSym(symbol: Symbol, elf_file: *Elf) elf.Elf64_Sym { switch (file_ptr) { .zig_module => |x| return x.elfSym(symbol.esym_index).*, .linker_defined => |x| return x.symtab.items[symbol.esym_index], - .object => |x| return x.symtab[symbol.esym_index], + inline else => |x| return x.symtab[symbol.esym_index], } } @@ -88,23 +88,18 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { - plt: bool = true, -}, elf_file: *Elf) u64 { - _ = elf_file; - _ = opts; - // if (symbol.flags.copy_rel) { - // return elf_file.sectionAddress(elf_file.copy_rel_sect_index.?) + symbol.value; - // } - // if (symbol.flags.plt and opts.plt) { - // const extra = symbol.getExtra(elf_file).?; - // if (!symbol.flags.is_canonical and symbol.flags.got) { - // // We have a non-lazy bound function pointer, use that! - // return elf_file.getPltGotEntryAddress(extra.plt_got); - // } - // // Lazy-bound function it is! - // return elf_file.getPltEntryAddress(extra.plt); - // } +pub fn address(symbol: Symbol, opts: struct { plt: bool = true }, elf_file: *Elf) u64 { + if (symbol.flags.has_copy_rel) { + return symbol.copyRelAddress(elf_file); + } + if (symbol.flags.has_plt and opts.plt) { + if (!symbol.flags.is_canonical and symbol.flags.has_got) { + // We have a non-lazy bound function pointer, use that! + return symbol.pltGotAddress(elf_file); + } + // Lazy-bound function it is! + return symbol.pltAddress(elf_file); + } return symbol.value; } @@ -115,48 +110,83 @@ pub fn gotAddress(symbol: Symbol, elf_file: *Elf) u64 { return entry.address(elf_file); } -const GetOrCreateGotEntryResult = struct { +pub fn pltGotAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!(symbol.flags.has_plt and symbol.flags.has_got)) return 0; + const extras = symbol.extra(elf_file).?; + const shdr = elf_file.shdrs.items[elf_file.plt_got_section_index.?]; + return shdr.sh_addr + extras.plt_got * 16; +} + +pub fn pltAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_plt) return 0; + const extras = symbol.extra(elf_file).?; + const shdr = elf_file.shdrs.items[elf_file.plt_section_index.?]; + return shdr.sh_addr + extras.plt * 16 + PltSection.preamble_size; +} + +pub fn gotPltAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_plt) return 0; + const extras = symbol.extra(elf_file).?; + const shdr = elf_file.shdrs.items[elf_file.got_plt_section_index.?]; + return shdr.sh_addr + extras.plt * 8 + GotPltSection.preamble_size; +} + +pub fn copyRelAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_copy_rel) return 0; + const shdr = elf_file.shdrs.items[elf_file.copy_rel_section_index.?]; + return shdr.sh_addr + symbol.value; +} + +pub fn tlsGdAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_tlsgd) return 0; + const extras = symbol.extra(elf_file).?; + const entry = elf_file.got.entries.items[extras.tlsgd]; + return entry.address(elf_file); +} + +pub fn gotTpAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_gottp) return 0; + const extras = symbol.extra(elf_file).?; + const entry = elf_file.got.entries.items[extras.gottp]; + return entry.address(elf_file); +} + +pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_tlsdesc) return 0; + const extras = symbol.extra(elf_file).?; + const entry = elf_file.got.entries.items[extras.tlsdesc]; + return entry.address(elf_file); +} + +const GetOrCreateZigGotEntryResult = struct { found_existing: bool, - index: GotSection.Index, + index: ZigGotSection.Index, }; -pub fn getOrCreateGotEntry(symbol: *Symbol, symbol_index: Index, elf_file: *Elf) !GetOrCreateGotEntryResult { - assert(symbol.flags.needs_got); - if (symbol.flags.has_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).?.got }; - const index = try elf_file.got.addGotSymbol(symbol_index, elf_file); - symbol.flags.has_got = true; +pub fn getOrCreateZigGotEntry(symbol: *Symbol, symbol_index: Index, elf_file: *Elf) !GetOrCreateZigGotEntryResult { + if (symbol.flags.has_zig_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).?.zig_got }; + const index = try elf_file.zig_got.addSymbol(symbol_index, elf_file); return .{ .found_existing = false, .index = index }; } -// pub fn tlsGdAddress(symbol: Symbol, elf_file: *Elf) u64 { -// if (!symbol.flags.tlsgd) return 0; -// const extra = symbol.getExtra(elf_file).?; -// return elf_file.getGotEntryAddress(extra.tlsgd); -// } - -// pub fn gotTpAddress(symbol: Symbol, elf_file: *Elf) u64 { -// if (!symbol.flags.gottp) return 0; -// const extra = symbol.getExtra(elf_file).?; -// return elf_file.getGotEntryAddress(extra.gottp); -// } - -// pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) u64 { -// if (!symbol.flags.tlsdesc) return 0; -// const extra = symbol.getExtra(elf_file).?; -// return elf_file.getGotEntryAddress(extra.tlsdesc); -// } - -// pub fn alignment(symbol: Symbol, elf_file: *Elf) !u64 { -// const file = symbol.getFile(elf_file) orelse return 0; -// const shared = file.shared; -// const s_sym = symbol.getSourceSymbol(elf_file); -// const shdr = shared.getShdrs()[s_sym.st_shndx]; -// const alignment = @max(1, shdr.sh_addralign); -// return if (s_sym.st_value == 0) -// alignment -// else -// @min(alignment, try std.math.powi(u64, 2, @ctz(s_sym.st_value))); -// } +pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) u64 { + if (!symbol.flags.has_zig_got) return 0; + const extras = symbol.extra(elf_file).?; + return elf_file.zig_got.entryAddress(extras.zig_got, elf_file); +} + +pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { + const file_ptr = symbol.file(elf_file) orelse return 0; + assert(file_ptr == .shared_object); + const shared_object = file_ptr.shared_object; + const esym = symbol.elfSym(elf_file); + const shdr = shared_object.shdrs.items[esym.st_shndx]; + const alignment = @max(1, shdr.sh_addralign); + return if (esym.st_value == 0) + alignment + else + @min(alignment, try std.math.powi(u64, 2, @ctz(esym.st_value))); +} pub fn addExtra(symbol: *Symbol, extras: Extra, elf_file: *Elf) !void { symbol.extra_index = try elf_file.addSymbolExtra(extras); @@ -180,22 +210,22 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { const st_bind: u8 = blk: { if (symbol.isLocal()) break :blk 0; if (symbol.flags.weak) break :blk elf.STB_WEAK; - // if (file_ptr == .shared) break :blk elf.STB_GLOBAL; + if (file_ptr == .shared_object) break :blk elf.STB_GLOBAL; break :blk esym.st_bind(); }; const st_shndx = blk: { - // if (symbol.flags.copy_rel) break :blk elf_file.copy_rel_sect_index.?; - // if (file_ptr == .shared or s_sym.st_shndx == elf.SHN_UNDEF) break :blk elf.SHN_UNDEF; + if (symbol.flags.has_copy_rel) break :blk elf_file.copy_rel_section_index.?; + if (file_ptr == .shared_object or esym.st_shndx == elf.SHN_UNDEF) break :blk elf.SHN_UNDEF; if (symbol.atom(elf_file) == null and file_ptr != .linker_defined) break :blk elf.SHN_ABS; break :blk symbol.outputShndx() orelse elf.SHN_UNDEF; }; const st_value = blk: { - // if (symbol.flags.copy_rel) break :blk symbol.address(.{}, elf_file); - // if (file_ptr == .shared or s_sym.st_shndx == elf.SHN_UNDEF) { - // if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); - // break :blk 0; - // } + if (symbol.flags.has_copy_rel) break :blk symbol.address(.{}, elf_file); + if (file_ptr == .shared_object or esym.st_shndx == elf.SHN_UNDEF) { + if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); + break :blk 0; + } if (st_shndx == elf.SHN_ABS) break :blk symbol.value; const shdr = &elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) @@ -251,9 +281,10 @@ fn formatName( switch (symbol.version_index & elf.VERSYM_VERSION) { elf.VER_NDX_LOCAL, elf.VER_NDX_GLOBAL => {}, else => { - unreachable; - // const shared = symbol.getFile(elf_file).?.shared; - // try writer.print("@{s}", .{shared.getVersionString(symbol.version_index)}); + const file_ptr = symbol.file(elf_file).?; + assert(file_ptr == .shared_object); + const shared_object = file_ptr.shared_object; + try writer.print("@{s}", .{shared_object.versionString(symbol.version_index)}); }, } } @@ -283,7 +314,7 @@ fn format2( try writer.writeAll(" : absolute"); } } else if (symbol.outputShndx()) |shndx| { - try writer.print(" : sect({d})", .{shndx}); + try writer.print(" : shdr({d})", .{shndx}); } if (symbol.atom(ctx.elf_file)) |atom_ptr| { try writer.print(" : atom({d})", .{atom_ptr.atom_index}); @@ -309,23 +340,25 @@ pub const Flags = packed struct { /// Whether this symbol is weak. weak: bool = false, - /// Whether the symbol makes into the output symtab or not. + /// Whether the symbol makes into the output symtab. output_symtab: bool = false, + /// Whether the symbol has entry in dynamic symbol table. + has_dynamic: bool = false, + /// Whether the symbol contains GOT indirection. needs_got: bool = false, has_got: bool = false, /// Whether the symbol contains PLT indirection. needs_plt: bool = false, - plt: bool = false, + has_plt: bool = false, /// Whether the PLT entry is canonical. is_canonical: bool = false, /// Whether the symbol contains COPYREL directive. - copy_rel: bool = false, + needs_copy_rel: bool = false, has_copy_rel: bool = false, - has_dynamic: bool = false, /// Whether the symbol contains TLSGD indirection. needs_tlsgd: bool = false, @@ -336,7 +369,11 @@ pub const Flags = packed struct { has_gottp: bool = false, /// Whether the symbol contains TLSDESC indirection. - tlsdesc: bool = false, + needs_tlsdesc: bool = false, + has_tlsdesc: bool = false, + + /// Whether the symbol contains .zig.got indirection. + has_zig_got: bool = false, }; pub const Extra = struct { @@ -348,6 +385,7 @@ pub const Extra = struct { tlsgd: u32 = 0, gottp: u32 = 0, tlsdesc: u32 = 0, + zig_got: u32 = 0, }; pub const Index = u32; @@ -361,8 +399,11 @@ const Atom = @import("Atom.zig"); const Elf = @import("../Elf.zig"); const File = @import("file.zig").File; const GotSection = synthetic_sections.GotSection; +const GotPltSection = synthetic_sections.GotPltSection; const LinkerDefined = @import("LinkerDefined.zig"); -// const Object = @import("Object.zig"); -// const SharedObject = @import("SharedObject.zig"); +const Object = @import("Object.zig"); +const PltSection = synthetic_sections.PltSection; +const SharedObject = @import("SharedObject.zig"); const Symbol = @This(); +const ZigGotSection = synthetic_sections.ZigGotSection; const ZigModule = @import("ZigModule.zig"); diff --git a/src/link/Elf/ZigModule.zig b/src/link/Elf/ZigModule.zig index c79680dbf5..4532d7b448 100644 --- a/src/link/Elf/ZigModule.zig +++ b/src/link/Elf/ZigModule.zig @@ -13,9 +13,11 @@ local_symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, global_symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, globals_lookup: std.AutoHashMapUnmanaged(u32, Symbol.Index) = .{}, -atoms: std.AutoArrayHashMapUnmanaged(Atom.Index, void) = .{}, +atoms: std.ArrayListUnmanaged(Atom.Index) = .{}, relocs: std.ArrayListUnmanaged(std.ArrayListUnmanaged(elf.Elf64_Rela)) = .{}, +num_dynrelocs: u32 = 0, + output_symtab_size: Elf.SymtabSize = .{}, pub fn deinit(self: *ZigModule, allocator: Allocator) void { @@ -56,7 +58,8 @@ pub fn addAtom(self: *ZigModule, elf_file: *Elf) !Symbol.Index { const symbol_index = try elf_file.addSymbol(); const esym_index = try self.addLocalEsym(gpa); - try self.atoms.putNoClobber(gpa, atom_index, {}); + const shndx = @as(u16, @intCast(self.atoms.items.len)); + try self.atoms.append(gpa, atom_index); try self.local_symbols.append(gpa, symbol_index); const atom_ptr = elf_file.atom(atom_index).?; @@ -67,10 +70,10 @@ pub fn addAtom(self: *ZigModule, elf_file: *Elf) !Symbol.Index { symbol_ptr.atom_index = atom_index; const esym = &self.local_esyms.items[esym_index]; - esym.st_shndx = atom_index; + esym.st_shndx = shndx; symbol_ptr.esym_index = esym_index; - const relocs_index = @as(Atom.Index, @intCast(self.relocs.items.len)); + const relocs_index = @as(u16, @intCast(self.relocs.items.len)); const relocs = try self.relocs.addOne(gpa); relocs.* = .{}; atom_ptr.relocs_section_index = relocs_index; @@ -78,6 +81,22 @@ pub fn addAtom(self: *ZigModule, elf_file: *Elf) !Symbol.Index { return symbol_index; } +/// TODO actually create fake input shdrs and return that instead. +pub fn inputShdr(self: ZigModule, atom_index: Atom.Index, elf_file: *Elf) Object.ElfShdr { + _ = self; + const shdr = shdr: { + const atom = elf_file.atom(atom_index) orelse break :shdr Elf.null_shdr; + const shndx = atom.outputShndx() orelse break :shdr Elf.null_shdr; + var shdr = elf_file.shdrs.items[shndx]; + shdr.sh_addr = 0; + shdr.sh_offset = 0; + shdr.sh_size = atom.size; + shdr.sh_addralign = atom.alignment.toByteUnits(1); + break :shdr shdr; + }; + return Object.ElfShdr.fromElf64Shdr(shdr) catch unreachable; +} + pub fn resolveSymbols(self: *ZigModule, elf_file: *Elf) void { for (self.globals(), 0..) |index, i| { const esym_index = @as(Symbol.Index, @intCast(i)) | 0x10000000; @@ -86,7 +105,7 @@ pub fn resolveSymbols(self: *ZigModule, elf_file: *Elf) void { if (esym.st_shndx == elf.SHN_UNDEF) continue; if (esym.st_shndx != elf.SHN_ABS and esym.st_shndx != elf.SHN_COMMON) { - const atom_index = esym.st_shndx; + const atom_index = self.atoms.items[esym.st_shndx]; const atom = elf_file.atom(atom_index) orelse continue; if (!atom.flags.alive) continue; } @@ -95,7 +114,7 @@ pub fn resolveSymbols(self: *ZigModule, elf_file: *Elf) void { if (self.asFile().symbolRank(esym, false) < global.symbolRank(elf_file)) { const atom_index = switch (esym.st_shndx) { elf.SHN_ABS, elf.SHN_COMMON => 0, - else => esym.st_shndx, + else => self.atoms.items[esym.st_shndx], }; const output_section_index = if (elf_file.atom(atom_index)) |atom| atom.outputShndx().? @@ -141,10 +160,12 @@ pub fn claimUnresolved(self: *ZigModule, elf_file: *Elf) void { } pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void { - for (self.atoms.keys()) |atom_index| { + for (self.atoms.items) |atom_index| { const atom = elf_file.atom(atom_index) orelse continue; if (!atom.flags.alive) continue; - if (try atom.scanRelocsRequiresCode(elf_file)) { + const shdr = atom.inputShdr(elf_file); + if (shdr.sh_type == elf.SHT_NOBITS) continue; + if (atom.scanRelocsRequiresCode(elf_file)) { // TODO ideally we don't have to fetch the code here. // Perhaps it would make sense to save the code until flushModule where we // would free all of generated code? @@ -272,7 +293,10 @@ pub fn codeAlloc(self: ZigModule, elf_file: *Elf, atom_index: Atom.Index) ![]u8 const code = try gpa.alloc(u8, size); errdefer gpa.free(code); const amt = try elf_file.base.file.?.preadAll(code, file_offset); - if (amt != code.len) return error.InputOutput; + if (amt != code.len) { + log.err("fetching code for {s} failed", .{atom.name(elf_file)}); + return error.InputOutput; + } return code; } @@ -324,7 +348,7 @@ fn formatAtoms( _ = unused_fmt_string; _ = options; try writer.writeAll(" atoms\n"); - for (ctx.self.atoms.keys()) |atom_index| { + for (ctx.self.atoms.items) |atom_index| { const atom = ctx.elf_file.atom(atom_index) orelse continue; try writer.print(" {}\n", .{atom.fmt(ctx.elf_file)}); } @@ -333,11 +357,13 @@ fn formatAtoms( const assert = std.debug.assert; const std = @import("std"); const elf = std.elf; +const log = std.log.scoped(.link); const Allocator = std.mem.Allocator; const Atom = @import("Atom.zig"); const Elf = @import("../Elf.zig"); const File = @import("file.zig").File; const Module = @import("../../Module.zig"); +const Object = @import("Object.zig"); const Symbol = @import("Symbol.zig"); const ZigModule = @This(); diff --git a/src/link/Elf/eh_frame.zig b/src/link/Elf/eh_frame.zig index 8c676504e2..468f673a9a 100644 --- a/src/link/Elf/eh_frame.zig +++ b/src/link/Elf/eh_frame.zig @@ -1,7 +1,7 @@ pub const Fde = struct { /// Includes 4byte size cell. - offset: u64, - size: u64, + offset: usize, + size: usize, cie_index: u32, rel_index: u32 = 0, rel_num: u32 = 0, @@ -20,9 +20,9 @@ pub const Fde = struct { return base + fde.out_offset; } - pub fn data(fde: Fde, elf_file: *Elf) error{Overflow}![]const u8 { + pub fn data(fde: Fde, elf_file: *Elf) []const u8 { const object = elf_file.file(fde.file_index).?.object; - const contents = try object.shdrContents(fde.input_section_index); + const contents = object.shdrContents(fde.input_section_index); return contents[fde.offset..][0..fde.calcSize()]; } @@ -32,24 +32,25 @@ pub const Fde = struct { } pub fn ciePointer(fde: Fde, elf_file: *Elf) u32 { - return std.mem.readIntLittle(u32, fde.data(elf_file)[4..8]); + const fde_data = fde.data(elf_file); + return std.mem.readIntLittle(u32, fde_data[4..8]); } - pub fn calcSize(fde: Fde) u64 { + pub fn calcSize(fde: Fde) usize { return fde.size + 4; } - pub fn atom(fde: Fde, elf_file: *Elf) error{Overflow}!*Atom { + pub fn atom(fde: Fde, elf_file: *Elf) *Atom { const object = elf_file.file(fde.file_index).?.object; - const rel = (try fde.relocs(elf_file))[0]; + const rel = fde.relocs(elf_file)[0]; const sym = object.symtab[rel.r_sym()]; const atom_index = object.atoms.items[sym.st_shndx]; return elf_file.atom(atom_index).?; } - pub fn relocs(fde: Fde, elf_file: *Elf) error{Overflow}![]align(1) const elf.Elf64_Rela { + pub fn relocs(fde: Fde, elf_file: *Elf) []align(1) const elf.Elf64_Rela { const object = elf_file.file(fde.file_index).?.object; - return (try object.getRelocs(fde.rel_section_index))[fde.rel_index..][0..fde.rel_num]; + return object.getRelocs(fde.rel_section_index)[fde.rel_index..][0..fde.rel_num]; } pub fn format( @@ -88,10 +89,7 @@ pub const Fde = struct { const fde = ctx.fde; const elf_file = ctx.elf_file; const base_addr = fde.address(elf_file); - const atom_name = if (fde.atom(elf_file)) |atom_ptr| - atom_ptr.name(elf_file) - else |_| - ""; + const atom_name = fde.atom(elf_file).name(elf_file); try writer.print("@{x} : size({x}) : cie({d}) : {s}", .{ base_addr + fde.out_offset, fde.calcSize(), @@ -104,8 +102,8 @@ pub const Fde = struct { pub const Cie = struct { /// Includes 4byte size cell. - offset: u64, - size: u64, + offset: usize, + size: usize, rel_index: u32 = 0, rel_num: u32 = 0, rel_section_index: u32 = 0, @@ -123,26 +121,26 @@ pub const Cie = struct { return base + cie.out_offset; } - pub fn data(cie: Cie, elf_file: *Elf) error{Overflow}![]const u8 { + pub fn data(cie: Cie, elf_file: *Elf) []const u8 { const object = elf_file.file(cie.file_index).?.object; - const contents = try object.shdrContents(cie.input_section_index); + const contents = object.shdrContents(cie.input_section_index); return contents[cie.offset..][0..cie.calcSize()]; } - pub fn calcSize(cie: Cie) u64 { + pub fn calcSize(cie: Cie) usize { return cie.size + 4; } - pub fn relocs(cie: Cie, elf_file: *Elf) error{Overflow}![]align(1) const elf.Elf64_Rela { + pub fn relocs(cie: Cie, elf_file: *Elf) []align(1) const elf.Elf64_Rela { const object = elf_file.file(cie.file_index).?.object; - return (try object.getRelocs(cie.rel_section_index))[cie.rel_index..][0..cie.rel_num]; + return object.getRelocs(cie.rel_section_index)[cie.rel_index..][0..cie.rel_num]; } - pub fn eql(cie: Cie, other: Cie, elf_file: *Elf) error{Overflow}!bool { - if (!std.mem.eql(u8, try cie.data(elf_file), try other.data(elf_file))) return false; + pub fn eql(cie: Cie, other: Cie, elf_file: *Elf) bool { + if (!std.mem.eql(u8, cie.data(elf_file), other.data(elf_file))) return false; - const cie_relocs = try cie.relocs(elf_file); - const other_relocs = try other.relocs(elf_file); + const cie_relocs = cie.relocs(elf_file); + const other_relocs = other.relocs(elf_file); if (cie_relocs.len != other_relocs.len) return false; for (cie_relocs, other_relocs) |cie_rel, other_rel| { @@ -152,8 +150,8 @@ pub const Cie = struct { const cie_object = elf_file.file(cie.file_index).?.object; const other_object = elf_file.file(other.file_index).?.object; - const cie_sym = cie_object.symbol(cie_rel.r_sym(), elf_file); - const other_sym = other_object.symbol(other_rel.r_sym(), elf_file); + const cie_sym = cie_object.symbols.items[cie_rel.r_sym()]; + const other_sym = other_object.symbols.items[other_rel.r_sym()]; if (!std.mem.eql(u8, std.mem.asBytes(&cie_sym), std.mem.asBytes(&other_sym))) return false; } return true; @@ -205,12 +203,12 @@ pub const Cie = struct { pub const Iterator = struct { data: []const u8, - pos: u64 = 0, + pos: usize = 0, pub const Record = struct { tag: enum { fde, cie }, - offset: u64, - size: u64, + offset: usize, + size: usize, }; pub fn next(it: *Iterator) !?Record { @@ -235,7 +233,7 @@ pub const Iterator = struct { }; pub fn calcEhFrameSize(elf_file: *Elf) !usize { - var offset: u64 = 0; + var offset: usize = 0; var cies = std.ArrayList(Cie).init(elf_file.base.allocator); defer cies.deinit(); @@ -285,7 +283,7 @@ pub fn calcEhFrameHdrSize(elf_file: *Elf) usize { } fn resolveReloc(rec: anytype, sym: *const Symbol, rel: elf.Elf64_Rela, elf_file: *Elf, contents: []u8) !void { - const offset = rel.r_offset - rec.offset; + const offset = std.math.cast(usize, rel.r_offset - rec.offset) orelse return error.Overflow; const P = @as(i64, @intCast(rec.address(elf_file) + offset)); const S = @as(i64, @intCast(sym.address(.{}, elf_file))); const A = rel.r_addend; @@ -319,11 +317,11 @@ pub fn writeEhFrame(elf_file: *Elf, writer: anytype) !void { for (object.cies.items) |cie| { if (!cie.alive) continue; - const contents = try gpa.dupe(u8, try cie.data(elf_file)); + const contents = try gpa.dupe(u8, cie.data(elf_file)); defer gpa.free(contents); - for (try cie.relocs(elf_file)) |rel| { - const sym = object.symbol(rel.r_sym(), elf_file); + for (cie.relocs(elf_file)) |rel| { + const sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); try resolveReloc(cie, sym, rel, elf_file, contents); } @@ -337,7 +335,7 @@ pub fn writeEhFrame(elf_file: *Elf, writer: anytype) !void { for (object.fdes.items) |fde| { if (!fde.alive) continue; - const contents = try gpa.dupe(u8, try fde.data(elf_file)); + const contents = try gpa.dupe(u8, fde.data(elf_file)); defer gpa.free(contents); std.mem.writeIntLittle( @@ -346,8 +344,8 @@ pub fn writeEhFrame(elf_file: *Elf, writer: anytype) !void { @as(i32, @truncate(@as(i64, @intCast(fde.out_offset + 4)) - @as(i64, @intCast(fde.cie(elf_file).out_offset)))), ); - for (try fde.relocs(elf_file)) |rel| { - const sym = object.symbol(rel.r_sym(), elf_file); + for (fde.relocs(elf_file)) |rel| { + const sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); try resolveReloc(fde, sym, rel, elf_file, contents); } @@ -395,10 +393,10 @@ pub fn writeEhFrameHdr(elf_file: *Elf, writer: anytype) !void { for (object.fdes.items) |fde| { if (!fde.alive) continue; - const relocs = try fde.relocs(elf_file); + const relocs = fde.relocs(elf_file); assert(relocs.len > 0); // Should this be an error? Things are completely broken anyhow if this trips... const rel = relocs[0]; - const sym = object.symbol(rel.r_sym(), elf_file); + const sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); const P = @as(i64, @intCast(fde.address(elf_file))); const S = @as(i64, @intCast(sym.address(.{}, elf_file))); const A = rel.r_addend; @@ -416,7 +414,7 @@ pub fn writeEhFrameHdr(elf_file: *Elf, writer: anytype) !void { try writer.writeAll(std.mem.sliceAsBytes(entries.items)); } -const eh_frame_hdr_header_size: u64 = 12; +const eh_frame_hdr_header_size: usize = 12; const EH_PE = struct { pub const absptr = 0x00; diff --git a/src/link/Elf/file.zig b/src/link/Elf/file.zig index a664061fda..f8258bb884 100644 --- a/src/link/Elf/file.zig +++ b/src/link/Elf/file.zig @@ -2,7 +2,7 @@ pub const File = union(enum) { zig_module: *ZigModule, linker_defined: *LinkerDefined, object: *Object, - // shared_object: *SharedObject, + shared_object: *SharedObject, pub fn index(file: File) Index { return switch (file) { @@ -26,7 +26,7 @@ pub const File = union(enum) { .zig_module => |x| try writer.print("{s}", .{x.path}), .linker_defined => try writer.writeAll("(linker defined)"), .object => |x| try writer.print("{}", .{x.fmtPath()}), - // .shared_object => |x| try writer.writeAll(x.path), + .shared_object => |x| try writer.writeAll(x.path), } } @@ -49,8 +49,7 @@ pub const File = union(enum) { pub fn symbolRank(file: File, sym: elf.Elf64_Sym, in_archive: bool) u32 { const base: u3 = blk: { if (sym.st_shndx == elf.SHN_COMMON) break :blk if (in_archive) 6 else 5; - // if (file == .shared or in_archive) break :blk switch (sym.st_bind()) { - if (in_archive) break :blk switch (sym.st_bind()) { + if (file == .shared_object or in_archive) break :blk switch (sym.st_bind()) { elf.STB_GLOBAL => 3, else => 4, }; @@ -92,7 +91,8 @@ pub const File = union(enum) { pub fn atoms(file: File) []const Atom.Index { return switch (file) { .linker_defined => unreachable, - .zig_module => |x| x.atoms.keys(), + .shared_object => unreachable, + .zig_module => |x| x.atoms.items, .object => |x| x.atoms.items, }; } @@ -100,6 +100,7 @@ pub const File = union(enum) { pub fn locals(file: File) []const Symbol.Index { return switch (file) { .linker_defined => unreachable, + .shared_object => unreachable, inline else => |x| x.locals(), }; } @@ -117,7 +118,7 @@ pub const File = union(enum) { zig_module: ZigModule, linker_defined: LinkerDefined, object: Object, - // shared_object: SharedObject, + shared_object: SharedObject, }; }; @@ -129,6 +130,6 @@ const Atom = @import("Atom.zig"); const Elf = @import("../Elf.zig"); const LinkerDefined = @import("LinkerDefined.zig"); const Object = @import("Object.zig"); -// const SharedObject = @import("SharedObject.zig"); +const SharedObject = @import("SharedObject.zig"); const Symbol = @import("Symbol.zig"); const ZigModule = @import("ZigModule.zig"); diff --git a/src/link/Elf/gc.zig b/src/link/Elf/gc.zig new file mode 100644 index 0000000000..26489d3d1a --- /dev/null +++ b/src/link/Elf/gc.zig @@ -0,0 +1,161 @@ +pub fn gcAtoms(elf_file: *Elf) !void { + var roots = std.ArrayList(*Atom).init(elf_file.base.allocator); + defer roots.deinit(); + try collectRoots(&roots, elf_file); + mark(roots, elf_file); + prune(elf_file); +} + +fn collectRoots(roots: *std.ArrayList(*Atom), elf_file: *Elf) !void { + if (elf_file.entry_index) |index| { + const global = elf_file.symbol(index); + try markSymbol(global, roots, elf_file); + } + + for (elf_file.objects.items) |index| { + for (elf_file.file(index).?.object.globals()) |global_index| { + const global = elf_file.symbol(global_index); + if (global.file(elf_file)) |file| { + if (file.index() == index and global.flags.@"export") + try markSymbol(global, roots, elf_file); + } + } + } + + for (elf_file.objects.items) |index| { + const object = elf_file.file(index).?.object; + + for (object.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (!atom.flags.alive) continue; + + const shdr = atom.inputShdr(elf_file); + const name = atom.name(elf_file); + const is_gc_root = blk: { + if (shdr.sh_flags & elf.SHF_GNU_RETAIN != 0) break :blk true; + if (shdr.sh_type == elf.SHT_NOTE) break :blk true; + if (shdr.sh_type == elf.SHT_PREINIT_ARRAY) break :blk true; + if (shdr.sh_type == elf.SHT_INIT_ARRAY) break :blk true; + if (shdr.sh_type == elf.SHT_FINI_ARRAY) break :blk true; + if (mem.startsWith(u8, name, ".ctors")) break :blk true; + if (mem.startsWith(u8, name, ".dtors")) break :blk true; + if (mem.startsWith(u8, name, ".init")) break :blk true; + if (mem.startsWith(u8, name, ".fini")) break :blk true; + if (Elf.isCIdentifier(name)) break :blk true; + break :blk false; + }; + if (is_gc_root and markAtom(atom)) try roots.append(atom); + if (shdr.sh_flags & elf.SHF_ALLOC == 0) atom.flags.visited = true; + } + + // Mark every atom referenced by CIE as alive. + for (object.cies.items) |cie| { + for (cie.relocs(elf_file)) |rel| { + const sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); + try markSymbol(sym, roots, elf_file); + } + } + } +} + +fn markSymbol(sym: *Symbol, roots: *std.ArrayList(*Atom), elf_file: *Elf) !void { + const atom = sym.atom(elf_file) orelse return; + if (markAtom(atom)) try roots.append(atom); +} + +fn markAtom(atom: *Atom) bool { + const already_visited = atom.flags.visited; + atom.flags.visited = true; + return atom.flags.alive and !already_visited; +} + +fn markLive(atom: *Atom, elf_file: *Elf) void { + if (@import("build_options").enable_logging) track_live_level.incr(); + + assert(atom.flags.visited); + const object = atom.file(elf_file).?.object; + + for (atom.fdes(elf_file)) |fde| { + for (fde.relocs(elf_file)[1..]) |rel| { + const target_sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); + const target_atom = target_sym.atom(elf_file) orelse continue; + target_atom.flags.alive = true; + gc_track_live_log.debug("{}marking live atom({d})", .{ track_live_level, target_atom.atom_index }); + if (markAtom(target_atom)) markLive(target_atom, elf_file); + } + } + + for (atom.relocs(elf_file)) |rel| { + const target_sym = elf_file.symbol(object.symbols.items[rel.r_sym()]); + const target_atom = target_sym.atom(elf_file) orelse continue; + target_atom.flags.alive = true; + gc_track_live_log.debug("{}marking live atom({d})", .{ track_live_level, target_atom.atom_index }); + if (markAtom(target_atom)) markLive(target_atom, elf_file); + } +} + +fn mark(roots: std.ArrayList(*Atom), elf_file: *Elf) void { + for (roots.items) |root| { + gc_track_live_log.debug("root atom({d})", .{root.atom_index}); + markLive(root, elf_file); + } +} + +fn prune(elf_file: *Elf) void { + for (elf_file.objects.items) |index| { + for (elf_file.file(index).?.object.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (atom.flags.alive and !atom.flags.visited) { + atom.flags.alive = false; + atom.markFdesDead(elf_file); + } + } + } +} + +pub fn dumpPrunedAtoms(elf_file: *Elf) !void { + const stderr = std.io.getStdErr().writer(); + for (elf_file.objects.items) |index| { + for (elf_file.file(index).?.object.atoms.items) |atom_index| { + const atom = elf_file.atom(atom_index) orelse continue; + if (!atom.flags.alive) + // TODO should we simply print to stderr? + try stderr.print("link: removing unused section '{s}' in file '{}'\n", .{ + atom.name(elf_file), + atom.file(elf_file).?.fmtPath(), + }); + } + } +} + +const Level = struct { + value: usize = 0, + + fn incr(self: *@This()) void { + self.value += 1; + } + + pub fn format( + self: *const @This(), + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = unused_fmt_string; + _ = options; + try writer.writeByteNTimes(' ', self.value); + } +}; + +var track_live_level: Level = .{}; + +const std = @import("std"); +const assert = std.debug.assert; +const elf = std.elf; +const gc_track_live_log = std.log.scoped(.gc_track_live); +const mem = std.mem; + +const Allocator = mem.Allocator; +const Atom = @import("Atom.zig"); +const Elf = @import("../Elf.zig"); +const Symbol = @import("Symbol.zig"); diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig index caffd401be..3524b981fd 100644 --- a/src/link/Elf/synthetic_sections.zig +++ b/src/link/Elf/synthetic_sections.zig @@ -1,11 +1,443 @@ +pub const DynamicSection = struct { + soname: ?u32 = null, + needed: std.ArrayListUnmanaged(u32) = .{}, + rpath: u32 = 0, + + pub fn deinit(dt: *DynamicSection, allocator: Allocator) void { + dt.needed.deinit(allocator); + } + + pub fn addNeeded(dt: *DynamicSection, shared: *SharedObject, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + const off = try elf_file.dynstrtab.insert(gpa, shared.soname()); + try dt.needed.append(gpa, off); + } + + pub fn setRpath(dt: *DynamicSection, rpath_list: []const []const u8, elf_file: *Elf) !void { + if (rpath_list.len == 0) return; + const gpa = elf_file.base.allocator; + var rpath = std.ArrayList(u8).init(gpa); + defer rpath.deinit(); + for (rpath_list, 0..) |path, i| { + if (i > 0) try rpath.append(':'); + try rpath.appendSlice(path); + } + dt.rpath = try elf_file.dynstrtab.insert(gpa, rpath.items); + } + + pub fn setSoname(dt: *DynamicSection, soname: []const u8, elf_file: *Elf) !void { + dt.soname = try elf_file.dynstrtab.insert(elf_file.base.allocator, soname); + } + + fn getFlags(dt: DynamicSection, elf_file: *Elf) ?u64 { + _ = dt; + var flags: u64 = 0; + if (elf_file.base.options.z_now) { + flags |= elf.DF_BIND_NOW; + } + for (elf_file.got.entries.items) |entry| switch (entry.tag) { + .gottp => { + flags |= elf.DF_STATIC_TLS; + break; + }, + else => {}, + }; + if (elf_file.has_text_reloc) { + flags |= elf.DF_TEXTREL; + } + return if (flags > 0) flags else null; + } + + fn getFlags1(dt: DynamicSection, elf_file: *Elf) ?u64 { + _ = dt; + var flags_1: u64 = 0; + if (elf_file.base.options.z_now) { + flags_1 |= elf.DF_1_NOW; + } + if (elf_file.base.options.pie) { + flags_1 |= elf.DF_1_PIE; + } + // if (elf_file.base.options.z_nodlopen) { + // flags_1 |= elf.DF_1_NOOPEN; + // } + return if (flags_1 > 0) flags_1 else null; + } + + pub fn size(dt: DynamicSection, elf_file: *Elf) usize { + var nentries: usize = 0; + nentries += dt.needed.items.len; // NEEDED + if (dt.soname != null) nentries += 1; // SONAME + if (dt.rpath > 0) nentries += 1; // RUNPATH + if (elf_file.sectionByName(".init") != null) nentries += 1; // INIT + if (elf_file.sectionByName(".fini") != null) nentries += 1; // FINI + if (elf_file.sectionByName(".init_array") != null) nentries += 2; // INIT_ARRAY + if (elf_file.sectionByName(".fini_array") != null) nentries += 2; // FINI_ARRAY + if (elf_file.rela_dyn_section_index != null) nentries += 3; // RELA + if (elf_file.rela_plt_section_index != null) nentries += 3; // JMPREL + if (elf_file.got_plt_section_index != null) nentries += 1; // PLTGOT + nentries += 1; // HASH + if (elf_file.gnu_hash_section_index != null) nentries += 1; // GNU_HASH + if (elf_file.has_text_reloc) nentries += 1; // TEXTREL + nentries += 1; // SYMTAB + nentries += 1; // SYMENT + nentries += 1; // STRTAB + nentries += 1; // STRSZ + if (elf_file.versym_section_index != null) nentries += 1; // VERSYM + if (elf_file.verneed_section_index != null) nentries += 2; // VERNEED + if (dt.getFlags(elf_file) != null) nentries += 1; // FLAGS + if (dt.getFlags1(elf_file) != null) nentries += 1; // FLAGS_1 + if (!elf_file.isDynLib()) nentries += 1; // DEBUG + nentries += 1; // NULL + return nentries * @sizeOf(elf.Elf64_Dyn); + } + + pub fn write(dt: DynamicSection, elf_file: *Elf, writer: anytype) !void { + // NEEDED + for (dt.needed.items) |off| { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_NEEDED, .d_val = off }); + } + + if (dt.soname) |off| { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_SONAME, .d_val = off }); + } + + // RUNPATH + // TODO add option in Options to revert to old RPATH tag + if (dt.rpath > 0) { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_RUNPATH, .d_val = dt.rpath }); + } + + // INIT + if (elf_file.sectionByName(".init")) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_INIT, .d_val = addr }); + } + + // FINI + if (elf_file.sectionByName(".fini")) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_FINI, .d_val = addr }); + } + + // INIT_ARRAY + if (elf_file.sectionByName(".init_array")) |shndx| { + const shdr = elf_file.shdrs.items[shndx]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_INIT_ARRAY, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_INIT_ARRAYSZ, .d_val = shdr.sh_size }); + } + + // FINI_ARRAY + if (elf_file.sectionByName(".fini_array")) |shndx| { + const shdr = elf_file.shdrs.items[shndx]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_FINI_ARRAY, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_FINI_ARRAYSZ, .d_val = shdr.sh_size }); + } + + // RELA + if (elf_file.rela_dyn_section_index) |shndx| { + const shdr = elf_file.shdrs.items[shndx]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_RELA, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_RELASZ, .d_val = shdr.sh_size }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_RELAENT, .d_val = shdr.sh_entsize }); + } + + // JMPREL + if (elf_file.rela_plt_section_index) |shndx| { + const shdr = elf_file.shdrs.items[shndx]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_JMPREL, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_PLTRELSZ, .d_val = shdr.sh_size }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_PLTREL, .d_val = elf.DT_RELA }); + } + + // PLTGOT + if (elf_file.got_plt_section_index) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_PLTGOT, .d_val = addr }); + } + + { + assert(elf_file.hash_section_index != null); + const addr = elf_file.shdrs.items[elf_file.hash_section_index.?].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_HASH, .d_val = addr }); + } + + if (elf_file.gnu_hash_section_index) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_GNU_HASH, .d_val = addr }); + } + + // TEXTREL + if (elf_file.has_text_reloc) { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_TEXTREL, .d_val = 0 }); + } + + // SYMTAB + SYMENT + { + assert(elf_file.dynsymtab_section_index != null); + const shdr = elf_file.shdrs.items[elf_file.dynsymtab_section_index.?]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_SYMTAB, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_SYMENT, .d_val = shdr.sh_entsize }); + } + + // STRTAB + STRSZ + { + assert(elf_file.dynstrtab_section_index != null); + const shdr = elf_file.shdrs.items[elf_file.dynstrtab_section_index.?]; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_STRTAB, .d_val = shdr.sh_addr }); + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_STRSZ, .d_val = shdr.sh_size }); + } + + // VERSYM + if (elf_file.versym_section_index) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_VERSYM, .d_val = addr }); + } + + // VERNEED + VERNEEDNUM + if (elf_file.verneed_section_index) |shndx| { + const addr = elf_file.shdrs.items[shndx].sh_addr; + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_VERNEED, .d_val = addr }); + try writer.writeStruct(elf.Elf64_Dyn{ + .d_tag = elf.DT_VERNEEDNUM, + .d_val = elf_file.verneed.verneed.items.len, + }); + } + + // FLAGS + if (dt.getFlags(elf_file)) |flags| { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_FLAGS, .d_val = flags }); + } + // FLAGS_1 + if (dt.getFlags1(elf_file)) |flags_1| { + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_FLAGS_1, .d_val = flags_1 }); + } + + // DEBUG + if (!elf_file.isDynLib()) try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_DEBUG, .d_val = 0 }); + + // NULL + try writer.writeStruct(elf.Elf64_Dyn{ .d_tag = elf.DT_NULL, .d_val = 0 }); + } +}; + +pub const ZigGotSection = struct { + entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, + output_symtab_size: Elf.SymtabSize = .{}, + flags: Flags = .{}, + + const Flags = packed struct { + needs_rela: bool = false, // TODO in prep for PIC/PIE and base relocations + dirty: bool = false, + }; + + pub const Index = u32; + + pub fn deinit(zig_got: *ZigGotSection, allocator: Allocator) void { + zig_got.entries.deinit(allocator); + } + + fn allocateEntry(zig_got: *ZigGotSection, allocator: Allocator) !Index { + try zig_got.entries.ensureUnusedCapacity(allocator, 1); + // TODO add free list + const index = @as(Index, @intCast(zig_got.entries.items.len)); + _ = zig_got.entries.addOneAssumeCapacity(); + zig_got.flags.dirty = true; + return index; + } + + pub fn addSymbol(zig_got: *ZigGotSection, sym_index: Symbol.Index, elf_file: *Elf) !Index { + const index = try zig_got.allocateEntry(elf_file.base.allocator); + const entry = &zig_got.entries.items[index]; + entry.* = sym_index; + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_zig_got = true; + if (elf_file.base.options.pic) { + zig_got.flags.needs_rela = true; + } + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.zig_got = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .zig_got = index }, elf_file); + return index; + } + + pub fn entryOffset(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 { + _ = zig_got; + const entry_size = elf_file.archPtrWidthBytes(); + const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; + return shdr.sh_offset + @as(u64, entry_size) * index; + } + + pub fn entryAddress(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 { + _ = zig_got; + const entry_size = elf_file.archPtrWidthBytes(); + const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; + return shdr.sh_addr + @as(u64, entry_size) * index; + } + + pub fn size(zig_got: ZigGotSection, elf_file: *Elf) usize { + return elf_file.archPtrWidthBytes() * zig_got.entries.items.len; + } + + pub fn writeOne(zig_got: *ZigGotSection, elf_file: *Elf, index: Index) !void { + if (zig_got.flags.dirty) { + const needed_size = zig_got.size(elf_file); + try elf_file.growAllocSection(elf_file.zig_got_section_index.?, needed_size); + zig_got.flags.dirty = false; + } + const entry_size: u16 = elf_file.archPtrWidthBytes(); + const endian = elf_file.base.options.target.cpu.arch.endian(); + const off = zig_got.entryOffset(index, elf_file); + const vaddr = zig_got.entryAddress(index, elf_file); + const entry = zig_got.entries.items[index]; + const value = elf_file.symbol(entry).value; + switch (entry_size) { + 2 => { + var buf: [2]u8 = undefined; + std.mem.writeInt(u16, &buf, @as(u16, @intCast(value)), endian); + try elf_file.base.file.?.pwriteAll(&buf, off); + }, + 4 => { + var buf: [4]u8 = undefined; + std.mem.writeInt(u32, &buf, @as(u32, @intCast(value)), endian); + try elf_file.base.file.?.pwriteAll(&buf, off); + }, + 8 => { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, value, endian); + try elf_file.base.file.?.pwriteAll(&buf, off); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.os.iovec_const = .{.{ + .iov_base = &buf, + .iov_len = buf.len, + }}; + var remote_vec: [1]std.os.iovec_const = .{.{ + .iov_base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), + .iov_len = buf.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.errno(rc)) { + .SUCCESS => assert(rc == buf.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } + } + }, + else => unreachable, + } + } + + pub fn writeAll(zig_got: ZigGotSection, elf_file: *Elf, writer: anytype) !void { + for (zig_got.entries.items) |entry| { + const symbol = elf_file.symbol(entry); + const value = symbol.address(.{ .plt = false }, elf_file); + try writeInt(value, elf_file, writer); + } + } + + pub fn numRela(zig_got: ZigGotSection) usize { + return zig_got.entries.items.len; + } + + pub fn addRela(zig_got: ZigGotSection, elf_file: *Elf) !void { + try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, zig_got.numRela()); + for (zig_got.entries.items) |entry| { + const symbol = elf_file.symbol(entry); + const offset = symbol.zigGotAddress(elf_file); + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .type = elf.R_X86_64_RELATIVE, + .addend = @intCast(symbol.address(.{ .plt = false }, elf_file)), + }); + } + } + + pub fn updateSymtabSize(zig_got: *ZigGotSection, elf_file: *Elf) void { + _ = elf_file; + zig_got.output_symtab_size.nlocals = @as(u32, @intCast(zig_got.entries.items.len)); + } + + pub fn updateStrtab(zig_got: ZigGotSection, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + for (zig_got.entries.items) |entry| { + const symbol_name = elf_file.symbol(entry).name(elf_file); + const name = try std.fmt.allocPrint(gpa, "{s}$ziggot", .{symbol_name}); + defer gpa.free(name); + _ = try elf_file.strtab.insert(gpa, name); + } + } + + pub fn writeSymtab(zig_got: ZigGotSection, elf_file: *Elf, ctx: anytype) !void { + const gpa = elf_file.base.allocator; + for (zig_got.entries.items, ctx.ilocal.., 0..) |entry, ilocal, index| { + const symbol = elf_file.symbol(entry); + const symbol_name = symbol.name(elf_file); + const name = try std.fmt.allocPrint(gpa, "{s}$ziggot", .{symbol_name}); + defer gpa.free(name); + const st_name = try elf_file.strtab.insert(gpa, name); + const st_value = zig_got.entryAddress(@intCast(index), elf_file); + const st_size = elf_file.archPtrWidthBytes(); + ctx.symtab[ilocal] = .{ + .st_name = st_name, + .st_info = elf.STT_OBJECT, + .st_other = 0, + .st_shndx = elf_file.zig_got_section_index.?, + .st_value = st_value, + .st_size = st_size, + }; + } + } + + const FormatCtx = struct { + zig_got: ZigGotSection, + elf_file: *Elf, + }; + + pub fn fmt(zig_got: ZigGotSection, elf_file: *Elf) std.fmt.Formatter(format2) { + return .{ .data = .{ .zig_got = zig_got, .elf_file = elf_file } }; + } + + pub fn format2( + ctx: FormatCtx, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + _ = unused_fmt_string; + try writer.writeAll(".zig.got\n"); + for (ctx.zig_got.entries.items, 0..) |entry, index| { + const symbol = ctx.elf_file.symbol(entry); + try writer.print(" {d}@0x{x} => {d}@0x{x} ({s})\n", .{ + index, + ctx.zig_got.entryAddress(@intCast(index), ctx.elf_file), + entry, + symbol.address(.{}, ctx.elf_file), + symbol.name(ctx.elf_file), + }); + } + } +}; + pub const GotSection = struct { entries: std.ArrayListUnmanaged(Entry) = .{}, - needs_rela: bool = false, - dirty: bool = false, output_symtab_size: Elf.SymtabSize = .{}, + tlsld_index: ?u32 = null, + flags: Flags = .{}, pub const Index = u32; + const Flags = packed struct { + needs_rela: bool = false, + needs_tlsld: bool = false, + }; + const Tag = enum { got, tlsld, @@ -48,7 +480,6 @@ pub const GotSection = struct { break :blk last.cell_index + @as(Index, @intCast(last.len())); } else 0; entry.* = .{ .tag = undefined, .symbol_index = undefined, .cell_index = cell_index }; - got.dirty = true; return index; } @@ -58,8 +489,12 @@ pub const GotSection = struct { entry.tag = .got; entry.symbol_index = sym_index; const symbol = elf_file.symbol(sym_index); - if (symbol.flags.import or symbol.isIFunc(elf_file) or (elf_file.base.options.pic and !symbol.isAbs(elf_file))) - got.needs_rela = true; + symbol.flags.has_got = true; + if (symbol.flags.import or symbol.isIFunc(elf_file) or + (elf_file.base.options.pic and !symbol.isAbs(elf_file))) + { + got.flags.needs_rela = true; + } if (symbol.extra(elf_file)) |extra| { var new_extra = extra; new_extra.got = index; @@ -68,44 +503,60 @@ pub const GotSection = struct { return index; } - // pub fn addTlsGdSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { - // const index = got.next_index; - // const symbol = elf_file.getSymbol(sym_index); - // if (symbol.flags.import or elf_file.options.output_mode == .lib) got.needs_rela = true; - // if (symbol.getExtra(elf_file)) |extra| { - // var new_extra = extra; - // new_extra.tlsgd = index; - // symbol.setExtra(new_extra, elf_file); - // } else try symbol.addExtra(.{ .tlsgd = index }, elf_file); - // try got.symbols.append(elf_file.base.allocator, .{ .tlsgd = sym_index }); - // got.next_index += 2; - // } - - // pub fn addGotTpSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { - // const index = got.next_index; - // const symbol = elf_file.getSymbol(sym_index); - // if (symbol.flags.import or elf_file.options.output_mode == .lib) got.needs_rela = true; - // if (symbol.getExtra(elf_file)) |extra| { - // var new_extra = extra; - // new_extra.gottp = index; - // symbol.setExtra(new_extra, elf_file); - // } else try symbol.addExtra(.{ .gottp = index }, elf_file); - // try got.symbols.append(elf_file.base.allocator, .{ .gottp = sym_index }); - // got.next_index += 1; - // } - - // pub fn addTlsDescSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { - // const index = got.next_index; - // const symbol = elf_file.getSymbol(sym_index); - // got.needs_rela = true; - // if (symbol.getExtra(elf_file)) |extra| { - // var new_extra = extra; - // new_extra.tlsdesc = index; - // symbol.setExtra(new_extra, elf_file); - // } else try symbol.addExtra(.{ .tlsdesc = index }, elf_file); - // try got.symbols.append(elf_file.base.allocator, .{ .tlsdesc = sym_index }); - // got.next_index += 2; - // } + pub fn addTlsLdSymbol(got: *GotSection, elf_file: *Elf) !void { + assert(got.flags.needs_tlsld); + const index = try got.allocateEntry(elf_file.base.allocator); + const entry = &got.entries.items[index]; + entry.tag = .tlsld; + entry.symbol_index = undefined; // unused + got.flags.needs_rela = true; + got.tlsld_index = index; + } + + pub fn addTlsGdSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = try got.allocateEntry(elf_file.base.allocator); + const entry = &got.entries.items[index]; + entry.tag = .tlsgd; + entry.symbol_index = sym_index; + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_tlsgd = true; + if (symbol.flags.import or elf_file.isDynLib()) got.flags.needs_rela = true; + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.tlsgd = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .tlsgd = index }, elf_file); + } + + pub fn addGotTpSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = try got.allocateEntry(elf_file.base.allocator); + const entry = &got.entries.items[index]; + entry.tag = .gottp; + entry.symbol_index = sym_index; + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_gottp = true; + if (symbol.flags.import or elf_file.isDynLib()) got.flags.needs_rela = true; + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.gottp = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .gottp = index }, elf_file); + } + + pub fn addTlsDescSymbol(got: *GotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = try got.allocateEntry(elf_file.base.allocator); + const entry = &got.entries.items[index]; + entry.tag = .tlsdesc; + entry.symbol_index = sym_index; + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_tlsdesc = true; + got.flags.needs_rela = true; + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.tlsdesc = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .tlsdesc = index }, elf_file); + } pub fn size(got: GotSection, elf_file: *Elf) usize { var s: usize = 0; @@ -115,285 +566,233 @@ pub const GotSection = struct { return s; } - pub fn writeEntry(got: *GotSection, elf_file: *Elf, index: Index) !void { - const entry_size: u16 = elf_file.archPtrWidthBytes(); - if (got.dirty) { - const needed_size = got.size(elf_file); - try elf_file.growAllocSection(elf_file.got_section_index.?, needed_size); - got.dirty = false; - } - const endian = elf_file.base.options.target.cpu.arch.endian(); - const entry = got.entries.items[index]; - const shdr = &elf_file.shdrs.items[elf_file.got_section_index.?]; - const off = shdr.sh_offset + @as(u64, entry_size) * entry.cell_index; - const vaddr = shdr.sh_addr + @as(u64, entry_size) * entry.cell_index; - const value = elf_file.symbol(entry.symbol_index).value; - switch (entry_size) { - 2 => { - var buf: [2]u8 = undefined; - std.mem.writeInt(u16, &buf, @as(u16, @intCast(value)), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 4 => { - var buf: [4]u8 = undefined; - std.mem.writeInt(u32, &buf, @as(u32, @intCast(value)), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 8 => { - var buf: [8]u8 = undefined; - std.mem.writeInt(u64, &buf, value, endian); - try elf_file.base.file.?.pwriteAll(&buf, off); + pub fn write(got: GotSection, elf_file: *Elf, writer: anytype) !void { + const is_dyn_lib = elf_file.isDynLib(); + const apply_relocs = true; // TODO add user option for this - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.os.iovec_const = .{.{ - .iov_base = &buf, - .iov_len = buf.len, - }}; - var remote_vec: [1]std.os.iovec_const = .{.{ - .iov_base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), - .iov_len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.errno(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, + for (got.entries.items) |entry| { + const symbol = switch (entry.tag) { + .tlsld => null, + inline else => elf_file.symbol(entry.symbol_index), + }; + switch (entry.tag) { + .got => { + const value = blk: { + const value = symbol.?.address(.{ .plt = false }, elf_file); + if (symbol.?.flags.import) break :blk 0; + if (symbol.?.isIFunc(elf_file)) + break :blk if (apply_relocs) value else 0; + if (elf_file.base.options.pic and !symbol.?.isAbs(elf_file)) + break :blk if (apply_relocs) value else 0; + break :blk value; + }; + try writeInt(value, elf_file, writer); + }, + .tlsld => { + try writeInt(if (is_dyn_lib) @as(u64, 0) else 1, elf_file, writer); + try writeInt(0, elf_file, writer); + }, + .tlsgd => { + if (symbol.?.flags.import) { + try writeInt(0, elf_file, writer); + try writeInt(0, elf_file, writer); + } else { + try writeInt(if (is_dyn_lib) @as(u64, 0) else 1, elf_file, writer); + const offset = symbol.?.address(.{}, elf_file) - elf_file.dtpAddress(); + try writeInt(offset, elf_file, writer); } - } - }, - else => unreachable, + }, + .gottp => { + if (symbol.?.flags.import) { + try writeInt(0, elf_file, writer); + } else if (is_dyn_lib) { + const offset = if (apply_relocs) + symbol.?.address(.{}, elf_file) - elf_file.tlsAddress() + else + 0; + try writeInt(offset, elf_file, writer); + } else { + const offset = @as(i64, @intCast(symbol.?.address(.{}, elf_file))) - + @as(i64, @intCast(elf_file.tpAddress())); + try writeInt(offset, elf_file, writer); + } + }, + .tlsdesc => { + try writeInt(0, elf_file, writer); + try writeInt(0, elf_file, writer); + }, + } } } - pub fn writeAllEntries(got: GotSection, elf_file: *Elf, writer: anytype) !void { - assert(!got.dirty); - const entry_size: u16 = elf_file.archPtrWidthBytes(); - const endian = elf_file.base.options.target.cpu.arch.endian(); + pub fn addRela(got: GotSection, elf_file: *Elf) !void { + const is_dyn_lib = elf_file.isDynLib(); + try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, got.numRela(elf_file)); + for (got.entries.items) |entry| { - const value = elf_file.symbol(entry.symbol_index).value; - switch (entry_size) { - 2 => try writer.writeInt(u16, @intCast(value), endian), - 4 => try writer.writeInt(u32, @intCast(value), endian), - 8 => try writer.writeInt(u64, @intCast(value), endian), - else => unreachable, + const symbol = switch (entry.tag) { + .tlsld => null, + inline else => elf_file.symbol(entry.symbol_index), + }; + const extra = if (symbol) |s| s.extra(elf_file).? else null; + + switch (entry.tag) { + .got => { + const offset = symbol.?.gotAddress(elf_file); + if (symbol.?.flags.import) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_GLOB_DAT, + }); + continue; + } + if (symbol.?.isIFunc(elf_file)) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .type = elf.R_X86_64_IRELATIVE, + .addend = @intCast(symbol.?.address(.{ .plt = false }, elf_file)), + }); + continue; + } + if (elf_file.base.options.pic and !symbol.?.isAbs(elf_file)) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .type = elf.R_X86_64_RELATIVE, + .addend = @intCast(symbol.?.address(.{ .plt = false }, elf_file)), + }); + } + }, + + .tlsld => { + if (is_dyn_lib) { + const offset = entry.address(elf_file); + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .type = elf.R_X86_64_DTPMOD64, + }); + } + }, + + .tlsgd => { + const offset = symbol.?.tlsGdAddress(elf_file); + if (symbol.?.flags.import) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_DTPMOD64, + }); + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset + 8, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_DTPOFF64, + }); + } else if (is_dyn_lib) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_DTPMOD64, + }); + } + }, + + .gottp => { + const offset = symbol.?.gotTpAddress(elf_file); + if (symbol.?.flags.import) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_TPOFF64, + }); + } else if (is_dyn_lib) { + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .type = elf.R_X86_64_TPOFF64, + .addend = @intCast(symbol.?.address(.{}, elf_file) - elf_file.tlsAddress()), + }); + } + }, + + .tlsdesc => { + const offset = symbol.?.tlsDescAddress(elf_file); + elf_file.addRelaDynAssumeCapacity(.{ + .offset = offset, + .sym = extra.?.dynamic, + .type = elf.R_X86_64_TLSDESC, + }); + }, } } } - // pub fn write(got: GotSection, elf_file: *Elf, writer: anytype) !void { - // const is_shared = elf_file.options.output_mode == .lib; - // const apply_relocs = elf_file.options.apply_dynamic_relocs; - - // for (got.symbols.items) |sym| { - // const symbol = elf_file.getSymbol(sym.getIndex()); - // switch (sym) { - // .got => { - // const value: u64 = blk: { - // const value = symbol.getAddress(.{ .plt = false }, elf_file); - // if (symbol.flags.import) break :blk 0; - // if (symbol.isIFunc(elf_file)) - // break :blk if (apply_relocs) value else 0; - // if (elf_file.options.pic and !symbol.isAbs(elf_file)) - // break :blk if (apply_relocs) value else 0; - // break :blk value; - // }; - // try writer.writeIntLittle(u64, value); - // }, - - // .tlsgd => { - // if (symbol.flags.import) { - // try writer.writeIntLittle(u64, 0); - // try writer.writeIntLittle(u64, 0); - // } else { - // try writer.writeIntLittle(u64, if (is_shared) @as(u64, 0) else 1); - // const offset = symbol.getAddress(.{}, elf_file) - elf_file.getDtpAddress(); - // try writer.writeIntLittle(u64, offset); - // } - // }, - - // .gottp => { - // if (symbol.flags.import) { - // try writer.writeIntLittle(u64, 0); - // } else if (is_shared) { - // const offset = if (apply_relocs) - // symbol.getAddress(.{}, elf_file) - elf_file.getTlsAddress() - // else - // 0; - // try writer.writeIntLittle(u64, offset); - // } else { - // const offset = @as(i64, @intCast(symbol.getAddress(.{}, elf_file))) - - // @as(i64, @intCast(elf_file.getTpAddress())); - // try writer.writeIntLittle(u64, @as(u64, @bitCast(offset))); - // } - // }, - - // .tlsdesc => { - // try writer.writeIntLittle(u64, 0); - // try writer.writeIntLittle(u64, 0); - // }, - // } - // } - - // if (got.emit_tlsld) { - // try writer.writeIntLittle(u64, if (is_shared) @as(u64, 0) else 1); - // try writer.writeIntLittle(u64, 0); - // } - // } - - // pub fn addRela(got: GotSection, elf_file: *Elf) !void { - // const is_shared = elf_file.options.output_mode == .lib; - // try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, got.numRela(elf_file)); - - // for (got.symbols.items) |sym| { - // const symbol = elf_file.getSymbol(sym.getIndex()); - // const extra = symbol.getExtra(elf_file).?; - - // switch (sym) { - // .got => { - // const offset = symbol.gotAddress(elf_file); - - // if (symbol.flags.import) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_GLOB_DAT, - // }); - // continue; - // } - - // if (symbol.isIFunc(elf_file)) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .type = elf.R_X86_64_IRELATIVE, - // .addend = @intCast(symbol.getAddress(.{ .plt = false }, elf_file)), - // }); - // continue; - // } - - // if (elf_file.options.pic and !symbol.isAbs(elf_file)) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .type = elf.R_X86_64_RELATIVE, - // .addend = @intCast(symbol.getAddress(.{ .plt = false }, elf_file)), - // }); - // } - // }, - - // .tlsgd => { - // const offset = symbol.getTlsGdAddress(elf_file); - // if (symbol.flags.import) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_DTPMOD64, - // }); - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset + 8, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_DTPOFF64, - // }); - // } else if (is_shared) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_DTPMOD64, - // }); - // } - // }, - - // .gottp => { - // const offset = symbol.getGotTpAddress(elf_file); - // if (symbol.flags.import) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_TPOFF64, - // }); - // } else if (is_shared) { - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .type = elf.R_X86_64_TPOFF64, - // .addend = @intCast(symbol.getAddress(.{}, elf_file) - elf_file.getTlsAddress()), - // }); - // } - // }, - - // .tlsdesc => { - // const offset = symbol.getTlsDescAddress(elf_file); - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .sym = extra.dynamic, - // .type = elf.R_X86_64_TLSDESC, - // }); - // }, - // } - // } - - // if (is_shared and got.emit_tlsld) { - // const offset = elf_file.getTlsLdAddress(); - // elf_file.addRelaDynAssumeCapacity(.{ - // .offset = offset, - // .type = elf.R_X86_64_DTPMOD64, - // }); - // } - // } - - // pub fn numRela(got: GotSection, elf_file: *Elf) usize { - // const is_shared = elf_file.options.output_mode == .lib; - // var num: usize = 0; - // for (got.symbols.items) |sym| { - // const symbol = elf_file.symbol(sym.index()); - // switch (sym) { - // .got => if (symbol.flags.import or - // symbol.isIFunc(elf_file) or (elf_file.options.pic and !symbol.isAbs(elf_file))) - // { - // num += 1; - // }, - - // .tlsgd => if (symbol.flags.import) { - // num += 2; - // } else if (is_shared) { - // num += 1; - // }, - - // .gottp => if (symbol.flags.import or is_shared) { - // num += 1; - // }, - - // .tlsdesc => num += 1, - // } - // } - // if (is_shared and got.emit_tlsld) num += 1; - // return num; - // } + pub fn numRela(got: GotSection, elf_file: *Elf) usize { + const is_dyn_lib = elf_file.isDynLib(); + var num: usize = 0; + for (got.entries.items) |entry| { + const symbol = switch (entry.tag) { + .tlsld => null, + inline else => elf_file.symbol(entry.symbol_index), + }; + switch (entry.tag) { + .got => if (symbol.?.flags.import or + symbol.?.isIFunc(elf_file) or (elf_file.base.options.pic and !symbol.?.isAbs(elf_file))) + { + num += 1; + }, + + .tlsld => if (is_dyn_lib) { + num += 1; + }, + + .tlsgd => if (symbol.?.flags.import) { + num += 2; + } else if (is_dyn_lib) { + num += 1; + }, + + .gottp => if (symbol.?.flags.import or is_dyn_lib) { + num += 1; + }, + + .tlsdesc => num += 1, + } + } + return num; + } pub fn updateSymtabSize(got: *GotSection, elf_file: *Elf) void { _ = elf_file; got.output_symtab_size.nlocals = @as(u32, @intCast(got.entries.items.len)); } + pub fn updateStrtab(got: GotSection, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + for (got.entries.items) |entry| { + const symbol_name = switch (entry.tag) { + .tlsld => "", + inline else => elf_file.symbol(entry.symbol_index).name(elf_file), + }; + const name = try std.fmt.allocPrint(gpa, "{s}${s}", .{ symbol_name, @tagName(entry.tag) }); + defer gpa.free(name); + _ = try elf_file.strtab.insert(gpa, name); + } + } + pub fn writeSymtab(got: GotSection, elf_file: *Elf, ctx: anytype) !void { const gpa = elf_file.base.allocator; for (got.entries.items, ctx.ilocal..) |entry, ilocal| { - const suffix = switch (entry.tag) { - .tlsld => "$tlsld", - .tlsgd => "$tlsgd", - .got => "$got", - .gottp => "$gottp", - .tlsdesc => "$tlsdesc", + const symbol = switch (entry.tag) { + .tlsld => null, + inline else => elf_file.symbol(entry.symbol_index), }; - const symbol = elf_file.symbol(entry.symbol_index); - const name = try std.fmt.allocPrint(gpa, "{s}{s}", .{ symbol.name(elf_file), suffix }); + const symbol_name = switch (entry.tag) { + .tlsld => "", + inline else => symbol.?.name(elf_file), + }; + const name = try std.fmt.allocPrint(gpa, "{s}${s}", .{ symbol_name, @tagName(entry.tag) }); defer gpa.free(name); const st_name = try elf_file.strtab.insert(gpa, name); - const st_value = switch (entry.tag) { - .got => symbol.gotAddress(elf_file), - else => unreachable, - }; + const st_value = entry.address(elf_file); const st_size: u64 = entry.len() * elf_file.archPtrWidthBytes(); ctx.symtab[ilocal] = .{ .st_name = st_name, @@ -437,12 +836,711 @@ pub const GotSection = struct { } }; +pub const PltSection = struct { + symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, + output_symtab_size: Elf.SymtabSize = .{}, + + pub const preamble_size = 32; + + pub fn deinit(plt: *PltSection, allocator: Allocator) void { + plt.symbols.deinit(allocator); + } + + pub fn addSymbol(plt: *PltSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = @as(u32, @intCast(plt.symbols.items.len)); + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_plt = true; + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.plt = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .plt = index }, elf_file); + try plt.symbols.append(elf_file.base.allocator, sym_index); + } + + pub fn size(plt: PltSection) usize { + return preamble_size + plt.symbols.items.len * 16; + } + + pub fn write(plt: PltSection, elf_file: *Elf, writer: anytype) !void { + const plt_addr = elf_file.shdrs.items[elf_file.plt_section_index.?].sh_addr; + const got_plt_addr = elf_file.shdrs.items[elf_file.got_plt_section_index.?].sh_addr; + var preamble = [_]u8{ + 0xf3, 0x0f, 0x1e, 0xfa, // endbr64 + 0x41, 0x53, // push r11 + 0xff, 0x35, 0x00, 0x00, 0x00, 0x00, // push qword ptr [rip] -> .got.plt[1] + 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [rip] -> .got.plt[2] + }; + var disp = @as(i64, @intCast(got_plt_addr + 8)) - @as(i64, @intCast(plt_addr + 8)) - 4; + mem.writeIntLittle(i32, preamble[8..][0..4], @as(i32, @intCast(disp))); + disp = @as(i64, @intCast(got_plt_addr + 16)) - @as(i64, @intCast(plt_addr + 14)) - 4; + mem.writeIntLittle(i32, preamble[14..][0..4], @as(i32, @intCast(disp))); + try writer.writeAll(&preamble); + try writer.writeByteNTimes(0xcc, preamble_size - preamble.len); + + for (plt.symbols.items, 0..) |sym_index, i| { + const sym = elf_file.symbol(sym_index); + const target_addr = sym.gotPltAddress(elf_file); + const source_addr = sym.pltAddress(elf_file); + disp = @as(i64, @intCast(target_addr)) - @as(i64, @intCast(source_addr + 12)) - 4; + var entry = [_]u8{ + 0xf3, 0x0f, 0x1e, 0xfa, // endbr64 + 0x41, 0xbb, 0x00, 0x00, 0x00, 0x00, // mov r11d, N + 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [rip] -> .got.plt[N] + }; + mem.writeIntLittle(i32, entry[6..][0..4], @as(i32, @intCast(i))); + mem.writeIntLittle(i32, entry[12..][0..4], @as(i32, @intCast(disp))); + try writer.writeAll(&entry); + } + } + + pub fn addRela(plt: PltSection, elf_file: *Elf) !void { + try elf_file.rela_plt.ensureUnusedCapacity(elf_file.base.allocator, plt.numRela()); + for (plt.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + assert(sym.flags.import); + const extra = sym.extra(elf_file).?; + const r_offset = sym.gotPltAddress(elf_file); + const r_sym: u64 = extra.dynamic; + const r_type: u32 = elf.R_X86_64_JUMP_SLOT; + elf_file.rela_plt.appendAssumeCapacity(.{ + .r_offset = r_offset, + .r_info = (r_sym << 32) | r_type, + .r_addend = 0, + }); + } + } + + pub fn numRela(plt: PltSection) usize { + return plt.symbols.items.len; + } + + pub fn updateSymtabSize(plt: *PltSection, elf_file: *Elf) void { + _ = elf_file; + plt.output_symtab_size.nlocals = @as(u32, @intCast(plt.symbols.items.len)); + } + + pub fn updateStrtab(plt: PltSection, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + for (plt.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + const name = try std.fmt.allocPrint(gpa, "{s}$plt", .{sym.name(elf_file)}); + defer gpa.free(name); + _ = try elf_file.strtab.insert(gpa, name); + } + } + + pub fn writeSymtab(plt: PltSection, elf_file: *Elf, ctx: anytype) !void { + const gpa = elf_file.base.allocator; + + var ilocal = ctx.ilocal; + for (plt.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + const name = try std.fmt.allocPrint(gpa, "{s}$plt", .{sym.name(elf_file)}); + defer gpa.free(name); + const st_name = try elf_file.strtab.insert(gpa, name); + ctx.symtab[ilocal] = .{ + .st_name = st_name, + .st_info = elf.STT_FUNC, + .st_other = 0, + .st_shndx = elf_file.plt_section_index.?, + .st_value = sym.pltAddress(elf_file), + .st_size = 16, + }; + ilocal += 1; + } + } +}; + +pub const GotPltSection = struct { + pub const preamble_size = 24; + + pub fn size(got_plt: GotPltSection, elf_file: *Elf) usize { + _ = got_plt; + return preamble_size + elf_file.plt.symbols.items.len * 8; + } + + pub fn write(got_plt: GotPltSection, elf_file: *Elf, writer: anytype) !void { + _ = got_plt; + { + // [0]: _DYNAMIC + const symbol = elf_file.symbol(elf_file.dynamic_index.?); + try writer.writeIntLittle(u64, symbol.value); + } + // [1]: 0x0 + // [2]: 0x0 + try writer.writeIntLittle(u64, 0x0); + try writer.writeIntLittle(u64, 0x0); + if (elf_file.plt_section_index) |shndx| { + const plt_addr = elf_file.shdrs.items[shndx].sh_addr; + for (0..elf_file.plt.symbols.items.len) |_| { + // [N]: .plt + try writer.writeIntLittle(u64, plt_addr); + } + } + } +}; + +pub const PltGotSection = struct { + symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, + output_symtab_size: Elf.SymtabSize = .{}, + + pub fn deinit(plt_got: *PltGotSection, allocator: Allocator) void { + plt_got.symbols.deinit(allocator); + } + + pub fn addSymbol(plt_got: *PltGotSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = @as(u32, @intCast(plt_got.symbols.items.len)); + const symbol = elf_file.symbol(sym_index); + symbol.flags.has_plt = true; + symbol.flags.has_got = true; + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.plt_got = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .plt_got = index }, elf_file); + try plt_got.symbols.append(elf_file.base.allocator, sym_index); + } + + pub fn size(plt_got: PltGotSection) usize { + return plt_got.symbols.items.len * 16; + } + + pub fn write(plt_got: PltGotSection, elf_file: *Elf, writer: anytype) !void { + for (plt_got.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + const target_addr = sym.gotAddress(elf_file); + const source_addr = sym.pltGotAddress(elf_file); + const disp = @as(i64, @intCast(target_addr)) - @as(i64, @intCast(source_addr + 6)) - 4; + var entry = [_]u8{ + 0xf3, 0x0f, 0x1e, 0xfa, // endbr64 + 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [rip] -> .got[N] + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + }; + mem.writeIntLittle(i32, entry[6..][0..4], @as(i32, @intCast(disp))); + try writer.writeAll(&entry); + } + } + + pub fn updateSymtabSize(plt_got: *PltGotSection, elf_file: *Elf) void { + _ = elf_file; + plt_got.output_symtab_size.nlocals = @as(u32, @intCast(plt_got.symbols.items.len)); + } + + pub fn updateStrtab(plt_got: PltGotSection, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + for (plt_got.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + const name = try std.fmt.allocPrint(gpa, "{s}$pltgot", .{sym.name(elf_file)}); + defer gpa.free(name); + _ = try elf_file.strtab.insert(gpa, name); + } + } + + pub fn writeSymtab(plt_got: PltGotSection, elf_file: *Elf, ctx: anytype) !void { + const gpa = elf_file.base.allocator; + var ilocal = ctx.ilocal; + for (plt_got.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + const name = try std.fmt.allocPrint(gpa, "{s}$pltgot", .{sym.name(elf_file)}); + defer gpa.free(name); + const st_name = try elf_file.strtab.insert(gpa, name); + ctx.symtab[ilocal] = .{ + .st_name = st_name, + .st_info = elf.STT_FUNC, + .st_other = 0, + .st_shndx = elf_file.plt_got_section_index.?, + .st_value = sym.pltGotAddress(elf_file), + .st_size = 16, + }; + ilocal += 1; + } + } +}; + +pub const CopyRelSection = struct { + symbols: std.ArrayListUnmanaged(Symbol.Index) = .{}, + + pub fn deinit(copy_rel: *CopyRelSection, allocator: Allocator) void { + copy_rel.symbols.deinit(allocator); + } + + pub fn addSymbol(copy_rel: *CopyRelSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const index = @as(u32, @intCast(copy_rel.symbols.items.len)); + const symbol = elf_file.symbol(sym_index); + symbol.flags.import = true; + symbol.flags.@"export" = true; + symbol.flags.has_copy_rel = true; + symbol.flags.weak = false; + + if (symbol.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.copy_rel = index; + symbol.setExtra(new_extra, elf_file); + } else try symbol.addExtra(.{ .copy_rel = index }, elf_file); + try copy_rel.symbols.append(elf_file.base.allocator, sym_index); + + const shared_object = symbol.file(elf_file).?.shared_object; + if (shared_object.aliases == null) { + try shared_object.initSymbolAliases(elf_file); + } + + const aliases = shared_object.symbolAliases(sym_index, elf_file); + for (aliases) |alias| { + if (alias == sym_index) continue; + const alias_sym = elf_file.symbol(alias); + alias_sym.flags.import = true; + alias_sym.flags.@"export" = true; + alias_sym.flags.has_copy_rel = true; + alias_sym.flags.needs_copy_rel = true; + alias_sym.flags.weak = false; + try elf_file.dynsym.addSymbol(alias, elf_file); + } + } + + pub fn updateSectionSize(copy_rel: CopyRelSection, shndx: u16, elf_file: *Elf) !void { + const shdr = &elf_file.shdrs.items[shndx]; + for (copy_rel.symbols.items) |sym_index| { + const symbol = elf_file.symbol(sym_index); + const shared_object = symbol.file(elf_file).?.shared_object; + const alignment = try symbol.dsoAlignment(elf_file); + symbol.value = mem.alignForward(u64, shdr.sh_size, alignment); + shdr.sh_addralign = @max(shdr.sh_addralign, alignment); + shdr.sh_size = symbol.value + symbol.elfSym(elf_file).st_size; + + const aliases = shared_object.symbolAliases(sym_index, elf_file); + for (aliases) |alias| { + if (alias == sym_index) continue; + const alias_sym = elf_file.symbol(alias); + alias_sym.value = symbol.value; + } + } + } + + pub fn addRela(copy_rel: CopyRelSection, elf_file: *Elf) !void { + try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, copy_rel.numRela()); + for (copy_rel.symbols.items) |sym_index| { + const sym = elf_file.symbol(sym_index); + assert(sym.flags.import and sym.flags.has_copy_rel); + const extra = sym.extra(elf_file).?; + elf_file.addRelaDynAssumeCapacity(.{ + .offset = sym.address(.{}, elf_file), + .sym = extra.dynamic, + .type = elf.R_X86_64_COPY, + }); + } + } + + pub fn numRela(copy_rel: CopyRelSection) usize { + return copy_rel.symbols.items.len; + } +}; + +pub const DynsymSection = struct { + entries: std.ArrayListUnmanaged(Entry) = .{}, + + pub const Entry = struct { + /// Index of the symbol which gets privilege of getting a dynamic treatment + symbol_index: Symbol.Index, + /// Offset into .dynstrtab + off: u32, + }; + + pub fn deinit(dynsym: *DynsymSection, allocator: Allocator) void { + dynsym.entries.deinit(allocator); + } + + pub fn addSymbol(dynsym: *DynsymSection, sym_index: Symbol.Index, elf_file: *Elf) !void { + const gpa = elf_file.base.allocator; + const index = @as(u32, @intCast(dynsym.entries.items.len + 1)); + const sym = elf_file.symbol(sym_index); + sym.flags.has_dynamic = true; + if (sym.extra(elf_file)) |extra| { + var new_extra = extra; + new_extra.dynamic = index; + sym.setExtra(new_extra, elf_file); + } else try sym.addExtra(.{ .dynamic = index }, elf_file); + const off = try elf_file.dynstrtab.insert(gpa, sym.name(elf_file)); + try dynsym.entries.append(gpa, .{ .symbol_index = sym_index, .off = off }); + } + + pub fn sort(dynsym: *DynsymSection, elf_file: *Elf) void { + const Sort = struct { + pub fn lessThan(ctx: *Elf, lhs: Entry, rhs: Entry) bool { + const lhs_sym = ctx.symbol(lhs.symbol_index); + const rhs_sym = ctx.symbol(rhs.symbol_index); + + if (lhs_sym.flags.@"export" != rhs_sym.flags.@"export") { + return rhs_sym.flags.@"export"; + } + + // TODO cache hash values + const nbuckets = ctx.gnu_hash.num_buckets; + const lhs_hash = GnuHashSection.hasher(lhs_sym.name(ctx)) % nbuckets; + const rhs_hash = GnuHashSection.hasher(rhs_sym.name(ctx)) % nbuckets; + + if (lhs_hash == rhs_hash) + return lhs_sym.extra(ctx).?.dynamic < rhs_sym.extra(ctx).?.dynamic; + return lhs_hash < rhs_hash; + } + }; + + var num_exports: u32 = 0; + for (dynsym.entries.items) |entry| { + const sym = elf_file.symbol(entry.symbol_index); + if (sym.flags.@"export") num_exports += 1; + } + + elf_file.gnu_hash.num_buckets = @divTrunc(num_exports, GnuHashSection.load_factor) + 1; + + std.mem.sort(Entry, dynsym.entries.items, elf_file, Sort.lessThan); + + for (dynsym.entries.items, 1..) |entry, index| { + const sym = elf_file.symbol(entry.symbol_index); + var extra = sym.extra(elf_file).?; + extra.dynamic = @as(u32, @intCast(index)); + sym.setExtra(extra, elf_file); + } + } + + pub fn size(dynsym: DynsymSection) usize { + return dynsym.count() * @sizeOf(elf.Elf64_Sym); + } + + pub fn count(dynsym: DynsymSection) u32 { + return @as(u32, @intCast(dynsym.entries.items.len + 1)); + } + + pub fn write(dynsym: DynsymSection, elf_file: *Elf, writer: anytype) !void { + try writer.writeStruct(Elf.null_sym); + for (dynsym.entries.items) |entry| { + const sym = elf_file.symbol(entry.symbol_index); + var out_sym: elf.Elf64_Sym = Elf.null_sym; + sym.setOutputSym(elf_file, &out_sym); + out_sym.st_name = entry.off; + try writer.writeStruct(out_sym); + } + } +}; + +pub const HashSection = struct { + buffer: std.ArrayListUnmanaged(u8) = .{}, + + pub fn deinit(hs: *HashSection, allocator: Allocator) void { + hs.buffer.deinit(allocator); + } + + pub fn generate(hs: *HashSection, elf_file: *Elf) !void { + if (elf_file.dynsym.count() == 1) return; + + const gpa = elf_file.base.allocator; + const nsyms = elf_file.dynsym.count(); + + var buckets = try gpa.alloc(u32, nsyms); + defer gpa.free(buckets); + @memset(buckets, 0); + + var chains = try gpa.alloc(u32, nsyms); + defer gpa.free(chains); + @memset(chains, 0); + + for (elf_file.dynsym.entries.items, 1..) |entry, i| { + const name = elf_file.dynstrtab.getAssumeExists(entry.off); + const hash = hasher(name) % buckets.len; + chains[@as(u32, @intCast(i))] = buckets[hash]; + buckets[hash] = @as(u32, @intCast(i)); + } + + try hs.buffer.ensureTotalCapacityPrecise(gpa, (2 + nsyms * 2) * 4); + hs.buffer.writer(gpa).writeIntLittle(u32, @as(u32, @intCast(nsyms))) catch unreachable; + hs.buffer.writer(gpa).writeIntLittle(u32, @as(u32, @intCast(nsyms))) catch unreachable; + hs.buffer.writer(gpa).writeAll(mem.sliceAsBytes(buckets)) catch unreachable; + hs.buffer.writer(gpa).writeAll(mem.sliceAsBytes(chains)) catch unreachable; + } + + pub inline fn size(hs: HashSection) usize { + return hs.buffer.items.len; + } + + pub fn hasher(name: [:0]const u8) u32 { + var h: u32 = 0; + var g: u32 = 0; + for (name) |c| { + h = (h << 4) + c; + g = h & 0xf0000000; + if (g > 0) h ^= g >> 24; + h &= ~g; + } + return h; + } +}; + +pub const GnuHashSection = struct { + num_buckets: u32 = 0, + num_bloom: u32 = 1, + num_exports: u32 = 0, + + pub const load_factor = 8; + pub const header_size = 16; + pub const bloom_shift = 26; + + fn getExports(elf_file: *Elf) []const DynsymSection.Entry { + const start = for (elf_file.dynsym.entries.items, 0..) |entry, i| { + const sym = elf_file.symbol(entry.symbol_index); + if (sym.flags.@"export") break i; + } else elf_file.dynsym.entries.items.len; + return elf_file.dynsym.entries.items[start..]; + } + + inline fn bitCeil(x: u64) u64 { + if (@popCount(x) == 1) return x; + return @as(u64, @intCast(@as(u128, 1) << (64 - @clz(x)))); + } + + pub fn calcSize(hash: *GnuHashSection, elf_file: *Elf) !void { + hash.num_exports = @as(u32, @intCast(getExports(elf_file).len)); + if (hash.num_exports > 0) { + const num_bits = hash.num_exports * 12; + hash.num_bloom = @as(u32, @intCast(bitCeil(@divTrunc(num_bits, 64)))); + } + } + + pub fn size(hash: GnuHashSection) usize { + return header_size + hash.num_bloom * 8 + hash.num_buckets * 4 + hash.num_exports * 4; + } + + pub fn write(hash: GnuHashSection, elf_file: *Elf, writer: anytype) !void { + const exports = getExports(elf_file); + const export_off = elf_file.dynsym.count() - hash.num_exports; + + var counting = std.io.countingWriter(writer); + const cwriter = counting.writer(); + + try cwriter.writeIntLittle(u32, hash.num_buckets); + try cwriter.writeIntLittle(u32, export_off); + try cwriter.writeIntLittle(u32, hash.num_bloom); + try cwriter.writeIntLittle(u32, bloom_shift); + + const gpa = elf_file.base.allocator; + const hashes = try gpa.alloc(u32, exports.len); + defer gpa.free(hashes); + const indices = try gpa.alloc(u32, exports.len); + defer gpa.free(indices); + + // Compose and write the bloom filter + const bloom = try gpa.alloc(u64, hash.num_bloom); + defer gpa.free(bloom); + @memset(bloom, 0); + + for (exports, 0..) |entry, i| { + const sym = elf_file.symbol(entry.symbol_index); + const h = hasher(sym.name(elf_file)); + hashes[i] = h; + indices[i] = h % hash.num_buckets; + const idx = @divTrunc(h, 64) % hash.num_bloom; + bloom[idx] |= @as(u64, 1) << @as(u6, @intCast(h % 64)); + bloom[idx] |= @as(u64, 1) << @as(u6, @intCast((h >> bloom_shift) % 64)); + } + + try cwriter.writeAll(mem.sliceAsBytes(bloom)); + + // Fill in the hash bucket indices + const buckets = try gpa.alloc(u32, hash.num_buckets); + defer gpa.free(buckets); + @memset(buckets, 0); + + for (0..hash.num_exports) |i| { + if (buckets[indices[i]] == 0) { + buckets[indices[i]] = @as(u32, @intCast(i + export_off)); + } + } + + try cwriter.writeAll(mem.sliceAsBytes(buckets)); + + // Finally, write the hash table + const table = try gpa.alloc(u32, hash.num_exports); + defer gpa.free(table); + @memset(table, 0); + + for (0..hash.num_exports) |i| { + const h = hashes[i]; + if (i == exports.len - 1 or indices[i] != indices[i + 1]) { + table[i] = h | 1; + } else { + table[i] = h & ~@as(u32, 1); + } + } + + try cwriter.writeAll(mem.sliceAsBytes(table)); + + assert(counting.bytes_written == hash.size()); + } + + pub fn hasher(name: [:0]const u8) u32 { + var h: u32 = 5381; + for (name) |c| { + h = (h << 5) +% h +% c; + } + return h; + } +}; + +pub const VerneedSection = struct { + verneed: std.ArrayListUnmanaged(elf.Elf64_Verneed) = .{}, + vernaux: std.ArrayListUnmanaged(elf.Elf64_Vernaux) = .{}, + index: elf.Elf64_Versym = elf.VER_NDX_GLOBAL + 1, + + pub fn deinit(vern: *VerneedSection, allocator: Allocator) void { + vern.verneed.deinit(allocator); + vern.vernaux.deinit(allocator); + } + + pub fn generate(vern: *VerneedSection, elf_file: *Elf) !void { + const dynsyms = elf_file.dynsym.entries.items; + var versyms = elf_file.versym.items; + + const VersionedSymbol = struct { + /// Index in the output version table + index: usize, + /// Index of the defining this symbol version shared object file + shared_object: File.Index, + /// Version index + version_index: elf.Elf64_Versym, + + fn soname(this: @This(), ctx: *Elf) []const u8 { + const shared_object = ctx.file(this.shared_object).?.shared_object; + return shared_object.soname(); + } + + fn versionString(this: @This(), ctx: *Elf) [:0]const u8 { + const shared_object = ctx.file(this.shared_object).?.shared_object; + return shared_object.versionString(this.version_index); + } + + pub fn lessThan(ctx: *Elf, lhs: @This(), rhs: @This()) bool { + if (lhs.shared_object == rhs.shared_object) return lhs.version_index < rhs.version_index; + return mem.lessThan(u8, lhs.soname(ctx), rhs.soname(ctx)); + } + }; + + const gpa = elf_file.base.allocator; + var verneed = std.ArrayList(VersionedSymbol).init(gpa); + defer verneed.deinit(); + try verneed.ensureTotalCapacity(dynsyms.len); + + for (dynsyms, 1..) |entry, i| { + const symbol = elf_file.symbol(entry.symbol_index); + if (symbol.flags.import and symbol.version_index & elf.VERSYM_VERSION > elf.VER_NDX_GLOBAL) { + const shared_object = symbol.file(elf_file).?.shared_object; + verneed.appendAssumeCapacity(.{ + .index = i, + .shared_object = shared_object.index, + .version_index = symbol.version_index, + }); + } + } + + mem.sort(VersionedSymbol, verneed.items, elf_file, VersionedSymbol.lessThan); + + var last = verneed.items[0]; + var last_verneed = try vern.addVerneed(last.soname(elf_file), elf_file); + var last_vernaux = try vern.addVernaux(last_verneed, last.versionString(elf_file), elf_file); + versyms[last.index] = last_vernaux.vna_other; + + for (verneed.items[1..]) |ver| { + if (ver.shared_object == last.shared_object) { + if (ver.version_index != last.version_index) { + last_vernaux = try vern.addVernaux(last_verneed, ver.versionString(elf_file), elf_file); + } + } else { + last_verneed = try vern.addVerneed(ver.soname(elf_file), elf_file); + last_vernaux = try vern.addVernaux(last_verneed, ver.versionString(elf_file), elf_file); + } + last = ver; + versyms[ver.index] = last_vernaux.vna_other; + } + + // Fixup offsets + var count: usize = 0; + var verneed_off: u32 = 0; + var vernaux_off: u32 = @as(u32, @intCast(vern.verneed.items.len)) * @sizeOf(elf.Elf64_Verneed); + for (vern.verneed.items, 0..) |*vsym, vsym_i| { + if (vsym_i < vern.verneed.items.len - 1) vsym.vn_next = @sizeOf(elf.Elf64_Verneed); + vsym.vn_aux = vernaux_off - verneed_off; + var inner_off: u32 = 0; + for (vern.vernaux.items[count..][0..vsym.vn_cnt], 0..) |*vaux, vaux_i| { + if (vaux_i < vsym.vn_cnt - 1) vaux.vna_next = @sizeOf(elf.Elf64_Vernaux); + inner_off += @sizeOf(elf.Elf64_Vernaux); + } + vernaux_off += inner_off; + verneed_off += @sizeOf(elf.Elf64_Verneed); + count += vsym.vn_cnt; + } + } + + fn addVerneed(vern: *VerneedSection, soname: []const u8, elf_file: *Elf) !*elf.Elf64_Verneed { + const gpa = elf_file.base.allocator; + const sym = try vern.verneed.addOne(gpa); + sym.* = .{ + .vn_version = 1, + .vn_cnt = 0, + .vn_file = try elf_file.dynstrtab.insert(gpa, soname), + .vn_aux = 0, + .vn_next = 0, + }; + return sym; + } + + fn addVernaux( + vern: *VerneedSection, + verneed_sym: *elf.Elf64_Verneed, + version: [:0]const u8, + elf_file: *Elf, + ) !elf.Elf64_Vernaux { + const gpa = elf_file.base.allocator; + const sym = try vern.vernaux.addOne(gpa); + sym.* = .{ + .vna_hash = HashSection.hasher(version), + .vna_flags = 0, + .vna_other = vern.index, + .vna_name = try elf_file.dynstrtab.insert(gpa, version), + .vna_next = 0, + }; + verneed_sym.vn_cnt += 1; + vern.index += 1; + return sym.*; + } + + pub fn size(vern: VerneedSection) usize { + return vern.verneed.items.len * @sizeOf(elf.Elf64_Verneed) + vern.vernaux.items.len * @sizeOf(elf.Elf64_Vernaux); + } + + pub fn write(vern: VerneedSection, writer: anytype) !void { + try writer.writeAll(mem.sliceAsBytes(vern.verneed.items)); + try writer.writeAll(mem.sliceAsBytes(vern.vernaux.items)); + } +}; + +fn writeInt(value: anytype, elf_file: *Elf, writer: anytype) !void { + const entry_size = elf_file.archPtrWidthBytes(); + const endian = elf_file.base.options.target.cpu.arch.endian(); + switch (entry_size) { + 2 => try writer.writeInt(u16, @intCast(value), endian), + 4 => try writer.writeInt(u32, @intCast(value), endian), + 8 => try writer.writeInt(u64, @intCast(value), endian), + else => unreachable, + } +} + const assert = std.debug.assert; const builtin = @import("builtin"); const elf = std.elf; +const mem = std.mem; const log = std.log.scoped(.link); const std = @import("std"); const Allocator = std.mem.Allocator; const Elf = @import("../Elf.zig"); +const File = @import("file.zig").File; +const SharedObject = @import("SharedObject.zig"); const Symbol = @import("Symbol.zig"); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index f54a370a26..425dffd6b1 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -707,6 +707,17 @@ fn accessLibPath( return true; } + noextension: { + test_path.clearRetainingCapacity(); + try test_path.writer().print("{s}" ++ sep ++ "{s}", .{ search_dir, lib_name }); + try checked_paths.append(try gpa.dupe(u8, test_path.items)); + fs.cwd().access(test_path.items, .{}) catch |err| switch (err) { + error.FileNotFound => break :noextension, + else => |e| return e, + }; + return true; + } + return false; } diff --git a/src/link/SpirV.zig b/src/link/SpirV.zig index d3906a13d7..325bb30fe0 100644 --- a/src/link/SpirV.zig +++ b/src/link/SpirV.zig @@ -45,9 +45,7 @@ const IdResult = spec.IdResult; base: link.File, -spv: SpvModule, -spv_arena: ArenaAllocator, -decl_link: codegen.DeclLinkMap, +object: codegen.Object, pub fn createEmpty(gpa: Allocator, options: link.Options) !*SpirV { const self = try gpa.create(SpirV); @@ -58,11 +56,8 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*SpirV { .file = null, .allocator = gpa, }, - .spv = undefined, - .spv_arena = ArenaAllocator.init(gpa), - .decl_link = codegen.DeclLinkMap.init(self.base.allocator), + .object = codegen.Object.init(gpa), }; - self.spv = SpvModule.init(gpa, self.spv_arena.allocator()); errdefer self.deinit(); // TODO: Figure out where to put all of these @@ -99,9 +94,7 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option } pub fn deinit(self: *SpirV) void { - self.spv.deinit(); - self.spv_arena.deinit(); - self.decl_link.deinit(); + self.object.deinit(); } pub fn updateFunc(self: *SpirV, module: *Module, func_index: InternPool.Index, air: Air, liveness: Liveness) !void { @@ -113,12 +106,7 @@ pub fn updateFunc(self: *SpirV, module: *Module, func_index: InternPool.Index, a const decl = module.declPtr(func.owner_decl); log.debug("lowering function {s}", .{module.intern_pool.stringToSlice(decl.name)}); - var decl_gen = codegen.DeclGen.init(self.base.allocator, module, &self.spv, &self.decl_link); - defer decl_gen.deinit(); - - if (try decl_gen.gen(func.owner_decl, air, liveness)) |msg| { - try module.failed_decls.put(module.gpa, func.owner_decl, msg); - } + try self.object.updateFunc(module, func_index, air, liveness); } pub fn updateDecl(self: *SpirV, module: *Module, decl_index: Module.Decl.Index) !void { @@ -129,12 +117,7 @@ pub fn updateDecl(self: *SpirV, module: *Module, decl_index: Module.Decl.Index) const decl = module.declPtr(decl_index); log.debug("lowering declaration {s}", .{module.intern_pool.stringToSlice(decl.name)}); - var decl_gen = codegen.DeclGen.init(self.base.allocator, module, &self.spv, &self.decl_link); - defer decl_gen.deinit(); - - if (try decl_gen.gen(decl_index, undefined, undefined)) |msg| { - try module.failed_decls.put(module.gpa, decl_index, msg); - } + try self.object.updateDecl(module, decl_index); } pub fn updateDeclExports( @@ -145,15 +128,9 @@ pub fn updateDeclExports( ) !void { const decl = mod.declPtr(decl_index); if (decl.val.isFuncBody(mod) and decl.ty.fnCallingConvention(mod) == .Kernel) { - // TODO: Unify with resolveDecl in spirv.zig. - const entry = try self.decl_link.getOrPut(decl_index); - if (!entry.found_existing) { - entry.value_ptr.* = try self.spv.allocDecl(.func); - } - const spv_decl_index = entry.value_ptr.*; - + const spv_decl_index = try self.object.resolveDecl(mod, decl_index); for (exports) |exp| { - try self.spv.declareEntryPoint(spv_decl_index, mod.intern_pool.stringToSlice(exp.opts.name)); + try self.object.spv.declareEntryPoint(spv_decl_index, mod.intern_pool.stringToSlice(exp.opts.name)); } } @@ -185,15 +162,19 @@ pub fn flushModule(self: *SpirV, comp: *Compilation, prog_node: *std.Progress.No sub_prog_node.activate(); defer sub_prog_node.end(); + const spv = &self.object.spv; + const target = comp.getTarget(); - try writeCapabilities(&self.spv, target); - try writeMemoryModel(&self.spv, target); + try writeCapabilities(spv, target); + try writeMemoryModel(spv, target); // We need to export the list of error names somewhere so that we can pretty-print them in the // executor. This is not really an important thing though, so we can just dump it in any old // nonsemantic instruction. For now, just put it in OpSourceExtension with a special name. - var error_info = std.ArrayList(u8).init(self.spv.arena); + var error_info = std.ArrayList(u8).init(self.object.gpa); + defer error_info.deinit(); + try error_info.appendSlice("zig_errors"); const module = self.base.options.module.?; for (module.global_error_set.keys()) |name_nts| { @@ -207,17 +188,17 @@ pub fn flushModule(self: *SpirV, comp: *Compilation, prog_node: *std.Progress.No defer self.base.allocator.free(escaped_name); try error_info.writer().print(":{s}", .{escaped_name}); } - try self.spv.sections.debug_strings.emit(self.spv.gpa, .OpSourceExtension, .{ + try spv.sections.debug_strings.emit(spv.gpa, .OpSourceExtension, .{ .extension = error_info.items, }); - try self.spv.flush(self.base.file.?); + try spv.flush(self.base.file.?); } fn writeCapabilities(spv: *SpvModule, target: std.Target) !void { // TODO: Integrate with a hypothetical feature system const caps: []const spec.Capability = switch (target.os.tag) { - .opencl => &.{ .Kernel, .Addresses, .Int8, .Int16, .Int64, .Float64, .GenericPointer }, + .opencl => &.{ .Kernel, .Addresses, .Int8, .Int16, .Int64, .Float64, .Float16, .GenericPointer }, .glsl450 => &.{.Shader}, .vulkan => &.{.Shader}, else => unreachable, // TODO @@ -249,7 +230,7 @@ fn writeMemoryModel(spv: *SpvModule, target: std.Target) !void { }; // TODO: Put this in a proper section. - try spv.sections.capabilities.emit(spv.gpa, .OpMemoryModel, .{ + try spv.sections.extensions.emit(spv.gpa, .OpMemoryModel, .{ .addressing_model = addressing_model, .memory_model = memory_model, }); diff --git a/src/main.zig b/src/main.zig index d30fe16dda..4c44076cd4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -420,6 +420,7 @@ const usage_build_generic = \\ --deps [dep],[dep],... Set dependency names for the root package \\ dep: [[import=]name] \\ --main-mod-path Set the directory of the root module + \\ --error-limit [num] Set the maximum amount of distinct error values \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fPIE Force-enable Position Independent Executable @@ -921,6 +922,8 @@ fn buildOutputType( var error_tracing: ?bool = null; var pdb_out_path: ?[]const u8 = null; var dwarf_format: ?std.dwarf.Format = null; + var error_limit: ?Module.ErrorInt = null; + // e.g. -m3dnow or -mno-outline-atomics. They correspond to std.Target llvm cpu feature names. // This array is populated by zig cc frontend and then has to be converted to zig-style // CPU features. @@ -938,6 +941,7 @@ fn buildOutputType( var rc_source_files = std.ArrayList(Compilation.RcSourceFile).init(arena); var rc_includes: Compilation.RcIncludes = .any; var res_files = std.ArrayList(Compilation.LinkObject).init(arena); + var manifest_file: ?[]const u8 = null; var link_objects = std.ArrayList(Compilation.LinkObject).init(arena); var framework_dirs = std.ArrayList([]const u8).init(arena); var frameworks: std.StringArrayHashMapUnmanaged(Framework) = .{}; @@ -1049,6 +1053,11 @@ fn buildOutputType( root_deps_str = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--main-mod-path")) { main_mod_path = args_iter.nextOrFatal(); + } else if (mem.eql(u8, arg, "--error-limit")) { + const next_arg = args_iter.nextOrFatal(); + error_limit = std.fmt.parseUnsigned(Module.ErrorInt, next_arg, 0) catch |err| { + fatal("unable to parse error limit '{s}': {s}", .{ next_arg, @errorName(err) }); + }; } else if (mem.eql(u8, arg, "-cflags")) { extra_cflags.shrinkRetainingCapacity(0); while (true) { @@ -1627,6 +1636,11 @@ fn buildOutputType( Compilation.classifyFileExt(arg)) { .object, .static_library, .shared_library => try link_objects.append(.{ .path = arg }), .res => try res_files.append(.{ .path = arg }), + .manifest => { + if (manifest_file) |other| { + fatal("only one manifest file can be specified, found '{s}' after '{s}'", .{ arg, other }); + } else manifest_file = arg; + }, .assembly, .assembly_with_cpp, .c, .cpp, .h, .ll, .bc, .m, .mm, .cu => { try c_source_files.append(.{ .src_path = arg, @@ -1647,6 +1661,9 @@ fn buildOutputType( } else root_src_file = arg; }, .def, .unknown => { + if (std.ascii.eqlIgnoreCase(".xml", std.fs.path.extension(arg))) { + std.log.warn("embedded manifest files must have the extension '.manifest'", .{}); + } fatal("unrecognized file extension of parameter '{s}'", .{arg}); }, } @@ -1734,6 +1751,11 @@ fn buildOutputType( .path = it.only_arg, .must_link = must_link, }), + .manifest => { + if (manifest_file) |other| { + fatal("only one manifest file can be specified, found '{s}' after previously specified manifest '{s}'", .{ it.only_arg, other }); + } else manifest_file = it.only_arg; + }, .def => { linker_module_definition_file = it.only_arg; }, @@ -2601,6 +2623,9 @@ fn buildOutputType( try link_objects.append(res_file); } } else { + if (manifest_file != null) { + fatal("manifest file is not allowed unless the target object format is coff (Windows/UEFI)", .{}); + } if (rc_source_files.items.len != 0) { fatal("rc files are not allowed unless the target object format is coff (Windows/UEFI)", .{}); } @@ -3418,6 +3443,7 @@ fn buildOutputType( .symbol_wrap_set = symbol_wrap_set, .c_source_files = c_source_files.items, .rc_source_files = rc_source_files.items, + .manifest_file = manifest_file, .rc_includes = rc_includes, .link_objects = link_objects.items, .framework_dirs = framework_dirs.items, @@ -3538,6 +3564,7 @@ fn buildOutputType( .reference_trace = reference_trace, .error_tracing = error_tracing, .pdb_out_path = pdb_out_path, + .error_limit = error_limit, }) catch |err| switch (err) { error.LibCUnavailable => { const target = target_info.target; @@ -5117,6 +5144,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .thread_pool = &thread_pool, .global_cache = global_cache_directory, .recursive = true, + .debug_hash = false, .work_around_btrfs_bug = work_around_btrfs_bug, }; defer job_queue.deinit(); @@ -6930,9 +6958,9 @@ fn accessFrameworkPath( ) !bool { const sep = fs.path.sep_str; - for (&[_][]const u8{ "tbd", "dylib" }) |ext| { + for (&[_][]const u8{ ".tbd", ".dylib", "" }) |ext| { test_path.clearRetainingCapacity(); - try test_path.writer().print("{s}" ++ sep ++ "{s}.framework" ++ sep ++ "{s}.{s}", .{ + try test_path.writer().print("{s}" ++ sep ++ "{s}.framework" ++ sep ++ "{s}{s}", .{ framework_dir_path, framework_name, framework_name, @@ -6965,6 +6993,7 @@ pub const usage_fetch = \\Options: \\ -h, --help Print this help and exit \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --debug-hash Print verbose hash information to stdout \\ ; @@ -6978,6 +7007,7 @@ fn cmdFetch( std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); + var debug_hash: bool = false; { var i: usize = 0; @@ -6993,6 +7023,9 @@ fn cmdFetch( i += 1; override_global_cache_dir = args[i]; continue; + } else if (mem.eql(u8, arg, "--debug-hash")) { + debug_hash = true; + continue; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } @@ -7031,6 +7064,7 @@ fn cmdFetch( .thread_pool = &thread_pool, .global_cache = global_cache_directory, .recursive = false, + .debug_hash = debug_hash, .work_around_btrfs_bug = work_around_btrfs_bug, }; defer job_queue.deinit(); diff --git a/src/translate_c/ast.zig b/src/translate_c/ast.zig index 8d4d6f833a..0381f58cf9 100644 --- a/src/translate_c/ast.zig +++ b/src/translate_c/ast.zig @@ -794,6 +794,7 @@ pub fn render(gpa: Allocator, nodes: []const Node) !std.zig.Ast { .nodes = ctx.nodes.toOwnedSlice(), .extra_data = try ctx.extra_data.toOwnedSlice(gpa), .errors = &.{}, + .mode = .zig, }; } diff --git a/src/type.zig b/src/type.zig index 5bdca27667..49582bd3c8 100644 --- a/src/type.zig +++ b/src/type.zig @@ -905,8 +905,11 @@ pub const Type = struct { .opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat), .error_union_type => |info| return abiAlignmentAdvancedErrorUnion(ty, mod, strat, info.payload_type.toType()), - // TODO revisit this when we have the concept of the error tag type - .error_set_type, .inferred_error_set_type => return .{ .scalar = .@"2" }, + .error_set_type, .inferred_error_set_type => { + const bits = mod.errorSetBits(); + if (bits == 0) return AbiAlignmentAdvanced{ .scalar = .@"1" }; + return .{ .scalar = intAbiAlignment(bits, target) }; + }, // represents machine code; not a pointer .func_type => |func_type| return .{ @@ -967,10 +970,11 @@ pub const Type = struct { else => return .{ .scalar = .@"16" }, }, - // TODO revisit this when we have the concept of the error tag type - .anyerror, - .adhoc_inferred_error_set, - => return .{ .scalar = .@"2" }, + .anyerror, .adhoc_inferred_error_set => { + const bits = mod.errorSetBits(); + if (bits == 0) return AbiAlignmentAdvanced{ .scalar = .@"1" }; + return .{ .scalar = intAbiAlignment(bits, target) }; + }, .void, .type, @@ -1284,8 +1288,11 @@ pub const Type = struct { .opt_type => return ty.abiSizeAdvancedOptional(mod, strat), - // TODO revisit this when we have the concept of the error tag type - .error_set_type, .inferred_error_set_type => return AbiSizeAdvanced{ .scalar = 2 }, + .error_set_type, .inferred_error_set_type => { + const bits = mod.errorSetBits(); + if (bits == 0) return AbiSizeAdvanced{ .scalar = 0 }; + return AbiSizeAdvanced{ .scalar = intAbiSize(bits, target) }; + }, .error_union_type => |error_union_type| { const payload_ty = error_union_type.payload_type.toType(); @@ -1379,10 +1386,11 @@ pub const Type = struct { .enum_literal, => return AbiSizeAdvanced{ .scalar = 0 }, - // TODO revisit this when we have the concept of the error tag type - .anyerror, - .adhoc_inferred_error_set, - => return AbiSizeAdvanced{ .scalar = 2 }, + .anyerror, .adhoc_inferred_error_set => { + const bits = mod.errorSetBits(); + if (bits == 0) return AbiSizeAdvanced{ .scalar = 0 }; + return AbiSizeAdvanced{ .scalar = intAbiSize(bits, target) }; + }, .prefetch_options => unreachable, // missing call to resolveTypeFields .export_options => unreachable, // missing call to resolveTypeFields @@ -1576,8 +1584,7 @@ pub const Type = struct { return (try abiSizeAdvanced(ty, mod, strat)).scalar * 8; }, - // TODO revisit this when we have the concept of the error tag type - .error_set_type, .inferred_error_set_type => return 16, + .error_set_type, .inferred_error_set_type => return mod.errorSetBits(), .error_union_type => { // Optionals and error unions are not packed so their bitsize @@ -1610,10 +1617,9 @@ pub const Type = struct { .bool => return 1, .void => return 0, - // TODO revisit this when we have the concept of the error tag type .anyerror, .adhoc_inferred_error_set, - => return 16, + => return mod.errorSetBits(), .anyopaque => unreachable, .type => unreachable, @@ -2172,8 +2178,7 @@ pub const Type = struct { while (true) switch (ty.toIntern()) { .anyerror_type, .adhoc_inferred_error_set_type => { - // TODO revisit this when error sets support custom int types - return .{ .signedness = .unsigned, .bits = 16 }; + return .{ .signedness = .unsigned, .bits = mod.errorSetBits() }; }, .usize_type => return .{ .signedness = .unsigned, .bits = target.ptrBitWidth() }, .isize_type => return .{ .signedness = .signed, .bits = target.ptrBitWidth() }, @@ -2192,8 +2197,9 @@ pub const Type = struct { .enum_type => |enum_type| ty = enum_type.tag_ty.toType(), .vector_type => |vector_type| ty = vector_type.child.toType(), - // TODO revisit this when error sets support custom int types - .error_set_type, .inferred_error_set_type => return .{ .signedness = .unsigned, .bits = 16 }, + .error_set_type, .inferred_error_set_type => { + return .{ .signedness = .unsigned, .bits = mod.errorSetBits() }; + }, .anon_struct_type => unreachable, diff --git a/src/value.zig b/src/value.zig index 2c4a88da17..15289de6b2 100644 --- a/src/value.zig +++ b/src/value.zig @@ -701,15 +701,20 @@ pub const Value = struct { } }, .ErrorSet => { - // TODO revisit this when we have the concept of the error tag type - const Int = u16; + const bits = mod.errorSetBits(); + const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); + const name = switch (ip.indexToKey(val.toIntern())) { .err => |err| err.name, .error_union => |error_union| error_union.val.err_name, else => unreachable, }; - const int = @as(Module.ErrorInt, @intCast(mod.global_error_set.getIndex(name).?)); - std.mem.writeInt(Int, buffer[0..@sizeOf(Int)], @as(Int, @intCast(int)), endian); + var bigint_buffer: BigIntSpace = undefined; + const bigint = BigIntMutable.init( + &bigint_buffer.limbs, + mod.global_error_set.getIndex(name).?, + ).toConst(); + bigint.writeTwosComplement(buffer[0..byte_count], endian); }, .Union => switch (ty.containerLayout(mod)) { .Auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already @@ -987,10 +992,12 @@ pub const Value = struct { } }, .ErrorSet => { - // TODO revisit this when we have the concept of the error tag type - const Int = u16; - const int = std.mem.readInt(Int, buffer[0..@sizeOf(Int)], endian); - const name = mod.global_error_set.keys()[@as(usize, @intCast(int))]; + const bits = mod.errorSetBits(); + const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); + const int = std.mem.readVarInt(u64, buffer[0..byte_count], endian); + const index = (int << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); + const name = mod.global_error_set.keys()[@intCast(index)]; + return (try mod.intern(.{ .err = .{ .ty = ty.toIntern(), .name = name, |