Merge pull request #17978 from ziglang/elf-x86-tls

x86_64+elf: TLS support

3 files changed, 185 insertions, 77 deletions

diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig
index 1040515de5..3dd35dfe63 100644
--- a/src/link/Elf/Symbol.zig
+++ b/src/link/Elf/Symbol.zig
@@ -399,6 +399,11 @@ pub const Flags = packed struct {
     /// Whether the symbol contains .zig.got indirection.
     needs_zig_got: bool = false,
     has_zig_got: bool = false,
+
+    /// Whether the symbol is a TLS variable.
+    /// TODO this is really not needed if only we operated on esyms between
+    /// codegen and ZigObject.
+    is_tls: bool = false,
 };
 
 pub const Extra = struct {
diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig
index f08e5bacbb..98bc779ee7 100644
--- a/src/link/Elf/ZigObject.zig
+++ b/src/link/Elf/ZigObject.zig
@@ -29,6 +29,9 @@ lazy_syms: LazySymbolTable = .{},
 /// Table of tracked Decls.
 decls: DeclTable = .{},
 
+/// TLS variables indexed by Atom.Index.
+tls_variables: TlsTable = .{},
+
 /// Table of unnamed constants associated with a parent `Decl`.
 /// We store them here so that we can free the constants whenever the `Decl`
 /// needs updating or is freed.
@@ -137,6 +140,11 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void {
         self.anon_decls.deinit(allocator);
     }
 
+    for (self.tls_variables.values()) |*tlv| {
+        tlv.deinit(allocator);
+    }
+    self.tls_variables.deinit(allocator);
+
     if (self.dwarf) |*dw| {
         dw.deinit();
     }
@@ -212,8 +220,6 @@ pub fn flushModule(self: *ZigObject, elf_file: *Elf) !void {
         self.saveDebugSectionsSizes(elf_file);
     }
 
-    try self.sortSymbols(elf_file);
-
     // The point of flushModule() is to commit changes, so in theory, nothing should
     // be dirty after this. However, it is possible for some things to remain
     // dirty because they fail to be written in the event of compile errors,
@@ -280,6 +286,7 @@ pub fn addAtom(self: *ZigObject, elf_file: *Elf) !Symbol.Index {
     self.local_esyms.items(.elf_sym)[esym_index].st_shndx = SHN_ATOM;
     symbol_ptr.esym_index = esym_index;
 
+    // TODO I'm thinking that maybe we shouldn' set this value unless it's actually needed?
     const relocs_index = @as(u32, @intCast(self.relocs.items.len));
     const relocs = try self.relocs.addOne(gpa);
     relocs.* = .{};
@@ -388,6 +395,19 @@ pub fn claimUnresolvedObject(self: ZigObject, elf_file: *Elf) void {
     }
 }
 
+pub fn allocateTlvAtoms(self: ZigObject, elf_file: *Elf) void {
+    for (self.tls_variables.keys(), self.tls_variables.values()) |atom_index, tlv| {
+        const atom = elf_file.atom(atom_index) orelse continue;
+        if (!atom.flags.alive) continue;
+        const local = elf_file.symbol(tlv.symbol_index);
+        const shdr = elf_file.shdrs.items[atom.output_section_index];
+        atom.value += shdr.sh_addr;
+        local.value = atom.value;
+
+        // TODO exported TLS vars
+    }
+}
+
 pub fn scanRelocs(self: *ZigObject, elf_file: *Elf, undefs: anytype) !void {
     for (self.atoms.items) |atom_index| {
         const atom = elf_file.atom(atom_index) orelse continue;
@@ -421,72 +441,6 @@ pub fn markLive(self: *ZigObject, elf_file: *Elf) void {
     }
 }
 
-fn sortSymbols(self: *ZigObject, elf_file: *Elf) error{OutOfMemory}!void {
-    _ = self;
-    _ = elf_file;
-    // const Entry = struct {
-    //     index: Symbol.Index,
-
-    //     const Ctx = struct {
-    //         zobj: ZigObject,
-    //         efile: *Elf,
-    //     };
-
-    //     pub fn lessThan(ctx: Ctx, lhs: @This(), rhs: @This()) bool {
-    //         const lhs_sym = ctx.efile.symbol(zobj.symbol(lhs.index));
-    //         const rhs_sym = ctx.efile.symbol(zobj.symbol(rhs.index));
-    //         if (lhs_sym.outputShndx() != null and rhs_sym.outputShndx() != null) {
-    //             if (lhs_sym.output_section_index == rhs_sym.output_section_index) {
-    //                 if (lhs_sym.value == rhs_sym.value) {
-    //                     return lhs_sym.name_offset < rhs_sym.name_offset;
-    //                 }
-    //                 return lhs_sym.value < rhs_sym.value;
-    //             }
-    //             return lhs_sym.output_section_index < rhs_sym.output_section_index;
-    //         }
-    //         if (lhs_sym.outputShndx() != null) {
-    //             if (rhs_sym.isAbs(ctx.efile)) return false;
-    //             return true;
-    //         }
-    //         return false;
-    //     }
-    // };
-
-    // const gpa = elf_file.base.allocator;
-
-    // {
-    //     const sorted = try gpa.alloc(Entry, self.local_symbols.items.len);
-    //     defer gpa.free(sorted);
-    //     for (0..self.local_symbols.items.len) |index| {
-    //         sorted[i] = .{ .index = @as(Symbol.Index, @intCast(index)) };
-    //     }
-    //     mem.sort(Entry, sorted, .{ .zobj = self, .efile = elf_file }, Entry.lessThan);
-
-    //     const backlinks = try gpa.alloc(Symbol.Index, sorted.len);
-    //     defer gpa.free(backlinks);
-    //     for (sorted, 0..) |entry, i| {
-    //         backlinks[entry.index] = @as(Symbol.Index, @intCast(i));
-    //     }
-
-    //     const local_symbols = try self.local_symbols.toOwnedSlice(gpa);
-    //     defer gpa.free(local_symbols);
-
-    //     try self.local_symbols.ensureTotalCapacityPrecise(gpa, local_symbols.len);
-    //     for (sorted) |entry| {
-    //         self.local_symbols.appendAssumeCapacity(local_symbols[entry.index]);
-    //     }
-
-    //     for (self.)
-    // }
-
-    // const sorted_globals = try gpa.alloc(Entry, self.global_symbols.items.len);
-    // defer gpa.free(sorted_globals);
-    // for (self.global_symbols.items, 0..) |index, i| {
-    //     sorted_globals[i] = .{ .index = index };
-    // }
-    // mem.sort(Entry, sorted_globals, elf_file, Entry.lessThan);
-}
-
 pub fn updateArSymtab(self: ZigObject, ar_symtab: *Archive.ArSymtab, elf_file: *Elf) error{OutOfMemory}!void {
     const gpa = elf_file.base.allocator;
 
@@ -537,7 +491,9 @@ pub fn addAtomsToRelaSections(self: ZigObject, elf_file: *Elf) !void {
     for (self.atoms.items) |atom_index| {
         const atom = elf_file.atom(atom_index) orelse continue;
         if (!atom.flags.alive) continue;
-        _ = atom.relocsShndx() orelse continue;
+        const rela_shndx = atom.relocsShndx() orelse continue;
+        // TODO this check will become obsolete when we rework our relocs mechanism at the ZigObject level
+        if (self.relocs.items[rela_shndx].items.len == 0) continue;
         const out_shndx = atom.outputShndx().?;
         const out_shdr = elf_file.shdrs.items[out_shndx];
         if (out_shdr.sh_type == elf.SHT_NOBITS) continue;
@@ -583,6 +539,13 @@ pub fn codeAlloc(self: ZigObject, elf_file: *Elf, atom_index: Atom.Index) ![]u8
     const atom = elf_file.atom(atom_index).?;
     assert(atom.file_index == self.index);
     const shdr = &elf_file.shdrs.items[atom.outputShndx().?];
+
+    if (shdr.sh_flags & elf.SHF_TLS != 0) {
+        const tlv = self.tls_variables.get(atom_index).?;
+        const code = try gpa.dupe(u8, tlv.code);
+        return code;
+    }
+
     const file_offset = shdr.sh_offset + atom.value - shdr.sh_addr;
     const size = std.math.cast(usize, atom.size) orelse return error.Overflow;
     const code = try gpa.alloc(u8, size);
@@ -705,7 +668,12 @@ pub fn getOrCreateMetadataForLazySymbol(
         },
     };
     switch (metadata.state.*) {
-        .unused => metadata.symbol_index.* = try self.addAtom(elf_file),
+        .unused => {
+            const symbol_index = try self.addAtom(elf_file);
+            const sym = elf_file.symbol(symbol_index);
+            sym.flags.needs_zig_got = true;
+            metadata.symbol_index.* = symbol_index;
+        },
         .pending_flush => return metadata.symbol_index.*,
         .flushed => {},
     }
@@ -760,20 +728,55 @@ pub fn getOrCreateMetadataForDecl(
 ) !Symbol.Index {
     const gop = try self.decls.getOrPut(elf_file.base.allocator, decl_index);
     if (!gop.found_existing) {
-        gop.value_ptr.* = .{ .symbol_index = try self.addAtom(elf_file) };
+        const single_threaded = elf_file.base.options.single_threaded;
+        const symbol_index = try self.addAtom(elf_file);
+        const mod = elf_file.base.options.module.?;
+        const decl = mod.declPtr(decl_index);
+        const sym = elf_file.symbol(symbol_index);
+        if (decl.getOwnedVariable(mod)) |variable| {
+            if (variable.is_threadlocal and !single_threaded) {
+                sym.flags.is_tls = true;
+            }
+        }
+        if (!sym.flags.is_tls) {
+            sym.flags.needs_zig_got = true;
+        }
+        gop.value_ptr.* = .{ .symbol_index = symbol_index };
     }
     return gop.value_ptr.symbol_index;
 }
 
-fn getDeclShdrIndex(self: *ZigObject, elf_file: *Elf, decl_index: Module.Decl.Index, code: []const u8) u16 {
+fn getDeclShdrIndex(
+    self: *ZigObject,
+    elf_file: *Elf,
+    decl: *const Module.Decl,
+    code: []const u8,
+) error{OutOfMemory}!u16 {
     _ = self;
     const mod = elf_file.base.options.module.?;
-    const decl = mod.declPtr(decl_index);
+    const single_threaded = elf_file.base.options.single_threaded;
     const shdr_index = switch (decl.ty.zigTypeTag(mod)) {
-        // TODO: what if this is a function pointer?
         .Fn => elf_file.zig_text_section_index.?,
         else => blk: {
             if (decl.getOwnedVariable(mod)) |variable| {
+                if (variable.is_threadlocal and !single_threaded) {
+                    const is_all_zeroes = for (code) |byte| {
+                        if (byte != 0) break false;
+                    } else true;
+                    if (is_all_zeroes) break :blk elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{
+                        .type = elf.SHT_NOBITS,
+                        .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
+                        .name = ".tbss",
+                        .offset = std.math.maxInt(u64),
+                    });
+
+                    break :blk elf_file.sectionByName(".tdata") orelse try elf_file.addSection(.{
+                        .type = elf.SHT_PROGBITS,
+                        .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS,
+                        .name = ".tdata",
+                        .offset = std.math.maxInt(u64),
+                    });
+                }
                 if (variable.is_const) break :blk elf_file.zig_data_rel_ro_section_index.?;
                 if (variable.init.toValue().isUndefDeep(mod)) {
                     const mode = elf_file.base.options.optimize_mode;
@@ -799,6 +802,7 @@ fn updateDeclCode(
     elf_file: *Elf,
     decl_index: Module.Decl.Index,
     sym_index: Symbol.Index,
+    shdr_index: u16,
     code: []const u8,
     stt_bits: u8,
 ) !void {
@@ -815,7 +819,6 @@ fn updateDeclCode(
     const esym = &self.local_esyms.items(.elf_sym)[sym.esym_index];
     const atom_ptr = sym.atom(elf_file).?;
 
-    const shdr_index = self.getDeclShdrIndex(elf_file, decl_index, code);
     sym.output_section_index = shdr_index;
     atom_ptr.output_section_index = shdr_index;
 
@@ -893,6 +896,58 @@ fn updateDeclCode(
     }
 }
 
+fn updateTlv(
+    self: *ZigObject,
+    elf_file: *Elf,
+    decl_index: Module.Decl.Index,
+    sym_index: Symbol.Index,
+    shndx: u16,
+    code: []const u8,
+) !void {
+    const gpa = elf_file.base.allocator;
+    const mod = elf_file.base.options.module.?;
+    const decl = mod.declPtr(decl_index);
+    const decl_name = mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod));
+
+    log.debug("updateTlv {s} ({*})", .{ decl_name, decl });
+
+    const required_alignment = decl.getAlignment(mod);
+
+    const sym = elf_file.symbol(sym_index);
+    const esym = &self.local_esyms.items(.elf_sym)[sym.esym_index];
+    const atom_ptr = sym.atom(elf_file).?;
+
+    sym.output_section_index = shndx;
+    atom_ptr.output_section_index = shndx;
+
+    sym.name_offset = try self.strtab.insert(gpa, decl_name);
+    atom_ptr.flags.alive = true;
+    atom_ptr.name_offset = sym.name_offset;
+    esym.st_name = sym.name_offset;
+    esym.st_info = elf.STT_TLS;
+    esym.st_size = code.len;
+
+    atom_ptr.alignment = required_alignment;
+    atom_ptr.size = code.len;
+
+    {
+        const gop = try self.tls_variables.getOrPut(gpa, atom_ptr.atom_index);
+        assert(!gop.found_existing); // TODO incremental updates
+        gop.value_ptr.* = .{ .symbol_index = sym_index };
+
+        // We only store the data for the TLV if it's non-zerofill.
+        if (elf_file.shdrs.items[shndx].sh_type != elf.SHT_NOBITS) {
+            gop.value_ptr.code = try gpa.dupe(u8, code);
+        }
+    }
+
+    {
+        const gop = try elf_file.output_sections.getOrPut(gpa, atom_ptr.output_section_index);
+        if (!gop.found_existing) gop.value_ptr.* = .{};
+        try gop.value_ptr.append(gpa, atom_ptr.atom_index);
+    }
+}
+
 pub fn updateFunc(
     self: *ZigObject,
     elf_file: *Elf,
@@ -947,7 +1002,10 @@ pub fn updateFunc(
             return;
         },
     };
-    try self.updateDeclCode(elf_file, decl_index, sym_index, code, elf.STT_FUNC);
+
+    const shndx = try self.getDeclShdrIndex(elf_file, decl, code);
+    try self.updateDeclCode(elf_file, decl_index, sym_index, shndx, code, elf.STT_FUNC);
+
     if (decl_state) |*ds| {
         const sym = elf_file.symbol(sym_index);
         try self.dwarf.?.commitDeclState(
@@ -1026,7 +1084,12 @@ pub fn updateDecl(
         },
     };
 
-    try self.updateDeclCode(elf_file, decl_index, sym_index, code, elf.STT_OBJECT);
+    const shndx = try self.getDeclShdrIndex(elf_file, decl, code);
+    if (elf_file.shdrs.items[shndx].sh_flags & elf.SHF_TLS != 0)
+        try self.updateTlv(elf_file, decl_index, sym_index, shndx, code)
+    else
+        try self.updateDeclCode(elf_file, decl_index, sym_index, shndx, code, elf.STT_OBJECT);
+
     if (decl_state) |*ds| {
         const sym = elf_file.symbol(sym_index);
         try self.dwarf.?.commitDeclState(
@@ -1454,11 +1517,21 @@ const DeclMetadata = struct {
     }
 };
 
+const TlsVariable = struct {
+    symbol_index: Symbol.Index,
+    code: []const u8 = &[0]u8{},
+
+    fn deinit(tlv: *TlsVariable, allocator: Allocator) void {
+        allocator.free(tlv.code);
+    }
+};
+
 const AtomList = std.ArrayListUnmanaged(Atom.Index);
 const UnnamedConstTable = std.AutoHashMapUnmanaged(Module.Decl.Index, std.ArrayListUnmanaged(Symbol.Index));
 const DeclTable = std.AutoHashMapUnmanaged(Module.Decl.Index, DeclMetadata);
 const AnonDeclTable = std.AutoHashMapUnmanaged(InternPool.Index, DeclMetadata);
 const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
+const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable);
 
 const assert = std.debug.assert;
 const builtin = @import("builtin");
diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig
index 7419709afc..47e72cb1e3 100644
--- a/src/link/Elf/synthetic_sections.zig
+++ b/src/link/Elf/synthetic_sections.zig
@@ -935,6 +935,36 @@ pub const PltSection = struct {
             ilocal += 1;
         }
     }
+
+    const FormatCtx = struct {
+        plt: PltSection,
+        elf_file: *Elf,
+    };
+
+    pub fn fmt(plt: PltSection, elf_file: *Elf) std.fmt.Formatter(format2) {
+        return .{ .data = .{ .plt = plt, .elf_file = elf_file } };
+    }
+
+    pub fn format2(
+        ctx: FormatCtx,
+        comptime unused_fmt_string: []const u8,
+        options: std.fmt.FormatOptions,
+        writer: anytype,
+    ) !void {
+        _ = options;
+        _ = unused_fmt_string;
+        try writer.writeAll("PLT\n");
+        for (ctx.plt.symbols.items, 0..) |symbol_index, i| {
+            const symbol = ctx.elf_file.symbol(symbol_index);
+            try writer.print("  {d}@0x{x} => {d}@0x{x} ({s})\n", .{
+                i,
+                symbol.pltAddress(ctx.elf_file),
+                symbol_index,
+                symbol.address(.{}, ctx.elf_file),
+                symbol.name(ctx.elf_file),
+            });
+        }
+    }
 };
 
 pub const GotPltSection = struct {