riscv: `@atomicRmw`

Now we generate debug undefined constants when the user asks for them to dedup across the function decl. This takes 2 instructions instead of 7 in the RISC-V backend.

TODO, we need to dedupe across function decl boundaries.

7 files changed, 342 insertions, 55 deletions

diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig
index f8b67a95d4..d5440f38bb 100644
--- a/src/arch/riscv64/CodeGen.zig
+++ b/src/arch/riscv64/CodeGen.zig
@@ -117,8 +117,9 @@ const MCValue = union(enum) {
     /// No more references to this value remain.
     /// The payload is the value of scope_generation at the point where the death occurred
     dead: u32,
-    /// The value is undefined.
-    undef,
+    /// The value is undefined. Contains a symbol index to an undefined constant. Null means
+    /// set the undefined value via immediate instead of a load.
+    undef: ?u32,
     /// A pointer-sized integer that fits in a register.
     /// If the type is a pointer, this is the pointer address in virtual address space.
     immediate: u64,
@@ -1045,6 +1046,7 @@ pub fn addExtraAssumeCapacity(func: *Func, extra: anytype) u32 {
 const required_features = [_]Target.riscv.Feature{
     .d,
     .m,
+    .a,
 };
 
 fn gen(func: *Func) !void {
@@ -1631,7 +1633,7 @@ fn computeFrameLayout(func: *Func) !FrameLayout {
 
     // The total frame size is calculated by the amount of s registers you need to save * 8, as each
     // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0
-    // register. Finally we align the frame size to the align of the base pointer.
+    // register. Finally we align the frame size to the alignment of the base pointer.
     const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)];
     const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)];
     const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)];
@@ -2110,7 +2112,7 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
                 });
             },
             .Int => {
-                const size = ty.bitSize(zcu);
+                const size = ty.bitSize(pt);
                 if (!math.isPowerOfTwo(size))
                     return func.fail("TODO: airNot non-pow 2 int size", .{});
 
@@ -3249,7 +3251,7 @@ fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void {
         const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, pt));
         const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, pt));
         const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, pt));
-        try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
+        try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null });
         const operand = try func.resolveInst(ty_op.operand);
         try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
         break :result .{ .load_frame = .{ .index = frame_index } };
@@ -5627,10 +5629,14 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
         .none,
         .dead,
         => unreachable,
-        .undef => {
+        .undef => |sym_index| {
             if (!func.wantSafety())
                 return;
 
+            if (sym_index) |index| {
+                return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } });
+            }
+
             switch (abi_size) {
                 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
                 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
@@ -5865,11 +5871,17 @@ fn genSetMem(
         .dead,
         .reserved_frame,
         => unreachable,
-        .undef => try func.genInlineMemset(
-            dst_ptr_mcv,
-            src_mcv,
-            .{ .immediate = abi_size },
-        ),
+        .undef => |sym_index| {
+            if (sym_index) |index| {
+                return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } });
+            }
+
+            try func.genInlineMemset(
+                dst_ptr_mcv,
+                src_mcv,
+                .{ .immediate = abi_size },
+            );
+        },
         .register_offset,
         .memory,
         .indirect,
@@ -6069,12 +6081,82 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {
 }
 
 fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
-    _ = inst;
-    return func.fail("TODO implement airCmpxchg for {}", .{func.target.cpu.arch});
+    const zcu = func.pt.zcu;
+    const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+    const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
+
+    const op = extra.op();
+    const order = extra.ordering();
+
+    const ptr_ty = func.typeOf(pl_op.operand);
+    const ptr_mcv = try func.resolveInst(pl_op.operand);
+
+    const val_ty = func.typeOf(extra.operand);
+    const val_size = val_ty.abiSize(func.pt);
+    const val_mcv = try func.resolveInst(extra.operand);
+
+    if (!math.isPowerOfTwo(val_size))
+        return func.fail("TODO: airAtomicRmw non-pow 2", .{});
+
+    switch (val_ty.zigTypeTag(zcu)) {
+        .Int => {},
+        inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
+        else => unreachable,
+    }
+
+    switch (val_size) {
+        1, 2 => return func.fail("TODO: airAtomicRmw Int {}", .{val_size}),
+        4, 8 => {},
+        else => unreachable,
+    }
+
+    const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
+    defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
+
+    const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);
+    defer if (val_lock) |lock| func.register_manager.unlockReg(lock);
+
+    const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
+    assert(result_mcv == .register); // should fit into 8 bytes
+
+    const aq, const rl = switch (order) {
+        .unordered => unreachable,
+        .monotonic => .{ false, false },
+        .acquire => .{ true, false },
+        .release => .{ false, true },
+        .acq_rel => .{ true, true },
+        .seq_cst => .{ true, true },
+    };
+
+    _ = try func.addInst(.{
+        .tag = .pseudo,
+        .ops = .pseudo_amo,
+        .data = .{ .amo = .{
+            .rd = result_mcv.register,
+            .rs1 = ptr_register,
+            .rs2 = val_register,
+            .aq = if (aq) .aq else .none,
+            .rl = if (rl) .rl else .none,
+            .op = switch (op) {
+                .Xchg => .SWAP,
+                .Add => .ADD,
+                .Sub => return func.fail("TODO: airAtomicRmw SUB", .{}),
+                .And => .AND,
+                .Nand => return func.fail("TODO: airAtomicRmw NAND", .{}),
+                .Or => .OR,
+                .Xor => .XOR,
+                .Max => .MAX,
+                .Min => .MIN,
+            },
+            .ty = val_ty,
+        } },
+    });
+
+    return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none });
 }
 
 fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
-    const zcu = func.bin_file.comp.module.?;
+    const zcu = func.pt.zcu;
     const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
     const order: std.builtin.AtomicOrder = atomic_load.order;
 
@@ -6083,6 +6165,7 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
     const ptr_mcv = try func.resolveInst(atomic_load.ptr);
 
     const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);
+    assert(result_mcv == .register); // should be less than 8 bytes
 
     if (order == .seq_cst) {
         _ = try func.addInst(.{
@@ -6535,19 +6618,40 @@ fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking {
 }
 
 fn genTypedValue(func: *Func, val: Value) InnerError!MCValue {
-    const pt = func.pt;
-    const zcu = pt.zcu;
+    const zcu = func.pt.zcu;
+    const gpa = func.gpa;
+
+    const owner_decl_index = zcu.funcOwnerDeclIndex(func.func_index);
+    const lf = func.bin_file;
+    const src_loc = func.src_loc;
+
+    if (val.isUndef(zcu)) {
+        const local_sym_index = lf.lowerUnnamedConst(func.pt, val, owner_decl_index) catch |err| {
+            const msg = try ErrorMsg.create(gpa, src_loc, "lowering unnamed undefined constant failed: {s}", .{@errorName(err)});
+            func.err_msg = msg;
+            return error.CodegenFail;
+        };
+        switch (lf.tag) {
+            .elf => {
+                const elf_file = lf.cast(link.File.Elf).?;
+                const local = elf_file.symbol(local_sym_index);
+                return MCValue{ .undef = local.esym_index };
+            },
+            else => unreachable,
+        }
+    }
+
     const result = try codegen.genTypedValue(
-        func.bin_file,
-        pt,
-        func.src_loc,
+        lf,
+        func.pt,
+        src_loc,
         val,
-        zcu.funcOwnerDeclIndex(func.func_index),
+        owner_decl_index,
     );
     const mcv: MCValue = switch (result) {
         .mcv => |mcv| switch (mcv) {
             .none => .none,
-            .undef => .undef,
+            .undef => unreachable,
             .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } },
             .immediate => |imm| .{ .immediate = imm },
             .memory => |addr| .{ .memory = addr },
diff --git a/src/arch/riscv64/Encoding.zig b/src/arch/riscv64/Encoding.zig
index 08044e644e..817decbd54 100644
--- a/src/arch/riscv64/Encoding.zig
+++ b/src/arch/riscv64/Encoding.zig
@@ -28,7 +28,7 @@ const OpCode = enum(u7) {
     NONE = 0b00000000,
 };
 
-const Fmt = enum(u2) {
+const FpFmt = enum(u2) {
     /// 32-bit single-precision
     S = 0b00,
     /// 64-bit double-precision
@@ -40,6 +40,11 @@ const Fmt = enum(u2) {
     Q = 0b11,
 };
 
+const AmoWidth = enum(u3) {
+    W = 0b010,
+    D = 0b011,
+};
+
 const Enc = struct {
     opcode: OpCode,
 
@@ -49,11 +54,15 @@ const Enc = struct {
             funct3: u3,
             funct7: u7,
         },
+        amo: struct {
+            funct5: u5,
+            width: AmoWidth,
+        },
         /// funct5 + rm + fmt
         fmt: struct {
             funct5: u5,
             rm: u3,
-            fmt: Fmt,
+            fmt: FpFmt,
         },
         /// funct3
         f: struct {
@@ -202,6 +211,27 @@ pub const Mnemonic = enum {
     // MISC
     fence,
 
+    // AMO
+    amoswapw,
+    amoaddw,
+    amoandw,
+    amoorw,
+    amoxorw,
+    amomaxw,
+    amominw,
+    amomaxuw,
+    amominuw,
+
+    amoswapd,
+    amoaddd,
+    amoandd,
+    amoord,
+    amoxord,
+    amomaxd,
+    amomind,
+    amomaxud,
+    amominud,
+
     pub fn encoding(mnem: Mnemonic) Enc {
         return switch (mnem) {
             // zig fmt: off
@@ -379,7 +409,34 @@ pub const Mnemonic = enum {
             // MISC_MEM
 
             .fence   => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
-        
+
+            // AMO
+
+            .amoaddw   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } },
+            .amoswapw  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } },
+            // LR.W
+            // SC.W
+            .amoxorw   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } },
+            .amoandw   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } },
+            .amoorw    => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } },
+            .amominw   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10000 } } },
+            .amomaxw   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10100 } } },
+            .amominuw  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } },
+            .amomaxuw  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } },
+
+            .amoaddd   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } },
+            .amoswapd  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } },
+            // LR.D
+            // SC.D
+            .amoxord   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } },
+            .amoandd   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } },
+            .amoord    => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } },
+            .amomind   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10000 } } },
+            .amomaxd   => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10100 } } },
+            .amominud  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } },
+            .amomaxud  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } },
+
+                    
 
             // zig fmt: on
         };
@@ -395,6 +452,7 @@ pub const InstEnc = enum {
     U,
     J,
     fence,
+    amo,
     /// extras that have unusual op counts
     system,
 
@@ -526,21 +584,43 @@ pub const InstEnc = enum {
 
             .fence,
             => .fence,
+
+            .amoswapw,
+            .amoaddw,
+            .amoandw,
+            .amoorw,
+            .amoxorw,
+            .amomaxw,
+            .amominw,
+            .amomaxuw,
+            .amominuw,
+
+            .amoswapd,
+            .amoaddd,
+            .amoandd,
+            .amoord,
+            .amoxord,
+            .amomaxd,
+            .amomind,
+            .amomaxud,
+            .amominud,
+            => .amo,
         };
     }
 
-    pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {
+    pub fn opsList(enc: InstEnc) [5]std.meta.FieldEnum(Operand) {
         return switch (enc) {
             // zig fmt: off
-            .R      => .{ .reg,     .reg,     .reg,  .none },
-            .R4     => .{ .reg,     .reg,     .reg,  .reg  },  
-            .I      => .{ .reg,     .reg,     .imm,  .none },
-            .S      => .{ .reg,     .reg,     .imm,  .none },
-            .B      => .{ .reg,     .reg,     .imm,  .none },
-            .U      => .{ .reg,     .imm,     .none, .none },
-            .J      => .{ .reg,     .imm,     .none, .none },
-            .system => .{ .none,    .none,    .none, .none },
-            .fence  => .{ .barrier, .barrier, .none, .none },
+            .R      => .{ .reg,     .reg,     .reg,  .none,    .none,   },
+            .R4     => .{ .reg,     .reg,     .reg,  .reg,     .none,   },  
+            .I      => .{ .reg,     .reg,     .imm,  .none,    .none,   },
+            .S      => .{ .reg,     .reg,     .imm,  .none,    .none,   },
+            .B      => .{ .reg,     .reg,     .imm,  .none,    .none,   },
+            .U      => .{ .reg,     .imm,     .none, .none,    .none,   },
+            .J      => .{ .reg,     .imm,     .none, .none,    .none,   },
+            .system => .{ .none,    .none,    .none, .none,    .none,   },
+            .fence  => .{ .barrier, .barrier, .none, .none,    .none,   },
+            .amo    => .{ .reg,     .reg,     .reg,  .barrier, .barrier },
             // zig fmt: on
         };
     }
@@ -611,19 +691,29 @@ pub const Data = union(InstEnc) {
         pred: u4,
         _ignored: u4 = 0,
     },
-    system: void,
+    amo: packed struct {
+        opcode: u7,
+        rd: u5,
+        funct3: u3,
+        rs1: u5,
+        rs2: u5,
+        rl: bool,
+        aq: bool,
+        funct5: u5,
+    },
+    system: u32,
+
+    comptime {
+        for (std.meta.fields(Data)) |field| {
+            assert(@bitSizeOf(field.type) == 32);
+        }
+    }
 
     pub fn toU32(self: Data) u32 {
         return switch (self) {
             // zig fmt: off
-            .R  => |v| @bitCast(v),
-            .R4 => |v| @bitCast(v),
-            .I  => |v| @bitCast(v),
-            .S  => |v| @bitCast(v),
             .B  => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
-            .U  => |v| @bitCast(v),
-            .J  => |v| @bitCast(v),
-            .fence => |v| @bitCast(v),
+            inline else => |v| @bitCast(v),
             .system => unreachable,
             // zig fmt: on
         };
@@ -792,6 +882,34 @@ pub const Data = union(InstEnc) {
                     },
                 };
             },
+            .amo => {
+                assert(ops.len == 5);
+
+                const rd = ops[0];
+                const rs1 = ops[1];
+                const rs2 = ops[2];
+                const rl = ops[3];
+                const aq = ops[4];
+
+                const ret: Data = .{
+                    .amo = .{
+                        .rd = rd.reg.encodeId(),
+                        .rs1 = rs1.reg.encodeId(),
+                        .rs2 = rs2.reg.encodeId(),
+
+                        // TODO: https://github.com/ziglang/zig/issues/20113
+                        .rl = if (rl.barrier == .rl) true else false,
+                        .aq = if (aq.barrier == .aq) true else false,
+
+                        .opcode = @intFromEnum(enc.opcode),
+                        .funct3 = @intFromEnum(enc.data.amo.width),
+                        .funct5 = enc.data.amo.funct5,
+                    },
+                };
+
+                std.debug.print("ret: {}, {}", .{ ret.amo.rl, rl.barrier == .rl });
+                return ret;
+            },
 
             else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
         }
diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig
index 56a0305562..b748cba8cf 100644
--- a/src/arch/riscv64/Lower.zig
+++ b/src/arch/riscv64/Lower.zig
@@ -412,6 +412,32 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
                 });
             },
 
+            .pseudo_amo => {
+                const amo = inst.data.amo;
+                const is_d = amo.ty.abiSize(pt) == 8;
+                const is_un = amo.ty.isUnsignedInt(pt.zcu);
+
+                const mnem: Encoding.Mnemonic = switch (amo.op) {
+                    // zig fmt: off
+                    .SWAP => if (is_d) .amoswapd  else .amoswapw,
+                    .ADD  => if (is_d) .amoaddd   else .amoaddw,
+                    .AND  => if (is_d) .amoandd   else .amoandw,
+                    .OR   => if (is_d) .amoord    else .amoorw,
+                    .XOR  => if (is_d) .amoxord   else .amoxorw,
+                    .MAX  => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
+                    .MIN  => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
+                    // zig fmt: on
+                };
+
+                try lower.emit(mnem, &.{
+                    .{ .reg = inst.data.amo.rd },
+                    .{ .reg = inst.data.amo.rs1 },
+                    .{ .reg = inst.data.amo.rs2 },
+                    .{ .barrier = inst.data.amo.rl },
+                    .{ .barrier = inst.data.amo.aq },
+                });
+            },
+
             else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),
         },
     }
diff --git a/src/arch/riscv64/Mir.zig b/src/arch/riscv64/Mir.zig
index e8567a3275..bb38478bcf 100644
--- a/src/arch/riscv64/Mir.zig
+++ b/src/arch/riscv64/Mir.zig
@@ -39,8 +39,6 @@ pub const Inst = struct {
         ecall,
         unimp,
 
-        fence,
-
         add,
         addw,
         sub,
@@ -82,6 +80,8 @@ pub const Inst = struct {
         sh,
         sb,
 
+        fence,
+
         // M extension
         mul,
         mulw,
@@ -136,6 +136,9 @@ pub const Inst = struct {
         fltd,
         fled,
 
+        /// A Extension Instructions
+        amo,
+
         /// A pseudo-instruction. Used for anything that isn't 1:1 with an
         /// assembly instruction.
         pseudo,
@@ -254,6 +257,16 @@ pub const Inst = struct {
             pred: Barrier,
             succ: Barrier,
         },
+
+        amo: struct {
+            rd: Register,
+            rs1: Register,
+            rs2: Register,
+            aq: Barrier,
+            rl: Barrier,
+            op: AmoOp,
+            ty: Type,
+        },
     };
 
     pub const Ops = enum {
@@ -343,6 +356,9 @@ pub const Inst = struct {
 
         /// IORW, IORW
         fence,
+
+        /// Ordering, Src, Addr, Dest
+        pseudo_amo,
     };
 
     // Make sure we don't accidentally make instructions bigger than expected.
@@ -379,9 +395,25 @@ pub const FrameLoc = struct {
 };
 
 pub const Barrier = enum(u4) {
+    // Fence
     r = 0b0001,
     w = 0b0010,
     rw = 0b0011,
+
+    // Amo
+    none,
+    aq,
+    rl,
+};
+
+pub const AmoOp = enum(u5) {
+    SWAP,
+    ADD,
+    AND,
+    OR,
+    XOR,
+    MAX,
+    MIN,
 };
 
 /// Returns the requested data, as well as the new index which is at the start of the
diff --git a/src/arch/riscv64/encoder.zig b/src/arch/riscv64/encoder.zig
index b219b709de..34d43ba782 100644
--- a/src/arch/riscv64/encoder.zig
+++ b/src/arch/riscv64/encoder.zig
@@ -1,6 +1,6 @@
 pub const Instruction = struct {
     encoding: Encoding,
-    ops: [4]Operand = .{.none} ** 4,
+    ops: [5]Operand = .{.none} ** 5,
 
     pub const Operand = union(enum) {
         none,
@@ -12,16 +12,18 @@ pub const Instruction = struct {
 
     pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
         const encoding = (try Encoding.findByMnemonic(mnemonic, ops)) orelse {
-            std.log.err("no encoding found for:  {s} [{s} {s} {s}]", .{
+            std.log.err("no encoding found for:  {s} [{s} {s} {s} {s} {s}]", .{
                 @tagName(mnemonic),
                 @tagName(if (ops.len > 0) ops[0] else .none),
                 @tagName(if (ops.len > 1) ops[1] else .none),
                 @tagName(if (ops.len > 2) ops[2] else .none),
+                @tagName(if (ops.len > 3) ops[3] else .none),
+                @tagName(if (ops.len > 4) ops[4] else .none),
             });
             return error.InvalidInstruction;
         };
 
-        var result_ops: [4]Operand = .{.none} ** 4;
+        var result_ops: [5]Operand = .{.none} ** 5;
         @memcpy(result_ops[0..ops.len], ops);
 
         return .{
diff --git a/src/codegen.zig b/src/codegen.zig
index 9e0f4db305..1658f63616 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -987,8 +987,9 @@ pub fn genTypedValue(
 
     log.debug("genTypedValue: val = {}", .{val.fmtValue(pt, null)});
 
-    if (val.isUndef(zcu))
+    if (val.isUndef(zcu)) {
         return GenResult.mcv(.undef);
+    }
 
     const owner_decl = zcu.declPtr(owner_decl_index);
     const namespace = zcu.namespacePtr(owner_decl.src_namespace);
diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig
index e8ec5eb492..8fee0d64df 100644
--- a/src/link/Elf/ZigObject.zig
+++ b/src/link/Elf/ZigObject.zig
@@ -540,8 +540,8 @@ inline fn isGlobal(index: Symbol.Index) bool {
 
 pub fn symbol(self: ZigObject, index: Symbol.Index) Symbol.Index {
     const actual_index = index & symbol_mask;
-    if (isGlobal(index)) return self.global_symbols.items[actual_index];
-    return self.local_symbols.items[actual_index];
+    if (isGlobal(index)) return self.globals()[actual_index];
+    return self.locals()[actual_index];
 }
 
 pub fn elfSym(self: *ZigObject, index: Symbol.Index) *elf.Elf64_Sym {
@@ -1334,11 +1334,15 @@ fn lowerConst(
 
     const sym_index = try self.addAtom(elf_file);
 
-    const res = try codegen.generateSymbol(&elf_file.base, pt, src_loc, val, &code_buffer, .{
-        .none = {},
-    }, .{
-        .parent_atom_index = sym_index,
-    });
+    const res = try codegen.generateSymbol(
+        &elf_file.base,
+        pt,
+        src_loc,
+        val,
+        &code_buffer,
+        .{ .none = {} },
+        .{ .parent_atom_index = sym_index },
+    );
     const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| return .{ .fail = em },