riscv: `@atomicLoad` and `@atomicStore`

5 files changed, 221 insertions, 50 deletions

diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig
index c5ae354f54..f8b67a95d4 100644
--- a/src/arch/riscv64/CodeGen.zig
+++ b/src/arch/riscv64/CodeGen.zig
@@ -2087,19 +2087,17 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
         const operand = try func.resolveInst(ty_op.operand);
         const ty = func.typeOf(ty_op.operand);
 
-        switch (ty.zigTypeTag(zcu)) {
-            .Bool => {
-                const operand_reg = blk: {
-                    if (operand == .register) break :blk operand.register;
-                    break :blk try func.copyToTmpRegister(ty, operand);
-                };
+        const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
+        defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
 
-                const dst_reg: Register =
-                    if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
-                    operand.register
-                else
-                    (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
+        const dst_reg: Register =
+            if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
+            operand.register
+        else
+            (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
 
+        switch (ty.zigTypeTag(zcu)) {
+            .Bool => {
                 _ = try func.addInst(.{
                     .tag = .pseudo,
                     .ops = .pseudo_not,
@@ -2110,12 +2108,34 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void {
                         },
                     },
                 });
+            },
+            .Int => {
+                const size = ty.bitSize(zcu);
+                if (!math.isPowerOfTwo(size))
+                    return func.fail("TODO: airNot non-pow 2 int size", .{});
 
-                break :result .{ .register = dst_reg };
+                switch (size) {
+                    32, 64 => {
+                        _ = try func.addInst(.{
+                            .tag = .xori,
+                            .ops = .rri,
+                            .data = .{
+                                .i_type = .{
+                                    .rd = dst_reg,
+                                    .rs1 = operand_reg,
+                                    .imm12 = Immediate.s(-1),
+                                },
+                            },
+                        });
+                    },
+                    8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}),
+                    else => unreachable,
+                }
             },
-            .Int => return func.fail("TODO: airNot ints", .{}),
             else => unreachable,
         }
+
+        break :result .{ .register = dst_reg };
     };
     return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
@@ -5600,17 +5620,24 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!
     const abi_size: u32 = @intCast(ty.abiSize(pt));
 
     if (abi_size > 8) return std.debug.panic("tried to set reg with size {}", .{abi_size});
-
     const dst_reg_class = reg.class();
 
     switch (src_mcv) {
-        .dead => unreachable,
-        .unreach, .none => return, // Nothing to do.
+        .unreach,
+        .none,
+        .dead,
+        => unreachable,
         .undef => {
             if (!func.wantSafety())
-                return; // The already existing value will do just fine.
-            // Write the debug undefined value.
-            return func.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa });
+                return;
+
+            switch (abi_size) {
+                1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
+                2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
+                3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }),
+                5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }),
+                else => unreachable,
+            }
         },
         .immediate => |unsigned_x| {
             assert(dst_reg_class == .int);
@@ -6047,14 +6074,82 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
 }
 
 fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
-    _ = inst;
-    return func.fail("TODO implement airAtomicLoad for {}", .{func.target.cpu.arch});
+    const zcu = func.bin_file.comp.module.?;
+    const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
+    const order: std.builtin.AtomicOrder = atomic_load.order;
+
+    const ptr_ty = func.typeOf(atomic_load.ptr);
+    const elem_ty = ptr_ty.childType(zcu);
+    const ptr_mcv = try func.resolveInst(atomic_load.ptr);
+
+    const result_mcv = try func.allocRegOrMem(elem_ty, inst, true);
+
+    if (order == .seq_cst) {
+        _ = try func.addInst(.{
+            .tag = .fence,
+            .ops = .fence,
+            .data = .{
+                .fence = .{
+                    .pred = .rw,
+                    .succ = .rw,
+                },
+            },
+        });
+    }
+
+    try func.load(result_mcv, ptr_mcv, ptr_ty);
+
+    switch (order) {
+        // Don't guarnetee other memory operations to be ordered after the load.
+        .unordered => {},
+        .monotonic => {},
+        // Make sure all previous reads happen before any reading or writing accurs.
+        .seq_cst, .acquire => {
+            _ = try func.addInst(.{
+                .tag = .fence,
+                .ops = .fence,
+                .data = .{
+                    .fence = .{
+                        .pred = .r,
+                        .succ = .rw,
+                    },
+                },
+            });
+        },
+        else => unreachable,
+    }
+
+    return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none });
 }
 
 fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
-    _ = inst;
-    _ = order;
-    return func.fail("TODO implement airAtomicStore for {}", .{func.target.cpu.arch});
+    const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
+
+    const ptr_ty = func.typeOf(bin_op.lhs);
+    const ptr_mcv = try func.resolveInst(bin_op.lhs);
+
+    const val_ty = func.typeOf(bin_op.rhs);
+    const val_mcv = try func.resolveInst(bin_op.rhs);
+
+    switch (order) {
+        .unordered, .monotonic => {},
+        .release, .seq_cst => {
+            _ = try func.addInst(.{
+                .tag = .fence,
+                .ops = .fence,
+                .data = .{
+                    .fence = .{
+                        .pred = .rw,
+                        .succ = .w,
+                    },
+                },
+            });
+        },
+        else => unreachable,
+    }
+
+    try func.store(ptr_mcv, val_mcv, ptr_ty, val_ty);
+    return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
 fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
diff --git a/src/arch/riscv64/Encoding.zig b/src/arch/riscv64/Encoding.zig
index b280b8a483..08044e644e 100644
--- a/src/arch/riscv64/Encoding.zig
+++ b/src/arch/riscv64/Encoding.zig
@@ -2,25 +2,30 @@ mnemonic: Mnemonic,
 data: Data,
 
 const OpCode = enum(u7) {
-    OP = 0b0110011,
+    LOAD = 0b0000011,
+    LOAD_FP = 0b0000111,
+    MISC_MEM = 0b0001111,
     OP_IMM = 0b0010011,
+    AUIPC = 0b0010111,
     OP_IMM_32 = 0b0011011,
-    OP_32 = 0b0111011,
-
-    BRANCH = 0b1100011,
-    LOAD = 0b0000011,
     STORE = 0b0100011,
-    SYSTEM = 0b1110011,
-
-    OP_FP = 0b1010011,
-    LOAD_FP = 0b0000111,
     STORE_FP = 0b0100111,
-
-    JALR = 0b1100111,
-    AUIPC = 0b0010111,
+    AMO = 0b0101111,
+    OP = 0b0110011,
+    OP_32 = 0b0111011,
     LUI = 0b0110111,
+    MADD = 0b1000011,
+    MSUB = 0b1000111,
+    NMSUB = 0b1001011,
+    NMADD = 0b1001111,
+    OP_FP = 0b1010011,
+    OP_IMM_64 = 0b1011011,
+    BRANCH = 0b1100011,
+    JALR = 0b1100111,
     JAL = 0b1101111,
-    NONE = 0b0000000,
+    SYSTEM = 0b1110011,
+    OP_64 = 0b1111011,
+    NONE = 0b00000000,
 };
 
 const Fmt = enum(u2) {
@@ -28,7 +33,9 @@ const Fmt = enum(u2) {
     S = 0b00,
     /// 64-bit double-precision
     D = 0b01,
-    _reserved = 0b10,
+
+    // H = 0b10, unused in the G extension
+
     /// 128-bit quad-precision
     Q = 0b11,
 };
@@ -192,6 +199,9 @@ pub const Mnemonic = enum {
     fsgnjnd,
     fsgnjxd,
 
+    // MISC
+    fence,
+
     pub fn encoding(mnem: Mnemonic) Enc {
         return switch (mnem) {
             // zig fmt: off
@@ -366,6 +376,10 @@ pub const Mnemonic = enum {
             
             .unimp   => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },
 
+            // MISC_MEM
+
+            .fence   => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
+        
 
             // zig fmt: on
         };
@@ -380,7 +394,7 @@ pub const InstEnc = enum {
     B,
     U,
     J,
-
+    fence,
     /// extras that have unusual op counts
     system,
 
@@ -509,20 +523,24 @@ pub const InstEnc = enum {
             .ebreak,
             .unimp,
             => .system,
+
+            .fence,
+            => .fence,
         };
     }
 
     pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) {
         return switch (enc) {
             // zig fmt: off
-            .R      => .{ .reg,  .reg,  .reg,  .none },
-            .R4     => .{ .reg,  .reg,  .reg,  .reg  },  
-            .I      => .{ .reg,  .reg,  .imm,  .none },
-            .S      => .{ .reg,  .reg,  .imm,  .none },
-            .B      => .{ .reg,  .reg,  .imm,  .none },
-            .U      => .{ .reg,  .imm,  .none, .none },
-            .J      => .{ .reg,  .imm,  .none, .none },
-            .system => .{ .none, .none, .none, .none },
+            .R      => .{ .reg,     .reg,     .reg,  .none },
+            .R4     => .{ .reg,     .reg,     .reg,  .reg  },  
+            .I      => .{ .reg,     .reg,     .imm,  .none },
+            .S      => .{ .reg,     .reg,     .imm,  .none },
+            .B      => .{ .reg,     .reg,     .imm,  .none },
+            .U      => .{ .reg,     .imm,     .none, .none },
+            .J      => .{ .reg,     .imm,     .none, .none },
+            .system => .{ .none,    .none,    .none, .none },
+            .fence  => .{ .barrier, .barrier, .none, .none },
             // zig fmt: on
         };
     }
@@ -584,6 +602,15 @@ pub const Data = union(InstEnc) {
         imm1_10: u10,
         imm20: u1,
     },
+    fence: packed struct {
+        opcode: u7,
+        rd: u5 = 0,
+        funct3: u3,
+        rs1: u5 = 0,
+        succ: u4,
+        pred: u4,
+        _ignored: u4 = 0,
+    },
     system: void,
 
     pub fn toU32(self: Data) u32 {
@@ -596,6 +623,7 @@ pub const Data = union(InstEnc) {
             .B  => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
             .U  => |v| @bitCast(v),
             .J  => |v| @bitCast(v),
+            .fence => |v| @bitCast(v),
             .system => unreachable,
             // zig fmt: on
         };
@@ -748,6 +776,22 @@ pub const Data = union(InstEnc) {
                     },
                 };
             },
+            .fence => {
+                assert(ops.len == 2);
+
+                const succ = ops[0];
+                const pred = ops[1];
+
+                return .{
+                    .fence = .{
+                        .succ = @intFromEnum(succ.barrier),
+                        .pred = @intFromEnum(pred.barrier),
+
+                        .opcode = @intFromEnum(enc.opcode),
+                        .funct3 = enc.data.f.funct3,
+                    },
+                };
+            },
 
             else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
         }
diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig
index f71311ff97..56a0305562 100644
--- a/src/arch/riscv64/Lower.zig
+++ b/src/arch/riscv64/Lower.zig
@@ -378,7 +378,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
                 const rr = inst.data.rr;
                 assert(rr.rs.class() == .int and rr.rd.class() == .int);
 
-                try lower.emit(.xori, &.{
+                // mask out any other bits that aren't the boolean
+                try lower.emit(.andi, &.{
+                    .{ .reg = rr.rs },
+                    .{ .reg = rr.rs },
+                    .{ .imm = Immediate.s(1) },
+                });
+
+                try lower.emit(.sltiu, &.{
                     .{ .reg = rr.rd },
                     .{ .reg = rr.rs },
                     .{ .imm = Immediate.s(1) },
@@ -447,6 +454,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.r_type.rs1 },
             .{ .reg = inst.data.r_type.rs2 },
         },
+        .fence => &.{
+            .{ .barrier = inst.data.fence.succ },
+            .{ .barrier = inst.data.fence.pred },
+        },
         else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
     });
 }
diff --git a/src/arch/riscv64/Mir.zig b/src/arch/riscv64/Mir.zig
index 80a533d880..e8567a3275 100644
--- a/src/arch/riscv64/Mir.zig
+++ b/src/arch/riscv64/Mir.zig
@@ -31,6 +31,7 @@ pub const Inst = struct {
         @"and",
         andi,
 
+        xori,
         xor,
         @"or",
 
@@ -38,6 +39,8 @@ pub const Inst = struct {
         ecall,
         unimp,
 
+        fence,
+
         add,
         addw,
         sub,
@@ -246,6 +249,11 @@ pub const Inst = struct {
             atom_index: u32,
             sym_index: u32,
         },
+
+        fence: struct {
+            pred: Barrier,
+            succ: Barrier,
+        },
     };
 
     pub const Ops = enum {
@@ -326,10 +334,15 @@ pub const Inst = struct {
         pseudo_spill_regs,
 
         pseudo_compare,
+
+        /// NOT operation on booleans. Does an `andi reg, reg, 1` to mask out any other bits from the boolean.
         pseudo_not,
 
         /// Generates an auipc + jalr pair, with a R_RISCV_CALL_PLT reloc
         pseudo_extern_fn_reloc,
+
+        /// IORW, IORW
+        fence,
     };
 
     // Make sure we don't accidentally make instructions bigger than expected.
@@ -365,6 +378,12 @@ pub const FrameLoc = struct {
     disp: i32,
 };
 
+pub const Barrier = enum(u4) {
+    r = 0b0001,
+    w = 0b0010,
+    rw = 0b0011,
+};
+
 /// Returns the requested data, as well as the new index which is at the start of the
 /// trailers for the object.
 pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {
diff --git a/src/arch/riscv64/encoder.zig b/src/arch/riscv64/encoder.zig
index 54d1549ebe..b219b709de 100644
--- a/src/arch/riscv64/encoder.zig
+++ b/src/arch/riscv64/encoder.zig
@@ -1,12 +1,13 @@
 pub const Instruction = struct {
     encoding: Encoding,
-    ops: [3]Operand = .{.none} ** 3,
+    ops: [4]Operand = .{.none} ** 4,
 
     pub const Operand = union(enum) {
         none,
         reg: Register,
         mem: Memory,
         imm: Immediate,
+        barrier: Mir.Barrier,
     };
 
     pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction {
@@ -20,7 +21,7 @@ pub const Instruction = struct {
             return error.InvalidInstruction;
         };
 
-        var result_ops: [3]Operand = .{.none} ** 3;
+        var result_ops: [4]Operand = .{.none} ** 4;
         @memcpy(result_ops[0..ops.len], ops);
 
         return .{
@@ -54,6 +55,7 @@ pub const Instruction = struct {
                 .reg => |reg| try writer.writeAll(@tagName(reg)),
                 .imm => |imm| try writer.print("{d}", .{imm.asSigned(64)}),
                 .mem => unreachable, // there is no "mem" operand in the actual instructions
+                .barrier => |barrier| try writer.writeAll(@tagName(barrier)),
             }
         }
     }