riscv: implement `@fence`

4 files changed, 75 insertions, 34 deletions

diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig
index 9ff9987014..6a073158e6 100644
--- a/src/arch/riscv64/CodeGen.zig
+++ b/src/arch/riscv64/CodeGen.zig
@@ -1318,7 +1318,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
             .breakpoint      => try func.airBreakpoint(),
             .ret_addr        => try func.airRetAddr(inst),
             .frame_addr      => try func.airFrameAddress(inst),
-            .fence           => try func.airFence(),
+            .fence           => try func.airFence(inst),
             .cond_br         => try func.airCondBr(inst),
             .dbg_stmt        => try func.airDbgStmt(inst),
             .fptrunc         => try func.airFptrunc(inst),
@@ -4238,9 +4238,28 @@ fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {
     return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
 }
 
-fn airFence(func: *Func) !void {
-    return func.fail("TODO implement fence() for {}", .{func.target.cpu.arch});
-    //return func.finishAirBookkeeping();
+fn airFence(func: *Func, inst: Air.Inst.Index) !void {
+    const order = func.air.instructions.items(.data)[@intFromEnum(inst)].fence;
+    const pred: Mir.Barrier, const succ: Mir.Barrier = switch (order) {
+        .unordered, .monotonic => unreachable,
+        .acquire => .{ .r, .rw },
+        .release => .{ .rw, .r },
+        .acq_rel => .{ .rw, .rw },
+        .seq_cst => .{ .rw, .rw },
+    };
+
+    _ = try func.addInst(.{
+        .tag = .pseudo,
+        .ops = .pseudo_fence,
+        .data = .{
+            .fence = .{
+                .pred = pred,
+                .succ = succ,
+                .fm = if (order == .acq_rel) .tso else .none,
+            },
+        },
+    });
+    return func.finishAirBookkeeping();
 }
 
 fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
@@ -6264,12 +6283,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
 
     if (order == .seq_cst) {
         _ = try func.addInst(.{
-            .tag = .fence,
-            .ops = .fence,
+            .tag = .pseudo,
+            .ops = .pseudo_fence,
             .data = .{
                 .fence = .{
                     .pred = .rw,
                     .succ = .rw,
+                    .fm = .none,
                 },
             },
         });
@@ -6284,12 +6304,13 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
         // Make sure all previous reads happen before any reading or writing accurs.
         .seq_cst, .acquire => {
             _ = try func.addInst(.{
-                .tag = .fence,
-                .ops = .fence,
+                .tag = .pseudo,
+                .ops = .pseudo_fence,
                 .data = .{
                     .fence = .{
                         .pred = .r,
                         .succ = .rw,
+                        .fm = .none,
                     },
                 },
             });
@@ -6313,12 +6334,13 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr
         .unordered, .monotonic => {},
         .release, .seq_cst => {
             _ = try func.addInst(.{
-                .tag = .fence,
-                .ops = .fence,
+                .tag = .pseudo,
+                .ops = .pseudo_fence,
                 .data = .{
                     .fence = .{
                         .pred = .rw,
                         .succ = .w,
+                        .fm = .none,
                     },
                 },
             });
diff --git a/src/arch/riscv64/Encoding.zig b/src/arch/riscv64/Encoding.zig
index 817decbd54..5f83e3de21 100644
--- a/src/arch/riscv64/Encoding.zig
+++ b/src/arch/riscv64/Encoding.zig
@@ -45,6 +45,11 @@ const AmoWidth = enum(u3) {
     D = 0b011,
 };
 
+const FenceMode = enum(u4) {
+    none = 0b0000,
+    tso = 0b1000,
+};
+
 const Enc = struct {
     opcode: OpCode,
 
@@ -58,6 +63,10 @@ const Enc = struct {
             funct5: u5,
             width: AmoWidth,
         },
+        fence: struct {
+            funct3: u3,
+            fm: FenceMode,
+        },
         /// funct5 + rm + fmt
         fmt: struct {
             funct5: u5,
@@ -210,6 +219,7 @@ pub const Mnemonic = enum {
 
     // MISC
     fence,
+    fencetso,
 
     // AMO
     amoswapw,
@@ -406,9 +416,12 @@ pub const Mnemonic = enum {
             
             .unimp   => .{ .opcode = .NONE, .data = .{ .f = .{ .funct3 = 0b000 } } },
 
+
             // MISC_MEM
 
-            .fence   => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } },
+            .fence    => .{ .opcode = .MISC_MEM, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .none } } },
+            .fencetso => .{ .opcode = .MISC_MEM, .data = .{ .fence = .{ .funct3 = 0b000, .fm = .tso  } } },
+
 
             // AMO
 
@@ -437,7 +450,6 @@ pub const Mnemonic = enum {
             .amomaxud  => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } },
 
                     
-
             // zig fmt: on
         };
     }
@@ -583,6 +595,7 @@ pub const InstEnc = enum {
             => .system,
 
             .fence,
+            .fencetso,
             => .fence,
 
             .amoswapw,
@@ -689,7 +702,7 @@ pub const Data = union(InstEnc) {
         rs1: u5 = 0,
         succ: u4,
         pred: u4,
-        _ignored: u4 = 0,
+        fm: u4,
     },
     amo: packed struct {
         opcode: u7,
@@ -711,11 +724,9 @@ pub const Data = union(InstEnc) {
 
     pub fn toU32(self: Data) u32 {
         return switch (self) {
-            // zig fmt: off
-            .B  => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31),
+            .fence => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.rd)) << 7) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.succ)) << 20) + (@as(u32, @intCast(v.pred)) << 24) + (@as(u32, @intCast(v.fm)) << 28),
             inline else => |v| @bitCast(v),
             .system => unreachable,
-            // zig fmt: on
         };
     }
 
@@ -869,16 +880,17 @@ pub const Data = union(InstEnc) {
             .fence => {
                 assert(ops.len == 2);
 
-                const succ = ops[0];
-                const pred = ops[1];
+                const succ = ops[0].barrier;
+                const pred = ops[1].barrier;
 
                 return .{
                     .fence = .{
-                        .succ = @intFromEnum(succ.barrier),
-                        .pred = @intFromEnum(pred.barrier),
+                        .succ = @intFromEnum(succ),
+                        .pred = @intFromEnum(pred),
 
                         .opcode = @intFromEnum(enc.opcode),
-                        .funct3 = enc.data.f.funct3,
+                        .funct3 = enc.data.fence.funct3,
+                        .fm = @intFromEnum(enc.data.fence.fm),
                     },
                 };
             },
@@ -891,7 +903,7 @@ pub const Data = union(InstEnc) {
                 const rl = ops[3];
                 const aq = ops[4];
 
-                const ret: Data = .{
+                return .{
                     .amo = .{
                         .rd = rd.reg.encodeId(),
                         .rs1 = rs1.reg.encodeId(),
@@ -906,9 +918,6 @@ pub const Data = union(InstEnc) {
                         .funct5 = enc.data.amo.funct5,
                     },
                 };
-
-                std.debug.print("ret: {}, {}", .{ ret.amo.rl, rl.barrier == .rl });
-                return ret;
             },
 
             else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}),
diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig
index bbad404e60..723c64b160 100644
--- a/src/arch/riscv64/Lower.zig
+++ b/src/arch/riscv64/Lower.zig
@@ -443,6 +443,18 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct {
                 });
             },
 
+            .pseudo_fence => {
+                const fence = inst.data.fence;
+
+                try lower.emit(switch (fence.fm) {
+                    .tso => .fencetso,
+                    .none => .fence,
+                }, &.{
+                    .{ .barrier = fence.succ },
+                    .{ .barrier = fence.pred },
+                });
+            },
+
             else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}),
         },
     }
@@ -485,10 +497,6 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.r_type.rs1 },
             .{ .reg = inst.data.r_type.rs2 },
         },
-        .fence => &.{
-            .{ .barrier = inst.data.fence.succ },
-            .{ .barrier = inst.data.fence.pred },
-        },
         else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}),
     });
 }
diff --git a/src/arch/riscv64/Mir.zig b/src/arch/riscv64/Mir.zig
index bb38478bcf..da872da067 100644
--- a/src/arch/riscv64/Mir.zig
+++ b/src/arch/riscv64/Mir.zig
@@ -80,8 +80,6 @@ pub const Inst = struct {
         sh,
         sb,
 
-        fence,
-
         // M extension
         mul,
         mulw,
@@ -256,6 +254,10 @@ pub const Inst = struct {
         fence: struct {
             pred: Barrier,
             succ: Barrier,
+            fm: enum {
+                none,
+                tso,
+            },
         },
 
         amo: struct {
@@ -355,7 +357,7 @@ pub const Inst = struct {
         pseudo_extern_fn_reloc,
 
         /// IORW, IORW
-        fence,
+        pseudo_fence,
 
         /// Ordering, Src, Addr, Dest
         pseudo_amo,
@@ -396,8 +398,8 @@ pub const FrameLoc = struct {
 
 pub const Barrier = enum(u4) {
     // Fence
-    r = 0b0001,
-    w = 0b0010,
+    w = 0b0001,
+    r = 0b0010,
     rw = 0b0011,
 
     // Amo