Merge pull request #18906 from jacobly0/x86_64-tests

x86_64: pass more tests

5 files changed, 1764 insertions, 322 deletions

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index f9a291f40b..4ca2ae44bb 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(
     });
 }
 
+fn asmRegisterRegisterMemoryRegister(
+    self: *Self,
+    tag: Mir.Inst.FixedTag,
+    reg1: Register,
+    reg2: Register,
+    m: Memory,
+    reg3: Register,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag[1],
+        .ops = .rrmr,
+        .data = .{ .rrrx = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
+            .r3 = reg3,
+            .payload = try self.addExtra(Mir.Memory.encode(m)),
+        } },
+    });
+}
+
 fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
     _ = try self.addInst(.{
         .tag = tag[1],
@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
     });
 }
 
+fn asmRegisterMemoryRegister(
+    self: *Self,
+    tag: Mir.Inst.FixedTag,
+    reg1: Register,
+    m: Memory,
+    reg2: Register,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag[1],
+        .ops = .rmr,
+        .data = .{ .rrx = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
+            .payload = try self.addExtra(Mir.Memory.encode(m)),
+        } },
+    });
+}
+
 fn asmRegisterMemoryImmediate(
     self: *Self,
     tag: Mir.Inst.FixedTag,
@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
 
     const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
     var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
-        if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
+        if (opts.update_tracking)
+    {} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
 
     var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
         stack.get(),
@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.comp.module.?;
     const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
     const dst_ty = self.typeOfIndex(inst);
-    const dst_bits = dst_ty.floatBits(self.target.*);
+    const dst_scalar_ty = dst_ty.scalarType(mod);
+    const dst_bits = dst_scalar_ty.floatBits(self.target.*);
     const src_ty = self.typeOf(ty_op.operand);
-    const src_bits = src_ty.floatBits(self.target.*);
+    const src_scalar_ty = src_ty.scalarType(mod);
+    const src_bits = src_scalar_ty.floatBits(self.target.*);
 
     const result = result: {
         if (switch (src_bits) {
@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
             },
             else => unreachable,
         }) {
+            if (dst_ty.isVector(mod)) break :result null;
             var callee_buf: ["__extend?f?f2".len]u8 = undefined;
             break :result try self.genCall(.{ .lib = .{
-                .return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(),
-                .param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()},
+                .return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(),
+                .param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()},
                 .callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{
                     floatCompilerRtAbiName(src_bits),
                     floatCompilerRtAbiName(dst_bits),
                 }) catch unreachable,
-            } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
+            } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});
         }
 
+        const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
         const src_mcv = try self.resolveInst(ty_op.operand);
         const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
             src_mcv
         else
             try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
-        const dst_reg = dst_mcv.getReg().?.to128();
+        const dst_reg = dst_mcv.getReg().?;
+        const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(mod), 16)));
         const dst_lock = self.register_manager.lockReg(dst_reg);
         defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
+        const vec_len = if (dst_ty.isVector(mod)) dst_ty.vectorLen(mod) else 1;
         if (src_bits == 16) {
             assert(self.hasFeature(.f16c));
             const mat_src_reg = if (src_mcv.isRegister())
                 src_mcv.getReg().?
             else
                 try self.copyToTmpRegister(src_ty, src_mcv);
-            try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());
+            try self.asmRegisterRegister(
+                .{ .v_ps, .cvtph2 },
+                dst_alias,
+                registerAlias(mat_src_reg, src_abi_size),
+            );
             switch (dst_bits) {
                 32 => {},
                 64 => try self.asmRegisterRegisterRegister(
                     .{ .v_sd, .cvtss2 },
-                    dst_reg,
-                    dst_reg,
-                    dst_reg,
+                    dst_alias,
+                    dst_alias,
+                    dst_alias,
                 ),
                 else => unreachable,
             }
         } else {
             assert(src_bits == 32 and dst_bits == 64);
-            if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
-                .{ .v_sd, .cvtss2 },
-                dst_reg,
-                dst_reg,
-                try src_mcv.mem(self, .dword),
-            ) else try self.asmRegisterRegisterRegister(
-                .{ .v_sd, .cvtss2 },
-                dst_reg,
-                dst_reg,
-                (if (src_mcv.isRegister())
-                    src_mcv.getReg().?
-                else
-                    try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
-            ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                .{ ._sd, .cvtss2 },
-                dst_reg,
-                try src_mcv.mem(self, .dword),
+            if (self.hasFeature(.avx)) switch (vec_len) {
+                1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
+                    .{ .v_sd, .cvtss2 },
+                    dst_alias,
+                    dst_alias,
+                    try src_mcv.mem(self, self.memSize(src_ty)),
+                ) else try self.asmRegisterRegisterRegister(
+                    .{ .v_sd, .cvtss2 },
+                    dst_alias,
+                    dst_alias,
+                    registerAlias(if (src_mcv.isRegister())
+                        src_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
+                ),
+                2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                    .{ .v_pd, .cvtps2 },
+                    dst_alias,
+                    try src_mcv.mem(self, self.memSize(src_ty)),
+                ) else try self.asmRegisterRegister(
+                    .{ .v_pd, .cvtps2 },
+                    dst_alias,
+                    registerAlias(if (src_mcv.isRegister())
+                        src_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
+                ),
+                else => break :result null,
+            } else if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                switch (vec_len) {
+                    1 => .{ ._sd, .cvtss2 },
+                    2 => .{ ._pd, .cvtps2 },
+                    else => break :result null,
+                },
+                dst_alias,
+                try src_mcv.mem(self, self.memSize(src_ty)),
             ) else try self.asmRegisterRegister(
-                .{ ._sd, .cvtss2 },
-                dst_reg,
-                (if (src_mcv.isRegister())
+                switch (vec_len) {
+                    1 => .{ ._sd, .cvtss2 },
+                    2 => .{ ._pd, .cvtps2 },
+                    else => break :result null,
+                },
+                dst_alias,
+                registerAlias(if (src_mcv.isRegister())
                     src_mcv.getReg().?
                 else
-                    try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
+                    try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
             );
         }
         break :result dst_mcv;
-    };
+    } orelse return self.fail("TODO implement airFpext from {} to {}", .{
+        src_ty.fmt(mod), dst_ty.fmt(mod),
+    });
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
     const mod = self.bin_file.comp.module.?;
     const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
-    const result: MCValue = result: {
-        const src_ty = self.typeOf(ty_op.operand);
-        const src_int_info = src_ty.intInfo(mod);
+    const src_ty = self.typeOf(ty_op.operand);
+    const dst_ty = self.typeOfIndex(inst);
 
-        const dst_ty = self.typeOfIndex(inst);
-        const dst_int_info = dst_ty.intInfo(mod);
-        const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
+    const result = @as(?MCValue, result: {
+        const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
 
-        const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
+        const src_int_info = src_ty.intInfo(mod);
+        const dst_int_info = dst_ty.intInfo(mod);
         const extend = switch (src_int_info.signedness) {
             .signed => dst_int_info,
             .unsigned => src_int_info,
         }.signedness;
 
         const src_mcv = try self.resolveInst(ty_op.operand);
+        if (dst_ty.isVector(mod)) {
+            const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
+            const max_abi_size = @max(dst_abi_size, src_abi_size);
+            if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;
+            const has_avx = self.hasFeature(.avx);
+
+            const dst_elem_abi_size = dst_ty.childType(mod).abiSize(mod);
+            const src_elem_abi_size = src_ty.childType(mod).abiSize(mod);
+            switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
+                .lt => {
+                    const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
+                        else => break :result null,
+                        1 => switch (src_elem_abi_size) {
+                            else => break :result null,
+                            2 => switch (dst_int_info.signedness) {
+                                .signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw },
+                                .unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
+                            },
+                        },
+                        2 => switch (src_elem_abi_size) {
+                            else => break :result null,
+                            4 => switch (dst_int_info.signedness) {
+                                .signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd },
+                                .unsigned => if (has_avx)
+                                    .{ .vp_w, .ackusd }
+                                else if (self.hasFeature(.sse4_1))
+                                    .{ .p_w, .ackusd }
+                                else
+                                    break :result null,
+                            },
+                        },
+                    };
+
+                    const dst_mcv: MCValue = if (src_mcv.isRegister() and
+                        self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                        src_mcv
+                    else if (has_avx and src_mcv.isRegister())
+                        .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+                    else
+                        try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv);
+                    const dst_reg = dst_mcv.getReg().?;
+                    const dst_alias = registerAlias(dst_reg, dst_abi_size);
+
+                    if (has_avx) try self.asmRegisterRegisterRegister(
+                        mir_tag,
+                        dst_alias,
+                        registerAlias(if (src_mcv.isRegister())
+                            src_mcv.getReg().?
+                        else
+                            dst_reg, src_abi_size),
+                        dst_alias,
+                    ) else try self.asmRegisterRegister(
+                        mir_tag,
+                        dst_alias,
+                        dst_alias,
+                    );
+                    break :result dst_mcv;
+                },
+                .eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                    break :result src_mcv
+                else {
+                    const dst_mcv = try self.allocRegOrMem(inst, true);
+                    try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
+                    break :result dst_mcv;
+                },
+                .gt => if (self.hasFeature(.sse4_1)) {
+                    const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) {
+                        else => break :result null,
+                        2 => if (has_avx) .vp_w else .p_w,
+                        4 => if (has_avx) .vp_d else .p_d,
+                        8 => if (has_avx) .vp_q else .p_q,
+                    }, switch (src_elem_abi_size) {
+                        else => break :result null,
+                        1 => switch (extend) {
+                            .signed => .movsxb,
+                            .unsigned => .movzxb,
+                        },
+                        2 => switch (extend) {
+                            .signed => .movsxw,
+                            .unsigned => .movzxw,
+                        },
+                        4 => switch (extend) {
+                            .signed => .movsxd,
+                            .unsigned => .movzxd,
+                        },
+                    } };
+
+                    const dst_mcv: MCValue = if (src_mcv.isRegister() and
+                        self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                        src_mcv
+                    else
+                        .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) };
+                    const dst_reg = dst_mcv.getReg().?;
+                    const dst_alias = registerAlias(dst_reg, dst_abi_size);
+
+                    if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                        mir_tag,
+                        dst_alias,
+                        try src_mcv.mem(self, self.memSize(src_ty)),
+                    ) else try self.asmRegisterRegister(
+                        mir_tag,
+                        dst_alias,
+                        registerAlias(if (src_mcv.isRegister())
+                            src_mcv.getReg().?
+                        else
+                            try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
+                    );
+                    break :result dst_mcv;
+                } else {
+                    const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
+                        else => break :result null,
+                        2 => switch (src_elem_abi_size) {
+                            else => break :result null,
+                            1 => .{ .p_, .unpcklbw },
+                        },
+                        4 => switch (src_elem_abi_size) {
+                            else => break :result null,
+                            2 => .{ .p_, .unpcklwd },
+                        },
+                        8 => switch (src_elem_abi_size) {
+                            else => break :result null,
+                            2 => .{ .p_, .unpckldq },
+                        },
+                    };
+
+                    const dst_mcv: MCValue = if (src_mcv.isRegister() and
+                        self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                        src_mcv
+                    else
+                        try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
+                    const dst_reg = dst_mcv.getReg().?;
+
+                    const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
+                    const ext_alias = registerAlias(ext_reg, src_abi_size);
+                    const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg);
+                    defer self.register_manager.unlockReg(ext_lock);
+
+                    try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias);
+                    switch (extend) {
+                        .signed => try self.asmRegisterRegister(
+                            .{ switch (src_elem_abi_size) {
+                                else => unreachable,
+                                1 => .p_b,
+                                2 => .p_w,
+                                4 => .p_d,
+                            }, .cmpgt },
+                            ext_alias,
+                            registerAlias(dst_reg, src_abi_size),
+                        ),
+                        .unsigned => {},
+                    }
+                    try self.asmRegisterRegister(
+                        mir_tag,
+                        registerAlias(dst_reg, dst_abi_size),
+                        registerAlias(ext_reg, dst_abi_size),
+                    );
+                    break :result dst_mcv;
+                },
+            }
+            @compileError("unreachable");
+        }
+
+        const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
+
         const src_storage_bits: u16 = switch (src_mcv) {
             .register, .register_offset => 64,
             .register_pair => 128,
@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
         };
 
         if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
-            .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }
+            .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }
         else
             dst_mcv;
 
         if (dst_mcv.isRegister()) {
             try self.truncateRegister(src_ty, dst_mcv.getReg().?);
-            break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
+            break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
         }
 
         const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
         );
 
         break :result dst_mcv;
-    };
+    }) orelse return self.fail("TODO implement airIntCast from {} to {}", .{
+        src_ty.fmt(mod), dst_ty.fmt(mod),
+    });
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
             src_mcv
         else if (dst_abi_size <= 8)
             try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
-        else if (dst_abi_size <= 16) dst: {
+        else if (dst_abi_size <= 16 and !dst_ty.isVector(mod)) dst: {
             const dst_regs =
                 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
             const dst_mcv: MCValue = .{ .register_pair = dst_regs };
@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
             try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
             break :dst dst_mcv;
         } else dst: {
-            const dst_mcv = try self.allocRegOrMem(inst, true);
-            try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
+            const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
+            try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
             break :dst dst_mcv;
         };
 
         if (dst_ty.zigTypeTag(mod) == .Vector) {
             assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));
-            const dst_info = dst_ty.childType(mod).intInfo(mod);
-            const src_info = src_ty.childType(mod).intInfo(mod);
-            const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_info.bits) {
-                8 => switch (src_info.bits) {
-                    16 => switch (dst_ty.vectorLen(mod)) {
+            const dst_elem_ty = dst_ty.childType(mod);
+            const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(mod));
+            const src_elem_ty = src_ty.childType(mod);
+            const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(mod));
+
+            const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) {
+                1 => switch (src_elem_abi_size) {
+                    2 => switch (dst_ty.vectorLen(mod)) {
                         1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
                         9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
                         else => null,
                     },
                     else => null,
                 },
-                16 => switch (src_info.bits) {
-                    32 => switch (dst_ty.vectorLen(mod)) {
+                2 => switch (src_elem_abi_size) {
+                    4 => switch (dst_ty.vectorLen(mod)) {
                         1...4 => if (self.hasFeature(.avx))
                             .{ .vp_w, .ackusd }
                         else if (self.hasFeature(.sse4_1))
@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
                 else => null,
             }) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});
 
-            const elem_ty = src_ty.childType(mod);
-            const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
+            const dst_info = dst_elem_ty.intInfo(mod);
+            const src_info = src_elem_ty.intInfo(mod);
+
+            const mask_val = try mod.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
 
             const splat_ty = try mod.vectorType(.{
                 .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
-                .child = elem_ty.ip_index,
+                .child = src_elem_ty.ip_index,
             });
             const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));
 
@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
                 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
             };
 
-            const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);
+            const dst_reg = dst_mcv.getReg().?;
+            const dst_alias = registerAlias(dst_reg, src_abi_size);
             if (self.hasFeature(.avx)) {
                 try self.asmRegisterRegisterMemory(
                     .{ .vp_, .@"and" },
-                    dst_reg,
-                    dst_reg,
+                    dst_alias,
+                    dst_alias,
                     try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
                 );
-                try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
+                if (src_abi_size > 16) {
+                    const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
+                    const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
+                    defer self.register_manager.unlockReg(temp_lock);
+
+                    try self.asmRegisterRegisterImmediate(
+                        .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
+                        registerAlias(temp_reg, dst_abi_size),
+                        dst_alias,
+                        Immediate.u(1),
+                    );
+                    try self.asmRegisterRegisterRegister(
+                        mir_tag,
+                        registerAlias(dst_reg, dst_abi_size),
+                        registerAlias(dst_reg, dst_abi_size),
+                        registerAlias(temp_reg, dst_abi_size),
+                    );
+                } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
             } else {
                 try self.asmRegisterMemory(
                     .{ .p_, .@"and" },
-                    dst_reg,
+                    dst_alias,
                     try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
                 );
-                try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
+                try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
             }
             break :result dst_mcv;
         }
@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
             if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
                 const slow_inc = self.hasFeature(.slow_incdec);
                 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
-                const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
+                const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
 
                 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
                 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
         switch (lhs_ty.zigTypeTag(mod)) {
             .Int => {
                 try self.spillRegisters(&.{.rcx});
-                try self.register_manager.getReg(.rcx, null);
+                try self.register_manager.getKnownReg(.rcx, null);
                 const lhs_mcv = try self.resolveInst(bin_op.lhs);
                 const rhs_mcv = try self.resolveInst(bin_op.rhs);
 
@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
 
             const dst_mcv: MCValue = .{ .register = .st0 };
             if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
-                try self.register_manager.getReg(.st0, inst);
+                try self.register_manager.getKnownReg(.st0, inst);
 
             try self.genCopy(ty, dst_mcv, src_mcv, .{});
             switch (tag) {
@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
                 },
                 else => {
                     const abi_size: u31 = @intCast(ty.abiSize(mod));
-                    const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
+                    const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
 
                     const tmp_regs =
                         try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(
                         try self.asmRegisterImmediate(
                             .{ ._, .@"and" },
                             .cl,
-                            Immediate.u(std.math.maxInt(u6)),
+                            Immediate.u(math.maxInt(u6)),
                         );
                         try self.asmRegisterImmediate(
                             .{ ._r, .sh },
@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(
                         try self.asmRegisterImmediate(
                             .{ ._, .@"and" },
                             .cl,
-                            Immediate.u(std.math.maxInt(u6)),
+                            Immediate.u(math.maxInt(u6)),
                         );
                         try self.asmRegisterImmediate(
                             .{ ._r, .sh },
@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(
                     }, .sh },
                     temp_regs[2].to64(),
                     temp_regs[3].to64(),
-                    Immediate.u(shift_imm & std.math.maxInt(u6)),
+                    Immediate.u(shift_imm & math.maxInt(u6)),
                 ),
                 else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
                     ._l => ._ld,
@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(
             .immediate => |shift_imm| try self.asmRegisterImmediate(
                 tag,
                 temp_regs[2].to64(),
-                Immediate.u(shift_imm & std.math.maxInt(u6)),
+                Immediate.u(shift_imm & math.maxInt(u6)),
             ),
             else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
         }
@@ -8794,7 +9059,7 @@ fn genShiftBinOp(
         lhs_ty.fmt(mod),
     });
 
-    try self.register_manager.getReg(.rcx, null);
+    try self.register_manager.getKnownReg(.rcx, null);
     const rcx_lock = self.register_manager.lockReg(.rcx);
     defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
 
@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(
         switch (tag) {
             .mul, .mul_wrap => {
                 const slow_inc = self.hasFeature(.slow_incdec);
-                const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
+                const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
 
                 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
                 const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(
                 .rem => maybe_inst,
                 else => null,
             };
-            try self.register_manager.getReg(.rax, track_inst_rax);
-            try self.register_manager.getReg(.rdx, track_inst_rdx);
+            try self.register_manager.getKnownReg(.rax, track_inst_rax);
+            try self.register_manager.getKnownReg(.rdx, track_inst_rdx);
 
             try self.genIntMulDivOpMir(switch (signedness) {
                 .signed => switch (tag) {
@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(
         },
 
         .mod => {
-            try self.register_manager.getReg(.rax, null);
-            try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null);
+            try self.register_manager.getKnownReg(.rax, null);
+            try self.register_manager.getKnownReg(
+                .rdx,
+                if (signedness == .unsigned) maybe_inst else null,
+            );
 
             switch (signedness) {
                 .signed => {
@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(
         },
 
         .div_floor => {
-            try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null);
-            try self.register_manager.getReg(.rdx, null);
+            try self.register_manager.getKnownReg(
+                .rax,
+                if (signedness == .unsigned) maybe_inst else null,
+            );
+            try self.register_manager.getKnownReg(.rdx, null);
 
             const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
                 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
@@ -9445,7 +9716,7 @@ fn genBinOp(
         .rem, .mod => unreachable,
         .max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(
             if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
-                try self.register_manager.getReg(.xmm0, null);
+                try self.register_manager.getKnownReg(.xmm0, null);
                 break :mask .xmm0;
             } else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
             abi_size,
@@ -10820,96 +11091,35 @@ fn genBinOp(
                 lhs_copy_reg.?,
                 mask_reg,
             ) else {
-                try self.asmRegisterRegister(
-                    @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
-                        .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                            32 => .{ ._ps, .@"and" },
-                            64 => .{ ._pd, .@"and" },
-                            16, 80, 128 => null,
-                            else => unreachable,
-                        },
-                        .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
-                            .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
-                                32 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...4 => .{ ._ps, .@"and" },
-                                    else => null,
-                                },
-                                64 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...2 => .{ ._pd, .@"and" },
-                                    else => null,
-                                },
-                                16, 80, 128 => null,
-                                else => unreachable,
-                            },
-                            else => unreachable,
-                        },
+                const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) {
+                    .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                        32 => ._ps,
+                        64 => ._pd,
+                        16, 80, 128 => null,
                         else => unreachable,
-                    }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
-                        @tagName(air_tag), lhs_ty.fmt(mod),
-                    }),
-                    dst_reg,
-                    mask_reg,
-                );
-                try self.asmRegisterRegister(
-                    @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
-                        .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                            32 => .{ ._ps, .andn },
-                            64 => .{ ._pd, .andn },
-                            16, 80, 128 => null,
-                            else => unreachable,
-                        },
-                        .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
-                            .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
-                                32 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...4 => .{ ._ps, .andn },
-                                    else => null,
-                                },
-                                64 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...2 => .{ ._pd, .andn },
-                                    else => null,
-                                },
-                                16, 80, 128 => null,
-                                else => unreachable,
+                    },
+                    .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
+                        .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
+                            32 => switch (lhs_ty.vectorLen(mod)) {
+                                1...4 => ._ps,
+                                else => null,
                             },
-                            else => unreachable,
-                        },
-                        else => unreachable,
-                    }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
-                        @tagName(air_tag), lhs_ty.fmt(mod),
-                    }),
-                    mask_reg,
-                    lhs_copy_reg.?,
-                );
-                try self.asmRegisterRegister(
-                    @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
-                        .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                            32 => .{ ._ps, .@"or" },
-                            64 => .{ ._pd, .@"or" },
-                            16, 80, 128 => null,
-                            else => unreachable,
-                        },
-                        .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
-                            .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
-                                32 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...4 => .{ ._ps, .@"or" },
-                                    else => null,
-                                },
-                                64 => switch (lhs_ty.vectorLen(mod)) {
-                                    1...2 => .{ ._pd, .@"or" },
-                                    else => null,
-                                },
-                                16, 80, 128 => null,
-                                else => unreachable,
+                            64 => switch (lhs_ty.vectorLen(mod)) {
+                                1...2 => ._pd,
+                                else => null,
                             },
+                            16, 80, 128 => null,
                             else => unreachable,
                         },
                         else => unreachable,
-                    }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
-                        @tagName(air_tag), lhs_ty.fmt(mod),
-                    }),
-                    dst_reg,
-                    mask_reg,
-                );
+                    },
+                    else => unreachable,
+                }) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
+                    @tagName(air_tag), lhs_ty.fmt(mod),
+                });
+                try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg);
+                try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?);
+                try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg);
             }
         },
         .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
@@ -12192,9 +12402,36 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
 fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
     const mod = self.bin_file.comp.module.?;
     const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
-    const ty = self.typeOf(bin_op.lhs);
+    var ty = self.typeOf(bin_op.lhs);
+    var null_compare: ?Mir.Inst.Index = null;
 
     const result: Condition = result: {
+        try self.spillEflagsIfOccupied();
+
+        const lhs_mcv = try self.resolveInst(bin_op.lhs);
+        const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
+            .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
+            .register_pair => |lhs_regs| locks: {
+                const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
+                break :locks .{ locks[0], locks[1] };
+            },
+            .register_offset => |lhs_ro| .{
+                self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
+                null,
+            },
+            else => .{null} ** 2,
+        };
+        defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const rhs_mcv = try self.resolveInst(bin_op.rhs);
+        const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
+            .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
+            .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
+            .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
+            else => .{null} ** 2,
+        };
+        defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
+
         switch (ty.zigTypeTag(mod)) {
             .Float => {
                 const float_bits = ty.floatBits(self.target.*);
@@ -12231,34 +12468,66 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                     };
                 }
             },
-            else => {},
-        }
+            .Optional => if (!ty.optionalReprIsPayload(mod)) {
+                const opt_ty = ty;
+                const opt_abi_size: u31 = @intCast(opt_ty.abiSize(mod));
+                ty = opt_ty.optionalChild(mod);
+                const payload_abi_size: u31 = @intCast(ty.abiSize(mod));
 
-        try self.spillEflagsIfOccupied();
+                const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
+                const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
+                defer self.register_manager.unlockReg(temp_lhs_lock);
 
-        const lhs_mcv = try self.resolveInst(bin_op.lhs);
-        const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
-            .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
-            .register_pair => |lhs_regs| locks: {
-                const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
-                break :locks .{ locks[0], locks[1] };
-            },
-            .register_offset => |lhs_ro| .{
-                self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
-                null,
-            },
-            else => .{null} ** 2,
-        };
-        defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
+                if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
+                    .{ ._, .mov },
+                    temp_lhs_reg.to8(),
+                    try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
+                ) else {
+                    try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
+                    try self.asmRegisterImmediate(
+                        .{ ._r, .sh },
+                        registerAlias(temp_lhs_reg, opt_abi_size),
+                        Immediate.u(payload_abi_size * 8),
+                    );
+                }
 
-        const rhs_mcv = try self.resolveInst(bin_op.rhs);
-        const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
-            .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
-            .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
-            .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
-            else => .{null} ** 2,
-        };
-        defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
+                const payload_compare = payload_compare: {
+                    if (rhs_mcv.isMemory()) {
+                        const rhs_mem =
+                            try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
+                        try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
+                        const payload_compare = try self.asmJccReloc(.nz, undefined);
+                        try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
+                        break :payload_compare payload_compare;
+                    }
+
+                    const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv);
+                    const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg);
+                    defer self.register_manager.unlockReg(temp_rhs_lock);
+
+                    try self.asmRegisterImmediate(
+                        .{ ._r, .sh },
+                        registerAlias(temp_rhs_reg, opt_abi_size),
+                        Immediate.u(payload_abi_size * 8),
+                    );
+                    try self.asmRegisterRegister(
+                        .{ ._, .@"test" },
+                        temp_lhs_reg.to8(),
+                        temp_rhs_reg.to8(),
+                    );
+                    const payload_compare = try self.asmJccReloc(.nz, undefined);
+                    try self.asmRegisterRegister(
+                        .{ ._, .cmp },
+                        temp_lhs_reg.to8(),
+                        temp_rhs_reg.to8(),
+                    );
+                    break :payload_compare payload_compare;
+                };
+                null_compare = try self.asmJmpReloc(undefined);
+                self.performReloc(payload_compare);
+            },
+            else => {},
+        }
 
         switch (ty.zigTypeTag(mod)) {
             else => {
@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
         }
     };
 
+    if (null_compare) |reloc| self.performReloc(reloc);
     self.eflags_inst = inst;
     return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
 }
@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
         } else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
             const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
             if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});
+            try self.genCopy(ty, args.items[index], input_mcv, .{});
             break :arg args.items[index];
         } else return self.fail("invalid constraint: '{s}'", .{constraint});
         if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
             label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
         } else continue;
 
-        var mnem_size: ?Memory.Size = null;
-        const mnem_tag = mnem: {
-            mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
-                .byte
-            else if (mem.endsWith(u8, mnem_str, "w"))
-                .word
-            else if (mem.endsWith(u8, mnem_str, "l"))
-                .dword
-            else if (mem.endsWith(u8, mnem_str, "q"))
-                .qword
-            else if (mem.endsWith(u8, mnem_str, "t"))
-                .tbyte
-            else
-                break :mnem null;
-            break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);
-        } orelse mnem: {
+        var mnem_size: ?Memory.Size = if (mem.endsWith(u8, mnem_str, "b"))
+            .byte
+        else if (mem.endsWith(u8, mnem_str, "w"))
+            .word
+        else if (mem.endsWith(u8, mnem_str, "l"))
+            .dword
+        else if (mem.endsWith(u8, mnem_str, "q") and
+            (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
+            .qword
+        else if (mem.endsWith(u8, mnem_str, "t"))
+            .tbyte
+        else
+            null;
+        const mnem_tag = while (true) break std.meta.stringToEnum(
+            Instruction.Mnemonic,
+            mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
+        ) orelse if (mnem_size) |_| {
             mnem_size = null;
-            break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
-        } orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
+            continue;
+        } else return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
         if (@as(?Memory.Size, switch (mnem_tag) {
             .clflush => .byte,
             .fldenv, .fnstenv, .fstenv => .none,
@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                     else => {},
                 },
                 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
-                    8 => switch (ty.vectorLen(mod)) {
-                        1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
-                            .insert = .{ .vp_b, .insr },
-                            .extract = .{ .vp_b, .extr },
-                        } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
-                            .insert = .{ .p_b, .insr },
-                            .extract = .{ .p_b, .extr },
-                        } },
-                        2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
-                            .insert = .{ .vp_w, .insr },
-                            .extract = .{ .vp_w, .extr },
-                        } } else .{ .insert_extract = .{
-                            .insert = .{ .p_w, .insr },
-                            .extract = .{ .p_w, .extr },
-                        } },
-                        3...4 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_d, .mov }
-                        else
-                            .{ ._d, .mov } },
-                        5...8 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        9...16 => return .{ .move = if (self.hasFeature(.avx))
+                    1...8 => switch (ty.vectorLen(mod)) {
+                        1...16 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         17...32 => if (self.hasFeature(.avx))
@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                                 .{ .v_, .movdqu } },
                         else => {},
                     },
-                    16 => switch (ty.vectorLen(mod)) {
-                        1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
-                            .insert = .{ .vp_w, .insr },
-                            .extract = .{ .vp_w, .extr },
-                        } } else .{ .insert_extract = .{
-                            .insert = .{ .p_w, .insr },
-                            .extract = .{ .p_w, .extr },
-                        } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_d, .mov }
-                        else
-                            .{ ._d, .mov } },
-                        3...4 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        5...8 => return .{ .move = if (self.hasFeature(.avx))
+                    9...16 => switch (ty.vectorLen(mod)) {
+                        1...8 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         9...16 => if (self.hasFeature(.avx))
@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                                 .{ .v_, .movdqu } },
                         else => {},
                     },
-                    32 => switch (ty.vectorLen(mod)) {
-                        1 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_d, .mov }
-                        else
-                            .{ ._d, .mov } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        3...4 => return .{ .move = if (self.hasFeature(.avx))
+                    17...32 => switch (ty.vectorLen(mod)) {
+                        1...4 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         5...8 => if (self.hasFeature(.avx))
@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                                 .{ .v_, .movdqu } },
                         else => {},
                     },
-                    64 => switch (ty.vectorLen(mod)) {
-                        1 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
+                    33...64 => switch (ty.vectorLen(mod)) {
+                        1...2 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         3...4 => if (self.hasFeature(.avx))
@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                                 .{ .v_, .movdqu } },
                         else => {},
                     },
-                    128 => switch (ty.vectorLen(mod)) {
+                    65...128 => switch (ty.vectorLen(mod)) {
                         1 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                                 .{ .v_, .movdqu } },
                         else => {},
                     },
-                    256 => switch (ty.vectorLen(mod)) {
+                    129...256 => switch (ty.vectorLen(mod)) {
                         1 => if (self.hasFeature(.avx))
                             return .{ .move = if (aligned)
                                 .{ .v_, .movdqa }
@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                 },
                 .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))
                     switch (ty.vectorLen(mod)) {
-                        1 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
+                        1...2 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         3...4 => if (self.hasFeature(.avx))
@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                     unreachable,
                 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
                     16 => switch (ty.vectorLen(mod)) {
-                        1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
-                            .insert = .{ .vp_w, .insr },
-                            .extract = .{ .vp_w, .extr },
-                        } } else .{ .insert_extract = .{
-                            .insert = .{ .p_w, .insr },
-                            .extract = .{ .p_w, .extr },
-                        } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_d, .mov }
-                        else
-                            .{ ._d, .mov } },
-                        3...4 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_q, .mov }
-                        else
-                            .{ ._q, .mov } },
-                        5...8 => return .{ .move = if (self.hasFeature(.avx))
+                        1...8 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
                         else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
                         9...16 => if (self.hasFeature(.avx))
@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                         else => {},
                     },
                     32 => switch (ty.vectorLen(mod)) {
-                        1 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_ss, .mov }
-                        else
-                            .{ ._ss, .mov } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_sd, .mov }
-                        else
-                            .{ ._sd, .mov } },
-                        3...4 => return .{ .move = if (self.hasFeature(.avx))
+                        1...4 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
                         else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
                         5...8 => if (self.hasFeature(.avx))
@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
                         else => {},
                     },
                     64 => switch (ty.vectorLen(mod)) {
-                        1 => return .{ .move = if (self.hasFeature(.avx))
-                            .{ .v_sd, .mov }
-                        else
-                            .{ ._sd, .mov } },
-                        2 => return .{ .move = if (self.hasFeature(.avx))
+                        1...2 => return .{ .move = if (self.hasFeature(.avx))
                             if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
                         else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
                         3...4 => if (self.hasFeature(.avx))
@@ -14633,7 +14825,7 @@ fn genSetReg(
                 ty,
                 dst_reg.class(),
                 self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
-                    std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
+                    math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
                 )),
             ),
             .lea_frame => .{ .move = .{ ._, .lea } },
@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
                     },
                     65...128 => switch (vector_len) {
                         else => null,
-                        1...2 => .{ .vp_i128, .broadcast },
+                        1...2 => .{ .v_i128, .broadcast },
                     },
                 }) orelse break :avx2;
 
@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
                     registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
                     try src_mcv.mem(self, self.memSize(scalar_ty)),
                 ) else {
-                    if (mir_tag[0] == .vp_i128) break :avx2;
+                    if (mir_tag[0] == .v_i128) break :avx2;
                     try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
                     try self.asmRegisterRegister(
                         mir_tag,
@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
                     .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
                     dst_alias,
                     dst_alias,
-                    Immediate.u(0),
+                    Immediate.u(0b00_00_00_00),
                 );
                 if (switch (scalar_bits) {
                     1...8 => vector_len > 4,
@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.comp.module.?;
     const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
     const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
-    _ = extra;
-    return self.fail("TODO implement airSelect for x86_64", .{});
-    //return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
+    const ty = self.typeOfIndex(inst);
+    const vec_len = ty.vectorLen(mod);
+    const elem_ty = ty.childType(mod);
+    const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
+    const abi_size: u32 = @intCast(ty.abiSize(mod));
+    const pred_ty = self.typeOf(pl_op.operand);
+
+    const result = result: {
+        const has_blend = self.hasFeature(.sse4_1);
+        const has_avx = self.hasFeature(.avx);
+        const need_xmm0 = has_blend and !has_avx;
+        const pred_mcv = try self.resolveInst(pl_op.operand);
+        const mask_reg = mask: {
+            switch (pred_mcv) {
+                .register => |pred_reg| switch (pred_reg.class()) {
+                    .general_purpose => {},
+                    .sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) {
+                        try self.register_manager.getKnownReg(.xmm0, null);
+                        try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{});
+                        break :mask .xmm0;
+                    } else break :mask if (has_blend)
+                        pred_reg
+                    else
+                        try self.copyToTmpRegister(pred_ty, pred_mcv),
+                    else => unreachable,
+                },
+                else => {},
+            }
+            const mask_reg: Register = if (need_xmm0) mask_reg: {
+                try self.register_manager.getKnownReg(.xmm0, null);
+                break :mask_reg .xmm0;
+            } else try self.register_manager.allocReg(null, abi.RegisterClass.sse);
+            const mask_alias = registerAlias(mask_reg, abi_size);
+            const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
+            defer self.register_manager.unlockReg(mask_lock);
+
+            const pred_fits_in_elem = vec_len <= elem_abi_size;
+            if (self.hasFeature(.avx2) and abi_size <= 32) {
+                if (pred_mcv.isRegister()) broadcast: {
+                    try self.asmRegisterRegister(
+                        .{ .v_d, .mov },
+                        mask_reg.to128(),
+                        pred_mcv.getReg().?.to32(),
+                    );
+                    if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister(
+                        .{ switch (elem_abi_size) {
+                            1 => .vp_b,
+                            2 => .vp_w,
+                            3...4 => .vp_d,
+                            5...8 => .vp_q,
+                            9...16 => {
+                                try self.asmRegisterRegisterRegisterImmediate(
+                                    .{ .v_f128, .insert },
+                                    mask_alias,
+                                    mask_alias,
+                                    mask_reg.to128(),
+                                    Immediate.u(1),
+                                );
+                                break :broadcast;
+                            },
+                            17...32 => break :broadcast,
+                            else => unreachable,
+                        }, .broadcast },
+                        mask_alias,
+                        mask_reg.to128(),
+                    );
+                } else try self.asmRegisterMemory(
+                    .{ switch (vec_len) {
+                        1...8 => .vp_b,
+                        9...16 => .vp_w,
+                        17...32 => .vp_d,
+                        else => unreachable,
+                    }, .broadcast },
+                    mask_alias,
+                    if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
+                        .base = .{ .reg = (try self.copyToTmpRegister(
+                            Type.usize,
+                            pred_mcv.address(),
+                        )).to64() },
+                        .mod = .{ .rm = .{ .size = .byte } },
+                    },
+                );
+            } else if (abi_size <= 16) broadcast: {
+                try self.asmRegisterRegister(
+                    .{ if (has_avx) .v_d else ._d, .mov },
+                    mask_alias,
+                    (if (pred_mcv.isRegister())
+                        pred_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(),
+                );
+                if (!pred_fits_in_elem or vec_len == 1) break :broadcast;
+                if (elem_abi_size <= 1) {
+                    if (has_avx) try self.asmRegisterRegisterRegister(
+                        .{ .vp_, .unpcklbw },
+                        mask_alias,
+                        mask_alias,
+                        mask_alias,
+                    ) else try self.asmRegisterRegister(
+                        .{ .p_, .unpcklbw },
+                        mask_alias,
+                        mask_alias,
+                    );
+                    if (abi_size <= 2) break :broadcast;
+                }
+                if (elem_abi_size <= 2) {
+                    try self.asmRegisterRegisterImmediate(
+                        .{ if (has_avx) .vp_w else .p_w, .shufl },
+                        mask_alias,
+                        mask_alias,
+                        Immediate.u(0b00_00_00_00),
+                    );
+                    if (abi_size <= 8) break :broadcast;
+                }
+                try self.asmRegisterRegisterImmediate(
+                    .{ if (has_avx) .vp_d else .p_d, .shuf },
+                    mask_alias,
+                    mask_alias,
+                    Immediate.u(switch (elem_abi_size) {
+                        1...2, 5...8 => 0b01_00_01_00,
+                        3...4 => 0b00_00_00_00,
+                        else => unreachable,
+                    }),
+                );
+            } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
+            const elem_bits: u16 = @intCast(elem_abi_size * 8);
+            const mask_elem_ty = try mod.intType(.unsigned, elem_bits);
+            const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() });
+            if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) {
+                var mask_elems: [32]InternPool.Index = undefined;
+                for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{
+                    .ty = mask_elem_ty.toIntern(),
+                    .storage = .{ .u64 = bit / elem_bits },
+                } });
+                const mask_mcv = try self.genTypedValue(.{
+                    .ty = mask_ty,
+                    .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
+                        .ty = mask_ty.toIntern(),
+                        .storage = .{ .elems = mask_elems[0..vec_len] },
+                    } })),
+                });
+                const mask_mem: Memory = .{
+                    .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
+                    .mod = .{ .rm = .{ .size = self.memSize(ty) } },
+                };
+                if (has_avx) try self.asmRegisterRegisterMemory(
+                    .{ .vp_b, .shuf },
+                    mask_alias,
+                    mask_alias,
+                    mask_mem,
+                ) else try self.asmRegisterMemory(
+                    .{ .p_b, .shuf },
+                    mask_alias,
+                    mask_mem,
+                );
+            } else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
+            {
+                var mask_elems: [32]InternPool.Index = undefined;
+                for (mask_elems[0..vec_len], 0..) |*elem, bit| elem.* = try mod.intern(.{ .int = .{
+                    .ty = mask_elem_ty.toIntern(),
+                    .storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
+                } });
+                const mask_mcv = try self.genTypedValue(.{
+                    .ty = mask_ty,
+                    .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
+                        .ty = mask_ty.toIntern(),
+                        .storage = .{ .elems = mask_elems[0..vec_len] },
+                    } })),
+                });
+                const mask_mem: Memory = .{
+                    .base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
+                    .mod = .{ .rm = .{ .size = self.memSize(ty) } },
+                };
+                if (has_avx) {
+                    try self.asmRegisterRegisterMemory(
+                        .{ .vp_, .@"and" },
+                        mask_alias,
+                        mask_alias,
+                        mask_mem,
+                    );
+                    try self.asmRegisterRegisterMemory(
+                        .{ .vp_d, .cmpeq },
+                        mask_alias,
+                        mask_alias,
+                        mask_mem,
+                    );
+                } else {
+                    try self.asmRegisterMemory(
+                        .{ .p_, .@"and" },
+                        mask_alias,
+                        mask_mem,
+                    );
+                    try self.asmRegisterMemory(
+                        .{ .p_d, .cmpeq },
+                        mask_alias,
+                        mask_mem,
+                    );
+                }
+            }
+            break :mask mask_reg;
+        };
+        const mask_alias = registerAlias(mask_reg, abi_size);
+        const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
+        defer self.register_manager.unlockReg(mask_lock);
+
+        const lhs_mcv = try self.resolveInst(extra.lhs);
+        const lhs_lock = switch (lhs_mcv) {
+            .register => |lhs_reg| self.register_manager.lockRegAssumeUnused(lhs_reg),
+            else => null,
+        };
+        defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const rhs_mcv = try self.resolveInst(extra.rhs);
+        const rhs_lock = switch (rhs_mcv) {
+            .register => |rhs_reg| self.register_manager.lockReg(rhs_reg),
+            else => null,
+        };
+        defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv;
+        const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand(
+            inst,
+            if (has_blend) extra.rhs else extra.lhs,
+            @intFromBool(has_blend),
+            reuse_mcv,
+        )) reuse_mcv else if (has_avx)
+            .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+        else
+            try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv);
+        const dst_reg = dst_mcv.getReg().?;
+        const dst_alias = registerAlias(dst_reg, abi_size);
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) {
+            else => null,
+            .Int => switch (abi_size) {
+                0 => unreachable,
+                1...16 => if (has_avx)
+                    .{ .vp_b, .blendv }
+                else if (has_blend)
+                    .{ .p_b, .blendv }
+                else
+                    .{ .p_, undefined },
+                17...32 => if (self.hasFeature(.avx2))
+                    .{ .vp_b, .blendv }
+                else
+                    null,
+                else => null,
+            },
+            .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
+                else => unreachable,
+                16, 80, 128 => null,
+                32 => switch (vec_len) {
+                    0 => unreachable,
+                    1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv },
+                    5...8 => if (has_avx) .{ .v_ps, .blendv } else null,
+                    else => null,
+                },
+                64 => switch (vec_len) {
+                    0 => unreachable,
+                    1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv },
+                    3...4 => if (has_avx) .{ .v_pd, .blendv } else null,
+                    else => null,
+                },
+            },
+        }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
+        if (has_avx) {
+            const rhs_alias = if (rhs_mcv.isRegister())
+                registerAlias(rhs_mcv.getReg().?, abi_size)
+            else rhs: {
+                try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
+                break :rhs dst_alias;
+            };
+            if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
+                mir_tag,
+                dst_alias,
+                rhs_alias,
+                try lhs_mcv.mem(self, self.memSize(ty)),
+                mask_alias,
+            ) else try self.asmRegisterRegisterRegisterRegister(
+                mir_tag,
+                dst_alias,
+                rhs_alias,
+                registerAlias(if (lhs_mcv.isRegister())
+                    lhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
+                mask_alias,
+            );
+        } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
+            mir_tag,
+            dst_alias,
+            try lhs_mcv.mem(self, self.memSize(ty)),
+            mask_alias,
+        ) else try self.asmRegisterRegisterRegister(
+            mir_tag,
+            dst_alias,
+            registerAlias(if (lhs_mcv.isRegister())
+                lhs_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
+            mask_alias,
+        ) else {
+            const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) {
+                else => null,
+                .Int => .p_,
+                .Float => switch (elem_ty.floatBits(self.target.*)) {
+                    32 => ._ps,
+                    64 => ._pd,
+                    16, 80, 128 => null,
+                    else => unreachable,
+                },
+            }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
+            try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
+            if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
+                .{ mir_fixes, .andn },
+                mask_alias,
+                try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
+            ) else try self.asmRegisterRegister(
+                .{ mir_fixes, .andn },
+                mask_alias,
+                if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(ty, rhs_mcv),
+            );
+            try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
+        }
+        break :result dst_mcv;
+    };
+    return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
 }
 
 fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.comp.module.?;
     const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
-    _ = ty_pl;
-    return self.fail("TODO implement airShuffle for x86_64", .{});
-    //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+    const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
+
+    const dst_ty = self.typeOfIndex(inst);
+    const elem_ty = dst_ty.childType(mod);
+    const elem_abi_size: u16 = @intCast(elem_ty.abiSize(mod));
+    const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
+    const lhs_ty = self.typeOf(extra.a);
+    const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
+    const rhs_ty = self.typeOf(extra.b);
+    const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));
+    const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
+
+    const ExpectedContents = [32]?i32;
+    var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
+        std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
+    const allocator = stack.get();
+
+    const mask_elems = try allocator.alloc(?i32, extra.mask_len);
+    defer allocator.free(mask_elems);
+    for (mask_elems, 0..) |*mask_elem, elem_index| {
+        const mask_elem_val =
+            Value.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;
+        mask_elem.* = if (mask_elem_val.isUndef(mod))
+            null
+        else
+            @intCast(mask_elem_val.toSignedInt(mod));
+    }
+
+    const has_avx = self.hasFeature(.avx);
+    const result = @as(?MCValue, result: {
+        for (mask_elems) |mask_elem| {
+            if (mask_elem) |_| break;
+        } else break :result try self.allocRegOrMem(inst, true);
+
+        for (mask_elems, 0..) |mask_elem, elem_index| {
+            if (mask_elem orelse continue != elem_index) break;
+        } else {
+            const lhs_mcv = try self.resolveInst(extra.a);
+            if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
+            const dst_mcv = try self.allocRegOrMem(inst, true);
+            try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
+            break :result dst_mcv;
+        }
+
+        for (mask_elems, 0..) |mask_elem, elem_index| {
+            if (~(mask_elem orelse continue) != elem_index) break;
+        } else {
+            const rhs_mcv = try self.resolveInst(extra.b);
+            if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
+            const dst_mcv = try self.allocRegOrMem(inst, true);
+            try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
+            break :result dst_mcv;
+        }
+
+        for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: {
+            if (elem_abi_size > 8) break :unpck;
+            if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
+                has_avx
+            else
+                self.hasFeature(.avx2)) 32 else 16)) break :unpck;
+
+            var sources = [1]?u1{null} ** 2;
+            for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index =
+                    math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
+                const elem_byte = (elem_index >> 1) * elem_abi_size;
+                if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) {
+                    .unpckl => 0b0000,
+                    .unpckh => 0b1000,
+                    else => unreachable,
+                }) | (elem_byte << 1 & 0b10000)) break :unpck;
+
+                const source = @intFromBool(mask_elem < 0);
+                if (sources[elem_index & 0b00001]) |prev_source| {
+                    if (source != prev_source) break :unpck;
+                } else sources[elem_index & 0b00001] = source;
+            }
+            if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
+
+            const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
+            const operand_tys = [2]Type{ lhs_ty, rhs_ty };
+            const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
+            const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
+
+            const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
+                lhs_mcv
+            else if (has_avx and lhs_mcv.isRegister())
+                .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+            else
+                try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
+            const dst_reg = dst_mcv.getReg().?;
+            const dst_alias = registerAlias(dst_reg, max_abi_size);
+
+            const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
+                (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
+                4 => if (has_avx) .v_ps else ._ps,
+                8 => if (has_avx) .v_pd else ._pd,
+                else => unreachable,
+            }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
+                .unpckl => switch (elem_abi_size) {
+                    1 => .unpcklbw,
+                    2 => .unpcklwd,
+                    4 => .unpckldq,
+                    8 => .unpcklqdq,
+                    else => unreachable,
+                },
+                .unpckh => switch (elem_abi_size) {
+                    1 => .unpckhbw,
+                    2 => .unpckhwd,
+                    4 => .unpckhdq,
+                    8 => .unpckhqdq,
+                    else => unreachable,
+                },
+                else => unreachable,
+            } };
+            if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(
+                mir_tag,
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+            ) else try self.asmRegisterRegisterRegister(
+                mir_tag,
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+            ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
+                mir_tag,
+                dst_alias,
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+            ) else try self.asmRegisterRegister(
+                mir_tag,
+                dst_alias,
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+            );
+            break :result dst_mcv;
+        }
+
+        pshufd: {
+            if (elem_abi_size != 4) break :pshufd;
+            if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
+
+            var control: u8 = 0b00_00_00_00;
+            var sources = [1]?u1{null} ** 1;
+            for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
+                if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
+
+                const source = @intFromBool(mask_elem < 0);
+                if (sources[0]) |prev_source| {
+                    if (source != prev_source) break :pshufd;
+                } else sources[(elem_index & 0b010) >> 1] = source;
+
+                const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
+                const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
+                if (elem_index & 0b100 == 0)
+                    control |= select
+                else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
+            }
+
+            const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
+            const operand_tys = [2]Type{ lhs_ty, rhs_ty };
+            const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
+
+            const dst_reg = if (src_mcv.isRegister() and
+                self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
+                src_mcv.getReg().?
+            else
+                try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
+            const dst_alias = registerAlias(dst_reg, max_abi_size);
+
+            if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                .{ if (has_avx) .vp_d else .p_d, .shuf },
+                dst_alias,
+                try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+                Immediate.u(control),
+            ) else try self.asmRegisterRegisterImmediate(
+                .{ if (has_avx) .vp_d else .p_d, .shuf },
+                dst_alias,
+                registerAlias(if (src_mcv.isRegister())
+                    src_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
+                Immediate.u(control),
+            );
+            break :result .{ .register = dst_reg };
+        }
+
+        shufps: {
+            if (elem_abi_size != 4) break :shufps;
+            if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
+
+            var control: u8 = 0b00_00_00_00;
+            var sources = [1]?u1{null} ** 2;
+            for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
+                if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
+
+                const source = @intFromBool(mask_elem < 0);
+                if (sources[(elem_index & 0b010) >> 1]) |prev_source| {
+                    if (source != prev_source) break :shufps;
+                } else sources[(elem_index & 0b010) >> 1] = source;
+
+                const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
+                const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
+                if (elem_index & 0b100 == 0)
+                    control |= select
+                else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
+            }
+            if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
+
+            const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
+            const operand_tys = [2]Type{ lhs_ty, rhs_ty };
+            const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
+            const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
+
+            const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
+                lhs_mcv
+            else if (has_avx and lhs_mcv.isRegister())
+                .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+            else
+                try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
+            const dst_reg = dst_mcv.getReg().?;
+            const dst_alias = registerAlias(dst_reg, max_abi_size);
+
+            if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                .{ .v_ps, .shuf },
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+                Immediate.u(control),
+            ) else try self.asmRegisterRegisterRegisterImmediate(
+                .{ .v_ps, .shuf },
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+                Immediate.u(control),
+            ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                .{ ._ps, .shuf },
+                dst_alias,
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+                Immediate.u(control),
+            ) else try self.asmRegisterRegisterImmediate(
+                .{ ._ps, .shuf },
+                dst_alias,
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+                Immediate.u(control),
+            );
+            break :result dst_mcv;
+        }
+
+        shufpd: {
+            if (elem_abi_size != 8) break :shufpd;
+            if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
+
+            var control: u4 = 0b0_0_0_0;
+            var sources = [1]?u1{null} ** 2;
+            for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
+                if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
+
+                const source = @intFromBool(mask_elem < 0);
+                if (sources[elem_index & 0b01]) |prev_source| {
+                    if (source != prev_source) break :shufpd;
+                } else sources[elem_index & 0b01] = source;
+
+                control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
+            }
+            if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
+
+            const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
+            const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
+            const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
+            const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
+
+            const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
+                lhs_mcv
+            else if (has_avx and lhs_mcv.isRegister())
+                .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+            else
+                try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
+            const dst_reg = dst_mcv.getReg().?;
+            const dst_alias = registerAlias(dst_reg, max_abi_size);
+
+            if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                .{ .v_pd, .shuf },
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+                Immediate.u(control),
+            ) else try self.asmRegisterRegisterRegisterImmediate(
+                .{ .v_pd, .shuf },
+                dst_alias,
+                registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+                Immediate.u(control),
+            ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                .{ ._pd, .shuf },
+                dst_alias,
+                try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
+                Immediate.u(control),
+            ) else try self.asmRegisterRegisterImmediate(
+                .{ ._pd, .shuf },
+                dst_alias,
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
+                Immediate.u(control),
+            );
+            break :result dst_mcv;
+        }
+
+        blend: {
+            if (elem_abi_size < 2) break :blend;
+            if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;
+            if (!self.hasFeature(.sse4_1)) break :blend;
+
+            var control: u8 = 0b0_0_0_0_0_0_0_0;
+            for (mask_elems, 0..) |maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index =
+                    math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
+                if (mask_elem_index != elem_index) break :blend;
+
+                const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
+                if (elem_index & 0b1000 == 0)
+                    control |= select
+                else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
+            }
+
+            if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
+                const expanded_control = switch (elem_abi_size) {
+                    4 => control,
+                    8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) |
+                        @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) |
+                        @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) |
+                        @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
+                    else => break :vpblendd,
+                };
+
+                const lhs_mcv = try self.resolveInst(extra.a);
+                const lhs_reg = if (lhs_mcv.isRegister())
+                    lhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(dst_ty, lhs_mcv);
+                const lhs_lock = self.register_manager.lockReg(lhs_reg);
+                defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
+
+                const rhs_mcv = try self.resolveInst(extra.b);
+                const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
+                if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                    .{ .vp_d, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    registerAlias(lhs_reg, dst_abi_size),
+                    try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                    Immediate.u(expanded_control),
+                ) else try self.asmRegisterRegisterRegisterImmediate(
+                    .{ .vp_d, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    registerAlias(lhs_reg, dst_abi_size),
+                    registerAlias(if (rhs_mcv.isRegister())
+                        rhs_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                    Immediate.u(expanded_control),
+                );
+                break :result .{ .register = dst_reg };
+            }
+
+            if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
+                const expanded_control = switch (elem_abi_size) {
+                    2 => control,
+                    4 => if (dst_abi_size <= 16 or
+                        @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
+                        @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) |
+                            @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) |
+                            @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) |
+                            @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
+                    else
+                        break :pblendw,
+                    8 => if (dst_abi_size <= 16 or
+                        @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
+                        @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) |
+                            @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
+                    else
+                        break :pblendw,
+                    16 => break :pblendw,
+                    else => unreachable,
+                };
+
+                const lhs_mcv = try self.resolveInst(extra.a);
+                const rhs_mcv = try self.resolveInst(extra.b);
+
+                const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                    self.reuseOperand(inst, extra.a, 0, lhs_mcv))
+                    lhs_mcv
+                else if (has_avx and lhs_mcv.isRegister())
+                    .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+                else
+                    try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
+                const dst_reg = dst_mcv.getReg().?;
+
+                if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                    .{ .vp_w, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    registerAlias(if (lhs_mcv.isRegister())
+                        lhs_mcv.getReg().?
+                    else
+                        dst_reg, dst_abi_size),
+                    try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                    Immediate.u(expanded_control),
+                ) else try self.asmRegisterRegisterRegisterImmediate(
+                    .{ .vp_w, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    registerAlias(if (lhs_mcv.isRegister())
+                        lhs_mcv.getReg().?
+                    else
+                        dst_reg, dst_abi_size),
+                    registerAlias(if (rhs_mcv.isRegister())
+                        rhs_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                    Immediate.u(expanded_control),
+                ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                    .{ .p_w, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                    Immediate.u(expanded_control),
+                ) else try self.asmRegisterRegisterImmediate(
+                    .{ .p_w, .blend },
+                    registerAlias(dst_reg, dst_abi_size),
+                    registerAlias(if (rhs_mcv.isRegister())
+                        rhs_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                    Immediate.u(expanded_control),
+                );
+                break :result .{ .register = dst_reg };
+            }
+
+            const expanded_control = switch (elem_abi_size) {
+                4, 8 => control,
+                16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) |
+                    @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
+                else => unreachable,
+            };
+
+            const lhs_mcv = try self.resolveInst(extra.a);
+            const rhs_mcv = try self.resolveInst(extra.b);
+
+            const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                self.reuseOperand(inst, extra.a, 0, lhs_mcv))
+                lhs_mcv
+            else if (has_avx and lhs_mcv.isRegister())
+                .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+            else
+                try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
+            const dst_reg = dst_mcv.getReg().?;
+
+            if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                switch (elem_abi_size) {
+                    4 => .{ .v_ps, .blend },
+                    8, 16 => .{ .v_pd, .blend },
+                    else => unreachable,
+                },
+                registerAlias(dst_reg, dst_abi_size),
+                registerAlias(if (lhs_mcv.isRegister())
+                    lhs_mcv.getReg().?
+                else
+                    dst_reg, dst_abi_size),
+                try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                Immediate.u(expanded_control),
+            ) else try self.asmRegisterRegisterRegisterImmediate(
+                switch (elem_abi_size) {
+                    4 => .{ .v_ps, .blend },
+                    8, 16 => .{ .v_pd, .blend },
+                    else => unreachable,
+                },
+                registerAlias(dst_reg, dst_abi_size),
+                registerAlias(if (lhs_mcv.isRegister())
+                    lhs_mcv.getReg().?
+                else
+                    dst_reg, dst_abi_size),
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                Immediate.u(expanded_control),
+            ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                switch (elem_abi_size) {
+                    4 => .{ ._ps, .blend },
+                    8, 16 => .{ ._pd, .blend },
+                    else => unreachable,
+                },
+                registerAlias(dst_reg, dst_abi_size),
+                try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                Immediate.u(expanded_control),
+            ) else try self.asmRegisterRegisterImmediate(
+                switch (elem_abi_size) {
+                    4 => .{ ._ps, .blend },
+                    8, 16 => .{ ._pd, .blend },
+                    else => unreachable,
+                },
+                registerAlias(dst_reg, dst_abi_size),
+                registerAlias(if (rhs_mcv.isRegister())
+                    rhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                Immediate.u(expanded_control),
+            );
+            break :result .{ .register = dst_reg };
+        }
+
+        blendv: {
+            if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
+                has_avx
+            else
+                self.hasFeature(.avx2)) 32 else 16)) break :blendv;
+
+            const select_mask_elem_ty = try mod.intType(.unsigned, elem_abi_size * 8);
+            const select_mask_ty = try mod.vectorType(.{
+                .len = @intCast(mask_elems.len),
+                .child = select_mask_elem_ty.toIntern(),
+            });
+            var select_mask_elems: [32]InternPool.Index = undefined;
+            for (
+                select_mask_elems[0..mask_elems.len],
+                mask_elems,
+                0..,
+            ) |*select_mask_elem, maybe_mask_elem, elem_index| {
+                const mask_elem = maybe_mask_elem orelse continue;
+                const mask_elem_index =
+                    math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
+                if (mask_elem_index != elem_index) break :blendv;
+
+                select_mask_elem.* = (if (mask_elem < 0)
+                    try select_mask_elem_ty.maxIntScalar(mod, select_mask_elem_ty)
+                else
+                    try select_mask_elem_ty.minIntScalar(mod, select_mask_elem_ty)).toIntern();
+            }
+            const select_mask_mcv = try self.genTypedValue(.{
+                .ty = select_mask_ty,
+                .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
+                    .ty = select_mask_ty.toIntern(),
+                    .storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
+                } })),
+            });
+
+            if (self.hasFeature(.sse4_1)) {
+                const mir_tag: Mir.Inst.FixedTag = .{
+                    if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
+                        (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
+                        4 => if (has_avx) .v_ps else ._ps,
+                        8 => if (has_avx) .v_pd else ._pd,
+                        else => unreachable,
+                    } else if (has_avx) .vp_b else .p_b,
+                    .blendv,
+                };
+
+                const select_mask_reg = if (!has_avx) reg: {
+                    try self.register_manager.getKnownReg(.xmm0, null);
+                    try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
+                    break :reg .xmm0;
+                } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
+                const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
+                const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
+                defer self.register_manager.unlockReg(select_mask_lock);
+
+                const lhs_mcv = try self.resolveInst(extra.a);
+                const rhs_mcv = try self.resolveInst(extra.b);
+
+                const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
+                    self.reuseOperand(inst, extra.a, 0, lhs_mcv))
+                    lhs_mcv
+                else if (has_avx and lhs_mcv.isRegister())
+                    .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
+                else
+                    try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
+                const dst_reg = dst_mcv.getReg().?;
+                const dst_alias = registerAlias(dst_reg, dst_abi_size);
+
+                if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
+                    mir_tag,
+                    dst_alias,
+                    if (lhs_mcv.isRegister())
+                        registerAlias(lhs_mcv.getReg().?, dst_abi_size)
+                    else
+                        dst_alias,
+                    try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                    select_mask_alias,
+                ) else try self.asmRegisterRegisterRegisterRegister(
+                    mir_tag,
+                    dst_alias,
+                    if (lhs_mcv.isRegister())
+                        registerAlias(lhs_mcv.getReg().?, dst_abi_size)
+                    else
+                        dst_alias,
+                    registerAlias(if (rhs_mcv.isRegister())
+                        rhs_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                    select_mask_alias,
+                ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
+                    mir_tag,
+                    dst_alias,
+                    try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+                    select_mask_alias,
+                ) else try self.asmRegisterRegisterRegister(
+                    mir_tag,
+                    dst_alias,
+                    registerAlias(if (rhs_mcv.isRegister())
+                        rhs_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
+                    select_mask_alias,
+                );
+                break :result dst_mcv;
+            }
+
+            const lhs_mcv = try self.resolveInst(extra.a);
+            const rhs_mcv = try self.resolveInst(extra.b);
+
+            const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
+                self.reuseOperand(inst, extra.b, 1, rhs_mcv))
+                rhs_mcv
+            else
+                try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
+            const dst_reg = dst_mcv.getReg().?;
+            const dst_alias = registerAlias(dst_reg, dst_abi_size);
+
+            const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
+            const mask_alias = registerAlias(mask_reg, dst_abi_size);
+            const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
+            defer self.register_manager.unlockReg(mask_lock);
+
+            const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
+                switch (elem_ty.floatBits(self.target.*)) {
+                    16, 80, 128 => .p_,
+                    32 => ._ps,
+                    64 => ._pd,
+                    else => unreachable,
+                }
+            else
+                .p_;
+            try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
+            if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
+                .{ mir_fixes, .andn },
+                mask_alias,
+                try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
+            ) else try self.asmRegisterRegister(
+                .{ mir_fixes, .andn },
+                mask_alias,
+                if (lhs_mcv.isRegister())
+                    lhs_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(dst_ty, lhs_mcv),
+            );
+            try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
+            break :result dst_mcv;
+        }
+
+        pshufb: {
+            if (max_abi_size > 16) break :pshufb;
+            if (!self.hasFeature(.ssse3)) break :pshufb;
+
+            const temp_regs =
+                try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
+            const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
+            defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
+
+            const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
+            try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
+
+            const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
+            try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
+
+            var lhs_mask_elems: [16]InternPool.Index = undefined;
+            for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {
+                const elem_index = byte_index / elem_abi_size;
+                lhs_mask_elem.* = try mod.intern(.{ .int = .{
+                    .ty = .u8_type,
+                    .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
+                        const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
+                        if (mask_elem < 0) break :elem 0b1_00_00000;
+                        const mask_elem_index: u31 = @intCast(mask_elem);
+                        const byte_off: u32 = @intCast(byte_index % elem_abi_size);
+                        break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
+                    } },
+                } });
+            }
+            const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
+            const lhs_mask_mcv = try self.genTypedValue(.{
+                .ty = lhs_mask_ty,
+                .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
+                    .ty = lhs_mask_ty.toIntern(),
+                    .storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
+                } })),
+            });
+            const lhs_mask_mem: Memory = .{
+                .base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
+                .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
+            };
+            if (has_avx) try self.asmRegisterRegisterMemory(
+                .{ .vp_b, .shuf },
+                lhs_temp_alias,
+                lhs_temp_alias,
+                lhs_mask_mem,
+            ) else try self.asmRegisterMemory(
+                .{ .p_b, .shuf },
+                lhs_temp_alias,
+                lhs_mask_mem,
+            );
+
+            var rhs_mask_elems: [16]InternPool.Index = undefined;
+            for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {
+                const elem_index = byte_index / elem_abi_size;
+                rhs_mask_elem.* = try mod.intern(.{ .int = .{
+                    .ty = .u8_type,
+                    .storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
+                        const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
+                        if (mask_elem >= 0) break :elem 0b1_00_00000;
+                        const mask_elem_index: u31 = @intCast(~mask_elem);
+                        const byte_off: u32 = @intCast(byte_index % elem_abi_size);
+                        break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
+                    } },
+                } });
+            }
+            const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
+            const rhs_mask_mcv = try self.genTypedValue(.{
+                .ty = rhs_mask_ty,
+                .val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
+                    .ty = rhs_mask_ty.toIntern(),
+                    .storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
+                } })),
+            });
+            const rhs_mask_mem: Memory = .{
+                .base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
+                .mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
+            };
+            if (has_avx) try self.asmRegisterRegisterMemory(
+                .{ .vp_b, .shuf },
+                rhs_temp_alias,
+                rhs_temp_alias,
+                rhs_mask_mem,
+            ) else try self.asmRegisterMemory(
+                .{ .p_b, .shuf },
+                rhs_temp_alias,
+                rhs_mask_mem,
+            );
+
+            if (has_avx) try self.asmRegisterRegisterRegister(
+                .{ switch (elem_ty.zigTypeTag(mod)) {
+                    else => break :result null,
+                    .Int => .vp_,
+                    .Float => switch (elem_ty.floatBits(self.target.*)) {
+                        32 => .v_ps,
+                        64 => .v_pd,
+                        16, 80, 128 => break :result null,
+                        else => unreachable,
+                    },
+                }, .@"or" },
+                lhs_temp_alias,
+                lhs_temp_alias,
+                rhs_temp_alias,
+            ) else try self.asmRegisterRegister(
+                .{ switch (elem_ty.zigTypeTag(mod)) {
+                    else => break :result null,
+                    .Int => .p_,
+                    .Float => switch (elem_ty.floatBits(self.target.*)) {
+                        32 => ._ps,
+                        64 => ._pd,
+                        16, 80, 128 => break :result null,
+                        else => unreachable,
+                    },
+                }, .@"or" },
+                lhs_temp_alias,
+                rhs_temp_alias,
+            );
+            break :result .{ .register = temp_regs[0] };
+        }
+
+        break :result null;
+    }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
+        lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),
+        Value.fromInterned(extra.mask).fmtValue(
+            Type.fromInterned(mod.intern_pool.typeOf(extra.mask)),
+            mod,
+        ),
+    });
+    return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
 }
 
 fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
             },
             .Array, .Vector => {
                 const elem_ty = result_ty.childType(mod);
-                if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) {
+                if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {
                     const result_size: u32 = @intCast(result_ty.abiSize(mod));
                     const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
                     try self.asmRegisterRegister(
@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
                 else => unreachable,
             },
             .float => Type.f32,
-            .float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
+            .float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),
             .sse => Type.f64,
             else => break,
         };
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 8b91a20a4f..c4bf71e233 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -324,16 +324,19 @@ pub const Mnemonic = enum {
     // SSE3
     movddup, movshdup, movsldup,
     // SSSE3
-    pabsb, pabsd, pabsw, palignr,
+    pabsb, pabsd, pabsw, palignr, pshufb,
     // SSE4.1
     blendpd, blendps, blendvpd, blendvps,
     extractps,
     insertps,
     packusdw,
+    pblendvb, pblendw,
     pcmpeqq,
     pextrb, pextrd, pextrq,
     pinsrb, pinsrd, pinsrq,
     pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
+    pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
+    pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
     pmulld,
     roundpd, roundps, roundsd, roundss,
     // SSE4.2
@@ -377,7 +380,8 @@ pub const Mnemonic = enum {
     vpabsb, vpabsd, vpabsw,
     vpackssdw, vpacksswb, vpackusdw, vpackuswb,
     vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
-    vpalignr, vpand, vpandn, vpclmulqdq,
+    vpalignr, vpand, vpandn,
+    vpblendvb, vpblendw, vpclmulqdq,
     vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,
     vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,
     vpextrb, vpextrd, vpextrq, vpextrw,
@@ -385,9 +389,11 @@ pub const Mnemonic = enum {
     vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
     vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
     vpmovmskb,
+    vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq,
+    vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq,
     vpmulhw, vpmulld, vpmullw,
     vpor,
-    vpshufd, vpshufhw, vpshuflw,
+    vpshufb, vpshufd, vpshufhw, vpshuflw,
     vpslld, vpslldq, vpsllq, vpsllw,
     vpsrad, vpsraq, vpsraw,
     vpsrld, vpsrldq, vpsrlq, vpsrlw,
@@ -409,7 +415,8 @@ pub const Mnemonic = enum {
     vfmadd132sd, vfmadd213sd, vfmadd231sd,
     vfmadd132ss, vfmadd213ss, vfmadd231ss,
     // AVX2
-    vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
+    vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw,
+    vextracti128, vinserti128, vpblendd,
     // zig fmt: on
 };
 
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 4e9c37e5aa..13b97b551a 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -477,8 +477,9 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
         .rri_s, .rri_u => inst.data.rri.fixes,
         .ri_s, .ri_u => inst.data.ri.fixes,
         .ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
-        .mrr, .rrm => inst.data.rrx.fixes,
+        .mrr, .rrm, .rmr => inst.data.rrx.fixes,
         .rmi, .mri => inst.data.rix.fixes,
+        .rrmr => inst.data.rrrx.fixes,
         .rrmi => inst.data.rrix.fixes,
         .mi_u, .mi_s => inst.data.x.fixes,
         .m => inst.data.x.fixes,
@@ -565,6 +566,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rx.r1 },
             .{ .mem = lower.mem(inst.data.rx.payload) },
         },
+        .rmr => &.{
+            .{ .reg = inst.data.rrx.r1 },
+            .{ .mem = lower.mem(inst.data.rrx.payload) },
+            .{ .reg = inst.data.rrx.r2 },
+        },
         .rmi => &.{
             .{ .reg = inst.data.rix.r1 },
             .{ .mem = lower.mem(inst.data.rix.payload) },
@@ -597,6 +603,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rrx.r2 },
             .{ .mem = lower.mem(inst.data.rrx.payload) },
         },
+        .rrmr => &.{
+            .{ .reg = inst.data.rrrx.r1 },
+            .{ .reg = inst.data.rrrx.r2 },
+            .{ .mem = lower.mem(inst.data.rrrx.payload) },
+            .{ .reg = inst.data.rrrx.r3 },
+        },
         .rrmi => &.{
             .{ .reg = inst.data.rrix.r1 },
             .{ .reg = inst.data.rrix.r2 },
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index dea9bb50cb..6cccb34b3e 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -230,6 +230,8 @@ pub const Inst = struct {
         v_d,
         /// VEX-Encoded ___ QuadWord
         v_q,
+        /// VEX-Encoded ___ Integer Data
+        v_i128,
         /// VEX-Encoded Packed ___
         vp_,
         /// VEX-Encoded Packed ___ Byte
@@ -242,8 +244,6 @@ pub const Inst = struct {
         vp_q,
         /// VEX-Encoded Packed ___ Double Quadword
         vp_dq,
-        /// VEX-Encoded Packed ___ Integer Data
-        vp_i128,
         /// VEX-Encoded ___ Scalar Single-Precision Values
         v_ss,
         /// VEX-Encoded ___ Packed Single-Precision Values
@@ -654,10 +654,19 @@ pub const Inst = struct {
         /// Variable blend scalar double-precision floating-point values
         blendv,
         /// Extract packed floating-point values
+        /// Extract packed integer values
         extract,
         /// Insert scalar single-precision floating-point value
         /// Insert packed floating-point values
         insert,
+        /// Packed move with sign extend
+        movsxb,
+        movsxd,
+        movsxw,
+        /// Packed move with zero extend
+        movzxb,
+        movzxd,
+        movzxw,
         /// Round packed single-precision floating-point values
         /// Round scalar single-precision floating-point value
         /// Round packed double-precision floating-point values
@@ -688,6 +697,7 @@ pub const Inst = struct {
         sha256rnds2,
 
         /// Load with broadcast floating-point data
+        /// Load integer and broadcast
         broadcast,
 
         /// Convert 16-bit floating-point values to single-precision floating-point values
@@ -762,8 +772,11 @@ pub const Inst = struct {
         /// Uses `imm` payload.
         rel,
         /// Register, memory operands.
-        /// Uses `rx` payload.
+        /// Uses `rx` payload with extra data of type `Memory`.
         rm,
+        /// Register, memory, register operands.
+        /// Uses `rrx` payload with extra data of type `Memory`.
+        rmr,
         /// Register, memory, immediate (word) operands.
         /// Uses `rix` payload with extra data of type `Memory`.
         rmi,
@@ -776,6 +789,9 @@ pub const Inst = struct {
         /// Register, register, memory.
         /// Uses `rrix` payload with extra data of type `Memory`.
         rrm,
+        /// Register, register, memory, register.
+        /// Uses `rrrx` payload with extra data of type `Memory`.
+        rrmr,
         /// Register, register, memory, immediate (byte) operands.
         /// Uses `rrix` payload with extra data of type `Memory`.
         rrmi,
@@ -953,6 +969,14 @@ pub const Inst = struct {
             r2: Register,
             payload: u32,
         },
+        /// Register, register, register, followed by Custom payload found in extra.
+        rrrx: struct {
+            fixes: Fixes = ._,
+            r1: Register,
+            r2: Register,
+            r3: Register,
+            payload: u32,
+        },
         /// Register, byte immediate, followed by Custom payload found in extra.
         rix: struct {
             fixes: Fixes = ._,
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 545e6b23ce..d4a7dcafe7 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{
 
     .{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },
 
+    .{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },
+
     // SSE4.1
     .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
 
@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{
 
     .{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },
 
+    .{ .pblendvb, .rm, &.{ .xmm, .xmm_m128        }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
+    .{ .pblendvb, .rm, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
+
+    .{ .pblendw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 },
+
     .{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },
 
     .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{
 
     .{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },
 
+    .{ .pmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 },
+    .{ .pmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 },
+    .{ .pmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 },
+    .{ .pmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 },
+    .{ .pmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 },
+    .{ .pmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 },
+
+    .{ .pmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 },
+    .{ .pmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 },
+    .{ .pmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 },
+    .{ .pmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 },
+    .{ .pmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 },
+    .{ .pmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 },
+
     .{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
 
     .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{
 
     .{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },
 
+    .{ .vpblendvb, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx },
+
+    .{ .vpblendw, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx },
+
     .{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },
 
     .{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },
@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{
     .{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
     .{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
 
+    .{ .vpmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx },
+
+    .{ .vpmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx },
+    .{ .vpmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx },
+
     .{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
 
     .{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{
 
     .{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
 
+    .{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
+
     .{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
 
     .{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{
     .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
     .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
 
+    .{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 },
+
+    .{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 },
+
     .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
     .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
     .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{
 
     .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
 
+    .{ .vpblendd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 },
+    .{ .vpblendd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 },
+
+    .{ .vpblendvb, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 },
+
+    .{ .vpblendw, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 },
+
     .{ .vpbroadcastb,    .rm, &.{ .xmm, .xmm_m8  }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
     .{ .vpbroadcastb,    .rm, &.{ .ymm, .xmm_m8  }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
     .{ .vpbroadcastw,    .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{
     .{ .vpbroadcastd,    .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
     .{ .vpbroadcastq,    .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
     .{ .vpbroadcastq,    .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
-    .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128    }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
+    .{ .vbroadcasti128,  .rm, &.{ .ymm, .m128    }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
 
     .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
     .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{
     .{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
     .{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
 
+    .{ .vpmovsxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovsxbd, .rm, &.{ .ymm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovsxbq, .rm, &.{ .ymm, .xmm_m32  }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovsxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovsxwq, .rm, &.{ .ymm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovsxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 },
+
+    .{ .vpmovzxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovzxbd, .rm, &.{ .ymm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovzxbq, .rm, &.{ .ymm, .xmm_m32  }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovzxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovzxwq, .rm, &.{ .ymm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpmovzxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 },
+
     .{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },
 
     .{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },
@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{
 
     .{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
 
+    .{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
     .{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
 
     .{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },