path: root/src/arch/aarch64/Emit.zig

//! This file contains the functionality for lowering AArch64 MIR into
//! machine code

const Emit = @This();
const std = @import("std");
const math = std.math;
const Mir = @import("Mir.zig");
const bits = @import("bits.zig");
const link = @import("../../link.zig");
const Module = @import("../../Module.zig");
const ErrorMsg = Module.ErrorMsg;
const assert = std.debug.assert;
const DW = std.dwarf;
const leb128 = std.leb;
const Instruction = bits.Instruction;
const Register = bits.Register;
const log = std.log.scoped(.aarch64_emit);
const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;

mir: Mir,
bin_file: *link.File,
debug_output: DebugInfoOutput,
target: *const std.Target,
err_msg: ?*ErrorMsg = null,
src_loc: Module.SrcLoc,
code: *std.ArrayList(u8),

prev_di_line: u32,
prev_di_column: u32,

/// Relative to the beginning of `code`.
prev_di_pc: usize,

/// The amount of stack space consumed by the saved callee-saved
/// registers in bytes
saved_regs_stack_space: u32,

/// The branch type of every branch
branch_types: std.AutoHashMapUnmanaged(Mir.Inst.Index, BranchType) = .{},

/// For every forward branch, maps the target instruction to a list of
/// branches which branch to this target instruction
branch_forward_origins: std.AutoHashMapUnmanaged(Mir.Inst.Index, std.ArrayListUnmanaged(Mir.Inst.Index)) = .{},

/// For backward branches: stores the code offset of the target
/// instruction
///
/// For forward branches: stores the code offset of the branch
/// instruction
code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},

/// The final stack frame size of the function (already aligned to the
/// respective stack alignment). Does not include prologue stack space.
stack_size: u32,

const InnerError = error{
    OutOfMemory,
    EmitFail,
};

const BranchType = enum {
    cbz,
    b_cond,
    unconditional_branch_immediate,

    fn default(tag: Mir.Inst.Tag) BranchType {
        return switch (tag) {
            .cbz => .cbz,
            .b, .bl => .unconditional_branch_immediate,
            .b_cond => .b_cond,
            else => unreachable,
        };
    }
};

pub fn emitMir(
    emit: *Emit,
) !void {
    const mir_tags = emit.mir.instructions.items(.tag);

    // Find smallest lowerings for branch instructions
    try emit.lowerBranches();

    // Emit machine code
    for (mir_tags) |tag, index| {
        const inst = @intCast(u32, index);
        switch (tag) {
            .add_immediate => try emit.mirAddSubtractImmediate(inst),
            .adds_immediate => try emit.mirAddSubtractImmediate(inst),
            .cmp_immediate => try emit.mirAddSubtractImmediate(inst),
            .sub_immediate => try emit.mirAddSubtractImmediate(inst),
            .subs_immediate => try emit.mirAddSubtractImmediate(inst),

            .asr_register => try emit.mirDataProcessing2Source(inst),
            .lsl_register => try emit.mirDataProcessing2Source(inst),
            .lsr_register => try emit.mirDataProcessing2Source(inst),
            .sdiv => try emit.mirDataProcessing2Source(inst),
            .udiv => try emit.mirDataProcessing2Source(inst),

            .asr_immediate => try emit.mirShiftImmediate(inst),
            .lsl_immediate => try emit.mirShiftImmediate(inst),
            .lsr_immediate => try emit.mirShiftImmediate(inst),

            .b_cond => try emit.mirConditionalBranchImmediate(inst),

            .b => try emit.mirBranch(inst),
            .bl => try emit.mirBranch(inst),

            .cbz => try emit.mirCompareAndBranch(inst),

            .blr => try emit.mirUnconditionalBranchRegister(inst),
            .ret => try emit.mirUnconditionalBranchRegister(inst),

            .brk => try emit.mirExceptionGeneration(inst),
            .svc => try emit.mirExceptionGeneration(inst),

            .call_extern => try emit.mirCallExtern(inst),

            .eor_immediate => try emit.mirLogicalImmediate(inst),
            .tst_immediate => try emit.mirLogicalImmediate(inst),

            .add_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
            .adds_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
            .cmp_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
            .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
            .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),

            .add_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
            .adds_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
            .sub_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
            .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
            .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),

            .cset => try emit.mirConditionalSelect(inst),

            .dbg_line => try emit.mirDbgLine(inst),

            .dbg_prologue_end => try emit.mirDebugPrologueEnd(),
            .dbg_epilogue_begin => try emit.mirDebugEpilogueBegin(),

            .and_shifted_register => try emit.mirLogicalShiftedRegister(inst),
            .eor_shifted_register => try emit.mirLogicalShiftedRegister(inst),
            .orr_shifted_register => try emit.mirLogicalShiftedRegister(inst),

            .load_memory_got => try emit.mirLoadMemoryPie(inst),
            .load_memory_direct => try emit.mirLoadMemoryPie(inst),
            .load_memory_ptr_got => try emit.mirLoadMemoryPie(inst),
            .load_memory_ptr_direct => try emit.mirLoadMemoryPie(inst),

            .ldp => try emit.mirLoadStoreRegisterPair(inst),
            .stp => try emit.mirLoadStoreRegisterPair(inst),

            .ldr_stack => try emit.mirLoadStoreStack(inst),
            .ldrb_stack => try emit.mirLoadStoreStack(inst),
            .ldrh_stack => try emit.mirLoadStoreStack(inst),
            .ldrsb_stack => try emit.mirLoadStoreStack(inst),
            .ldrsh_stack => try emit.mirLoadStoreStack(inst),
            .str_stack => try emit.mirLoadStoreStack(inst),
            .strb_stack => try emit.mirLoadStoreStack(inst),
            .strh_stack => try emit.mirLoadStoreStack(inst),

            .ldr_stack_argument => try emit.mirLoadStackArgument(inst),
            .ldr_ptr_stack_argument => try emit.mirLoadStackArgument(inst),
            .ldrb_stack_argument => try emit.mirLoadStackArgument(inst),
            .ldrh_stack_argument => try emit.mirLoadStackArgument(inst),
            .ldrsb_stack_argument => try emit.mirLoadStackArgument(inst),
            .ldrsh_stack_argument => try emit.mirLoadStackArgument(inst),

            .ldr_register => try emit.mirLoadStoreRegisterRegister(inst),
            .ldrb_register => try emit.mirLoadStoreRegisterRegister(inst),
            .ldrh_register => try emit.mirLoadStoreRegisterRegister(inst),
            .str_register => try emit.mirLoadStoreRegisterRegister(inst),
            .strb_register => try emit.mirLoadStoreRegisterRegister(inst),
            .strh_register => try emit.mirLoadStoreRegisterRegister(inst),

            .ldr_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .ldrb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .ldrh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .ldrsb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .ldrsh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .ldrsw_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .str_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .strb_immediate => try emit.mirLoadStoreRegisterImmediate(inst),
            .strh_immediate => try emit.mirLoadStoreRegisterImmediate(inst),

            .mov_register => try emit.mirMoveRegister(inst),
            .mov_to_from_sp => try emit.mirMoveRegister(inst),
            .mvn => try emit.mirMoveRegister(inst),

            .movk => try emit.mirMoveWideImmediate(inst),
            .movz => try emit.mirMoveWideImmediate(inst),

            .msub => try emit.mirDataProcessing3Source(inst),
            .mul => try emit.mirDataProcessing3Source(inst),
            .smulh => try emit.mirDataProcessing3Source(inst),
            .smull => try emit.mirDataProcessing3Source(inst),
            .umulh => try emit.mirDataProcessing3Source(inst),
            .umull => try emit.mirDataProcessing3Source(inst),

            .nop => try emit.mirNop(),

            .push_regs => try emit.mirPushPopRegs(inst),
            .pop_regs => try emit.mirPushPopRegs(inst),

            .sbfx,
            .ubfx,
            => try emit.mirBitfieldExtract(inst),

            .sxtb,
            .sxth,
            .sxtw,
            .uxtb,
            .uxth,
            => try emit.mirExtend(inst),
        }
    }
}

pub fn deinit(emit: *Emit) void {
    var iter = emit.branch_forward_origins.valueIterator();
    while (iter.next()) |origin_list| {
        origin_list.deinit(emit.bin_file.allocator);
    }

    emit.branch_types.deinit(emit.bin_file.allocator);
    emit.branch_forward_origins.deinit(emit.bin_file.allocator);
    emit.code_offset_mapping.deinit(emit.bin_file.allocator);
    emit.* = undefined;
}

fn optimalBranchType(emit: *Emit, tag: Mir.Inst.Tag, offset: i64) !BranchType {
    assert(offset & 0b11 == 0);

    switch (tag) {
        .cbz => {
            if (std.math.cast(i19, @shrExact(offset, 2))) |_| {
                return BranchType.cbz;
            } else {
                return emit.fail("TODO support cbz branches larger than +-1 MiB", .{});
            }
        },
        .b, .bl => {
            if (std.math.cast(i26, @shrExact(offset, 2))) |_| {
                return BranchType.unconditional_branch_immediate;
            } else {
                return emit.fail("TODO support unconditional branches larger than +-128 MiB", .{});
            }
        },
        .b_cond => {
            if (std.math.cast(i19, @shrExact(offset, 2))) |_| {
                return BranchType.b_cond;
            } else {
                return emit.fail("TODO support conditional branches larger than +-1 MiB", .{});
            }
        },
        else => unreachable,
    }
}

fn instructionSize(emit: *Emit, inst: Mir.Inst.Index) usize {
    const tag = emit.mir.instructions.items(.tag)[inst];

    if (isBranch(tag)) {
        switch (emit.branch_types.get(inst).?) {
            .cbz,
            .unconditional_branch_immediate,
            .b_cond,
            => return 4,
        }
    }

    switch (tag) {
        .load_memory_direct => return 3 * 4,
        .load_memory_got,
        .load_memory_ptr_got,
        .load_memory_ptr_direct,
        => return 2 * 4,
        .pop_regs, .push_regs => {
            const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;
            const number_of_regs = @popCount(reg_list);
            const number_of_insts = std.math.divCeil(u6, number_of_regs, 2) catch unreachable;
            return number_of_insts * 4;
        },
        .call_extern => return 4,
        .dbg_line,
        .dbg_epilogue_begin,
        .dbg_prologue_end,
        => return 0,
        else => return 4,
    }
}

fn isBranch(tag: Mir.Inst.Tag) bool {
    return switch (tag) {
        .cbz,
        .b,
        .bl,
        .b_cond,
        => true,
        else => false,
    };
}

fn branchTarget(emit: *Emit, inst: Mir.Inst.Index) Mir.Inst.Index {
    const tag = emit.mir.instructions.items(.tag)[inst];

    switch (tag) {
        .cbz => return emit.mir.instructions.items(.data)[inst].r_inst.inst,
        .b, .bl => return emit.mir.instructions.items(.data)[inst].inst,
        .b_cond => return emit.mir.instructions.items(.data)[inst].inst_cond.inst,
        else => unreachable,
    }
}

fn lowerBranches(emit: *Emit) !void {
    const mir_tags = emit.mir.instructions.items(.tag);
    const allocator = emit.bin_file.allocator;

    // First pass: Note down all branches and their target
    // instructions, i.e. populate branch_types,
    // branch_forward_origins, and code_offset_mapping
    //
    // TODO optimization opportunity: do this in codegen while
    // generating MIR
    for (mir_tags) |tag, index| {
        const inst = @intCast(u32, index);
        if (isBranch(tag)) {
            const target_inst = emit.branchTarget(inst);

            // Remember this branch instruction
            try emit.branch_types.put(allocator, inst, BranchType.default(tag));

            // Forward branches require some extra stuff: We only
            // know their offset once we arrive at the target
            // instruction. Therefore, we need to be able to
            // access the branch instruction when we visit the
            // target instruction in order to manipulate its type
            // etc.
            if (target_inst > inst) {
                // Remember the branch instruction index
                try emit.code_offset_mapping.put(allocator, inst, 0);

                if (emit.branch_forward_origins.getPtr(target_inst)) |origin_list| {
                    try origin_list.append(allocator, inst);
                } else {
                    var origin_list: std.ArrayListUnmanaged(Mir.Inst.Index) = .{};
                    try origin_list.append(allocator, inst);
                    try emit.branch_forward_origins.put(allocator, target_inst, origin_list);
                }
            }

            // Remember the target instruction index so that we
            // update the real code offset in all future passes
            //
            // putNoClobber may not be used as the put operation
            // may clobber the entry when multiple branches branch
            // to the same target instruction
            try emit.code_offset_mapping.put(allocator, target_inst, 0);
        }
    }

    // Further passes: Until all branches are lowered, interate
    // through all instructions and calculate new offsets and
    // potentially new branch types
    var all_branches_lowered = false;
    while (!all_branches_lowered) {
        all_branches_lowered = true;
        var current_code_offset: usize = 0;

        for (mir_tags) |tag, index| {
            const inst = @intCast(u32, index);

            // If this instruction contained in the code offset
            // mapping (when it is a target of a branch or if it is a
            // forward branch), update the code offset
            if (emit.code_offset_mapping.getPtr(inst)) |offset| {
                offset.* = current_code_offset;
            }

            // If this instruction is a backward branch, calculate the
            // offset, which may potentially update the branch type
            if (isBranch(tag)) {
                const target_inst = emit.branchTarget(inst);
                if (target_inst < inst) {
                    const target_offset = emit.code_offset_mapping.get(target_inst).?;
                    const offset = @intCast(i64, target_offset) - @intCast(i64, current_code_offset);
                    const branch_type = emit.branch_types.getPtr(inst).?;
                    const optimal_branch_type = try emit.optimalBranchType(tag, offset);
                    if (branch_type.* != optimal_branch_type) {
                        branch_type.* = optimal_branch_type;
                        all_branches_lowered = false;
                    }

                    log.debug("lowerBranches: branch {} has offset {}", .{ inst, offset });
                }
            }

            // If this instruction is the target of one or more
            // forward branches, calculate the offset, which may
            // potentially update the branch type
            if (emit.branch_forward_origins.get(inst)) |origin_list| {
                for (origin_list.items) |forward_branch_inst| {
                    const branch_tag = emit.mir.instructions.items(.tag)[forward_branch_inst];
                    const forward_branch_inst_offset = emit.code_offset_mapping.get(forward_branch_inst).?;
                    const offset = @intCast(i64, current_code_offset) - @intCast(i64, forward_branch_inst_offset);
                    const branch_type = emit.branch_types.getPtr(forward_branch_inst).?;
                    const optimal_branch_type = try emit.optimalBranchType(branch_tag, offset);
                    if (branch_type.* != optimal_branch_type) {
                        branch_type.* = optimal_branch_type;
                        all_branches_lowered = false;
                    }

                    log.debug("lowerBranches: branch {} has offset {}", .{ forward_branch_inst, offset });
                }
            }

            // Increment code offset
            current_code_offset += emit.instructionSize(inst);
        }
    }
}

fn writeInstruction(emit: *Emit, instruction: Instruction) !void {
    const endian = emit.target.cpu.arch.endian();
    std.mem.writeInt(u32, try emit.code.addManyAsArray(4), instruction.toU32(), endian);
}

fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
    @setCold(true);
    assert(emit.err_msg == null);
    emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
    return error.EmitFail;
}

fn dbgAdvancePCAndLine(self: *Emit, line: u32, column: u32) !void {
    const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
    const delta_pc: usize = self.code.items.len - self.prev_di_pc;
    switch (self.debug_output) {
        .dwarf => |dw| {
            // TODO Look into using the DWARF special opcodes to compress this data.
            // It lets you emit single-byte opcodes that add different numbers to
            // both the PC and the line number at the same time.
            const dbg_line = &dw.dbg_line;
            try dbg_line.ensureUnusedCapacity(11);
            dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
            leb128.writeULEB128(dbg_line.writer(), delta_pc) catch unreachable;
            if (delta_line != 0) {
                dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
                leb128.writeILEB128(dbg_line.writer(), delta_line) catch unreachable;
            }
            dbg_line.appendAssumeCapacity(DW.LNS.copy);
            self.prev_di_pc = self.code.items.len;
            self.prev_di_line = line;
            self.prev_di_column = column;
            self.prev_di_pc = self.code.items.len;
        },
        .plan9 => |dbg_out| {
            if (delta_pc <= 0) return; // only do this when the pc changes
            // we have already checked the target in the linker to make sure it is compatable
            const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;

            // increasing the line number
            try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
            // increasing the pc
            const d_pc_p9 = @intCast(i64, delta_pc) - quant;
            if (d_pc_p9 > 0) {
                // minus one because if its the last one, we want to leave space to change the line which is one quanta
                try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
                if (dbg_out.pcop_change_index.*) |pci|
                    dbg_out.dbg_line.items[pci] += 1;
                dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
            } else if (d_pc_p9 == 0) {
                // we don't need to do anything, because adding the quant does it for us
            } else unreachable;
            if (dbg_out.start_line.* == null)
                dbg_out.start_line.* = self.prev_di_line;
            dbg_out.end_line.* = line;
            // only do this if the pc changed
            self.prev_di_line = line;
            self.prev_di_column = column;
            self.prev_di_pc = self.code.items.len;
        },
        .none => {},
    }
}

fn mirAddSubtractImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    switch (tag) {
        .add_immediate,
        .adds_immediate,
        .sub_immediate,
        .subs_immediate,
        => {
            const rr_imm12_sh = emit.mir.instructions.items(.data)[inst].rr_imm12_sh;
            const rd = rr_imm12_sh.rd;
            const rn = rr_imm12_sh.rn;
            const imm12 = rr_imm12_sh.imm12;
            const sh = rr_imm12_sh.sh == 1;

            switch (tag) {
                .add_immediate => try emit.writeInstruction(Instruction.add(rd, rn, imm12, sh)),
                .adds_immediate => try emit.writeInstruction(Instruction.adds(rd, rn, imm12, sh)),
                .sub_immediate => try emit.writeInstruction(Instruction.sub(rd, rn, imm12, sh)),
                .subs_immediate => try emit.writeInstruction(Instruction.subs(rd, rn, imm12, sh)),
                else => unreachable,
            }
        },
        .cmp_immediate => {
            const r_imm12_sh = emit.mir.instructions.items(.data)[inst].r_imm12_sh;
            const rn = r_imm12_sh.rn;
            const imm12 = r_imm12_sh.imm12;
            const sh = r_imm12_sh.sh == 1;
            const zr: Register = switch (rn.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };

            try emit.writeInstruction(Instruction.subs(zr, rn, imm12, sh));
        },
        else => unreachable,
    }
}

fn mirDataProcessing2Source(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rrr = emit.mir.instructions.items(.data)[inst].rrr;
    const rd = rrr.rd;
    const rn = rrr.rn;
    const rm = rrr.rm;

    switch (tag) {
        .asr_register => try emit.writeInstruction(Instruction.asrRegister(rd, rn, rm)),
        .lsl_register => try emit.writeInstruction(Instruction.lslRegister(rd, rn, rm)),
        .lsr_register => try emit.writeInstruction(Instruction.lsrRegister(rd, rn, rm)),
        .sdiv => try emit.writeInstruction(Instruction.sdiv(rd, rn, rm)),
        .udiv => try emit.writeInstruction(Instruction.udiv(rd, rn, rm)),
        else => unreachable,
    }
}

fn mirShiftImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rr_shift = emit.mir.instructions.items(.data)[inst].rr_shift;
    const rd = rr_shift.rd;
    const rn = rr_shift.rn;
    const shift = rr_shift.shift;

    switch (tag) {
        .asr_immediate => try emit.writeInstruction(Instruction.asrImmediate(rd, rn, shift)),
        .lsl_immediate => try emit.writeInstruction(Instruction.lslImmediate(rd, rn, shift)),
        .lsr_immediate => try emit.writeInstruction(Instruction.lsrImmediate(rd, rn, shift)),
        else => unreachable,
    }
}

fn mirConditionalBranchImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const inst_cond = emit.mir.instructions.items(.data)[inst].inst_cond;

    const offset = @intCast(i64, emit.code_offset_mapping.get(inst_cond.inst).?) - @intCast(i64, emit.code.items.len);
    const branch_type = emit.branch_types.get(inst).?;
    log.debug("mirConditionalBranchImmediate: {} offset={}", .{ inst, offset });

    switch (branch_type) {
        .b_cond => switch (tag) {
            .b_cond => try emit.writeInstruction(Instruction.bCond(inst_cond.cond, @intCast(i21, offset))),
            else => unreachable,
        },
        else => unreachable,
    }
}

fn mirBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const target_inst = emit.mir.instructions.items(.data)[inst].inst;

    log.debug("branch {}(tag: {}) -> {}(tag: {})", .{
        inst,
        tag,
        target_inst,
        emit.mir.instructions.items(.tag)[target_inst],
    });

    const offset = @intCast(i64, emit.code_offset_mapping.get(target_inst).?) - @intCast(i64, emit.code.items.len);
    const branch_type = emit.branch_types.get(inst).?;
    log.debug("mirBranch: {} offset={}", .{ inst, offset });

    switch (branch_type) {
        .unconditional_branch_immediate => switch (tag) {
            .b => try emit.writeInstruction(Instruction.b(@intCast(i28, offset))),
            .bl => try emit.writeInstruction(Instruction.bl(@intCast(i28, offset))),
            else => unreachable,
        },
        else => unreachable,
    }
}

fn mirCompareAndBranch(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const r_inst = emit.mir.instructions.items(.data)[inst].r_inst;

    const offset = @intCast(i64, emit.code_offset_mapping.get(r_inst.inst).?) - @intCast(i64, emit.code.items.len);
    const branch_type = emit.branch_types.get(inst).?;
    log.debug("mirCompareAndBranch: {} offset={}", .{ inst, offset });

    switch (branch_type) {
        .cbz => switch (tag) {
            .cbz => try emit.writeInstruction(Instruction.cbz(r_inst.rt, @intCast(i21, offset))),
            else => unreachable,
        },
        else => unreachable,
    }
}

fn mirUnconditionalBranchRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const reg = emit.mir.instructions.items(.data)[inst].reg;

    switch (tag) {
        .blr => try emit.writeInstruction(Instruction.blr(reg)),
        .ret => try emit.writeInstruction(Instruction.ret(reg)),
        else => unreachable,
    }
}

fn mirExceptionGeneration(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const imm16 = emit.mir.instructions.items(.data)[inst].imm16;

    switch (tag) {
        .brk => try emit.writeInstruction(Instruction.brk(imm16)),
        .svc => try emit.writeInstruction(Instruction.svc(imm16)),
        else => unreachable,
    }
}

fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const dbg_line_column = emit.mir.instructions.items(.data)[inst].dbg_line_column;

    switch (tag) {
        .dbg_line => try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column),
        else => unreachable,
    }
}

fn mirDebugPrologueEnd(self: *Emit) !void {
    switch (self.debug_output) {
        .dwarf => |dw| {
            try dw.dbg_line.append(DW.LNS.set_prologue_end);
            try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
        },
        .plan9 => {},
        .none => {},
    }
}

fn mirDebugEpilogueBegin(self: *Emit) !void {
    switch (self.debug_output) {
        .dwarf => |dw| {
            try dw.dbg_line.append(DW.LNS.set_epilogue_begin);
            try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
        },
        .plan9 => {},
        .none => {},
    }
}

fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) !void {
    assert(emit.mir.instructions.items(.tag)[inst] == .call_extern);
    const relocation = emit.mir.instructions.items(.data)[inst].relocation;

    if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
        const offset = blk: {
            const offset = @intCast(u32, emit.code.items.len);
            // bl
            try emit.writeInstruction(Instruction.bl(0));
            break :blk offset;
        };
        // Add relocation to the decl.
        const atom = macho_file.atom_by_index_table.get(relocation.atom_index).?;
        const target = macho_file.getGlobalByIndex(relocation.sym_index);
        try atom.relocs.append(emit.bin_file.allocator, .{
            .offset = offset,
            .target = target,
            .addend = 0,
            .subtractor = null,
            .pcrel = true,
            .length = 2,
            .@"type" = @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_BRANCH26),
        });
    } else {
        return emit.fail("Implement call_extern for linking backends != MachO", .{});
    }
}

fn mirLogicalImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rr_bitmask = emit.mir.instructions.items(.data)[inst].rr_bitmask;
    const rd = rr_bitmask.rd;
    const rn = rr_bitmask.rn;
    const imms = rr_bitmask.imms;
    const immr = rr_bitmask.immr;
    const n = rr_bitmask.n;

    switch (tag) {
        .eor_immediate => try emit.writeInstruction(Instruction.eorImmediate(rd, rn, imms, immr, n)),
        .tst_immediate => {
            const zr: Register = switch (rd.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };
            try emit.writeInstruction(Instruction.andsImmediate(zr, rn, imms, immr, n));
        },
        else => unreachable,
    }
}

fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    switch (tag) {
        .add_shifted_register,
        .adds_shifted_register,
        .sub_shifted_register,
        .subs_shifted_register,
        => {
            const rrr_imm6_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_shift;
            const rd = rrr_imm6_shift.rd;
            const rn = rrr_imm6_shift.rn;
            const rm = rrr_imm6_shift.rm;
            const shift = rrr_imm6_shift.shift;
            const imm6 = rrr_imm6_shift.imm6;

            switch (tag) {
                .add_shifted_register => try emit.writeInstruction(Instruction.addShiftedRegister(rd, rn, rm, shift, imm6)),
                .adds_shifted_register => try emit.writeInstruction(Instruction.addsShiftedRegister(rd, rn, rm, shift, imm6)),
                .sub_shifted_register => try emit.writeInstruction(Instruction.subShiftedRegister(rd, rn, rm, shift, imm6)),
                .subs_shifted_register => try emit.writeInstruction(Instruction.subsShiftedRegister(rd, rn, rm, shift, imm6)),
                else => unreachable,
            }
        },
        .cmp_shifted_register => {
            const rr_imm6_shift = emit.mir.instructions.items(.data)[inst].rr_imm6_shift;
            const rn = rr_imm6_shift.rn;
            const rm = rr_imm6_shift.rm;
            const shift = rr_imm6_shift.shift;
            const imm6 = rr_imm6_shift.imm6;
            const zr: Register = switch (rn.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };

            try emit.writeInstruction(Instruction.subsShiftedRegister(zr, rn, rm, shift, imm6));
        },
        else => unreachable,
    }
}

fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    switch (tag) {
        .add_extended_register,
        .adds_extended_register,
        .sub_extended_register,
        .subs_extended_register,
        => {
            const rrr_extend_shift = emit.mir.instructions.items(.data)[inst].rrr_extend_shift;
            const rd = rrr_extend_shift.rd;
            const rn = rrr_extend_shift.rn;
            const rm = rrr_extend_shift.rm;
            const ext_type = rrr_extend_shift.ext_type;
            const imm3 = rrr_extend_shift.imm3;

            switch (tag) {
                .add_extended_register => try emit.writeInstruction(Instruction.addExtendedRegister(rd, rn, rm, ext_type, imm3)),
                .adds_extended_register => try emit.writeInstruction(Instruction.addsExtendedRegister(rd, rn, rm, ext_type, imm3)),
                .sub_extended_register => try emit.writeInstruction(Instruction.subExtendedRegister(rd, rn, rm, ext_type, imm3)),
                .subs_extended_register => try emit.writeInstruction(Instruction.subsExtendedRegister(rd, rn, rm, ext_type, imm3)),
                else => unreachable,
            }
        },
        .cmp_extended_register => {
            const rr_extend_shift = emit.mir.instructions.items(.data)[inst].rr_extend_shift;
            const rn = rr_extend_shift.rn;
            const rm = rr_extend_shift.rm;
            const ext_type = rr_extend_shift.ext_type;
            const imm3 = rr_extend_shift.imm3;
            const zr: Register = switch (rn.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };

            try emit.writeInstruction(Instruction.subsExtendedRegister(zr, rn, rm, ext_type, imm3));
        },
        else => unreachable,
    }
}

fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    switch (tag) {
        .cset => {
            const r_cond = emit.mir.instructions.items(.data)[inst].r_cond;
            const zr: Register = switch (r_cond.rd.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };
            try emit.writeInstruction(Instruction.csinc(r_cond.rd, zr, zr, r_cond.cond.negate()));
        },
        else => unreachable,
    }
}

fn mirLogicalShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rrr_imm6_logical_shift = emit.mir.instructions.items(.data)[inst].rrr_imm6_logical_shift;
    const rd = rrr_imm6_logical_shift.rd;
    const rn = rrr_imm6_logical_shift.rn;
    const rm = rrr_imm6_logical_shift.rm;
    const shift = rrr_imm6_logical_shift.shift;
    const imm6 = rrr_imm6_logical_shift.imm6;

    switch (tag) {
        .and_shifted_register => try emit.writeInstruction(Instruction.andShiftedRegister(rd, rn, rm, shift, imm6)),
        .eor_shifted_register => try emit.writeInstruction(Instruction.eorShiftedRegister(rd, rn, rm, shift, imm6)),
        .orr_shifted_register => try emit.writeInstruction(Instruction.orrShiftedRegister(rd, rn, rm, shift, imm6)),
        else => unreachable,
    }
}

fn mirLoadMemoryPie(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const payload = emit.mir.instructions.items(.data)[inst].payload;
    const data = emit.mir.extraData(Mir.LoadMemoryPie, payload).data;
    const reg = @intToEnum(Register, data.register);

    // PC-relative displacement to the entry in memory.
    // adrp
    const offset = @intCast(u32, emit.code.items.len);
    try emit.writeInstruction(Instruction.adrp(reg.to64(), 0));

    switch (tag) {
        .load_memory_got => {
            // ldr reg, reg, offset
            try emit.writeInstruction(Instruction.ldr(
                reg,
                reg.to64(),
                Instruction.LoadStoreOffset.imm(0),
            ));
        },
        .load_memory_direct => {
            // We cannot load the offset directly as it may not be aligned properly.
            // For example, load for 64bit register will require the target address offset
            // to be 8-byte aligned, while the value might have non-8-byte natural alignment,
            // meaning the linker might have put it at a non-8-byte aligned address. To circumvent
            // this, we use `adrp, add` to form the address value which we then dereference with
            // `ldr`.
            // Note that this can potentially be optimised out by the codegen/linker if the
            // target address is appropriately aligned.
            // add reg, reg, offset
            try emit.writeInstruction(Instruction.add(reg.to64(), reg.to64(), 0, false));
            // ldr reg, reg, offset
            try emit.writeInstruction(Instruction.ldr(
                reg,
                reg.to64(),
                Instruction.LoadStoreOffset.imm(0),
            ));
        },
        .load_memory_ptr_got,
        .load_memory_ptr_direct,
        => {
            // add reg, reg, offset
            try emit.writeInstruction(Instruction.add(reg, reg, 0, false));
        },
        else => unreachable,
    }

    if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
        const atom = macho_file.atom_by_index_table.get(data.atom_index).?;
        // Page reloc for adrp instruction.
        try atom.relocs.append(emit.bin_file.allocator, .{
            .offset = offset,
            .target = .{ .sym_index = data.sym_index, .file = null },
            .addend = 0,
            .subtractor = null,
            .pcrel = true,
            .length = 2,
            .@"type" = switch (tag) {
                .load_memory_got,
                .load_memory_ptr_got,
                => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGE21),
                .load_memory_direct,
                .load_memory_ptr_direct,
                => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_PAGE21),
                else => unreachable,
            },
        });
        // Pageoff reloc for adrp instruction.
        try atom.relocs.append(emit.bin_file.allocator, .{
            .offset = offset + 4,
            .target = .{ .sym_index = data.sym_index, .file = null },
            .addend = 0,
            .subtractor = null,
            .pcrel = false,
            .length = 2,
            .@"type" = switch (tag) {
                .load_memory_got,
                .load_memory_ptr_got,
                => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGEOFF12),
                .load_memory_direct,
                .load_memory_ptr_direct,
                => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_PAGEOFF12),
                else => unreachable,
            },
        });
    } else {
        return emit.fail("TODO implement load_memory for PIE GOT indirection on this platform", .{});
    }
}

fn mirLoadStoreRegisterPair(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const load_store_register_pair = emit.mir.instructions.items(.data)[inst].load_store_register_pair;
    const rt = load_store_register_pair.rt;
    const rt2 = load_store_register_pair.rt2;
    const rn = load_store_register_pair.rn;
    const offset = load_store_register_pair.offset;

    switch (tag) {
        .stp => try emit.writeInstruction(Instruction.stp(rt, rt2, rn, offset)),
        .ldp => try emit.writeInstruction(Instruction.ldp(rt, rt2, rn, offset)),
        else => unreachable,
    }
}

fn mirLoadStackArgument(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
    const rt = load_store_stack.rt;

    const raw_offset = emit.stack_size + emit.saved_regs_stack_space + load_store_stack.offset;
    switch (tag) {
        .ldr_ptr_stack_argument => {
            const offset = if (math.cast(u12, raw_offset)) |imm| imm else {
                return emit.fail("TODO load stack argument ptr with larger offset", .{});
            };

            switch (tag) {
                .ldr_ptr_stack_argument => try emit.writeInstruction(Instruction.add(rt, .sp, offset, false)),
                else => unreachable,
            }
        },
        .ldrb_stack_argument, .ldrsb_stack_argument => {
            const offset = if (math.cast(u12, raw_offset)) |imm| Instruction.LoadStoreOffset.imm(imm) else {
                return emit.fail("TODO load stack argument byte with larger offset", .{});
            };

            switch (tag) {
                .ldrb_stack_argument => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
                .ldrsb_stack_argument => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
                else => unreachable,
            }
        },
        .ldrh_stack_argument, .ldrsh_stack_argument => {
            assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
            const offset = if (math.cast(u12, @divExact(raw_offset, 2))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
                return emit.fail("TODO load stack argument halfword with larger offset", .{});
            };

            switch (tag) {
                .ldrh_stack_argument => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
                .ldrsh_stack_argument => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
                else => unreachable,
            }
        },
        .ldr_stack_argument => {
            const alignment: u32 = switch (rt.size()) {
                32 => 4,
                64 => 8,
                else => unreachable,
            };

            assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
            const offset = if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| Instruction.LoadStoreOffset.imm(imm) else {
                return emit.fail("TODO load stack argument with larger offset", .{});
            };

            switch (tag) {
                .ldr_stack_argument => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
                else => unreachable,
            }
        },
        else => unreachable,
    }
}

fn mirLoadStoreStack(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const load_store_stack = emit.mir.instructions.items(.data)[inst].load_store_stack;
    const rt = load_store_stack.rt;

    const raw_offset = emit.stack_size - load_store_stack.offset;
    const offset = switch (tag) {
        .ldrb_stack, .ldrsb_stack, .strb_stack => blk: {
            if (math.cast(u12, raw_offset)) |imm| {
                break :blk Instruction.LoadStoreOffset.imm(imm);
            } else {
                return emit.fail("TODO load/store stack byte with larger offset", .{});
            }
        },
        .ldrh_stack, .ldrsh_stack, .strh_stack => blk: {
            assert(std.mem.isAlignedGeneric(u32, raw_offset, 2)); // misaligned stack entry
            if (math.cast(u12, @divExact(raw_offset, 2))) |imm| {
                break :blk Instruction.LoadStoreOffset.imm(imm);
            } else {
                return emit.fail("TODO load/store stack halfword with larger offset", .{});
            }
        },
        .ldr_stack, .str_stack => blk: {
            const alignment: u32 = switch (rt.size()) {
                32 => 4,
                64 => 8,
                else => unreachable,
            };

            assert(std.mem.isAlignedGeneric(u32, raw_offset, alignment)); // misaligned stack entry
            if (math.cast(u12, @divExact(raw_offset, alignment))) |imm| {
                break :blk Instruction.LoadStoreOffset.imm(imm);
            } else {
                return emit.fail("TODO load/store stack with larger offset", .{});
            }
        },
        else => unreachable,
    };

    switch (tag) {
        .ldr_stack => try emit.writeInstruction(Instruction.ldr(rt, .sp, offset)),
        .ldrb_stack => try emit.writeInstruction(Instruction.ldrb(rt, .sp, offset)),
        .ldrh_stack => try emit.writeInstruction(Instruction.ldrh(rt, .sp, offset)),
        .ldrsb_stack => try emit.writeInstruction(Instruction.ldrsb(rt, .sp, offset)),
        .ldrsh_stack => try emit.writeInstruction(Instruction.ldrsh(rt, .sp, offset)),
        .str_stack => try emit.writeInstruction(Instruction.str(rt, .sp, offset)),
        .strb_stack => try emit.writeInstruction(Instruction.strb(rt, .sp, offset)),
        .strh_stack => try emit.writeInstruction(Instruction.strh(rt, .sp, offset)),
        else => unreachable,
    }
}

fn mirLoadStoreRegisterImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const load_store_register_immediate = emit.mir.instructions.items(.data)[inst].load_store_register_immediate;
    const rt = load_store_register_immediate.rt;
    const rn = load_store_register_immediate.rn;
    const offset = Instruction.LoadStoreOffset{ .immediate = load_store_register_immediate.offset };

    switch (tag) {
        .ldr_immediate => try emit.writeInstruction(Instruction.ldr(rt, rn, offset)),
        .ldrb_immediate => try emit.writeInstruction(Instruction.ldrb(rt, rn, offset)),
        .ldrh_immediate => try emit.writeInstruction(Instruction.ldrh(rt, rn, offset)),
        .ldrsb_immediate => try emit.writeInstruction(Instruction.ldrsb(rt, rn, offset)),
        .ldrsh_immediate => try emit.writeInstruction(Instruction.ldrsh(rt, rn, offset)),
        .ldrsw_immediate => try emit.writeInstruction(Instruction.ldrsw(rt, rn, offset)),
        .str_immediate => try emit.writeInstruction(Instruction.str(rt, rn, offset)),
        .strb_immediate => try emit.writeInstruction(Instruction.strb(rt, rn, offset)),
        .strh_immediate => try emit.writeInstruction(Instruction.strh(rt, rn, offset)),
        else => unreachable,
    }
}

fn mirLoadStoreRegisterRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const load_store_register_register = emit.mir.instructions.items(.data)[inst].load_store_register_register;
    const rt = load_store_register_register.rt;
    const rn = load_store_register_register.rn;
    const offset = Instruction.LoadStoreOffset{ .register = load_store_register_register.offset };

    switch (tag) {
        .ldr_register => try emit.writeInstruction(Instruction.ldr(rt, rn, offset)),
        .ldrb_register => try emit.writeInstruction(Instruction.ldrb(rt, rn, offset)),
        .ldrh_register => try emit.writeInstruction(Instruction.ldrh(rt, rn, offset)),
        .str_register => try emit.writeInstruction(Instruction.str(rt, rn, offset)),
        .strb_register => try emit.writeInstruction(Instruction.strb(rt, rn, offset)),
        .strh_register => try emit.writeInstruction(Instruction.strh(rt, rn, offset)),
        else => unreachable,
    }
}

fn mirMoveRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    switch (tag) {
        .mov_register => {
            const rr = emit.mir.instructions.items(.data)[inst].rr;
            const zr: Register = switch (rr.rd.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };

            try emit.writeInstruction(Instruction.orrShiftedRegister(rr.rd, zr, rr.rn, .lsl, 0));
        },
        .mov_to_from_sp => {
            const rr = emit.mir.instructions.items(.data)[inst].rr;
            try emit.writeInstruction(Instruction.add(rr.rd, rr.rn, 0, false));
        },
        .mvn => {
            const rr_imm6_logical_shift = emit.mir.instructions.items(.data)[inst].rr_imm6_logical_shift;
            const rd = rr_imm6_logical_shift.rd;
            const rm = rr_imm6_logical_shift.rm;
            const shift = rr_imm6_logical_shift.shift;
            const imm6 = rr_imm6_logical_shift.imm6;
            const zr: Register = switch (rd.size()) {
                32 => .wzr,
                64 => .xzr,
                else => unreachable,
            };

            try emit.writeInstruction(Instruction.ornShiftedRegister(rd, zr, rm, shift, imm6));
        },
        else => unreachable,
    }
}

fn mirMoveWideImmediate(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const r_imm16_sh = emit.mir.instructions.items(.data)[inst].r_imm16_sh;

    switch (tag) {
        .movz => try emit.writeInstruction(Instruction.movz(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)),
        .movk => try emit.writeInstruction(Instruction.movk(r_imm16_sh.rd, r_imm16_sh.imm16, @as(u6, r_imm16_sh.hw) << 4)),
        else => unreachable,
    }
}

fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];

    switch (tag) {
        .mul,
        .smulh,
        .smull,
        .umulh,
        .umull,
        => {
            const rrr = emit.mir.instructions.items(.data)[inst].rrr;
            switch (tag) {
                .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
                .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),
                .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),
                .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),
                .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),
                else => unreachable,
            }
        },
        .msub => {
            const rrrr = emit.mir.instructions.items(.data)[inst].rrrr;
            switch (tag) {
                .msub => try emit.writeInstruction(Instruction.msub(rrrr.rd, rrrr.rn, rrrr.rm, rrrr.ra)),
                else => unreachable,
            }
        },
        else => unreachable,
    }
}

fn mirNop(emit: *Emit) !void {
    try emit.writeInstruction(Instruction.nop());
}

fn mirPushPopRegs(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const reg_list = emit.mir.instructions.items(.data)[inst].reg_list;

    if (reg_list & @as(u32, 1) << 31 != 0) return emit.fail("xzr is not a valid register for {}", .{tag});

    // sp must be aligned at all times, so we only use stp and ldp
    // instructions for minimal instruction count. However, if we do
    // not have an even number of registers, we use str and ldr
    const number_of_regs = @popCount(reg_list);

    switch (tag) {
        .pop_regs => {
            var i: u6 = 32;
            var count: u6 = 0;
            var other_reg: Register = undefined;
            while (i > 0) : (i -= 1) {
                const reg = @intToEnum(Register, i - 1);
                if (reg_list & @as(u32, 1) << @intCast(u5, reg.id()) != 0) {
                    if (count % 2 == 0) {
                        if (count == number_of_regs - 1) {
                            try emit.writeInstruction(Instruction.ldr(
                                reg,
                                .sp,
                                Instruction.LoadStoreOffset.imm_post_index(16),
                            ));
                        } else {
                            other_reg = reg;
                        }
                    } else {
                        try emit.writeInstruction(Instruction.ldp(
                            reg,
                            other_reg,
                            .sp,
                            Instruction.LoadStorePairOffset.post_index(16),
                        ));
                    }
                    count += 1;
                }
            }
            assert(count == number_of_regs);
        },
        .push_regs => {
            var i: u6 = 0;
            var count: u6 = 0;
            var other_reg: Register = undefined;
            while (i < 32) : (i += 1) {
                const reg = @intToEnum(Register, i);
                if (reg_list & @as(u32, 1) << @intCast(u5, reg.id()) != 0) {
                    if (count % 2 == 0) {
                        if (count == number_of_regs - 1) {
                            try emit.writeInstruction(Instruction.str(
                                reg,
                                .sp,
                                Instruction.LoadStoreOffset.imm_pre_index(-16),
                            ));
                        } else {
                            other_reg = reg;
                        }
                    } else {
                        try emit.writeInstruction(Instruction.stp(
                            other_reg,
                            reg,
                            .sp,
                            Instruction.LoadStorePairOffset.pre_index(-16),
                        ));
                    }
                    count += 1;
                }
            }
            assert(count == number_of_regs);
        },
        else => unreachable,
    }
}

fn mirBitfieldExtract(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rr_lsb_width = emit.mir.instructions.items(.data)[inst].rr_lsb_width;
    const rd = rr_lsb_width.rd;
    const rn = rr_lsb_width.rn;
    const lsb = rr_lsb_width.lsb;
    const width = rr_lsb_width.width;

    switch (tag) {
        .sbfx => try emit.writeInstruction(Instruction.sbfx(rd, rn, lsb, width)),
        .ubfx => try emit.writeInstruction(Instruction.ubfx(rd, rn, lsb, width)),
        else => unreachable,
    }
}

fn mirExtend(emit: *Emit, inst: Mir.Inst.Index) !void {
    const tag = emit.mir.instructions.items(.tag)[inst];
    const rr = emit.mir.instructions.items(.data)[inst].rr;

    switch (tag) {
        .sxtb => try emit.writeInstruction(Instruction.sxtb(rr.rd, rr.rn)),
        .sxth => try emit.writeInstruction(Instruction.sxth(rr.rd, rr.rn)),
        .sxtw => try emit.writeInstruction(Instruction.sxtw(rr.rd, rr.rn)),
        .uxtb => try emit.writeInstruction(Instruction.uxtb(rr.rd, rr.rn)),
        .uxth => try emit.writeInstruction(Instruction.uxth(rr.rd, rr.rn)),
        else => unreachable,
    }
}