diff options
| author | Andrew Kelley <andrew@ziglang.org> | 2021-02-25 21:04:23 -0700 |
|---|---|---|
| committer | Andrew Kelley <andrew@ziglang.org> | 2021-02-25 21:04:23 -0700 |
| commit | 0b58b617998b79a765b54f88fbe90ca2798b3d3e (patch) | |
| tree | ca6cc4b6bcc2b93166d196049ee49416afe781ad /src/codegen.zig | |
| parent | dc325669e360f7a9dfa24f85a62fa386529dade6 (diff) | |
| parent | fd208d9d5913a0929e444deb97b91092c427bb14 (diff) | |
| download | zig-0b58b617998b79a765b54f88fbe90ca2798b3d3e.tar.gz zig-0b58b617998b79a765b54f88fbe90ca2798b3d3e.zip | |
Merge remote-tracking branch 'origin/master' into llvm12
Conflicts:
* src/clang.zig
* src/llvm.zig
- this file got moved to src/llvm/bindings.zig in master branch so I
had to put the new LLVM arch/os enum tags into it.
* lib/std/target.zig, src/stage1/target.cpp
- haiku had an inconsistency with its default target ABI, gnu vs
eabi. In this commit we make it gnu in both places to match the
latest changes by @hoanga.
* src/translate_c.zig
Diffstat (limited to 'src/codegen.zig')
| -rw-r--r-- | src/codegen.zig | 1257 |
1 files changed, 918 insertions, 339 deletions
diff --git a/src/codegen.zig b/src/codegen.zig index 3b0a383a71..57fd732b42 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -9,7 +9,7 @@ const TypedValue = @import("TypedValue.zig"); const link = @import("link.zig"); const Module = @import("Module.zig"); const Compilation = @import("Compilation.zig"); -const ErrorMsg = Compilation.ErrorMsg; +const ErrorMsg = Module.ErrorMsg; const Target = std.Target; const Allocator = mem.Allocator; const trace = @import("tracy.zig").trace; @@ -42,6 +42,11 @@ pub const Reloc = union(enum) { /// To perform the reloc, write 32-bit signed little-endian integer /// which is a relative jump, based on the address following the reloc. rel32: usize, + /// A branch in the ARM instruction set + arm_branch: struct { + pos: usize, + cond: @import("codegen/arm.zig").Condition, + }, }; pub const Result = union(enum) { @@ -69,7 +74,7 @@ pub const DebugInfoOutput = union(enum) { pub fn generateSymbol( bin_file: *link.File, - src: usize, + src_loc: Module.SrcLoc, typed_value: TypedValue, code: *std.ArrayList(u8), debug_output: DebugInfoOutput, @@ -82,67 +87,67 @@ pub fn generateSymbol( switch (bin_file.options.target.cpu.arch) { .wasm32 => unreachable, // has its own code path .wasm64 => unreachable, // has its own code path - .arm => return Function(.arm).generateSymbol(bin_file, src, typed_value, code, debug_output), - .armeb => return Function(.armeb).generateSymbol(bin_file, src, typed_value, code, debug_output), - .aarch64 => return Function(.aarch64).generateSymbol(bin_file, src, typed_value, code, debug_output), - .aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src, typed_value, code, debug_output), - .aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.arc => return Function(.arc).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.avr => return Function(.avr).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.mips => return Function(.mips).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.mips64 => return Function(.mips64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.msp430 => return Function(.msp430).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.r600 => return Function(.r600).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src, typed_value, code, debug_output), - .riscv64 => return Function(.riscv64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.sparc => return Function(.sparc).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.s390x => return Function(.s390x).generateSymbol(bin_file, src, typed_value, code, debug_output), - .spu_2 => return Function(.spu_2).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.tce => return Function(.tce).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.tcele => return Function(.tcele).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.thumb => return Function(.thumb).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.i386 => return Function(.i386).generateSymbol(bin_file, src, typed_value, code, debug_output), - .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.xcore => return Function(.xcore).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.le32 => return Function(.le32).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.le64 => return Function(.le64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.amdil => return Function(.amdil).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.hsail => return Function(.hsail).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.spir => return Function(.spir).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.spir64 => return Function(.spir64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.shave => return Function(.shave).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.lanai => return Function(.lanai).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src, typed_value, code, debug_output), - //.ve => return Function(.ve).generateSymbol(bin_file, src, typed_value, code, debug_output), + .arm => return Function(.arm).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .armeb => return Function(.armeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .aarch64 => return Function(.aarch64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.arc => return Function(.arc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.avr => return Function(.avr).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.mips => return Function(.mips).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.mips64 => return Function(.mips64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.msp430 => return Function(.msp430).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.r600 => return Function(.r600).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .riscv64 => return Function(.riscv64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.sparc => return Function(.sparc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.s390x => return Function(.s390x).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .spu_2 => return Function(.spu_2).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.tce => return Function(.tce).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.tcele => return Function(.tcele).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.thumb => return Function(.thumb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.i386 => return Function(.i386).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.xcore => return Function(.xcore).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.le32 => return Function(.le32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.le64 => return Function(.le64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.amdil => return Function(.amdil).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.hsail => return Function(.hsail).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.spir => return Function(.spir).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.spir64 => return Function(.spir64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.shave => return Function(.shave).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.lanai => return Function(.lanai).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), + //.ve => return Function(.ve).generateSymbol(bin_file, src_loc, typed_value, code, debug_output), else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."), } }, .Array => { // TODO populate .debug_info for the array - if (typed_value.val.cast(Value.Payload.Bytes)) |payload| { + if (typed_value.val.castTag(.bytes)) |payload| { if (typed_value.ty.sentinel()) |sentinel| { try code.ensureCapacity(code.items.len + payload.data.len + 1); code.appendSliceAssumeCapacity(payload.data); const prev_len = code.items.len; - switch (try generateSymbol(bin_file, src, .{ + switch (try generateSymbol(bin_file, src_loc, .{ .ty = typed_value.ty.elemType(), .val = sentinel, }, code, debug_output)) { @@ -160,7 +165,7 @@ pub fn generateSymbol( return Result{ .fail = try ErrorMsg.create( bin_file.allocator, - src, + src_loc, "TODO implement generateSymbol for more kinds of arrays", .{}, ), @@ -168,8 +173,8 @@ pub fn generateSymbol( }, .Pointer => { // TODO populate .debug_info for the pointer - if (typed_value.val.cast(Value.Payload.DeclRef)) |payload| { - const decl = payload.decl; + if (typed_value.val.castTag(.decl_ref)) |payload| { + const decl = payload.data; if (decl.analysis != .complete) return error.AnalysisFail; // TODO handle the dependency of this symbol on the decl's vaddr. // If the decl changes vaddr, then this symbol needs to get regenerated. @@ -195,7 +200,7 @@ pub fn generateSymbol( return Result{ .fail = try ErrorMsg.create( bin_file.allocator, - src, + src_loc, "TODO implement generateSymbol for pointer {}", .{typed_value.val}, ), @@ -212,7 +217,7 @@ pub fn generateSymbol( return Result{ .fail = try ErrorMsg.create( bin_file.allocator, - src, + src_loc, "TODO implement generateSymbol for int type '{}'", .{typed_value.ty}, ), @@ -222,8 +227,8 @@ pub fn generateSymbol( return Result{ .fail = try ErrorMsg.create( bin_file.allocator, - src, - "TODO implement generateSymbol for type '{}'", + src_loc, + "TODO implement generateSymbol for type '{s}'", .{@tagName(t)}, ), }; @@ -254,7 +259,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { ret_mcv: MCValue, fn_type: Type, arg_index: usize, - src: usize, + src_loc: Module.SrcLoc, stack_align: u32, /// Byte offset within the source file. @@ -283,6 +288,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { /// The key must be canonical register. registers: std.AutoHashMapUnmanaged(Register, *ir.Inst) = .{}, free_registers: FreeRegInt = math.maxInt(FreeRegInt), + /// Tracks all registers allocated in the course of this function + allocated_registers: FreeRegInt = 0, /// Maps offset to what is stored there. stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{}, @@ -379,7 +386,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { const index = reg.allocIndex() orelse return; const ShiftInt = math.Log2Int(FreeRegInt); const shift = @intCast(ShiftInt, index); - self.free_registers &= ~(@as(FreeRegInt, 1) << shift); + const mask = @as(FreeRegInt, 1) << shift; + self.free_registers &= ~mask; + self.allocated_registers |= mask; } fn markRegFree(self: *Self, reg: Register) void { @@ -397,7 +406,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (free_index >= callee_preserved_regs.len) { return null; } - self.free_registers &= ~(@as(FreeRegInt, 1) << free_index); + const mask = @as(FreeRegInt, 1) << free_index; + self.free_registers &= ~mask; + self.allocated_registers |= mask; const reg = callee_preserved_regs[free_index]; self.registers.putAssumeCapacityNoClobber(reg, inst); log.debug("alloc {} => {*}", .{ reg, inst }); @@ -423,7 +434,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { fn generateSymbol( bin_file: *link.File, - src: usize, + src_loc: Module.SrcLoc, typed_value: TypedValue, code: *std.ArrayList(u8), debug_output: DebugInfoOutput, @@ -432,7 +443,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { @panic("Attempted to compile for architecture that was disabled by build configuration"); } - const module_fn = typed_value.val.cast(Value.Payload.Function).?.func; + const module_fn = typed_value.val.castTag(.function).?.data; const fn_type = module_fn.owner_decl.typed_value.most_recent.typed_value.ty; @@ -445,19 +456,22 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { try branch_stack.append(.{}); const src_data: struct { lbrace_src: usize, rbrace_src: usize, source: []const u8 } = blk: { - if (module_fn.owner_decl.scope.cast(Module.Scope.Container)) |container_scope| { - const tree = container_scope.file_scope.contents.tree; - const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?; - const block = fn_proto.getBodyNode().?.castTag(.Block).?; - const lbrace_src = tree.token_locs[block.lbrace].start; - const rbrace_src = tree.token_locs[block.rbrace].start; - break :blk .{ .lbrace_src = lbrace_src, .rbrace_src = rbrace_src, .source = tree.source }; - } else if (module_fn.owner_decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| { - const byte_off = zir_module.contents.module.decls[module_fn.owner_decl.src_index].inst.src; - break :blk .{ .lbrace_src = byte_off, .rbrace_src = byte_off, .source = zir_module.source.bytes }; - } else { - unreachable; - } + const container_scope = module_fn.owner_decl.container; + const tree = container_scope.file_scope.tree; + const node_tags = tree.nodes.items(.tag); + const node_datas = tree.nodes.items(.data); + const token_starts = tree.tokens.items(.start); + + const fn_decl = tree.rootDecls()[module_fn.owner_decl.src_index]; + assert(node_tags[fn_decl] == .fn_decl); + const block = node_datas[fn_decl].rhs; + const lbrace_src = token_starts[tree.firstToken(block)]; + const rbrace_src = token_starts[tree.lastToken(block)]; + break :blk .{ + .lbrace_src = lbrace_src, + .rbrace_src = rbrace_src, + .source = tree.source, + }; }; var function = Self{ @@ -473,7 +487,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .fn_type = fn_type, .arg_index = 0, .branch_stack = &branch_stack, - .src = src, + .src_loc = src_loc, .stack_align = undefined, .prev_di_pc = 0, .prev_di_src = src_data.lbrace_src, @@ -484,7 +498,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { defer function.stack.deinit(bin_file.allocator); defer function.exitlude_jump_relocs.deinit(bin_file.allocator); - var call_info = function.resolveCallingConventionValues(src, fn_type) catch |err| switch (err) { + var call_info = function.resolveCallingConventionValues(src_loc.byte_offset, fn_type) catch |err| switch (err) { error.CodegenFail => return Result{ .fail = function.err_msg.? }, else => |e| return e, }; @@ -527,24 +541,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { self.code.items.len += 4; try self.dbgSetPrologueEnd(); - try self.genBody(self.mod_fn.analysis.success); + try self.genBody(self.mod_fn.body); const stack_end = self.max_end_stack; if (stack_end > math.maxInt(i32)) - return self.fail(self.src, "too much stack used in call parameters", .{}); + return self.failSymbol("too much stack used in call parameters", .{}); const aligned_stack_end = mem.alignForward(stack_end, self.stack_align); mem.writeIntLittle(u32, self.code.items[reloc_index..][0..4], @intCast(u32, aligned_stack_end)); if (self.code.items.len >= math.maxInt(i32)) { - return self.fail(self.src, "unable to perform relocation: jump too far", .{}); + return self.failSymbol("unable to perform relocation: jump too far", .{}); } - for (self.exitlude_jump_relocs.items) |jmp_reloc| { + if (self.exitlude_jump_relocs.items.len == 1) { + self.code.items.len -= 5; + } else for (self.exitlude_jump_relocs.items) |jmp_reloc| { const amt = self.code.items.len - (jmp_reloc + 4); - // If it wouldn't jump at all, elide it. - if (amt == 0) { - self.code.items.len -= 5; - continue; - } const s32_amt = @intCast(i32, amt); mem.writeIntLittle(i32, self.code.items[jmp_reloc..][0..4], s32_amt); } @@ -571,7 +582,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }); } else { try self.dbgSetPrologueEnd(); - try self.genBody(self.mod_fn.analysis.success); + try self.genBody(self.mod_fn.body); try self.dbgSetEpilogueBegin(); } }, @@ -581,22 +592,36 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // push {fp, lr} // mov fp, sp // sub sp, sp, #reloc - writeInt(u32, try self.code.addManyAsArray(4), Instruction.push(.al, .{ .fp, .lr }).toU32()); - writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32()); - const backpatch_reloc = self.code.items.len; - try self.code.resize(backpatch_reloc + 4); + const prologue_reloc = self.code.items.len; + try self.code.resize(prologue_reloc + 12); + writeInt(u32, self.code.items[prologue_reloc + 4 ..][0..4], Instruction.mov(.al, .fp, Instruction.Operand.reg(.sp, Instruction.Operand.Shift.none)).toU32()); try self.dbgSetPrologueEnd(); - try self.genBody(self.mod_fn.analysis.success); + try self.genBody(self.mod_fn.body); + + // Backpatch push callee saved regs + var saved_regs = Instruction.RegisterList{ + .r11 = true, // fp + .r14 = true, // lr + }; + inline for (callee_preserved_regs) |reg, i| { + const ShiftInt = math.Log2Int(FreeRegInt); + const shift = @intCast(ShiftInt, i); + const mask = @as(FreeRegInt, 1) << shift; + if (self.allocated_registers & mask != 0) { + @field(saved_regs, @tagName(reg)) = true; + } + } + writeInt(u32, self.code.items[prologue_reloc..][0..4], Instruction.stmdb(.al, .sp, true, saved_regs).toU32()); // Backpatch stack offset const stack_end = self.max_end_stack; const aligned_stack_end = mem.alignForward(stack_end, self.stack_align); if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) |op| { - writeInt(u32, self.code.items[backpatch_reloc..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32()); + writeInt(u32, self.code.items[prologue_reloc + 8 ..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32()); } else { - return self.fail(self.src, "TODO ARM: allow larger stacks", .{}); + return self.failSymbol("TODO ARM: allow larger stacks", .{}); } try self.dbgSetEpilogueBegin(); @@ -610,8 +635,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // jump self.code.items.len -= 4; } else for (self.exitlude_jump_relocs.items) |jmp_reloc| { - const amt = self.code.items.len - (jmp_reloc + 4); - if (amt == 0) { + const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8); + if (amt == -4) { // This return is at the end of the // code block. We can't just delete // the space because there may be @@ -622,24 +647,104 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (math.cast(i26, amt)) |offset| { writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32()); } else |err| { - return self.fail(self.src, "exitlude jump is too large", .{}); + return self.failSymbol("exitlude jump is too large", .{}); } } } + // Epilogue: pop callee saved registers (swap lr with pc in saved_regs) + saved_regs.r14 = false; // lr + saved_regs.r15 = true; // pc + // mov sp, fp // pop {fp, pc} writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .sp, Instruction.Operand.reg(.fp, Instruction.Operand.Shift.none)).toU32()); - writeInt(u32, try self.code.addManyAsArray(4), Instruction.pop(.al, .{ .fp, .pc }).toU32()); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldm(.al, .sp, true, saved_regs).toU32()); + } else { + try self.dbgSetPrologueEnd(); + try self.genBody(self.mod_fn.body); + try self.dbgSetEpilogueBegin(); + } + }, + .aarch64, .aarch64_be, .aarch64_32 => { + const cc = self.fn_type.fnCallingConvention(); + if (cc != .Naked) { + // TODO Finish function prologue and epilogue for aarch64. + + // stp fp, lr, [sp, #-16]! + // mov fp, sp + // sub sp, sp, #reloc + writeInt(u32, try self.code.addManyAsArray(4), Instruction.stp( + .x29, + .x30, + Register.sp, + Instruction.LoadStorePairOffset.pre_index(-16), + ).toU32()); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.x29, .xzr, 0, false).toU32()); + const backpatch_reloc = self.code.items.len; + try self.code.resize(backpatch_reloc + 4); + + try self.dbgSetPrologueEnd(); + + try self.genBody(self.mod_fn.body); + + // Backpatch stack offset + const stack_end = self.max_end_stack; + const aligned_stack_end = mem.alignForward(stack_end, self.stack_align); + if (math.cast(u12, aligned_stack_end)) |size| { + writeInt(u32, self.code.items[backpatch_reloc..][0..4], Instruction.sub(.xzr, .xzr, size, false).toU32()); + } else |_| { + return self.failSymbol("TODO AArch64: allow larger stacks", .{}); + } + + try self.dbgSetEpilogueBegin(); + + // exitlude jumps + if (self.exitlude_jump_relocs.items.len == 1) { + // There is only one relocation. Hence, + // this relocation must be at the end of + // the code. Therefore, we can just delete + // the space initially reserved for the + // jump + self.code.items.len -= 4; + } else for (self.exitlude_jump_relocs.items) |jmp_reloc| { + const amt = @intCast(i32, self.code.items.len) - @intCast(i32, jmp_reloc + 8); + if (amt == -4) { + // This return is at the end of the + // code block. We can't just delete + // the space because there may be + // other jumps we already relocated to + // the address. Instead, insert a nop + writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.nop().toU32()); + } else { + if (math.cast(i28, amt)) |offset| { + writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(offset).toU32()); + } else |err| { + return self.failSymbol("exitlude jump is too large", .{}); + } + } + } + + // ldp fp, lr, [sp], #16 + writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldp( + .x29, + .x30, + Register.sp, + Instruction.LoadStorePairOffset.post_index(16), + ).toU32()); + // add sp, sp, #stack_size + writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.xzr, .xzr, @intCast(u12, aligned_stack_end), false).toU32()); + // ret lr + writeInt(u32, try self.code.addManyAsArray(4), Instruction.ret(null).toU32()); } else { try self.dbgSetPrologueEnd(); - try self.genBody(self.mod_fn.analysis.success); + try self.genBody(self.mod_fn.body); try self.dbgSetEpilogueBegin(); } }, else => { try self.dbgSetPrologueEnd(); - try self.genBody(self.mod_fn.analysis.success); + try self.genBody(self.mod_fn.body); try self.dbgSetEpilogueBegin(); }, } @@ -764,12 +869,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .arg => return self.genArg(inst.castTag(.arg).?), .assembly => return self.genAsm(inst.castTag(.assembly).?), .bitcast => return self.genBitCast(inst.castTag(.bitcast).?), + .bit_and => return self.genBitAnd(inst.castTag(.bit_and).?), + .bit_or => return self.genBitOr(inst.castTag(.bit_or).?), .block => return self.genBlock(inst.castTag(.block).?), .br => return self.genBr(inst.castTag(.br).?), + .br_block_flat => return self.genBrBlockFlat(inst.castTag(.br_block_flat).?), .breakpoint => return self.genBreakpoint(inst.src), - .brvoid => return self.genBrVoid(inst.castTag(.brvoid).?), - .booland => return self.genBoolOp(inst.castTag(.booland).?), - .boolor => return self.genBoolOp(inst.castTag(.boolor).?), + .br_void => return self.genBrVoid(inst.castTag(.br_void).?), + .bool_and => return self.genBoolOp(inst.castTag(.bool_and).?), + .bool_or => return self.genBoolOp(inst.castTag(.bool_or).?), .call => return self.genCall(inst.castTag(.call).?), .cmp_lt => return self.genCmp(inst.castTag(.cmp_lt).?, .lt), .cmp_lte => return self.genCmp(inst.castTag(.cmp_lte).?, .lte), @@ -782,9 +890,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .dbg_stmt => return self.genDbgStmt(inst.castTag(.dbg_stmt).?), .floatcast => return self.genFloatCast(inst.castTag(.floatcast).?), .intcast => return self.genIntCast(inst.castTag(.intcast).?), - .isnonnull => return self.genIsNonNull(inst.castTag(.isnonnull).?), - .isnull => return self.genIsNull(inst.castTag(.isnull).?), - .iserr => return self.genIsErr(inst.castTag(.iserr).?), + .is_non_null => return self.genIsNonNull(inst.castTag(.is_non_null).?), + .is_non_null_ptr => return self.genIsNonNullPtr(inst.castTag(.is_non_null_ptr).?), + .is_null => return self.genIsNull(inst.castTag(.is_null).?), + .is_null_ptr => return self.genIsNullPtr(inst.castTag(.is_null_ptr).?), + .is_err => return self.genIsErr(inst.castTag(.is_err).?), + .is_err_ptr => return self.genIsErrPtr(inst.castTag(.is_err_ptr).?), .load => return self.genLoad(inst.castTag(.load).?), .loop => return self.genLoop(inst.castTag(.loop).?), .not => return self.genNot(inst.castTag(.not).?), @@ -796,9 +907,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .sub => return self.genSub(inst.castTag(.sub).?), .switchbr => return self.genSwitch(inst.castTag(.switchbr).?), .unreach => return MCValue{ .unreach = {} }, - .unwrap_optional => return self.genUnwrapOptional(inst.castTag(.unwrap_optional).?), + .optional_payload => return self.genOptionalPayload(inst.castTag(.optional_payload).?), + .optional_payload_ptr => return self.genOptionalPayloadPtr(inst.castTag(.optional_payload_ptr).?), + .unwrap_errunion_err => return self.genUnwrapErrErr(inst.castTag(.unwrap_errunion_err).?), + .unwrap_errunion_payload => return self.genUnwrapErrPayload(inst.castTag(.unwrap_errunion_payload).?), + .unwrap_errunion_err_ptr => return self.genUnwrapErrErrPtr(inst.castTag(.unwrap_errunion_err_ptr).?), + .unwrap_errunion_payload_ptr => return self.genUnwrapErrPayloadPtr(inst.castTag(.unwrap_errunion_payload_ptr).?), .wrap_optional => return self.genWrapOptional(inst.castTag(.wrap_optional).?), + .wrap_errunion_payload => return self.genWrapErrUnionPayload(inst.castTag(.wrap_errunion_payload).?), + .wrap_errunion_err => return self.genWrapErrUnionErr(inst.castTag(.wrap_errunion_err).?), .varptr => return self.genVarPtr(inst.castTag(.varptr).?), + .xor => return self.genXor(inst.castTag(.xor).?), } } @@ -855,7 +974,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { /// Copies a value to a register without tracking the register. The register is not considered /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. - fn copyToTmpRegister(self: *Self, src: usize, mcv: MCValue) !Register { + fn copyToTmpRegister(self: *Self, src: usize, ty: Type, mcv: MCValue) !Register { const reg = self.findUnusedReg() orelse b: { // We'll take over the first register. Move the instruction that was previously // there to a stack allocation. @@ -872,7 +991,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { break :b reg; }; - try self.genSetReg(src, reg, mcv); + try self.genSetReg(src, ty, reg, mcv); return reg; } @@ -899,7 +1018,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { break :b reg; }; - try self.genSetReg(reg_owner.src, reg, mcv); + try self.genSetReg(reg_owner.src, reg_owner.ty, reg, mcv); return MCValue{ .register = reg }; } @@ -1009,15 +1128,89 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - fn genUnwrapOptional(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + fn genBitAnd(self: *Self, inst: *ir.Inst.BinOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .bit_and), + else => return self.fail(inst.base.src, "TODO implement bitwise and for {}", .{self.target.cpu.arch}), + } + } + + fn genBitOr(self: *Self, inst: *ir.Inst.BinOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .bit_or), + else => return self.fail(inst.base.src, "TODO implement bitwise or for {}", .{self.target.cpu.arch}), + } + } + + fn genXor(self: *Self, inst: *ir.Inst.BinOp) !MCValue { // No side effects, so if it's unreferenced, do nothing. if (inst.base.isUnused()) return MCValue.dead; switch (arch) { - else => return self.fail(inst.base.src, "TODO implement unwrap optional for {}", .{self.target.cpu.arch}), + .arm, .armeb => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .xor), + else => return self.fail(inst.base.src, "TODO implement xor for {}", .{self.target.cpu.arch}), } } + fn genOptionalPayload(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement .optional_payload for {}", .{self.target.cpu.arch}), + } + } + + fn genOptionalPayloadPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}), + } + } + + fn genUnwrapErrErr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement unwrap error union error for {}", .{self.target.cpu.arch}), + } + } + + fn genUnwrapErrPayload(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement unwrap error union payload for {}", .{self.target.cpu.arch}), + } + } + // *(E!T) -> E + fn genUnwrapErrErrPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch}), + } + } + // *(E!T) -> *T + fn genUnwrapErrPayloadPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch}), + } + } fn genWrapOptional(self: *Self, inst: *ir.Inst.UnOp) !MCValue { const optional_ty = inst.base.ty; @@ -1034,6 +1227,27 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + /// T to E!T + fn genWrapErrUnionPayload(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement wrap errunion payload for {}", .{self.target.cpu.arch}), + } + } + + /// E to E!T + fn genWrapErrUnionErr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + + switch (arch) { + else => return self.fail(inst.base.src, "TODO implement wrap errunion error for {}", .{self.target.cpu.arch}), + } + } fn genVarPtr(self: *Self, inst: *ir.Inst.VarPtr) !MCValue { // No side effects, so if it's unreferenced, do nothing. if (inst.base.isUnused()) @@ -1172,38 +1386,31 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { const rhs = try self.resolveInst(op_rhs); // Destination must be a register - // Source may be register, memory or an immediate - // - // So there are two options: (lhs is src and rhs is dest) - // or (rhs is src and lhs is dest) - const lhs_is_dest = blk: { - if (self.reuseOperand(inst, 0, lhs)) { - break :blk true; - } else if (self.reuseOperand(inst, 1, rhs)) { - break :blk false; - } else { - break :blk lhs == .register; - } - }; - var dst_mcv: MCValue = undefined; - var src_mcv: MCValue = undefined; - var src_inst: *ir.Inst = undefined; - if (lhs_is_dest) { + var lhs_mcv: MCValue = undefined; + var rhs_mcv: MCValue = undefined; + if (self.reuseOperand(inst, 0, lhs)) { // LHS is the destination // RHS is the source - src_inst = op_rhs; - src_mcv = rhs; - dst_mcv = if (lhs != .register) try self.copyToNewRegister(inst, lhs) else lhs; - } else { + lhs_mcv = if (lhs != .register) try self.copyToNewRegister(inst, lhs) else lhs; + rhs_mcv = rhs; + dst_mcv = lhs_mcv; + } else if (self.reuseOperand(inst, 1, rhs)) { // RHS is the destination // LHS is the source - src_inst = op_lhs; - src_mcv = lhs; - dst_mcv = if (rhs != .register) try self.copyToNewRegister(inst, rhs) else rhs; + lhs_mcv = lhs; + rhs_mcv = if (rhs != .register) try self.copyToNewRegister(inst, rhs) else rhs; + dst_mcv = rhs_mcv; + } else { + // TODO save 1 copy instruction by directly allocating the destination register + // LHS is the destination + // RHS is the source + lhs_mcv = try self.copyToNewRegister(inst, lhs); + rhs_mcv = rhs; + dst_mcv = lhs_mcv; } - try self.genArmBinOpCode(inst.src, dst_mcv.register, src_mcv, lhs_is_dest, op); + try self.genArmBinOpCode(inst.src, dst_mcv.register, lhs_mcv, rhs_mcv, op); return dst_mcv; } @@ -1211,11 +1418,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { self: *Self, src: usize, dst_reg: Register, - src_mcv: MCValue, - lhs_is_dest: bool, + lhs_mcv: MCValue, + rhs_mcv: MCValue, op: ir.Inst.Tag, ) !void { - const operand = switch (src_mcv) { + assert(lhs_mcv == .register or lhs_mcv == .register); + + const swap_lhs_and_rhs = rhs_mcv == .register and lhs_mcv != .register; + const op1 = if (swap_lhs_and_rhs) rhs_mcv.register else lhs_mcv.register; + const op2 = if (swap_lhs_and_rhs) lhs_mcv else rhs_mcv; + + const operand = switch (op2) { .none => unreachable, .undef => unreachable, .dead, .unreach => unreachable, @@ -1229,36 +1442,37 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // Load immediate into register if it doesn't fit // as an operand break :blk Instruction.Operand.fromU32(@intCast(u32, imm)) orelse - Instruction.Operand.reg(try self.copyToTmpRegister(src, src_mcv), Instruction.Operand.Shift.none); + Instruction.Operand.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), op2), Instruction.Operand.Shift.none); }, - .register => |src_reg| Instruction.Operand.reg(src_reg, Instruction.Operand.Shift.none), + .register => |reg| Instruction.Operand.reg(reg, Instruction.Operand.Shift.none), .stack_offset, .embedded_in_code, .memory, - => Instruction.Operand.reg(try self.copyToTmpRegister(src, src_mcv), Instruction.Operand.Shift.none), + => Instruction.Operand.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), op2), Instruction.Operand.Shift.none), }; switch (op) { .add => { - // TODO runtime safety checks (overflow) - writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, dst_reg, operand).toU32()); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.add(.al, dst_reg, op1, operand).toU32()); }, .sub => { - // TODO runtime safety checks (underflow) - if (lhs_is_dest) { - writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, dst_reg, operand).toU32()); + if (swap_lhs_and_rhs) { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, op1, operand).toU32()); } else { - writeInt(u32, try self.code.addManyAsArray(4), Instruction.rsb(.al, dst_reg, dst_reg, operand).toU32()); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.sub(.al, dst_reg, op1, operand).toU32()); } }, - .booland => { - writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, dst_reg, operand).toU32()); + .bool_and, .bit_and => { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.@"and"(.al, dst_reg, op1, operand).toU32()); + }, + .bool_or, .bit_or => { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, op1, operand).toU32()); }, - .boolor => { - writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr(.al, dst_reg, dst_reg, operand).toU32()); + .not, .xor => { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, op1, operand).toU32()); }, - .not => { - writeInt(u32, try self.code.addManyAsArray(4), Instruction.eor(.al, dst_reg, dst_reg, operand).toU32()); + .cmp_eq => { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, op1, operand).toU32()); }, else => unreachable, // not a binary instruction } @@ -1320,7 +1534,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { switch (src_mcv) { .immediate => |imm| { if (imm > math.maxInt(u31)) { - src_mcv = MCValue{ .register = try self.copyToTmpRegister(src_inst.src, src_mcv) }; + src_mcv = MCValue{ .register = try self.copyToTmpRegister(src_inst.src, Type.initTag(.u64), src_mcv) }; } }, else => {}, @@ -1351,7 +1565,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .register => |dst_reg| { switch (src_mcv) { .none => unreachable, - .undef => try self.genSetReg(src, dst_reg, .undef), + .undef => try self.genSetReg(src, dst_ty, dst_reg, .undef), .dead, .unreach => unreachable, .ptr_stack_offset => unreachable, .ptr_embedded_in_code => unreachable, @@ -1443,32 +1657,49 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - fn genArg(self: *Self, inst: *ir.Inst.Arg) !MCValue { - if (FreeRegInt == u0) { - return self.fail(inst.base.src, "TODO implement Register enum for {}", .{self.target.cpu.arch}); - } - if (inst.base.isUnused()) - return MCValue.dead; - - try self.registers.ensureCapacity(self.gpa, self.registers.count() + 1); - - const result = self.args[self.arg_index]; - self.arg_index += 1; - + fn genArgDbgInfo(self: *Self, inst: *ir.Inst.Arg, mcv: MCValue) !void { const name_with_null = inst.name[0 .. mem.lenZ(inst.name) + 1]; - switch (result) { + + switch (mcv) { .register => |reg| { - self.registers.putAssumeCapacityNoClobber(toCanonicalReg(reg), &inst.base); - self.markRegUsed(reg); + // Copy arg to stack for better debugging + const ty = inst.base.ty; + const abi_size = math.cast(u32, ty.abiSize(self.target.*)) catch { + return self.fail(inst.base.src, "type '{}' too big to fit into stack frame", .{ty}); + }; + const abi_align = ty.abiAlignment(self.target.*); + const stack_offset = try self.allocMem(&inst.base, abi_size, abi_align); + try self.genSetStack(inst.base.src, ty, stack_offset, MCValue{ .register = reg }); + const adjusted_stack_offset = math.negateCast(stack_offset + abi_size) catch { + return self.fail(inst.base.src, "Stack offset too large for arguments", .{}); + }; switch (self.debug_output) { .dwarf => |dbg_out| { - try dbg_out.dbg_info.ensureCapacity(dbg_out.dbg_info.items.len + 8 + name_with_null.len); - dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter); - dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT_location, DW.FORM_exprloc - 1, // ULEB128 dwarf expression length - reg.dwarfLocOp(), - }); + switch (arch) { + .arm, .armeb => { + try dbg_out.dbg_info.append(link.File.Elf.abbrev_parameter); + + // Get length of the LEB128 stack offset + var counting_writer = std.io.countingWriter(std.io.null_writer); + leb128.writeILEB128(counting_writer.writer(), adjusted_stack_offset) catch unreachable; + + // DW.AT_location, DW.FORM_exprloc + // ULEB128 dwarf expression length + try leb128.writeULEB128(dbg_out.dbg_info.writer(), counting_writer.bytes_written + 1); + try dbg_out.dbg_info.append(DW.OP_breg11); + try leb128.writeILEB128(dbg_out.dbg_info.writer(), adjusted_stack_offset); + }, + else => { + try dbg_out.dbg_info.ensureCapacity(dbg_out.dbg_info.items.len + 3); + dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter); + dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT_location, DW.FORM_exprloc + 1, // ULEB128 dwarf expression length + reg.dwarfLocOp(), + }); + }, + } + try dbg_out.dbg_info.ensureCapacity(dbg_out.dbg_info.items.len + 5 + name_with_null.len); try self.addDbgInfoTypeReloc(inst.base.ty); // DW.AT_type, DW.FORM_ref4 dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT_name, DW.FORM_string }, @@ -1477,6 +1708,29 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }, else => {}, } + } + + fn genArg(self: *Self, inst: *ir.Inst.Arg) !MCValue { + const arg_index = self.arg_index; + self.arg_index += 1; + + if (FreeRegInt == u0) { + return self.fail(inst.base.src, "TODO implement Register enum for {}", .{self.target.cpu.arch}); + } + + const result = self.args[arg_index]; + try self.genArgDbgInfo(inst, result); + + if (inst.base.isUnused()) + return MCValue.dead; + + switch (result) { + .register => |reg| { + try self.registers.putNoClobber(self.gpa, toCanonicalReg(reg), &inst.base); + self.markRegUsed(reg); + }, + else => {}, + } return result; } @@ -1521,7 +1775,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { switch (mc_arg) { .none => continue, .register => |reg| { - try self.genSetReg(arg.src, reg, arg_mcv); + try self.genSetReg(arg.src, arg.ty, reg, arg_mcv); // TODO interact with the register allocator to mark the instruction as moved. }, .stack_offset => { @@ -1546,9 +1800,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (inst.func.value()) |func_value| { + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); @@ -1564,6 +1818,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { try self.code.ensureCapacity(self.code.items.len + 7); self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 }); mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), got_addr); + } else if (func_value.castTag(.extern_fn)) |_| { + return self.fail(inst.base.src, "TODO implement calling extern functions", .{}); } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1574,9 +1830,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .riscv64 => { if (info.args.len > 0) return self.fail(inst.base.src, "TODO implement fn args for {}", .{self.target.cpu.arch}); - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (inst.func.value()) |func_value| { + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); @@ -1588,8 +1844,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { else unreachable; - try self.genSetReg(inst.base.src, .ra, .{ .memory = got_addr }); + try self.genSetReg(inst.base.src, Type.initTag(.usize), .ra, .{ .memory = got_addr }); mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.jalr(.ra, 0, .ra).toU32()); + } else if (func_value.castTag(.extern_fn)) |_| { + return self.fail(inst.base.src, "TODO implement calling extern functions", .{}); } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1598,12 +1856,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } }, .spu_2 => { - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { + if (inst.func.value()) |func_value| { if (info.args.len != 0) { return self.fail(inst.base.src, "TODO implement call with more than 0 parameters", .{}); } - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: { const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?]; break :blk @intCast(u16, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * 2); @@ -1634,6 +1892,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { else => return self.fail(inst.base.src, "TODO implement fn call with non-void return value", .{}), } } + } else if (func_value.castTag(.extern_fn)) |_| { + return self.fail(inst.base.src, "TODO implement calling extern functions", .{}); } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1657,7 +1917,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .compare_flags_signed => unreachable, .compare_flags_unsigned => unreachable, .register => |reg| { - try self.genSetReg(arg.src, reg, arg_mcv); + try self.genSetReg(arg.src, arg.ty, reg, arg_mcv); // TODO interact with the register allocator to mark the instruction as moved. }, .stack_offset => { @@ -1672,9 +1932,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (inst.func.value()) |func_value| { + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: { @@ -1685,7 +1945,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { else unreachable; - try self.genSetReg(inst.base.src, .lr, .{ .memory = got_addr }); + try self.genSetReg(inst.base.src, Type.initTag(.usize), .lr, .{ .memory = got_addr }); // TODO: add Instruction.supportedOn // function for ARM @@ -1695,6 +1955,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, .lr, Instruction.Operand.reg(.pc, Instruction.Operand.Shift.none)).toU32()); writeInt(u32, try self.code.addManyAsArray(4), Instruction.bx(.al, .lr).toU32()); } + } else if (func_value.castTag(.extern_fn)) |_| { + return self.fail(inst.base.src, "TODO implement calling extern functions", .{}); } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1718,7 +1980,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .compare_flags_signed => unreachable, .compare_flags_unsigned => unreachable, .register => |reg| { - try self.genSetReg(arg.src, reg, arg_mcv); + try self.genSetReg(arg.src, arg.ty, reg, arg_mcv); // TODO interact with the register allocator to mark the instruction as moved. }, .stack_offset => { @@ -1733,9 +1995,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (inst.func.value()) |func_value| { + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: { @@ -1746,9 +2008,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { else unreachable; - try self.genSetReg(inst.base.src, .x30, .{ .memory = got_addr }); + try self.genSetReg(inst.base.src, Type.initTag(.usize), .x30, .{ .memory = got_addr }); writeInt(u32, try self.code.addManyAsArray(4), Instruction.blr(.x30).toU32()); + } else if (func_value.castTag(.extern_fn)) |_| { + return self.fail(inst.base.src, "TODO implement calling extern functions", .{}); } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1767,7 +2031,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { switch (mc_arg) { .none => continue, .register => |reg| { - try self.genSetReg(arg.src, reg, arg_mcv); + try self.genSetReg(arg.src, arg.ty, reg, arg_mcv); // TODO interact with the register allocator to mark the instruction as moved. }, .stack_offset => { @@ -1792,25 +2056,58 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - if (inst.func.cast(ir.Inst.Constant)) |func_inst| { - if (func_inst.val.cast(Value.Payload.Function)) |func_val| { - const func = func_val.func; + if (inst.func.value()) |func_value| { + if (func_value.castTag(.function)) |func_payload| { + const func = func_payload.data; const text_segment = &macho_file.load_commands.items[macho_file.text_segment_cmd_index.?].Segment; const got = &text_segment.sections.items[macho_file.got_section_index.?]; const got_addr = got.addr + func.owner_decl.link.macho.offset_table_index * @sizeOf(u64); switch (arch) { .x86_64 => { - try self.genSetReg(inst.base.src, .rax, .{ .memory = got_addr }); + try self.genSetReg(inst.base.src, Type.initTag(.u32), .rax, .{ .memory = got_addr }); // callq *%rax self.code.appendSliceAssumeCapacity(&[2]u8{ 0xff, 0xd0 }); }, .aarch64 => { - try self.genSetReg(inst.base.src, .x30, .{ .memory = got_addr }); + try self.genSetReg(inst.base.src, Type.initTag(.u32), .x30, .{ .memory = got_addr }); // blr x30 writeInt(u32, try self.code.addManyAsArray(4), Instruction.blr(.x30).toU32()); }, else => unreachable, // unsupported architecture on MachO } + } else if (func_value.castTag(.extern_fn)) |func_payload| { + const decl = func_payload.data; + const decl_name = try std.fmt.allocPrint(self.bin_file.allocator, "_{s}", .{decl.name}); + defer self.bin_file.allocator.free(decl_name); + const already_defined = macho_file.extern_lazy_symbols.contains(decl_name); + const symbol: u32 = if (macho_file.extern_lazy_symbols.getIndex(decl_name)) |index| + @intCast(u32, index) + else + try macho_file.addExternSymbol(decl_name); + const start = self.code.items.len; + const len: usize = blk: { + switch (arch) { + .x86_64 => { + // callq + try self.code.ensureCapacity(self.code.items.len + 5); + self.code.appendSliceAssumeCapacity(&[5]u8{ 0xe8, 0x0, 0x0, 0x0, 0x0 }); + break :blk 5; + }, + .aarch64 => { + // bl + writeInt(u32, try self.code.addManyAsArray(4), 0); + break :blk 4; + }, + else => unreachable, // unsupported architecture on MachO + } + }; + try macho_file.stub_fixups.append(self.bin_file.allocator, .{ + .symbol = symbol, + .already_defined = already_defined, + .start = start, + .len = len, + }); + // We mark the space and fix it up later. } else { return self.fail(inst.base.src, "TODO implement calling bitcasted functions", .{}); } @@ -1875,8 +2172,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4); }, .aarch64 => { - // TODO: relocations - writeInt(u32, try self.code.addManyAsArray(4), Instruction.ret(null).toU32()); + // Just add space for an instruction, patch this later + try self.code.resize(self.code.items.len + 4); + try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4); }, else => return self.fail(src, "TODO implement return for {}", .{self.target.cpu.arch}), } @@ -1920,6 +2218,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .unsigned => MCValue{ .compare_flags_unsigned = op }, }; }, + .arm, .armeb => { + const lhs = try self.resolveInst(inst.lhs); + const rhs = try self.resolveInst(inst.rhs); + + const src_mcv = rhs; + const dst_mcv = if (lhs != .register) try self.copyToNewRegister(&inst.base, lhs) else lhs; + + try self.genArmBinOpCode(inst.base.src, dst_mcv.register, dst_mcv, src_mcv, .cmp_eq); + const info = inst.lhs.ty.intInfo(self.target.*); + return switch (info.signedness) { + .signed => MCValue{ .compare_flags_signed = op }, + .unsigned => MCValue{ .compare_flags_unsigned = op }, + }; + }, else => return self.fail(inst.base.src, "TODO implement cmp for {}", .{self.target.cpu.arch}), } } @@ -1976,13 +2288,44 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }); break :blk 0x84; }, - else => return self.fail(inst.base.src, "TODO implement condbr {} when condition is {}", .{ self.target.cpu.arch, @tagName(cond) }), + else => return self.fail(inst.base.src, "TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }), }; self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode }); const reloc = Reloc{ .rel32 = self.code.items.len }; self.code.items.len += 4; break :reloc reloc; }, + .arm, .armeb => reloc: { + const condition: Condition = switch (cond) { + .compare_flags_signed => |cmp_op| blk: { + // Here we map to the opposite condition because the jump is to the false branch. + const condition = Condition.fromCompareOperatorSigned(cmp_op); + break :blk condition.negate(); + }, + .compare_flags_unsigned => |cmp_op| blk: { + // Here we map to the opposite condition because the jump is to the false branch. + const condition = Condition.fromCompareOperatorUnsigned(cmp_op); + break :blk condition.negate(); + }, + .register => |reg| blk: { + // cmp reg, 1 + // bne ... + const op = Instruction.Operand.imm(1, 0); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.cmp(.al, reg, op).toU32()); + break :blk .ne; + }, + else => return self.fail(inst.base.src, "TODO implement condbr {} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }), + }; + + const reloc = Reloc{ + .arm_branch = .{ + .pos = self.code.items.len, + .cond = condition, + }, + }; + try self.code.resize(self.code.items.len + 4); + break :reloc reloc; + }, else => return self.fail(inst.base.src, "TODO implement condbr {}", .{self.target.cpu.arch}), }; @@ -2042,7 +2385,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { try parent_branch.inst_table.ensureCapacity(self.gpa, parent_branch.inst_table.items().len + else_branch.inst_table.items().len); for (else_branch.inst_table.items()) |else_entry| { - const canon_mcv = if (saved_then_branch.inst_table.remove(else_entry.key)) |then_entry| blk: { + const canon_mcv = if (saved_then_branch.inst_table.swapRemove(else_entry.key)) |then_entry| blk: { // The instruction's MCValue is overridden in both branches. parent_branch.inst_table.putAssumeCapacity(else_entry.key, then_entry.value); if (else_entry.value == .dead) { @@ -2105,6 +2448,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn genIsNullPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + return self.fail(inst.base.src, "TODO load the operand and call genIsNull", .{}); + } + fn genIsNonNull(self: *Self, inst: *ir.Inst.UnOp) !MCValue { // Here you can specialize this instruction if it makes sense to, otherwise the default // will call genIsNull and invert the result. @@ -2113,12 +2460,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn genIsNonNullPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + return self.fail(inst.base.src, "TODO load the operand and call genIsNonNull", .{}); + } + fn genIsErr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { switch (arch) { else => return self.fail(inst.base.src, "TODO implement iserr for {}", .{self.target.cpu.arch}), } } + fn genIsErrPtr(self: *Self, inst: *ir.Inst.UnOp) !MCValue { + return self.fail(inst.base.src, "TODO load the operand and call genIsErr", .{}); + } + fn genLoop(self: *Self, inst: *ir.Inst.Loop) !MCValue { // A loop is a setup to be able to jump back to the beginning. const start_index = self.code.items.len; @@ -2142,12 +2497,19 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } }, .arm, .armeb => { - if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len))) |delta| { + if (math.cast(i26, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) |delta| { writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(.al, delta).toU32()); } else |err| { return self.fail(src, "TODO: enable larger branch offset", .{}); } }, + .aarch64, .aarch64_be, .aarch64_32 => { + if (math.cast(i28, @intCast(i32, index) - @intCast(i32, self.code.items.len + 8))) |delta| { + writeInt(u32, try self.code.addManyAsArray(4), Instruction.b(delta).toU32()); + } else |err| { + return self.fail(src, "TODO: enable larger branch offset", .{}); + } + }, else => return self.fail(src, "TODO implement jump for {}", .{self.target.cpu.arch}), } } @@ -2192,20 +2554,30 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { return self.fail(src, "unable to perform relocation: jump too far", .{}); mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt); }, + .arm_branch => |info| { + switch (arch) { + .arm, .armeb => { + const amt = @intCast(i32, self.code.items.len) - @intCast(i32, info.pos + 8); + if (math.cast(i26, amt)) |delta| { + writeInt(u32, self.code.items[info.pos..][0..4], Instruction.b(info.cond, delta).toU32()); + } else |_| { + return self.fail(src, "TODO: enable larger branch offset", .{}); + } + }, + else => unreachable, // attempting to perfrom an ARM relocation on a non-ARM target arch + } + }, } } + fn genBrBlockFlat(self: *Self, inst: *ir.Inst.BrBlockFlat) !MCValue { + try self.genBody(inst.body); + const last = inst.body.instructions[inst.body.instructions.len - 1]; + return self.br(inst.base.src, inst.block, last); + } + fn genBr(self: *Self, inst: *ir.Inst.Br) !MCValue { - if (inst.operand.ty.hasCodeGenBits()) { - const operand = try self.resolveInst(inst.operand); - const block_mcv = @bitCast(MCValue, inst.block.codegen.mcv); - if (block_mcv == .none) { - inst.block.codegen.mcv = @bitCast(AnyMCValue, operand); - } else { - try self.setRegOrMem(inst.base.src, inst.block.base.ty, block_mcv, operand); - } - } - return self.brVoid(inst.base.src, inst.block); + return self.br(inst.base.src, inst.block, inst.operand); } fn genBrVoid(self: *Self, inst: *ir.Inst.BrVoid) !MCValue { @@ -2218,20 +2590,33 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { switch (arch) { .x86_64 => switch (inst.base.tag) { // lhs AND rhs - .booland => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 4, 0x20), + .bool_and => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 4, 0x20), // lhs OR rhs - .boolor => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 1, 0x08), + .bool_or => return try self.genX8664BinMath(&inst.base, inst.lhs, inst.rhs, 1, 0x08), else => unreachable, // Not a boolean operation }, .arm, .armeb => switch (inst.base.tag) { - .booland => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .booland), - .boolor => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .boolor), + .bool_and => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .bool_and), + .bool_or => return try self.genArmBinOp(&inst.base, inst.lhs, inst.rhs, .bool_or), else => unreachable, // Not a boolean operation }, else => return self.fail(inst.base.src, "TODO implement boolean operations for {}", .{self.target.cpu.arch}), } } + fn br(self: *Self, src: usize, block: *ir.Inst.Block, operand: *ir.Inst) !MCValue { + if (operand.ty.hasCodeGenBits()) { + const operand_mcv = try self.resolveInst(operand); + const block_mcv = @bitCast(MCValue, block.codegen.mcv); + if (block_mcv == .none) { + block.codegen.mcv = @bitCast(AnyMCValue, operand_mcv); + } else { + try self.setRegOrMem(src, block.base.ty, block_mcv, operand_mcv); + } + } + return self.brVoid(src, block); + } + fn brVoid(self: *Self, src: usize, block: *ir.Inst.Block) !MCValue { // Emit a jump with a relocation. It will be patched up after the block ends. try block.codegen.relocs.ensureCapacity(self.gpa, block.codegen.relocs.items.len + 1); @@ -2245,6 +2630,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // Leave the jump offset undefined block.codegen.relocs.appendAssumeCapacity(.{ .rel32 = self.code.items.len - 4 }); }, + .arm, .armeb => { + try self.code.resize(self.code.items.len + 4); + block.codegen.relocs.appendAssumeCapacity(.{ + .arm_branch = .{ + .pos = self.code.items.len - 4, + .cond = .al, + }, + }); + }, else => return self.fail(src, "TODO implement brvoid for {}", .{self.target.cpu.arch}), } return .none; @@ -2270,13 +2664,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .arm, .armeb => { for (inst.inputs) |input, i| { if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input}); + return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input}); } const reg_name = input[1 .. input.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); const arg = try self.resolveInst(inst.args[i]); - try self.genSetReg(inst.base.src, reg, arg); + try self.genSetReg(inst.base.src, inst.args[i].ty, reg, arg); } if (mem.eql(u8, inst.asm_source, "svc #0")) { @@ -2287,11 +2681,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (inst.output) |output| { if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output}); + return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output}); } const reg_name = output[2 .. output.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); return MCValue{ .register = reg }; } else { return MCValue.none; @@ -2300,39 +2694,30 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .aarch64 => { for (inst.inputs) |input, i| { if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input}); + return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input}); } const reg_name = input[1 .. input.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); const arg = try self.resolveInst(inst.args[i]); - try self.genSetReg(inst.base.src, reg, arg); + try self.genSetReg(inst.base.src, inst.args[i].ty, reg, arg); } - // TODO move this to lib/std/{elf, macho}.zig, etc. - const is_syscall_inst = switch (self.bin_file.tag) { - .macho => mem.eql(u8, inst.asm_source, "svc #0x80"), - .elf => mem.eql(u8, inst.asm_source, "svc #0"), - else => |tag| return self.fail(inst.base.src, "TODO implement aarch64 support for other syscall instructions for file format: '{}'", .{tag}), - }; - if (is_syscall_inst) { - const imm16: u16 = switch (self.bin_file.tag) { - .macho => 0x80, - .elf => 0, - else => unreachable, - }; - mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.svc(imm16).toU32()); + if (mem.eql(u8, inst.asm_source, "svc #0")) { + mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.svc(0x0).toU32()); + } else if (mem.eql(u8, inst.asm_source, "svc #0x80")) { + mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.svc(0x80).toU32()); } else { return self.fail(inst.base.src, "TODO implement support for more aarch64 assembly instructions", .{}); } if (inst.output) |output| { if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output}); + return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output}); } const reg_name = output[2 .. output.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); return MCValue{ .register = reg }; } else { return MCValue.none; @@ -2341,13 +2726,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .riscv64 => { for (inst.inputs) |input, i| { if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input}); + return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input}); } const reg_name = input[1 .. input.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); const arg = try self.resolveInst(inst.args[i]); - try self.genSetReg(inst.base.src, reg, arg); + try self.genSetReg(inst.base.src, inst.args[i].ty, reg, arg); } if (mem.eql(u8, inst.asm_source, "ecall")) { @@ -2358,11 +2743,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (inst.output) |output| { if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output}); + return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output}); } const reg_name = output[2 .. output.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); return MCValue{ .register = reg }; } else { return MCValue.none; @@ -2371,13 +2756,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .x86_64, .i386 => { for (inst.inputs) |input, i| { if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input}); + return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input}); } const reg_name = input[1 .. input.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); const arg = try self.resolveInst(inst.args[i]); - try self.genSetReg(inst.base.src, reg, arg); + try self.genSetReg(inst.base.src, inst.args[i].ty, reg, arg); } if (mem.eql(u8, inst.asm_source, "syscall")) { @@ -2388,11 +2773,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (inst.output) |output| { if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') { - return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output}); + return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output}); } const reg_name = output[2 .. output.len - 1]; const reg = parseRegName(reg_name) orelse - return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name}); + return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name}); return MCValue{ .register = reg }; } else { return MCValue.none; @@ -2439,7 +2824,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { fn setRegOrMem(self: *Self, src: usize, ty: Type, loc: MCValue, val: MCValue) !void { switch (loc) { .none => return, - .register => |reg| return self.genSetReg(src, reg, val), + .register => |reg| return self.genSetReg(src, ty, reg, val), .stack_offset => |off| return self.genSetStack(src, ty, off, val), .memory => { return self.fail(src, "TODO implement setRegOrMem for memory", .{}); @@ -2474,30 +2859,43 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { return self.fail(src, "TODO implement set stack variable with compare flags value (signed)", .{}); }, .immediate => { - const reg = try self.copyToTmpRegister(src, mcv); + const reg = try self.copyToTmpRegister(src, ty, mcv); return self.genSetStack(src, ty, stack_offset, MCValue{ .register = reg }); }, .embedded_in_code => |code_offset| { return self.fail(src, "TODO implement set stack variable from embedded_in_code", .{}); }, .register => |reg| { - // TODO: strh - const offset = if (stack_offset <= math.maxInt(u12)) blk: { - break :blk Instruction.Offset.imm(@intCast(u12, stack_offset)); - } else Instruction.Offset.reg(try self.copyToTmpRegister(src, MCValue{ .immediate = stack_offset }), 0); - const abi_size = ty.abiSize(self.target.*); + const adj_off = stack_offset + abi_size; + switch (abi_size) { - 1 => writeInt(u32, try self.code.addManyAsArray(4), Instruction.strb(.al, reg, .fp, .{ - .offset = offset, - .positive = false, - }).toU32()), - 2 => return self.fail(src, "TODO implement strh", .{}), - 4 => writeInt(u32, try self.code.addManyAsArray(4), Instruction.str(.al, reg, .fp, .{ - .offset = offset, - .positive = false, - }).toU32()), - else => return self.fail(src, "TODO a type of size {} is not allowed in a register", .{abi_size}), + 1, 4 => { + const offset = if (math.cast(u12, adj_off)) |imm| blk: { + break :blk Instruction.Offset.imm(imm); + } else |_| Instruction.Offset.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0); + const str = switch (abi_size) { + 1 => Instruction.strb, + 4 => Instruction.str, + else => unreachable, + }; + + writeInt(u32, try self.code.addManyAsArray(4), str(.al, reg, .fp, .{ + .offset = offset, + .positive = false, + }).toU32()); + }, + 2 => { + const offset = if (adj_off <= math.maxInt(u8)) blk: { + break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off)); + } else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), MCValue{ .immediate = adj_off })); + + writeInt(u32, try self.code.addManyAsArray(4), Instruction.strh(.al, reg, .fp, .{ + .offset = offset, + .positive = false, + }).toU32()); + }, + else => return self.fail(src, "TODO implement storing other types abi_size={}", .{abi_size}), } }, .memory => |vaddr| { @@ -2507,7 +2905,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (stack_offset == off) return; // Copy stack variable to itself; nothing to do. - const reg = try self.copyToTmpRegister(src, mcv); + const reg = try self.copyToTmpRegister(src, ty, mcv); return self.genSetStack(src, ty, stack_offset, MCValue{ .register = reg }); }, }, @@ -2596,7 +2994,70 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (stack_offset == off) return; // Copy stack variable to itself; nothing to do. - const reg = try self.copyToTmpRegister(src, mcv); + const reg = try self.copyToTmpRegister(src, ty, mcv); + return self.genSetStack(src, ty, stack_offset, MCValue{ .register = reg }); + }, + }, + .aarch64, .aarch64_be, .aarch64_32 => switch (mcv) { + .dead => unreachable, + .ptr_stack_offset => unreachable, + .ptr_embedded_in_code => unreachable, + .unreach, .none => return, // Nothing to do. + .undef => { + if (!self.wantSafety()) + return; // The already existing value will do just fine. + // TODO Upgrade this to a memset call when we have that available. + switch (ty.abiSize(self.target.*)) { + 1 => return self.genSetStack(src, ty, stack_offset, .{ .immediate = 0xaa }), + 2 => return self.genSetStack(src, ty, stack_offset, .{ .immediate = 0xaaaa }), + 4 => return self.genSetStack(src, ty, stack_offset, .{ .immediate = 0xaaaaaaaa }), + 8 => return self.genSetStack(src, ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), + else => return self.fail(src, "TODO implement memset", .{}), + } + }, + .compare_flags_unsigned => |op| { + return self.fail(src, "TODO implement set stack variable with compare flags value (unsigned)", .{}); + }, + .compare_flags_signed => |op| { + return self.fail(src, "TODO implement set stack variable with compare flags value (signed)", .{}); + }, + .immediate => { + const reg = try self.copyToTmpRegister(src, ty, mcv); + return self.genSetStack(src, ty, stack_offset, MCValue{ .register = reg }); + }, + .embedded_in_code => |code_offset| { + return self.fail(src, "TODO implement set stack variable from embedded_in_code", .{}); + }, + .register => |reg| { + const abi_size = ty.abiSize(self.target.*); + const adj_off = stack_offset + abi_size; + + switch (abi_size) { + 4, 8 => { + const offset = if (math.cast(i9, adj_off)) |imm| + Instruction.LoadStoreOffset.imm_post_index(-imm) + else |_| Instruction.LoadStoreOffset.reg(try self.copyToTmpRegister(src, Type.initTag(.u64), MCValue{ .immediate = adj_off })); + const rn: Register = switch (arch) { + .aarch64, .aarch64_be => .x29, + .aarch64_32 => .w29, + else => unreachable, + }; + + writeInt(u32, try self.code.addManyAsArray(4), Instruction.str(reg, rn, .{ + .offset = offset, + }).toU32()); + }, + else => return self.fail(src, "TODO implement storing other types abi_size={}", .{abi_size}), + } + }, + .memory => |vaddr| { + return self.fail(src, "TODO implement set stack variable from memory vaddr", .{}); + }, + .stack_offset => |off| { + if (stack_offset == off) + return; // Copy stack variable to itself; nothing to do. + + const reg = try self.copyToTmpRegister(src, ty, mcv); return self.genSetStack(src, ty, stack_offset, MCValue{ .register = reg }); }, }, @@ -2604,7 +3065,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } - fn genSetReg(self: *Self, src: usize, reg: Register, mcv: MCValue) InnerError!void { + fn genSetReg(self: *Self, src: usize, ty: Type, reg: Register, mcv: MCValue) InnerError!void { switch (arch) { .arm, .armeb => switch (mcv) { .dead => unreachable, @@ -2615,7 +3076,23 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (!self.wantSafety()) return; // The already existing value will do just fine. // Write the debug undefined value. - return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaa }); + return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaa }); + }, + .compare_flags_unsigned, + .compare_flags_signed, + => |op| { + const condition = switch (mcv) { + .compare_flags_unsigned => Condition.fromCompareOperatorUnsigned(op), + .compare_flags_signed => Condition.fromCompareOperatorSigned(op), + else => unreachable, + }; + + // mov reg, 0 + // moveq reg, 1 + const zero = Instruction.Operand.imm(0, 0); + const one = Instruction.Operand.imm(1, 0); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(.al, reg, zero).toU32()); + writeInt(u32, try self.code.addManyAsArray(4), Instruction.mov(condition, reg, one).toU32()); }, .immediate => |x| { if (x > math.maxInt(u32)) return self.fail(src, "ARM registers are 32-bit wide", .{}); @@ -2664,29 +3141,40 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .memory => |addr| { // The value is in memory at a hard-coded address. // If the type is a pointer, it means the pointer address is at this memory location. - try self.genSetReg(src, reg, .{ .immediate = addr }); + try self.genSetReg(src, ty, reg, .{ .immediate = addr }); writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, reg, .{ .offset = Instruction.Offset.none }).toU32()); }, .stack_offset => |unadjusted_off| { - // TODO: ldrh // TODO: maybe addressing from sp instead of fp - const offset = if (unadjusted_off <= math.maxInt(u12)) blk: { - break :blk Instruction.Offset.imm(@intCast(u12, unadjusted_off)); - } else Instruction.Offset.reg(try self.copyToTmpRegister(src, MCValue{ .immediate = unadjusted_off }), 0); + const abi_size = ty.abiSize(self.target.*); + const adj_off = unadjusted_off + abi_size; - // TODO: supply type information to genSetReg as we do to genSetStack - // const abi_size = ty.abiSize(self.target.*); - const abi_size = 4; switch (abi_size) { - 1 => writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrb(.al, reg, .fp, .{ - .offset = offset, - .positive = false, - }).toU32()), - 2 => return self.fail(src, "TODO implement strh", .{}), - 4 => writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldr(.al, reg, .fp, .{ - .offset = offset, - .positive = false, - }).toU32()), + 1, 4 => { + const offset = if (adj_off <= math.maxInt(u12)) blk: { + break :blk Instruction.Offset.imm(@intCast(u12, adj_off)); + } else Instruction.Offset.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0); + const ldr = switch (abi_size) { + 1 => Instruction.ldrb, + 4 => Instruction.ldr, + else => unreachable, + }; + + writeInt(u32, try self.code.addManyAsArray(4), ldr(.al, reg, .fp, .{ + .offset = offset, + .positive = false, + }).toU32()); + }, + 2 => { + const offset = if (adj_off <= math.maxInt(u8)) blk: { + break :blk Instruction.ExtraLoadStoreOffset.imm(@intCast(u8, adj_off)); + } else Instruction.ExtraLoadStoreOffset.reg(try self.copyToTmpRegister(src, Type.initTag(.u32), MCValue{ .immediate = adj_off })); + + writeInt(u32, try self.code.addManyAsArray(4), Instruction.ldrh(.al, reg, .fp, .{ + .offset = offset, + .positive = false, + }).toU32()); + }, else => return self.fail(src, "TODO a type of size {} is not allowed in a register", .{abi_size}), } }, @@ -2702,8 +3190,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { return; // The already existing value will do just fine. // Write the debug undefined value. switch (reg.size()) { - 32 => return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaa }), - 64 => return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), + 32 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaa }), + 64 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), else => unreachable, // unexpected register size } }, @@ -2724,7 +3212,19 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.movk(reg, @intCast(u16, x >> 48), 48).toU32()); } }, - .register => return self.fail(src, "TODO implement genSetReg for aarch64 {}", .{mcv}), + .register => |src_reg| { + // If the registers are the same, nothing to do. + if (src_reg.id() == reg.id()) + return; + + // mov reg, src_reg + writeInt(u32, try self.code.addManyAsArray(4), Instruction.orr( + reg, + .xzr, + src_reg, + Instruction.Shift.none, + ).toU32()); + }, .memory => |addr| { if (self.bin_file.options.pie) { // For MachO, the binary, with the exception of object files, has to be a PIE. @@ -2754,13 +3254,16 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // mov r, x0 mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.orr( reg, + .xzr, .x0, - Instruction.RegisterShift.none(), + Instruction.Shift.none, ).toU32()); // ldr x28, [sp], #16 mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.ldr(.x28, .{ - .rn = Register.sp, - .offset = Instruction.LoadStoreOffset.imm_post_index(16), + .register = .{ + .rn = Register.sp, + .offset = Instruction.LoadStoreOffset.imm_post_index(16), + }, }).toU32()); } else { // stp x0, x28, [sp, #-16] @@ -2786,8 +3289,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // mov r, x0 mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.orr( reg, + .xzr, .x0, - Instruction.RegisterShift.none(), + Instruction.Shift.none, ).toU32()); // ldp x0, x28, [sp, #16] mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.ldp( @@ -2800,8 +3304,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } else { // The value is in memory at a hard-coded address. // If the type is a pointer, it means the pointer address is at this memory location. - try self.genSetReg(src, reg, .{ .immediate = addr }); - mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.ldr(reg, .{ .rn = reg }).toU32()); + try self.genSetReg(src, Type.initTag(.usize), reg, .{ .immediate = addr }); + mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.ldr(reg, .{ .register = .{ .rn = reg } }).toU32()); } }, else => return self.fail(src, "TODO implement genSetReg for aarch64 {}", .{mcv}), @@ -2815,7 +3319,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (!self.wantSafety()) return; // The already existing value will do just fine. // Write the debug undefined value. - return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }); + return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }); }, .immediate => |unsigned_x| { const x = @bitCast(i64, unsigned_x); @@ -2840,7 +3344,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .memory => |addr| { // The value is in memory at a hard-coded address. // If the type is a pointer, it means the pointer address is at this memory location. - try self.genSetReg(src, reg, .{ .immediate = addr }); + try self.genSetReg(src, ty, reg, .{ .immediate = addr }); mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.ld(reg, 0, reg).toU32()); // LOAD imm=[i12 offset = 0], rs1 = @@ -2859,10 +3363,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { return; // The already existing value will do just fine. // Write the debug undefined value. switch (reg.size()) { - 8 => return self.genSetReg(src, reg, .{ .immediate = 0xaa }), - 16 => return self.genSetReg(src, reg, .{ .immediate = 0xaaaa }), - 32 => return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaa }), - 64 => return self.genSetReg(src, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), + 8 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaa }), + 16 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaa }), + 32 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaa }), + 64 => return self.genSetReg(src, ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }), else => unreachable, } }, @@ -3076,7 +3580,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { assert(id3 != 4 and id3 != 5); // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue. - try self.genSetReg(src, reg, MCValue{ .immediate = x }); + try self.genSetReg(src, ty, reg, MCValue{ .immediate = x }); // Now, the register contains the address of the value to load into it // Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant. @@ -3175,7 +3679,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { // This immediate is unsigned. const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed)); if (imm >= math.maxInt(U)) { - return MCValue{ .register = try self.copyToTmpRegister(inst.src, mcv) }; + return MCValue{ .register = try self.copyToTmpRegister(inst.src, Type.initTag(.usize), mcv) }; } }, else => {}, @@ -3190,20 +3694,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { const ptr_bytes: u64 = @divExact(ptr_bits, 8); switch (typed_value.ty.zigTypeTag()) { .Pointer => { - if (typed_value.val.cast(Value.Payload.DeclRef)) |payload| { + if (typed_value.val.castTag(.decl_ref)) |payload| { if (self.bin_file.cast(link.File.Elf)) |elf_file| { - const decl = payload.decl; + const decl = payload.data; const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?]; const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes; return MCValue{ .memory = got_addr }; } else if (self.bin_file.cast(link.File.MachO)) |macho_file| { - const decl = payload.decl; + const decl = payload.data; const text_segment = &macho_file.load_commands.items[macho_file.text_segment_cmd_index.?].Segment; const got = &text_segment.sections.items[macho_file.got_section_index.?]; const got_addr = got.addr + decl.link.macho.offset_table_index * ptr_bytes; return MCValue{ .memory = got_addr }; } else if (self.bin_file.cast(link.File.Coff)) |coff_file| { - const decl = payload.decl; + const decl = payload.data; const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes; return MCValue{ .memory = got_addr }; } else { @@ -3229,7 +3733,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { if (typed_value.val.isNull()) return MCValue{ .immediate = 0 }; - var buf: Type.Payload.PointerSimple = undefined; + var buf: Type.Payload.ElemType = undefined; return self.genTypedValue(src, .{ .ty = typed_value.ty.optionalChild(&buf), .val = typed_value.val, @@ -3289,20 +3793,25 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { for (param_types) |ty, i| { switch (ty.zigTypeTag()) { .Bool, .Int => { - const param_size = @intCast(u32, ty.abiSize(self.target.*)); - if (next_int_reg >= c_abi_int_param_regs.len) { - result.args[i] = .{ .stack_offset = next_stack_offset }; - next_stack_offset += param_size; + if (!ty.hasCodeGenBits()) { + assert(cc != .C); + result.args[i] = .{ .none = {} }; } else { - const aliased_reg = registerAlias( - c_abi_int_param_regs[next_int_reg], - param_size, - ); - result.args[i] = .{ .register = aliased_reg }; - next_int_reg += 1; + const param_size = @intCast(u32, ty.abiSize(self.target.*)); + if (next_int_reg >= c_abi_int_param_regs.len) { + result.args[i] = .{ .stack_offset = next_stack_offset }; + next_stack_offset += param_size; + } else { + const aliased_reg = registerAlias( + c_abi_int_param_regs[next_int_reg], + param_size, + ); + result.args[i] = .{ .register = aliased_reg }; + next_int_reg += 1; + } } }, - else => return self.fail(src, "TODO implement function parameters of type {}", .{@tagName(ty.zigTypeTag())}), + else => return self.fail(src, "TODO implement function parameters of type {s}", .{@tagName(ty.zigTypeTag())}), } } result.stack_byte_count = next_stack_offset; @@ -3326,10 +3835,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { var nsaa: u32 = 0; // Next stacked argument address for (param_types) |ty, i| { - if (ty.abiAlignment(self.target.*) == 8) { - // Round up NCRN to the next even number - ncrn += ncrn % 2; - } + if (ty.abiAlignment(self.target.*) == 8) + ncrn = std.mem.alignForwardGeneric(usize, ncrn, 2); const param_size = @intCast(u32, ty.abiSize(self.target.*)); if (std.math.divCeil(u32, param_size, 4) catch unreachable <= 4 - ncrn) { @@ -3343,6 +3850,56 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { return self.fail(src, "TODO MCValues split between registers and stack", .{}); } else { ncrn = 4; + if (ty.abiAlignment(self.target.*) == 8) + nsaa = std.mem.alignForwardGeneric(u32, nsaa, 8); + + result.args[i] = .{ .stack_offset = nsaa }; + nsaa += param_size; + } + } + + result.stack_byte_count = nsaa; + result.stack_align = 4; + }, + else => return self.fail(src, "TODO implement function parameters for {} on arm", .{cc}), + } + }, + .aarch64 => { + switch (cc) { + .Naked => { + assert(result.args.len == 0); + result.return_value = .{ .unreach = {} }; + result.stack_byte_count = 0; + result.stack_align = 1; + return result; + }, + .Unspecified, .C => { + // ARM64 Procedure Call Standard + var ncrn: usize = 0; // Next Core Register Number + var nsaa: u32 = 0; // Next stacked argument address + + for (param_types) |ty, i| { + // We round up NCRN only for non-Apple platforms which allow the 16-byte aligned + // values to spread across odd-numbered registers. + if (ty.abiAlignment(self.target.*) == 16 and !self.target.isDarwin()) { + // Round up NCRN to the next even number + ncrn += ncrn % 2; + } + + const param_size = @intCast(u32, ty.abiSize(self.target.*)); + if (std.math.divCeil(u32, param_size, 8) catch unreachable <= 8 - ncrn) { + if (param_size <= 8) { + result.args[i] = .{ .register = c_abi_int_param_regs[ncrn] }; + ncrn += 1; + } else { + return self.fail(src, "TODO MCValues with multiple registers", .{}); + } + } else if (ncrn < 8 and nsaa == 0) { + return self.fail(src, "TODO MCValues split between registers and stack", .{}); + } else { + ncrn = 8; + // TODO Apple allows the arguments on the stack to be non-8-byte aligned provided + // that the entire stack space consumed by the arguments is 8-byte aligned. if (ty.abiAlignment(self.target.*) == 8) { if (nsaa % 8 != 0) { nsaa += 8 - (nsaa % 8); @@ -3355,9 +3912,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } result.stack_byte_count = nsaa; - result.stack_align = 4; + result.stack_align = 16; }, - else => return self.fail(src, "TODO implement function parameters for {} on arm", .{cc}), + else => return self.fail(src, "TODO implement function parameters for {} on aarch64", .{cc}), } }, else => if (param_types.len != 0) @@ -3390,6 +3947,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { }, else => return self.fail(src, "TODO implement function return values for {}", .{cc}), }, + .aarch64 => switch (cc) { + .Naked => unreachable, + .Unspecified, .C => { + const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*)); + if (ret_ty_size <= 8) { + result.return_value = .{ .register = c_abi_int_return_regs[0] }; + } else { + return self.fail(src, "TODO support more return types for ARM backend", .{}); + } + }, + else => return self.fail(src, "TODO implement function return values for {}", .{cc}), + }, else => return self.fail(src, "TODO implement codegen return values for {}", .{self.target.cpu.arch}), } return result; @@ -3408,7 +3977,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { fn fail(self: *Self, src: usize, comptime format: []const u8, args: anytype) InnerError { @setCold(true); assert(self.err_msg == null); - self.err_msg = try ErrorMsg.create(self.bin_file.allocator, src, format, args); + self.err_msg = try ErrorMsg.create(self.bin_file.allocator, .{ + .file_scope = self.src_loc.file_scope, + .byte_offset = src, + }, format, args); + return error.CodegenFail; + } + + fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError { + @setCold(true); + assert(self.err_msg == null); + self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args); return error.CodegenFail; } |