From 46f4a97d051e630dfae75e3856c81ffd1c7811e3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 1 Dec 2022 18:10:36 -0700 Subject: Revert "cbe: write more instructions inline" This reverts commit f8b779c114a5fcb82f08168912f2300d7027a2fd. --- src/codegen/c.zig | 133 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 56 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 7fd76a6962..6f48117af7 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -395,27 +395,6 @@ pub const Function = struct { .shl, => try airBinBuiltinCall(f, inst, "shlw", .Bits), .shl_exact => try airBinOp(f, inst, "<<", "shl", .None), .not => try airNot (f, inst), - - .is_err => try airIsErr(f, inst, false, "!="), - .is_non_err => try airIsErr(f, inst, false, "=="), - .is_err_ptr => try airIsErr(f, inst, true, "!="), - .is_non_err_ptr => try airIsErr(f, inst, true, "=="), - - .is_null => try airIsNull(f, inst, "==", false), - .is_non_null => try airIsNull(f, inst, "!=", false), - .is_null_ptr => try airIsNull(f, inst, "==", true), - .is_non_null_ptr => try airIsNull(f, inst, "!=", true), - - .get_union_tag => try airGetUnionTag(f, inst), - .clz => try airUnBuiltinCall(f, inst, "clz", .Bits), - .ctz => try airUnBuiltinCall(f, inst, "ctz", .Bits), - .popcount => try airUnBuiltinCall(f, inst, "popcount", .Bits), - .byte_swap => try airUnBuiltinCall(f, inst, "byte_swap", .Bits), - .bit_reverse => try airUnBuiltinCall(f, inst, "bit_reverse", .Bits), - .tag_name => try airTagName(f, inst), - .error_name => try airErrorName(f, inst), - - .ptrtoint => try airPtrToInt(f, inst), else => unreachable, // zig fmt: on } @@ -443,7 +422,6 @@ pub const Function = struct { try w.writeByte('.'); return f.writeCValue(w, member, .Other); }, - .inline_index => unreachable, // Use resolveInstNoInline else => return f.object.dg.writeCValueMember(w, c_value, member), } } @@ -2729,15 +2707,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .optional_payload_ptr_set => try airOptionalPayloadPtrSet(f, inst), .wrap_optional => try airWrapOptional(f, inst), - .is_err => CValue{ .inline_index = inst }, - .is_non_err => CValue{ .inline_index = inst }, - .is_err_ptr => CValue{ .inline_index = inst }, - .is_non_err_ptr => CValue{ .inline_index = inst }, + .is_err => try airIsErr(f, inst, false, "!="), + .is_non_err => try airIsErr(f, inst, false, "=="), + .is_err_ptr => try airIsErr(f, inst, true, "!="), + .is_non_err_ptr => try airIsErr(f, inst, true, "=="), - .is_null => CValue{ .inline_index = inst }, - .is_non_null => CValue{ .inline_index = inst }, - .is_null_ptr => CValue{ .inline_index = inst }, - .is_non_null_ptr => CValue{ .inline_index = inst }, + .is_null => try airIsNull(f, inst, "==", false), + .is_non_null => try airIsNull(f, inst, "!=", false), + .is_null_ptr => try airIsNull(f, inst, "==", true), + .is_non_null_ptr => try airIsNull(f, inst, "!=", true), .alloc => try airAlloc(f, inst), .ret_ptr => try airRetPtr(f, inst), @@ -2765,14 +2743,14 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .memset => try airMemset(f, inst), .memcpy => try airMemcpy(f, inst), .set_union_tag => try airSetUnionTag(f, inst), - .get_union_tag => CValue{ .inline_index = inst }, - .clz => CValue{ .inline_index = inst }, - .ctz => CValue{ .inline_index = inst }, - .popcount => CValue{ .inline_index = inst }, - .byte_swap => CValue{ .inline_index = inst }, - .bit_reverse => CValue{ .inline_index = inst }, - .tag_name => CValue{ .inline_index = inst }, - .error_name => CValue{ .inline_index = inst }, + .get_union_tag => try airGetUnionTag(f, inst), + .clz => try airUnBuiltinCall(f, inst, "clz", .Bits), + .ctz => try airUnBuiltinCall(f, inst, "ctz", .Bits), + .popcount => try airUnBuiltinCall(f, inst, "popcount", .Bits), + .byte_swap => try airUnBuiltinCall(f, inst, "byte_swap", .Bits), + .bit_reverse => try airUnBuiltinCall(f, inst, "bit_reverse", .Bits), + .tag_name => try airTagName(f, inst), + .error_name => try airErrorName(f, inst), .splat => try airSplat(f, inst), .select => try airSelect(f, inst), .shuffle => try airShuffle(f, inst), @@ -2808,7 +2786,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .fpext, => try airFloatCast(f, inst), - .ptrtoint => CValue{ .inline_index = inst }, + .ptrtoint => try airPtrToInt(f, inst), .atomic_store_unordered => try airAtomicStore(f, inst, toMemoryOrder(.Unordered)), .atomic_store_monotonic => try airAtomicStore(f, inst, toMemoryOrder(.Monotonic)), @@ -2889,7 +2867,7 @@ fn airSliceField(f: *Function, inst: Air.Inst.Index, is_ptr: bool, field_name: [ const inst_ty = f.air.typeOfIndex(inst); const ty_op = f.air.instructions.items(.data)[inst].ty_op; - const operand = try f.resolveInstNoInline(ty_op.operand); + const operand = try f.resolveInst(ty_op.operand); const writer = f.object.writer(); const local = try f.allocLocal(inst_ty, .Const); try writer.writeAll(" = "); @@ -4425,11 +4403,16 @@ fn airIsNull( inst: Air.Inst.Index, operator: []const u8, is_ptr: bool, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) + return CValue.none; + const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); const operand = try f.resolveInst(un_op); + const local = try f.allocLocal(Type.initTag(.bool), .Const); + try writer.writeAll(" = "); try if (is_ptr) f.writeCValueDeref(writer, operand) else f.writeCValue(writer, operand, .Other); const operand_ty = f.air.typeOf(un_op); @@ -4457,6 +4440,8 @@ fn airIsNull( try writer.writeAll(operator); try writer.writeByte(' '); try f.object.dg.renderValue(writer, rhs.ty, rhs.val, .Other); + try writer.writeAll(";\n"); + return local; } fn airOptionalPayload(f: *Function, inst: Air.Inst.Index) !CValue { @@ -5038,15 +5023,21 @@ fn airWrapErrUnionPay(f: *Function, inst: Air.Inst.Index) !CValue { return local; } -fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const u8) !void { +fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const u8) !CValue { + if (f.liveness.isUnused(inst)) + return CValue.none; + const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); const operand = try f.resolveInst(un_op); const operand_ty = f.air.typeOf(un_op); + const local = try f.allocLocal(Type.initTag(.bool), .Const); const err_union_ty = if (is_ptr) operand_ty.childType() else operand_ty; const payload_ty = err_union_ty.errorUnionPayload(); const error_ty = err_union_ty.errorUnionSet(); + try writer.writeAll(" = "); + if (!error_ty.errorSetIsEmpty()) if (payload_ty.hasRuntimeBits()) if (is_ptr) @@ -5061,6 +5052,8 @@ fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const try writer.writeAll(operator); try writer.writeByte(' '); try f.object.dg.renderValue(writer, error_ty, Value.zero, .Other); + try writer.writeAll(";\n"); + return local; } fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { @@ -5134,16 +5127,21 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue { return local; } -fn airPtrToInt(f: *Function, inst: Air.Inst.Index) !void { +fn airPtrToInt(f: *Function, inst: Air.Inst.Index) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst_ty, .Const); const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); const operand = try f.resolveInst(un_op); - try writer.writeAll("("); + try writer.writeAll(" = ("); try f.renderTypecast(writer, inst_ty); try writer.writeByte(')'); try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); + return local; } fn airUnBuiltinCall( @@ -5151,19 +5149,24 @@ fn airUnBuiltinCall( inst: Air.Inst.Index, operation: []const u8, info: BuiltinInfo, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + + const inst_ty = f.air.typeOfIndex(inst); const operand = f.air.instructions.items(.data)[inst].ty_op.operand; const operand_ty = f.air.typeOf(operand); + const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); - try writer.writeAll("zig_"); + try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); try writer.writeByte('('); try f.writeCValue(writer, try f.resolveInst(operand), .FunctionArgument); try f.object.dg.renderBuiltinInfo(writer, operand_ty, info); - try writer.writeAll(")"); + try writer.writeAll(");\n"); + return local; } fn airBinBuiltinCall( @@ -5436,7 +5439,12 @@ fn airSetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { return CValue.none; } -fn airGetUnionTag(f: *Function, inst: Air.Inst.Index) !void { +fn airGetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { + if (f.liveness.isUnused(inst)) + return CValue.none; + + const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst_ty, .Const); const ty_op = f.air.instructions.items(.data)[inst].ty_op; const un_ty = f.air.typeOf(ty_op.operand); const writer = f.object.writer(); @@ -5444,31 +5452,44 @@ fn airGetUnionTag(f: *Function, inst: Air.Inst.Index) !void { const target = f.object.dg.module.getTarget(); const layout = un_ty.unionGetLayout(target); - assert(layout.tag_size != 0); + if (layout.tag_size == 0) return CValue.none; + try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Other); - try writer.writeAll(".tag"); + try writer.writeAll(".tag;\n"); + return local; } -fn airTagName(f: *Function, inst: Air.Inst.Index) !void { +fn airTagName(f: *Function, inst: Air.Inst.Index) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const un_op = f.air.instructions.items(.data)[inst].un_op; + const inst_ty = f.air.typeOfIndex(inst); const enum_ty = f.air.typeOf(un_op); const operand = try f.resolveInst(un_op); const writer = f.object.writer(); - try writer.print("{s}(", .{try f.object.dg.getTagNameFn(enum_ty)}); + const local = try f.allocLocal(inst_ty, .Const); + try writer.print(" = {s}(", .{try f.object.dg.getTagNameFn(enum_ty)}); try f.writeCValue(writer, operand, .Other); - try writer.writeAll(")"); + try writer.writeAll(");\n"); + + return local; } -fn airErrorName(f: *Function, inst: Air.Inst.Index) !void { +fn airErrorName(f: *Function, inst: Air.Inst.Index) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); + const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(un_op); + const local = try f.allocLocal(inst_ty, .Const); - try writer.writeAll("zig_errorName["); + try writer.writeAll(" = zig_errorName["); try f.writeCValue(writer, operand, .Other); - try writer.writeAll("]"); + try writer.writeAll("];\n"); + return local; } fn airSplat(f: *Function, inst: Air.Inst.Index) !CValue { -- cgit v1.2.3 From 2a0efbee56b03d9deafaa7918433ea10b46305c9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 1 Dec 2022 18:14:54 -0700 Subject: Revert "cbe: reduce amount of temporary locals" This reverts commit 15cc83e27ae8a1740d9b7e2ec14044903979a832. --- src/codegen/c.zig | 291 ++++++++++++++++++++++++------------------------------ 1 file changed, 128 insertions(+), 163 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 6f48117af7..4737e3eefa 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -50,8 +50,6 @@ pub const CValue = union(enum) { /// Render these bytes literally. /// TODO make this a [*:0]const u8 to save memory bytes: []const u8, - /// Index of an instruction that should later be rendered inline. - inline_index: Air.Inst.Index, }; const BlockData = struct { @@ -81,7 +79,6 @@ const ValueRenderLocation = enum { FunctionArgument, Initializer, Other, - condition, }; const BuiltinInfo = enum { @@ -281,19 +278,6 @@ pub const Function = struct { return result; } - fn resolveInstNoInline(f: *Function, inst: Air.Inst.Ref) !CValue { - const operand = try f.resolveInst(inst); - if (operand != .inline_index) return operand; - - const inst_ty = f.air.typeOf(inst); - const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); - try writer.writeAll(" = "); - try f.writeCValueInline(operand.inline_index); - try writer.writeAll(";\n"); - return local; - } - fn wantSafety(f: *Function) bool { return switch (f.object.dg.module.optimizeMode()) { .Debug, .ReleaseSafe => true, @@ -329,74 +313,10 @@ pub const Function = struct { .constant => |inst| { const ty = f.air.typeOf(inst); const val = f.air.value(inst).?; - try f.object.dg.renderValue(w, ty, val, location); - }, - .undef => |ty| try f.object.dg.renderValue(w, ty, Value.undef, location), - .inline_index => |node| { - if (location != .condition) try w.writeByte('('); - try f.writeCValueInline(node); - if (location != .condition) try w.writeByte(')'); - }, - else => try f.object.dg.writeCValue(w, c_value), - } - } - - const E = error{ OutOfMemory, AnalysisFail }; - - fn writeCValueInline(f: *Function, inst: Air.Inst.Index) E!void { - switch (f.air.instructions.items(.tag)[inst]) { - // zig fmt: off - // TODO use a different strategy for add, sub, mul, div - // that communicates to the optimizer that wrapping is UB. - .add => try airBinOp(f, inst, "+", "add", .None), - .sub => try airBinOp(f, inst, "-", "sub", .None), - .mul => try airBinOp(f, inst, "*", "mul", .None), - - .div_float => try airBinBuiltinCall(f, inst, "div", .None), - - .div_trunc, .div_exact => try airBinOp(f, inst, "/", "div_trunc", .None), - .rem => { - const bin_op = f.air.instructions.items(.data)[inst].bin_op; - const lhs_ty = f.air.typeOf(bin_op.lhs); - // For binary operations @TypeOf(lhs)==@TypeOf(rhs), - // so we only check one. - if (lhs_ty.isInt()) - try airBinOp(f, inst, "%", "rem", .None) - else - try airBinFloatOp(f, inst, "fmod"); + return f.object.dg.renderValue(w, ty, val, location); }, - .div_floor => try airBinBuiltinCall(f, inst, "div_floor", .None), - .mod => try airBinBuiltinCall(f, inst, "mod", .None), - - .addwrap => try airBinBuiltinCall(f, inst, "addw", .Bits), - .subwrap => try airBinBuiltinCall(f, inst, "subw", .Bits), - .mulwrap => try airBinBuiltinCall(f, inst, "mulw", .Bits), - - .add_sat => try airBinBuiltinCall(f, inst, "adds", .Bits), - .sub_sat => try airBinBuiltinCall(f, inst, "subs", .Bits), - .mul_sat => try airBinBuiltinCall(f, inst, "muls", .Bits), - .shl_sat => try airBinBuiltinCall(f, inst, "shls", .Bits), - - .min => try airMinMax(f, inst, '<', "fmin"), - .max => try airMinMax(f, inst, '>', "fmax"), - - .cmp_gt => try airCmpOp(f, inst, ">", "gt"), - .cmp_gte => try airCmpOp(f, inst, ">=", "ge"), - .cmp_lt => try airCmpOp(f, inst, "<", "lt"), - .cmp_lte => try airCmpOp(f, inst, "<=", "le"), - - .cmp_eq => try airEquality(f, inst, "((", "==", "eq"), - .cmp_neq => try airEquality(f, inst, "!((", "!=", "ne"), - - .bool_and, .bit_and => try airBinOp(f, inst, "&", "and", .None), - .bool_or, .bit_or => try airBinOp(f, inst, "|", "or", .None), - .xor => try airBinOp(f, inst, "^", "xor", .None), - .shr, .shr_exact => try airBinBuiltinCall(f, inst, "shr", .None), - .shl, => try airBinBuiltinCall(f, inst, "shlw", .Bits), - .shl_exact => try airBinOp(f, inst, "<<", "shl", .None), - .not => try airNot (f, inst), - else => unreachable, - // zig fmt: on + .undef => |ty| return f.object.dg.renderValue(w, ty, Value.undef, location), + else => return f.object.dg.writeCValue(w, c_value), } } @@ -2245,7 +2165,7 @@ pub const DeclGen = struct { fn writeCValue(dg: *DeclGen, w: anytype, c_value: CValue) !void { switch (c_value) { - .none, .inline_index => unreachable, + .none => unreachable, .local => |i| return w.print("t{d}", .{i}), .local_ref => |i| return w.print("&t{d}", .{i}), .constant => unreachable, @@ -2264,7 +2184,7 @@ pub const DeclGen = struct { fn writeCValueDeref(dg: *DeclGen, w: anytype, c_value: CValue) !void { switch (c_value) { - .none, .inline_index => unreachable, + .none => unreachable, .local => |i| return w.print("(*t{d})", .{i}), .local_ref => |i| return w.print("t{d}", .{i}), .constant => unreachable, @@ -2294,7 +2214,7 @@ pub const DeclGen = struct { fn writeCValueDerefMember(dg: *DeclGen, writer: anytype, c_value: CValue, member: CValue) !void { switch (c_value) { - .none, .constant, .field, .undef, .inline_index => unreachable, + .none, .constant, .field, .undef => unreachable, .local, .arg, .decl, .identifier, .bytes => { try dg.writeCValue(writer, c_value); try writer.writeAll("->"); @@ -2633,26 +2553,37 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .ptr_add => try airPtrAddSub(f, inst, '+'), .ptr_sub => try airPtrAddSub(f, inst, '-'), - .add => CValue{ .inline_index = inst }, - .sub => CValue{ .inline_index = inst }, - .mul => CValue{ .inline_index = inst }, + // TODO use a different strategy for add, sub, mul, div + // that communicates to the optimizer that wrapping is UB. + .add => try airBinOp(f, inst, "+", "add", .None), + .sub => try airBinOp(f, inst, "-", "sub", .None), + .mul => try airBinOp(f, inst, "*", "mul", .None), .neg => try airFloatNeg(f, inst), - .div_float => CValue{ .inline_index = inst }, + .div_float => try airBinBuiltinCall(f, inst, "div", .None), - .div_trunc, .div_exact => CValue{ .inline_index = inst }, - .rem => CValue{ .inline_index = inst }, - .div_floor => CValue{ .inline_index = inst }, - .mod => CValue{ .inline_index = inst }, + .div_trunc, .div_exact => try airBinOp(f, inst, "/", "div_trunc", .None), + .rem => blk: { + const bin_op = f.air.instructions.items(.data)[inst].bin_op; + const lhs_ty = f.air.typeOf(bin_op.lhs); + // For binary operations @TypeOf(lhs)==@TypeOf(rhs), + // so we only check one. + break :blk if (lhs_ty.isInt()) + try airBinOp(f, inst, "%", "rem", .None) + else + try airBinFloatOp(f, inst, "fmod"); + }, + .div_floor => try airBinBuiltinCall(f, inst, "div_floor", .None), + .mod => try airBinBuiltinCall(f, inst, "mod", .None), - .addwrap => CValue{ .inline_index = inst }, - .subwrap => CValue{ .inline_index = inst }, - .mulwrap => CValue{ .inline_index = inst }, + .addwrap => try airBinBuiltinCall(f, inst, "addw", .Bits), + .subwrap => try airBinBuiltinCall(f, inst, "subw", .Bits), + .mulwrap => try airBinBuiltinCall(f, inst, "mulw", .Bits), - .add_sat => CValue{ .inline_index = inst }, - .sub_sat => CValue{ .inline_index = inst }, - .mul_sat => CValue{ .inline_index = inst }, - .shl_sat => CValue{ .inline_index = inst }, + .add_sat => try airBinBuiltinCall(f, inst, "adds", .Bits), + .sub_sat => try airBinBuiltinCall(f, inst, "subs", .Bits), + .mul_sat => try airBinBuiltinCall(f, inst, "muls", .Bits), + .shl_sat => try airBinBuiltinCall(f, inst, "shls", .Bits), .sqrt, .sin, @@ -2677,30 +2608,30 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .mul_with_overflow => try airOverflow(f, inst, "mul", .Bits), .shl_with_overflow => try airOverflow(f, inst, "shl", .Bits), - .min => CValue{ .inline_index = inst }, - .max => CValue{ .inline_index = inst }, + .min => try airMinMax(f, inst, '<', "fmin"), + .max => try airMinMax(f, inst, '>', "fmax"), .slice => try airSlice(f, inst), - .cmp_gt => CValue{ .inline_index = inst }, - .cmp_gte => CValue{ .inline_index = inst }, - .cmp_lt => CValue{ .inline_index = inst }, - .cmp_lte => CValue{ .inline_index = inst }, + .cmp_gt => try airCmpOp(f, inst, ">", "gt"), + .cmp_gte => try airCmpOp(f, inst, ">=", "ge"), + .cmp_lt => try airCmpOp(f, inst, "<", "lt"), + .cmp_lte => try airCmpOp(f, inst, "<=", "le"), - .cmp_eq => CValue{ .inline_index = inst }, - .cmp_neq => CValue{ .inline_index = inst }, + .cmp_eq => try airEquality(f, inst, "((", "==", "eq"), + .cmp_neq => try airEquality(f, inst, "!((", "!=", "ne"), .cmp_vector => return f.fail("TODO: C backend: implement cmp_vector", .{}), .cmp_lt_errors_len => try airCmpLtErrorsLen(f, inst), // bool_and and bool_or are non-short-circuit operations - .bool_and, .bit_and => CValue{ .inline_index = inst }, - .bool_or, .bit_or => CValue{ .inline_index = inst }, - .xor => CValue{ .inline_index = inst }, - .shr, .shr_exact => CValue{ .inline_index = inst }, - .shl, => CValue{ .inline_index = inst }, - .shl_exact => CValue{ .inline_index = inst }, - .not => CValue{ .inline_index = inst }, + .bool_and, .bit_and => try airBinOp(f, inst, "&", "and", .None), + .bool_or, .bit_or => try airBinOp(f, inst, "|", "or", .None), + .xor => try airBinOp(f, inst, "^", "xor", .None), + .shr, .shr_exact => try airBinBuiltinCall(f, inst, "shr", .None), + .shl, => try airBinBuiltinCall(f, inst, "shlw", .Bits), + .shl_exact => try airBinOp(f, inst, "<<", "shl", .None), + .not => try airNot (f, inst), .optional_payload => try airOptionalPayload(f, inst), .optional_payload_ptr => try airOptionalPayloadPtr(f, inst), @@ -3449,21 +3380,22 @@ fn airOverflow(f: *Function, inst: Air.Inst.Index, operation: []const u8, info: return local; } -fn airNot(f: *Function, inst: Air.Inst.Index) !void { +fn airNot(f: *Function, inst: Air.Inst.Index) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; const op = try f.resolveInst(ty_op.operand); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst_ty, .Const); - const target = f.object.dg.module.getTarget(); - if (inst_ty.bitSize(target) > 64) {} - - try writer.writeByte('('); - try f.renderTypecast(writer, inst_ty); - try writer.writeByte(')'); + try writer.writeAll(" = "); try writer.writeByte(if (inst_ty.tag() == .bool) '!' else '~'); try f.writeCValue(writer, op, .Other); + try writer.writeAll(";\n"); + + return local; } fn airBinOp( @@ -3472,54 +3404,62 @@ fn airBinOp( operator: []const u8, operation: []const u8, info: BuiltinInfo, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if ((operand_ty.isInt() and operand_ty.bitSize(target) > 64) or operand_ty.isRuntimeFloat()) - return airBinBuiltinCall(f, inst, operation, info); + return try airBinBuiltinCall(f, inst, operation, info); const inst_ty = f.air.typeOfIndex(inst); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); const writer = f.object.writer(); - try writer.writeByte('('); - try f.renderTypecast(writer, inst_ty); - try writer.writeAll(")("); + const local = try f.allocLocal(inst_ty, .Const); + + try writer.writeAll(" = "); try f.writeCValue(writer, lhs, .Other); try writer.writeByte(' '); try writer.writeAll(operator); try writer.writeByte(' '); try f.writeCValue(writer, rhs, .Other); - try writer.writeByte(')'); + try writer.writeAll(";\n"); + + return local; } -fn airCmpOp( - f: *Function, - inst: Air.Inst.Index, - operator: []const u8, - operation: []const u8, -) !void { +fn airCmpOp(f: *Function, inst: Air.Inst.Index, operator: []const u8, operation: []const u8) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if (operand_ty.isInt() and operand_ty.bitSize(target) > 64) - return airCmpBuiltinCall(f, inst, operator, "cmp"); + return try airCmpBuiltinCall(f, inst, operator, "cmp"); if (operand_ty.isRuntimeFloat()) - return airCmpBuiltinCall(f, inst, operator, operation); + return try airCmpBuiltinCall(f, inst, operator, operation); + const inst_ty = f.air.typeOfIndex(inst); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); const writer = f.object.writer(); + const local = try f.allocLocal(inst_ty, .Const); + + try writer.writeAll(" = "); try f.writeCValue(writer, lhs, .Other); try writer.writeByte(' '); try writer.writeAll(operator); try writer.writeByte(' '); try f.writeCValue(writer, rhs, .Other); + try writer.writeAll(";\n"); + + return local; } fn airEquality( @@ -3528,20 +3468,27 @@ fn airEquality( negate_prefix: []const u8, operator: []const u8, operation: []const u8, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if (operand_ty.isInt() and operand_ty.bitSize(target) > 64) - return airCmpBuiltinCall(f, inst, operator, "cmp"); + return try airCmpBuiltinCall(f, inst, operator, "cmp"); if (operand_ty.isRuntimeFloat()) - return airCmpBuiltinCall(f, inst, operator, operation); + return try airCmpBuiltinCall(f, inst, operator, operation); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); const writer = f.object.writer(); + const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst_ty, .Const); + + try writer.writeAll(" = "); + if (operand_ty.zigTypeTag() == .Optional and !operand_ty.isPtrLikeOptional()) { // (A && B) || (C && (A == B)) // A = lhs.is_null ; B = rhs.is_null ; C = rhs.payload == lhs.payload @@ -3558,8 +3505,9 @@ fn airEquality( try f.writeCValue(writer, lhs, .Other); try writer.writeAll(".is_null == "); try f.writeCValue(writer, rhs, .Other); - try writer.writeAll(".is_null))"); - return; + try writer.writeAll(".is_null));\n"); + + return local; } try f.writeCValue(writer, lhs, .Other); @@ -3567,6 +3515,9 @@ fn airEquality( try writer.writeAll(operator); try writer.writeByte(' '); try f.writeCValue(writer, rhs, .Other); + try writer.writeAll(";\n"); + + return local; } fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { @@ -3620,23 +3571,26 @@ fn airPtrAddSub(f: *Function, inst: Air.Inst.Index, operator: u8) !CValue { return local; } -fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []const u8) !void { +fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []const u8) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + const bin_op = f.air.instructions.items(.data)[inst].bin_op; const inst_ty = f.air.typeOfIndex(inst); const target = f.object.dg.module.getTarget(); if (inst_ty.isInt() and inst_ty.bitSize(target) > 64) - return airBinBuiltinCall(f, inst, operation[1..], .None); + return try airBinBuiltinCall(f, inst, operation[1..], .None); if (inst_ty.isRuntimeFloat()) - return airBinFloatOp(f, inst, operation); + return try airBinFloatOp(f, inst, operation); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); const writer = f.object.writer(); + const local = try f.allocLocal(inst_ty, .Const); // (lhs <> rhs) ? lhs : rhs - try writer.writeAll("("); + try writer.writeAll(" = ("); try f.writeCValue(writer, lhs, .Other); try writer.writeByte(' '); try writer.writeByte(operator); @@ -3646,6 +3600,9 @@ fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []cons try f.writeCValue(writer, lhs, .Other); try writer.writeAll(" : "); try f.writeCValue(writer, rhs, .Other); + try writer.writeAll(";\n"); + + return local; } fn airSlice(f: *Function, inst: Air.Inst.Index) !CValue { @@ -3961,7 +3918,7 @@ fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue { if (f.liveness.isUnused(inst) or !src_ty.hasRuntimeBits()) return CValue.none; const ty_op = f.air.instructions.items(.data)[inst].ty_op; - const operand = try f.resolveInstNoInline(ty_op.operand); + const operand = try f.resolveInst(ty_op.operand); const dest_ty = f.air.typeOf(ty_op.operand); const target = f.object.dg.module.getTarget(); @@ -4067,7 +4024,7 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue { const body = f.air.extra[loop.end..][0..loop.data.body_len]; const writer = f.object.writer(); try writer.writeAll("while ("); - try f.object.dg.renderValue(writer, Type.bool, Value.true, .condition); + try f.object.dg.renderValue(writer, Type.bool, Value.true, .Other); try writer.writeAll(") "); try genBody(f, body); try writer.writeByte('\n'); @@ -4083,7 +4040,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); try writer.writeAll("if ("); - try f.writeCValue(writer, cond, .condition); + try f.writeCValue(writer, cond, .Other); try writer.writeAll(") "); try genBody(f, then_body); try writer.writeAll(" else "); @@ -5174,12 +5131,16 @@ fn airBinBuiltinCall( inst: Air.Inst.Index, operation: []const u8, info: BuiltinInfo, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + + const inst_ty = f.air.typeOfIndex(inst); const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); + const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); - try writer.writeAll("zig_"); + try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); @@ -5188,7 +5149,8 @@ fn airBinBuiltinCall( try writer.writeAll(", "); try f.writeCValue(writer, try f.resolveInst(bin_op.rhs), .FunctionArgument); try f.object.dg.renderBuiltinInfo(writer, operand_ty, info); - try writer.writeAll(")"); + try writer.writeAll(");\n"); + return local; } fn airCmpBuiltinCall( @@ -5196,12 +5158,16 @@ fn airCmpBuiltinCall( inst: Air.Inst.Index, operator: []const u8, operation: []const u8, -) !void { +) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; + + const inst_ty = f.air.typeOfIndex(inst); const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); + const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); - try writer.writeAll("zig_"); + try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); @@ -5209,7 +5175,8 @@ fn airCmpBuiltinCall( try f.writeCValue(writer, try f.resolveInst(bin_op.lhs), .FunctionArgument); try writer.writeAll(", "); try f.writeCValue(writer, try f.resolveInst(bin_op.rhs), .FunctionArgument); - try writer.print(") {s} {}", .{ operator, try f.fmtIntLiteral(Type.initTag(.i32), Value.zero) }); + try writer.print(") {s} {};\n", .{ operator, try f.fmtIntLiteral(Type.initTag(.i32), Value.zero) }); + return local; } fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue { @@ -5946,18 +5913,15 @@ fn airUnFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CVal return local; } -fn airBinFloatOp( - f: *Function, - inst: Air.Inst.Index, - operation: []const u8, -) !void { +fn airBinFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CValue { + if (f.liveness.isUnused(inst)) return CValue.none; const bin_op = f.air.instructions.items(.data)[inst].bin_op; const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); - - try writer.writeAll("zig_libc_name_"); + const local = try f.allocLocal(inst_ty, .Const); + try writer.writeAll(" = zig_libc_name_"); try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty); try writer.writeByte('('); try writer.writeAll(operation); @@ -5965,7 +5929,8 @@ fn airBinFloatOp( try f.writeCValue(writer, lhs, .FunctionArgument); try writer.writeAll(", "); try f.writeCValue(writer, rhs, .FunctionArgument); - try writer.writeAll(")"); + try writer.writeAll(");\n"); + return local; } fn airMulAdd(f: *Function, inst: Air.Inst.Index) !CValue { -- cgit v1.2.3 From 8d8b2c834d87494fc6c7c0386c04206a4c134801 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 2 Dec 2022 23:07:03 -0700 Subject: CBE: exploit Liveness analysis to reuse locals --- build.sh | 9 + src/codegen/c.zig | 1660 ++++++++++++++++++++++++++++++++++++----------------- src/link/C.zig | 11 +- test.sh | 20 + 4 files changed, 1153 insertions(+), 547 deletions(-) create mode 100755 build.sh create mode 100755 test.sh (limited to 'src/codegen') diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..e462514281 --- /dev/null +++ b/build.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e +if [ "x$1" != x--debug ]; then + cmake -GNinja -S. -Bbuild -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_C_COMPILER:FILEPATH=clang -DCMAKE_CXX_COMPILER:FILEPATH=clang++ -DZIG_NO_LIB:BOOL=ON + cmake --build build + cmake --install build +fi +build/stage3/bin/zig build -p debug -Dno-lib -Denable-stage1 -Denable-llvm -freference-trace +#build/stage3/bin/zig build -p only-c -Dno-lib -Donly-c diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 4737e3eefa..49d6b9ec18 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -30,10 +30,9 @@ const BigInt = std.math.big.int; pub const CValue = union(enum) { none: void, - /// Index into local_names - local: usize, - /// Index into local_names, but take the address. - local_ref: usize, + local: LocalIndex, + /// Address of a local. + local_ref: LocalIndex, /// A constant instruction, to be rendered inline. constant: Air.Inst.Ref, /// Index into the parameters @@ -70,6 +69,15 @@ pub const TypedefMap = std.ArrayHashMap( true, ); +const Local = struct { + ty: Type, + alignment: u32, +}; + +const LocalIndex = u16; +const LocalsList = std.ArrayListUnmanaged(LocalIndex); +const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true); + const FormatTypeAsCIdentContext = struct { ty: Type, mod: *Module, @@ -251,10 +259,23 @@ pub const Function = struct { value_map: CValueMap, blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{}, next_arg_index: usize = 0, - next_local_index: usize = 0, next_block_index: usize = 0, object: Object, func: *Module.Fn, + /// All the locals, to be emitted at the top of the function. + locals: std.ArrayListUnmanaged(Local) = .{}, + /// Which locals are available for reuse, based on Type. + free_locals: LocalsMap = .{}, + /// Locals which will not be freed by Liveness. This is used after a + /// Function body is lowered in order to make `free_locals` have 100% of + /// the locals within so that it can be used to render the block of + /// variable declarations at the top of a function, sorted descending by + /// type alignment. + allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, void) = .{}, + + fn tyHashCtx(f: Function) Type.HashContext32 { + return .{ .mod = f.object.dg.module }; + } fn resolveInst(f: *Function, inst: Air.Inst.Ref) !CValue { const gop = try f.value_map.getOrPut(inst); @@ -265,9 +286,10 @@ pub const Function = struct { const result = if (lowersToArray(ty, f.object.dg.module.getTarget())) result: { const writer = f.object.code_header.writer(); - const decl_c_value = f.allocLocalValue(); + const alignment = 0; + const decl_c_value = try f.allocLocalValue(ty, alignment); try writer.writeAll("static "); - try f.object.dg.renderTypeAndName(writer, ty, decl_c_value, .Const, 0, .Complete); + try f.object.dg.renderTypeAndName(writer, ty, decl_c_value, .Const, alignment, .Complete); try writer.writeAll(" = "); try f.object.dg.renderValue(writer, ty, val, .Initializer); try writer.writeAll(";\n "); @@ -285,27 +307,37 @@ pub const Function = struct { }; } - fn allocLocalValue(f: *Function) CValue { - const result = f.next_local_index; - f.next_local_index += 1; - return .{ .local = result }; + /// Skips the reuse logic. + fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue { + const gpa = f.object.dg.gpa; + try f.locals.append(gpa, .{ + .ty = ty, + .alignment = alignment, + }); + return .{ .local = @intCast(LocalIndex, f.locals.items.len - 1) }; } - fn allocLocal(f: *Function, ty: Type, mutability: Mutability) !CValue { - return f.allocAlignedLocal(ty, mutability, 0); + fn allocLocal(f: *Function, inst: Air.Inst.Index, ty: Type) !CValue { + const result = try f.allocAlignedLocal(ty, .Mut, 0); + log.debug("%{d}: allocating t{d}", .{ inst, result.local }); + return result; } + /// Only allocates the local; does not print anything. fn allocAlignedLocal(f: *Function, ty: Type, mutability: Mutability, alignment: u32) !CValue { - const local_value = f.allocLocalValue(); - try f.object.dg.renderTypeAndName( - f.object.writer(), - ty, - local_value, - mutability, - alignment, - .Complete, - ); - return local_value; + _ = mutability; + + if (f.free_locals.getPtrContext(ty, f.tyHashCtx())) |locals_list| { + for (locals_list.items) |local_index, i| { + const local = f.locals.items[local_index]; + if (local.alignment >= alignment) { + _ = locals_list.swapRemove(i); + return CValue{ .local = local_index }; + } + } + } + + return try f.allocLocalValue(ty, alignment); } fn writeCValue(f: *Function, w: anytype, c_value: CValue, location: ValueRenderLocation) !void { @@ -375,6 +407,20 @@ pub const Function = struct { fn fmtIntLiteral(f: *Function, ty: Type, val: Value) !std.fmt.Formatter(formatIntLiteral) { return f.object.dg.fmtIntLiteral(ty, val); } + + pub fn deinit(f: *Function, gpa: mem.Allocator) void { + f.allocs.deinit(gpa); + f.locals.deinit(gpa); + f.free_locals.deinit(gpa); + f.blocks.deinit(gpa); + f.value_map.deinit(); + f.object.code.deinit(); + for (f.object.dg.typedefs.values()) |typedef| { + gpa.free(typedef.rendered); + } + f.object.dg.typedefs.deinit(); + f.object.dg.fwd_decl.deinit(); + } }; /// This data is available when outputting .c code for a `Module`. @@ -2400,12 +2446,13 @@ pub fn genFunc(f: *Function) !void { defer tracy.end(); const o = &f.object; + const gpa = o.dg.gpa; const tv: TypedValue = .{ .ty = o.dg.decl.ty, .val = o.dg.decl.val, }; - o.code_header = std.ArrayList(u8).init(f.object.dg.gpa); + o.code_header = std.ArrayList(u8).init(gpa); defer o.code_header.deinit(); const is_global = o.dg.declIsGlobal(tv); @@ -2432,6 +2479,50 @@ pub fn genFunc(f: *Function) !void { try o.indent_writer.insertNewline(); + // Take advantage of the free_locals map to bucket locals per type. All + // locals corresponding to AIR instructions should be in there due to + // Liveness analysis, however, locals from alloc instructions will be + // missing. These are added now to complete the map. Then we can sort by + // alignment, descending. + for (f.allocs.keys()) |local_index| { + const local = f.locals.items[local_index]; + log.debug("inserting local {d} into free_locals", .{local_index}); + const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); + if (!gop.found_existing) { + gop.value_ptr.* = .{}; + } + try gop.value_ptr.append(gpa, local_index); + } + + const SortContext = struct { + target: std.Target, + keys: []const Type, + + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + const a_ty = ctx.keys[a_index]; + const b_ty = ctx.keys[b_index]; + return b_ty.abiAlignment(ctx.target) < a_ty.abiAlignment(ctx.target); + } + }; + const target = o.dg.module.getTarget(); + f.free_locals.sort(SortContext{ .target = target, .keys = f.free_locals.keys() }); + + const w = o.code_header.writer(); + for (f.free_locals.values()) |list| { + for (list.items) |local_index| { + const local = f.locals.items[local_index]; + try o.dg.renderTypeAndName( + w, + local.ty, + .{ .local = local_index }, + .Mut, + local.alignment, + .Complete, + ); + try w.writeAll(";\n "); + } + } + // If we have a header to insert, append the body to the header // and then return the result, freeing the body. if (o.code_header.items.len > empty_header_len) { @@ -2585,20 +2676,19 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .mul_sat => try airBinBuiltinCall(f, inst, "muls", .Bits), .shl_sat => try airBinBuiltinCall(f, inst, "shls", .Bits), - .sqrt, - .sin, - .cos, - .tan, - .exp, - .exp2, - .log, - .log2, - .log10, - .fabs, - .floor, - .ceil, - .round, - => |tag| try airUnFloatOp(f, inst, @tagName(tag)), + .sqrt => try airUnFloatOp(f, inst, "sqrt"), + .sin => try airUnFloatOp(f, inst, "sin"), + .cos => try airUnFloatOp(f, inst, "cos"), + .tan => try airUnFloatOp(f, inst, "tan"), + .exp => try airUnFloatOp(f, inst, "exp"), + .exp2 => try airUnFloatOp(f, inst, "exp2"), + .log => try airUnFloatOp(f, inst, "log"), + .log2 => try airUnFloatOp(f, inst, "log2"), + .log10 => try airUnFloatOp(f, inst, "log10"), + .fabs => try airUnFloatOp(f, inst, "fabs"), + .floor => try airUnFloatOp(f, inst, "floor"), + .ceil => try airUnFloatOp(f, inst, "ceil"), + .round => try airUnFloatOp(f, inst, "round"), .trunc_float => try airUnFloatOp(f, inst, "trunc"), .mul_add => try airMulAdd(f, inst), @@ -2786,6 +2876,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, .error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}), // zig fmt: on }; + if (result_value == .local) { + log.debug("map %{d} to t{d}", .{ inst, result_value.local }); + } switch (result_value) { .none => {}, else => try f.value_map.putNoClobber(Air.indexToRef(inst), result_value), @@ -2794,13 +2887,19 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, } fn airSliceField(f: *Function, inst: Air.Inst.Index, is_ptr: bool, field_name: []const u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (is_ptr) { try writer.writeByte('&'); @@ -2815,22 +2914,29 @@ fn airPtrElemVal(f: *Function, inst: Air.Inst.Index) !CValue { const bin_op = f.air.instructions.items(.data)[inst].bin_op; const ptr_ty = f.air.typeOf(bin_op.lhs); if ((!ptr_ty.isVolatilePtr() and f.liveness.isUnused(inst)) or - !inst_ty.hasRuntimeBitsIgnoreComptime()) return CValue.none; + !inst_ty.hasRuntimeBitsIgnoreComptime()) + { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } const ptr = try f.resolveInst(bin_op.lhs); const index = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(inst_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); + const local = try f.allocLocal(inst, inst_ty); const writer = f.object.writer(); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - } else try writer.writeAll(" = "); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + } try f.writeCValue(writer, ptr, .Other); try writer.writeByte('['); try f.writeCValue(writer, index, .Other); @@ -2845,19 +2951,28 @@ fn airPtrElemVal(f: *Function, inst: Air.Inst.Index) !CValue { } fn airPtrElemPtr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const bin_op = f.air.extraData(Air.Bin, ty_pl.payload).data; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const ptr_ty = f.air.typeOf(bin_op.lhs); const child_ty = ptr_ty.childType(); const ptr = try f.resolveInst(bin_op.lhs); - if (!child_ty.hasRuntimeBitsIgnoreComptime()) return ptr; + if (!child_ty.hasRuntimeBitsIgnoreComptime()) { + if (f.liveness.operandDies(inst, 1)) try die(f, inst, bin_op.rhs); + return ptr; + } const index = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); - const local = try f.allocLocal(f.air.typeOfIndex(inst), .Const); + const local = try f.allocLocal(inst, f.air.typeOfIndex(inst)); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = &("); if (ptr_ty.ptrSize() == .One) { // It's a pointer to an array, so we need to de-reference. @@ -2876,22 +2991,29 @@ fn airSliceElemVal(f: *Function, inst: Air.Inst.Index) !CValue { const bin_op = f.air.instructions.items(.data)[inst].bin_op; const slice_ty = f.air.typeOf(bin_op.lhs); if ((!slice_ty.isVolatilePtr() and f.liveness.isUnused(inst)) or - !inst_ty.hasRuntimeBitsIgnoreComptime()) return CValue.none; + !inst_ty.hasRuntimeBitsIgnoreComptime()) + { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } const slice = try f.resolveInst(bin_op.lhs); const index = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(inst_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); + const local = try f.allocLocal(inst, inst_ty); const writer = f.object.writer(); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - } else try writer.writeAll(" = "); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + } try f.writeCValue(writer, slice, .Other); try writer.writeAll(".ptr["); try f.writeCValue(writer, index, .Other); @@ -2906,23 +3028,28 @@ fn airSliceElemVal(f: *Function, inst: Air.Inst.Index) !CValue { } fn airSliceElemPtr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const bin_op = f.air.extraData(Air.Bin, ty_pl.payload).data; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const slice_ty = f.air.typeOf(bin_op.lhs); const child_ty = slice_ty.elemType2(); const slice = try f.resolveInst(bin_op.lhs); + const index = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); - const local = try f.allocLocal(f.air.typeOfIndex(inst), .Const); + const local = try f.allocLocal(inst, f.air.typeOfIndex(inst)); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (child_ty.hasRuntimeBitsIgnoreComptime()) try writer.writeByte('&'); try f.writeCValue(writer, slice, .Other); try writer.writeAll(".ptr"); if (child_ty.hasRuntimeBitsIgnoreComptime()) { - const index = try f.resolveInst(bin_op.rhs); try writer.writeByte('['); try f.writeCValue(writer, index, .Other); try writer.writeByte(']'); @@ -2932,24 +3059,30 @@ fn airSliceElemPtr(f: *Function, inst: Air.Inst.Index) !CValue { } fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue { + const bin_op = f.air.instructions.items(.data)[inst].bin_op; const inst_ty = f.air.typeOfIndex(inst); - if (f.liveness.isUnused(inst) or !inst_ty.hasRuntimeBitsIgnoreComptime()) return CValue.none; + if (f.liveness.isUnused(inst) or !inst_ty.hasRuntimeBitsIgnoreComptime()) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } - const bin_op = f.air.instructions.items(.data)[inst].bin_op; const array = try f.resolveInst(bin_op.lhs); const index = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(inst_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); + const local = try f.allocLocal(inst, inst_ty); const writer = f.object.writer(); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - } else try writer.writeAll(" = "); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + } try f.writeCValue(writer, array, .Other); try writer.writeByte('['); try f.writeCValue(writer, index, .Other); @@ -2964,36 +3097,36 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue { } fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue { - const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const elem_type = inst_ty.elemType(); - const mutability: Mutability = if (inst_ty.isConstPtr()) .Const else .Mut; if (!elem_type.isFnOrHasRuntimeBitsIgnoreComptime()) { return CValue{ .undef = inst_ty }; } + const mutability: Mutability = if (inst_ty.isConstPtr()) .Const else .Mut; const target = f.object.dg.module.getTarget(); - // First line: the variable used as data storage. const local = try f.allocAlignedLocal(elem_type, mutability, inst_ty.ptrAlignment(target)); - try writer.writeAll(";\n"); - + log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.local }); + const gpa = f.object.dg.module.gpa; + try f.allocs.put(gpa, local.local, {}); return CValue{ .local_ref = local.local }; } fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue { - const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const elem_ty = inst_ty.elemType(); - if (!elem_ty.hasRuntimeBitsIgnoreComptime()) { + if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime()) { return CValue{ .undef = inst_ty }; } - // First line: the variable used as data storage. - const local = try f.allocLocal(elem_ty, .Mut); - try writer.writeAll(";\n"); - + const mutability: Mutability = if (inst_ty.isConstPtr()) .Const else .Mut; + const target = f.object.dg.module.getTarget(); + const local = try f.allocAlignedLocal(elem_ty, mutability, inst_ty.ptrAlignment(target)); + log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.local }); + const gpa = f.object.dg.module.gpa; + try f.allocs.put(gpa, local.local, {}); return CValue{ .local_ref = local.local }; } @@ -3009,21 +3142,25 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { const src_ty = ptr_info.pointee_type; if (!src_ty.hasRuntimeBitsIgnoreComptime() or - !ptr_info.@"volatile" and f.liveness.isUnused(inst)) + (!ptr_info.@"volatile" and f.liveness.isUnused(inst))) + { + try reap(f, inst, &.{ty_op.operand}); return CValue.none; + } + + const operand = try f.resolveInst(ty_op.operand); + + try reap(f, inst, &.{ty_op.operand}); const target = f.object.dg.module.getTarget(); const is_aligned = ptr_info.@"align" == 0 or ptr_info.@"align" >= src_ty.abiAlignment(target); const is_array = lowersToArray(src_ty, target); const need_memcpy = !is_aligned or is_array; - const operand = try f.resolveInst(ty_op.operand); const writer = f.object.writer(); - // We need to initialize arrays and unaligned loads with a memcpy so they must be mutable. - const local = try f.allocLocal(src_ty, if (need_memcpy) .Mut else .Const); + const local = try f.allocLocal(inst, src_ty); if (need_memcpy) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); if (!is_array) try writer.writeByte('&'); try f.writeCValue(writer, local, .FunctionArgument); @@ -3057,6 +3194,7 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { }; const field_ty = Type.initPayload(&field_pl.base); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, src_ty); try writer.writeAll(")zig_wrap_"); @@ -3071,6 +3209,7 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { try f.object.dg.renderBuiltinInfo(writer, field_ty, .Bits); try writer.writeByte(')'); } else { + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValueDeref(writer, operand); } @@ -3090,9 +3229,9 @@ fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue { if (lowered_ret_ty.hasRuntimeBitsIgnoreComptime()) { var deref = is_ptr; const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const ret_val = if (lowersToArray(ret_ty, target)) ret_val: { - const array_local = try f.allocLocal(lowered_ret_ty, .Mut); - try writer.writeAll(";\n"); + const array_local = try f.allocLocal(inst, lowered_ret_ty); try writer.writeAll("memcpy("); try f.writeCValueMember(writer, array_local, .{ .field = 0 }); try writer.writeAll(", "); @@ -3113,23 +3252,30 @@ fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue { else try f.writeCValue(writer, ret_val, .Other); try writer.writeAll(";\n"); - } else if (f.object.dg.decl.ty.fnCallingConvention() != .Naked) { - // Not even allowed to return void in a naked function. - try writer.writeAll("return;\n"); + } else { + try reap(f, inst, &.{un_op}); + if (f.object.dg.decl.ty.fnCallingConvention() != .Naked) { + // Not even allowed to return void in a naked function. + try writer.writeAll("return;\n"); + } } return CValue.none; } fn airIntCast(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); return CValue.none; + } - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); - + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, inst_ty); try writer.writeByte(')'); @@ -3139,17 +3285,22 @@ fn airIntCast(f: *Function, inst: Air.Inst.Index) !CValue { } fn airTrunc(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const writer = f.object.writer(); - const operand = try f.resolveInst(ty_op.operand); + const local = try f.allocLocal(inst, inst_ty); const target = f.object.dg.module.getTarget(); const dest_int_info = inst_ty.intInfo(target); const dest_bits = dest_int_info.bits; + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, inst_ty); try writer.writeByte(')'); @@ -3191,20 +3342,24 @@ fn airTrunc(f: *Function, inst: Air.Inst.Index) !CValue { } fn airBoolToInt(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) - return CValue.none; const un_op = f.air.instructions.items(.data)[inst].un_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const operand = try f.resolveInst(un_op); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Other); try writer.writeAll(";\n"); return local; } -fn airStoreUndefined(f: *Function, lhs_child_ty: Type, dest_ptr: CValue) !CValue { +fn storeUndefined(f: *Function, lhs_child_ty: Type, dest_ptr: CValue) !CValue { if (f.wantSafety()) { const writer = f.object.writer(); try writer.writeAll("memset("); @@ -3220,18 +3375,23 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue { // *a = b; const bin_op = f.air.instructions.items(.data)[inst].bin_op; const ptr_info = f.air.typeOf(bin_op.lhs).ptrInfo().data; - if (!ptr_info.pointee_type.hasRuntimeBitsIgnoreComptime()) return CValue.none; + if (!ptr_info.pointee_type.hasRuntimeBitsIgnoreComptime()) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } const ptr_val = try f.resolveInst(bin_op.lhs); const src_ty = f.air.typeOf(bin_op.rhs); const src_val = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + // TODO Sema should emit a different instruction when the store should // possibly do the safety 0xaa bytes for undefined. const src_val_is_undefined = if (f.air.value(bin_op.rhs)) |v| v.isUndefDeep() else false; if (src_val_is_undefined) - return try airStoreUndefined(f, ptr_info.pointee_type, ptr_val); + return try storeUndefined(f, ptr_info.pointee_type, ptr_val); const target = f.object.dg.module.getTarget(); const is_aligned = ptr_info.@"align" == 0 or @@ -3249,7 +3409,8 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue { // so work around this by initializing into new local. // TODO this should be done by manually initializing elements of the dest array const array_src = if (src_val == .constant) blk: { - const new_local = try f.allocLocal(src_ty, .Const); + const new_local = try f.allocLocal(inst, src_ty); + try f.writeCValue(writer, new_local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, src_val, .Initializer); try writer.writeAll(";\n"); @@ -3265,6 +3426,9 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(", sizeof("); try f.renderTypecast(writer, src_ty); try writer.writeAll("))"); + if (src_val == .constant) { + try freeLocal(f, inst, array_src.local, 0); + } } else if (ptr_info.host_size != 0) { const host_bits = ptr_info.host_size * 8; var host_pl = Type.Payload.Bits{ .base = .{ .tag = .int_unsigned }, .data = host_bits }; @@ -3330,22 +3494,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index) !CValue { } fn airOverflow(f: *Function, inst: Air.Inst.Index, operation: []const u8, info: BuiltinInfo) !CValue { - if (f.liveness.isUnused(inst)) - return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const bin_op = f.air.extraData(Air.Bin, ty_pl.payload).data; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const inst_ty = f.air.typeOfIndex(inst); const vector_ty = f.air.typeOf(bin_op.lhs); const scalar_ty = vector_ty.scalarType(); const w = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Mut); - try w.writeAll(";\n"); + const local = try f.allocLocal(inst, inst_ty); switch (vector_ty.zigTypeTag()) { .Vector => { @@ -3381,15 +3547,20 @@ fn airOverflow(f: *Function, inst: Air.Inst.Index, operation: []const u8, info: } fn airNot(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + const op = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try writer.writeByte(if (inst_ty.tag() == .bool) '!' else '~'); try f.writeCValue(writer, op, .Other); @@ -3405,22 +3576,24 @@ fn airBinOp( operation: []const u8, info: BuiltinInfo, ) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const bin_op = f.air.instructions.items(.data)[inst].bin_op; - const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if ((operand_ty.isInt() and operand_ty.bitSize(target) > 64) or operand_ty.isRuntimeFloat()) return try airBinBuiltinCall(f, inst, operation, info); - const inst_ty = f.air.typeOfIndex(inst); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); - const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + + if (f.liveness.isUnused(inst)) return CValue.none; + + const inst_ty = f.air.typeOfIndex(inst); + const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, lhs, .Other); try writer.writeByte(' '); @@ -3433,24 +3606,28 @@ fn airBinOp( } fn airCmpOp(f: *Function, inst: Air.Inst.Index, operator: []const u8, operation: []const u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const bin_op = f.air.instructions.items(.data)[inst].bin_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if (operand_ty.isInt() and operand_ty.bitSize(target) > 64) - return try airCmpBuiltinCall(f, inst, operator, "cmp"); + return try cmpBuiltinCall(f, inst, operator, "cmp"); if (operand_ty.isRuntimeFloat()) - return try airCmpBuiltinCall(f, inst, operator, operation); + return try cmpBuiltinCall(f, inst, operator, operation); const inst_ty = f.air.typeOfIndex(inst); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); - + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, lhs, .Other); try writer.writeByte(' '); @@ -3469,24 +3646,28 @@ fn airEquality( operator: []const u8, operation: []const u8, ) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const bin_op = f.air.instructions.items(.data)[inst].bin_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const operand_ty = f.air.typeOf(bin_op.lhs); const target = f.object.dg.module.getTarget(); if (operand_ty.isInt() and operand_ty.bitSize(target) > 64) - return try airCmpBuiltinCall(f, inst, operator, "cmp"); + return try cmpBuiltinCall(f, inst, operator, "cmp"); if (operand_ty.isRuntimeFloat()) - return try airCmpBuiltinCall(f, inst, operator, operation); + return try cmpBuiltinCall(f, inst, operator, operation); const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (operand_ty.zigTypeTag() == .Optional and !operand_ty.isPtrLikeOptional()) { @@ -3521,14 +3702,20 @@ fn airEquality( } fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Other); try writer.print(" < sizeof({ }) / sizeof(*{0 });\n", .{fmtIdent("zig_errorName")}); @@ -3536,16 +3723,18 @@ fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { } fn airPtrAddSub(f: *Function, inst: Air.Inst.Index, operator: u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const bin_op = f.air.extraData(Air.Bin, ty_pl.payload).data; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); - const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); const elem_ty = switch (inst_ty.ptrSize()) { .One => blk: { const array_ty = inst_ty.childType(); @@ -3554,8 +3743,12 @@ fn airPtrAddSub(f: *Function, inst: Air.Inst.Index, operator: u8) !CValue { else => inst_ty.childType(), }; - // We must convert to and from integer types to prevent UB if the operation results in a NULL pointer, - // or if LHS is NULL. The operation is only UB if the result is NULL and then dereferenced. + // We must convert to and from integer types to prevent UB if the operation + // results in a NULL pointer, or if LHS is NULL. The operation is only UB + // if the result is NULL and then dereferenced. + const local = try f.allocLocal(inst, inst_ty); + const writer = f.object.writer(); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, inst_ty); try writer.writeAll(")(((uintptr_t)"); @@ -3572,10 +3765,13 @@ fn airPtrAddSub(f: *Function, inst: Air.Inst.Index, operator: u8) !CValue { } fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []const u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const bin_op = f.air.instructions.items(.data)[inst].bin_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const inst_ty = f.air.typeOfIndex(inst); const target = f.object.dg.module.getTarget(); if (inst_ty.isInt() and inst_ty.bitSize(target) > 64) @@ -3585,10 +3781,11 @@ fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []cons const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); - + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); // (lhs <> rhs) ? lhs : rhs try writer.writeAll(" = ("); try f.writeCValue(writer, lhs, .Other); @@ -3606,17 +3803,22 @@ fn airMinMax(f: *Function, inst: Air.Inst.Index, operator: u8, operation: []cons } fn airSlice(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const bin_op = f.air.extraData(Air.Bin, ty_pl.payload).data; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + const ptr = try f.resolveInst(bin_op.lhs); const len = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = {("); var buf: Type.SlicePtrFieldTypeBuffer = undefined; try f.renderTypecast(writer, inst_ty.slicePtrFieldType(&buf)); @@ -3634,8 +3836,7 @@ fn airCall( inst: Air.Inst.Index, modifier: std.builtin.CallOptions.Modifier, ) !CValue { - // Not even allowed to call panic in a naked function. - if (f.object.dg.decl.ty.fnCallingConvention() == .Naked) return .none; + const gpa = f.object.dg.gpa; switch (modifier) { .auto => {}, @@ -3647,6 +3848,21 @@ fn airCall( const pl_op = f.air.instructions.items(.data)[inst].pl_op; const extra = f.air.extraData(Air.Call, pl_op.payload); const args = @ptrCast([]const Air.Inst.Ref, f.air.extra[extra.end..][0..extra.data.args_len]); + + const resolved_args = try gpa.alloc(CValue, args.len); + defer gpa.free(resolved_args); + for (args) |arg, i| { + resolved_args[i] = try f.resolveInst(arg); + } + + const callee = try f.resolveInst(pl_op.operand); + + { + var bt = iterateBigTomb(f, inst); + try bt.feed(pl_op.operand); + for (args) |arg| try bt.feed(arg); + } + const callee_ty = f.air.typeOf(pl_op.operand); const fn_ty = switch (callee_ty.zigTypeTag()) { .Fn => callee_ty, @@ -3668,7 +3884,8 @@ fn airCall( try writer.writeByte(')'); break :r .none; } else r: { - const local = try f.allocLocal(lowered_ret_ty, .Const); + const local = try f.allocLocal(inst, lowered_ret_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); break :r local; }; @@ -3694,13 +3911,12 @@ fn airCall( break :callee; } // Fall back to function pointer call. - const callee = try f.resolveInst(pl_op.operand); try f.writeCValue(writer, callee, .Other); } try writer.writeByte('('); var args_written: usize = 0; - for (args) |arg| { + for (args) |arg, arg_i| { const ty = f.air.typeOf(arg); if (!ty.hasRuntimeBitsIgnoreComptime()) continue; if (args_written != 0) { @@ -3715,23 +3931,28 @@ fn airCall( if (ty.isVolatilePtr()) try writer.writeAll(" volatile"); try writer.writeAll(" *)"); } - try f.writeCValue(writer, try f.resolveInst(arg), .FunctionArgument); + try f.writeCValue(writer, resolved_args[arg_i], .FunctionArgument); args_written += 1; } try writer.writeAll(");\n"); - if (result_local == .none or !lowersToArray(ret_ty, target)) return result_local; + const result = r: { + if (result_local == .none or !lowersToArray(ret_ty, target)) + break :r result_local; - const array_local = try f.allocLocal(ret_ty, .Mut); - try writer.writeAll(";\n"); - try writer.writeAll("memcpy("); - try f.writeCValue(writer, array_local, .FunctionArgument); - try writer.writeAll(", "); - try f.writeCValueMember(writer, result_local, .{ .field = 0 }); - try writer.writeAll(", sizeof("); - try f.renderTypecast(writer, ret_ty); - try writer.writeAll("));\n"); - return array_local; + const array_local = try f.allocLocal(inst, ret_ty); + try writer.writeAll("memcpy("); + try f.writeCValue(writer, array_local, .FunctionArgument); + try writer.writeAll(", "); + try f.writeCValueMember(writer, result_local, .{ .field = 0 }); + try writer.writeAll(", sizeof("); + try f.renderTypecast(writer, ret_ty); + try writer.writeAll("));\n"); + try freeLocal(f, inst, result_local.local, 0); + break :r array_local; + }; + + return result; } fn airDbgStmt(f: *Function, inst: Air.Inst.Index) !CValue { @@ -3763,6 +3984,7 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue { const name = f.air.nullTerminatedString(pl_op.payload); const operand = try f.resolveInst(pl_op.operand); _ = operand; + try reap(f, inst, &.{pl_op.operand}); const writer = f.object.writer(); try writer.print("/* var:{s} */\n", .{name}); return CValue.none; @@ -3778,12 +4000,10 @@ fn airBlock(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const result = if (inst_ty.tag() != .void and !f.liveness.isUnused(inst)) blk: { - // allocate a location for the result - const local = try f.allocLocal(inst_ty, .Mut); - try writer.writeAll(";\n"); - break :blk local; - } else CValue{ .none = {} }; + const result = if (inst_ty.tag() != .void and !f.liveness.isUnused(inst)) + try f.allocLocal(inst, inst_ty) + else + CValue{ .none = {} }; try f.blocks.putNoClobber(f.object.dg.gpa, inst, .{ .block_id = block_id, @@ -3799,32 +4019,30 @@ fn airBlock(f: *Function, inst: Air.Inst.Index) !CValue { fn airTry(f: *Function, inst: Air.Inst.Index) !CValue { const pl_op = f.air.instructions.items(.data)[inst].pl_op; - const err_union = try f.resolveInst(pl_op.operand); const extra = f.air.extraData(Air.Try, pl_op.payload); const body = f.air.extra[extra.end..][0..extra.data.body_len]; const err_union_ty = f.air.typeOf(pl_op.operand); - const result_ty = f.air.typeOfIndex(inst); - return lowerTry(f, err_union, body, err_union_ty, false, result_ty); + return lowerTry(f, inst, pl_op.operand, body, err_union_ty, false); } fn airTryPtr(f: *Function, inst: Air.Inst.Index) !CValue { const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const extra = f.air.extraData(Air.TryPtr, ty_pl.payload); - const err_union_ptr = try f.resolveInst(extra.data.ptr); const body = f.air.extra[extra.end..][0..extra.data.body_len]; const err_union_ty = f.air.typeOf(extra.data.ptr).childType(); - const result_ty = f.air.typeOfIndex(inst); - return lowerTry(f, err_union_ptr, body, err_union_ty, true, result_ty); + return lowerTry(f, inst, extra.data.ptr, body, err_union_ty, true); } fn lowerTry( f: *Function, - err_union: CValue, + inst: Air.Inst.Index, + operand: Air.Inst.Ref, body: []const Air.Inst.Index, err_union_ty: Type, operand_is_ptr: bool, - result_ty: Type, ) !CValue { + const err_union = try f.resolveInst(operand); + const result_ty = f.air.typeOfIndex(inst); const writer = f.object.writer(); const payload_ty = err_union_ty.errorUnionPayload(); const payload_has_bits = payload_ty.hasRuntimeBitsIgnoreComptime(); @@ -3837,6 +4055,10 @@ fn lowerTry( else try f.writeCValue(writer, err_union, .Other); } else { + // Reap the operand so that it can be reused inside genBody. + // Remember we must avoid calling reap() twice for the same operand + // in this function. + try reap(f, inst, &.{operand}); if (operand_is_ptr or isByRef(err_union_ty)) try f.writeCValueDerefMember(writer, err_union, .{ .identifier = "error" }) else @@ -3858,7 +4080,9 @@ fn lowerTry( const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(payload_ty, target); - const local = try f.allocLocal(result_ty, if (is_array) .Mut else .Const); + try reap(f, inst, &.{operand}); + const local = try f.allocLocal(inst, result_ty); + try f.writeCValue(writer, local, .Other); if (is_array) { try writer.writeAll(";\n"); try writer.writeAll("memcpy("); @@ -3888,6 +4112,7 @@ fn airBr(f: *Function, inst: Air.Inst.Index) !CValue { // If result is .none then the value of the block is unused. if (result != .none) { const operand = try f.resolveInst(branch.operand); + try reap(f, inst, &.{branch.operand}); const operand_ty = f.air.typeOf(branch.operand); const target = f.object.dg.module.getTarget(); @@ -3912,40 +4137,50 @@ fn airBr(f: *Function, inst: Air.Inst.Index) !CValue { } fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue { - const src_ty = f.air.typeOfIndex(inst); + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + const dest_ty = f.air.typeOfIndex(inst); // No IgnoreComptime until Sema stops giving us garbage Air. // https://github.com/ziglang/zig/issues/13410 - if (f.liveness.isUnused(inst) or !src_ty.hasRuntimeBits()) return CValue.none; + if (f.liveness.isUnused(inst) or !dest_ty.hasRuntimeBits()) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); - const dest_ty = f.air.typeOf(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); + const operand_ty = f.air.typeOf(ty_op.operand); const target = f.object.dg.module.getTarget(); + const writer = f.object.writer(); + + const local = try f.allocLocal(inst, dest_ty); - if (dest_ty.isAbiInt() and src_ty.isAbiInt()) { - const src_info = src_ty.intInfo(target); - const dest_info = dest_ty.intInfo(target); - if (std.meta.eql(src_info, dest_info)) { - return operand; + if (operand_ty.isAbiInt() and dest_ty.isAbiInt()) { + const src_info = dest_ty.intInfo(target); + const dest_info = operand_ty.intInfo(target); + if (src_info.signedness == dest_info.signedness and + src_info.bits == dest_info.bits) + { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); + return local; } } - const writer = f.object.writer(); - if (src_ty.isPtrAtRuntime() and dest_ty.isPtrAtRuntime()) { - const local = try f.allocLocal(src_ty, .Const); + if (dest_ty.isPtrAtRuntime() and operand_ty.isPtrAtRuntime()) { + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); - try f.renderTypecast(writer, src_ty); + try f.renderTypecast(writer, dest_ty); try writer.writeByte(')'); try f.writeCValue(writer, operand, .Other); try writer.writeAll(";\n"); return local; } - const local = try f.allocLocal(src_ty, .Mut); - try writer.writeAll(";\n"); - const operand_lval = if (operand == .constant) blk: { - const operand_local = try f.allocLocal(dest_ty, .Const); + const operand_local = try f.allocLocal(inst, operand_ty); + try f.writeCValue(writer, operand_local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Initializer); try writer.writeAll(";\n"); @@ -3957,20 +4192,24 @@ fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(", &"); try f.writeCValue(writer, operand_lval, .Other); try writer.writeAll(", sizeof("); - try f.renderTypecast(writer, src_ty); + try f.renderTypecast(writer, dest_ty); try writer.writeAll("));\n"); // Ensure padding bits have the expected value. - if (src_ty.isAbiInt()) { + if (dest_ty.isAbiInt()) { try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_wrap_"); - try f.object.dg.renderTypeForBuiltinFnName(writer, src_ty); + try f.object.dg.renderTypeForBuiltinFnName(writer, dest_ty); try writer.writeByte('('); try f.writeCValue(writer, local, .Other); - try f.object.dg.renderBuiltinInfo(writer, src_ty, .Bits); + try f.object.dg.renderBuiltinInfo(writer, dest_ty, .Bits); try writer.writeAll(");\n"); } + if (operand == .constant) { + try freeLocal(f, inst, operand_lval.local, 0); + } + return local; } @@ -3982,7 +4221,8 @@ fn airBreakpoint(writer: anytype) !CValue { fn airRetAddr(f: *Function, inst: Air.Inst.Index) !CValue { if (f.liveness.isUnused(inst)) return CValue.none; const writer = f.object.writer(); - const local = try f.allocLocal(Type.usize, .Const); + const local = try f.allocLocal(inst, Type.usize); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, Type.usize); try writer.writeAll(")zig_return_address();\n"); @@ -3992,7 +4232,8 @@ fn airRetAddr(f: *Function, inst: Air.Inst.Index) !CValue { fn airFrameAddress(f: *Function, inst: Air.Inst.Index) !CValue { if (f.liveness.isUnused(inst)) return CValue.none; const writer = f.object.writer(); - const local = try f.allocLocal(Type.usize, .Const); + const local = try f.allocLocal(inst, Type.usize); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, Type.usize); try writer.writeAll(")zig_frame_address();\n"); @@ -4023,9 +4264,7 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue { const loop = f.air.extraData(Air.Block, ty_pl.payload); const body = f.air.extra[loop.end..][0..loop.data.body_len]; const writer = f.object.writer(); - try writer.writeAll("while ("); - try f.object.dg.renderValue(writer, Type.bool, Value.true, .Other); - try writer.writeAll(") "); + try writer.writeAll("for (;;) "); try genBody(f, body); try writer.writeByte('\n'); return CValue.none; @@ -4034,16 +4273,29 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue { fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { const pl_op = f.air.instructions.items(.data)[inst].pl_op; const cond = try f.resolveInst(pl_op.operand); + try reap(f, inst, &.{pl_op.operand}); const extra = f.air.extraData(Air.CondBr, pl_op.payload); const then_body = f.air.extra[extra.end..][0..extra.data.then_body_len]; const else_body = f.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]; const writer = f.object.writer(); + // Keep using the original for the then branch; use a clone of the value + // map for the else branch. + const gpa = f.object.dg.gpa; + var cloned_map = try f.value_map.clone(); + defer cloned_map.deinit(); + var cloned_frees = try f.free_locals.clone(gpa); + defer cloned_frees.deinit(gpa); + try writer.writeAll("if ("); try f.writeCValue(writer, cond, .Other); try writer.writeAll(") "); try genBody(f, then_body); try writer.writeAll(" else "); + f.value_map.deinit(); + f.value_map = cloned_map.move(); + f.free_locals.deinit(gpa); + f.free_locals = cloned_frees.move(); try genBody(f, else_body); try f.object.indent_writer.insertNewline(); @@ -4053,6 +4305,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { const pl_op = f.air.instructions.items(.data)[inst].pl_op; const condition = try f.resolveInst(pl_op.operand); + try reap(f, inst, &.{pl_op.operand}); const condition_ty = f.air.typeOf(pl_op.operand); const switch_br = f.air.extraData(Air.SwitchBr, pl_op.payload); const writer = f.object.writer(); @@ -4071,6 +4324,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(") {"); f.object.indent_writer.pushIndent(); + const gpa = f.object.dg.gpa; var extra_index: usize = switch_br.end; var case_i: u32 = 0; while (case_i < switch_br.data.cases_len) : (case_i += 1) { @@ -4090,8 +4344,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { try f.object.dg.renderValue(writer, condition_ty, f.air.value(item).?, .Other); try writer.writeAll(": "); } + // On the final iteration we do not clone the map. This ensures that + // lowering proceeds after the switch_br taking into account the + // mutations to the liveness information. // The case body must be noreturn so we don't need to insert a break. - try genBody(f, case_body); + if (case_i < switch_br.data.cases_len - 1) { + const old_value_map = f.value_map; + f.value_map = try old_value_map.clone(); + const old_free_locals = f.free_locals; + f.free_locals = try f.free_locals.clone(gpa); + defer { + f.value_map.deinit(); + f.free_locals.deinit(gpa); + f.value_map = old_value_map; + f.free_locals = old_free_locals; + } + try genBody(f, case_body); + } else { + try genBody(f, case_body); + } } const else_body = f.air.extra[extra_index..][0..switch_br.data.else_body_len]; @@ -4124,235 +4395,269 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { const inputs = @ptrCast([]const Air.Inst.Ref, f.air.extra[extra_i..][0..extra.data.inputs_len]); extra_i += inputs.len; - if (!is_volatile and f.liveness.isUnused(inst)) return CValue.none; - - const writer = f.object.writer(); - const inst_ty = f.air.typeOfIndex(inst); - const local = if (inst_ty.hasRuntimeBitsIgnoreComptime()) local: { - const local = try f.allocLocal(inst_ty, .Mut); - if (f.wantSafety()) { - try writer.writeAll(" = "); - try f.writeCValue(writer, .{ .undef = inst_ty }, .Initializer); - } - try writer.writeAll(";\n"); - break :local local; - } else .none; - - const locals_begin = f.next_local_index; - const constraints_extra_begin = extra_i; - for (outputs) |output| { - const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(extra_bytes, 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (constraint.len < 2 or constraint[0] != '=' or - (constraint[1] == '{' and constraint[constraint.len - 1] != '}')) - { - return f.fail("CBE: constraint not supported: '{s}'", .{constraint}); - } + const result: CValue = r: { + if (!is_volatile and f.liveness.isUnused(inst)) break :r CValue.none; - const is_reg = constraint[1] == '{'; - if (is_reg) { - const output_ty = if (output == .none) inst_ty else f.air.typeOf(output).childType(); - try writer.writeAll("register "); - _ = try f.allocLocal(output_ty, .Mut); - try writer.writeAll(" __asm(\""); - try writer.writeAll(constraint["={".len .. constraint.len - "}".len]); - try writer.writeAll("\")"); + const writer = f.object.writer(); + const inst_ty = f.air.typeOfIndex(inst); + const local = if (inst_ty.hasRuntimeBitsIgnoreComptime()) local: { + // TODO free this after using it + const local = try f.allocLocal(inst, inst_ty); if (f.wantSafety()) { + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); - try f.writeCValue(writer, .{ .undef = output_ty }, .Initializer); + try f.writeCValue(writer, .{ .undef = inst_ty }, .Initializer); + try writer.writeAll(";\n"); + } + break :local local; + } else .none; + + const locals_begin = @intCast(LocalIndex, f.locals.items.len); + const constraints_extra_begin = extra_i; + for (outputs) |output| { + const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (constraint.len < 2 or constraint[0] != '=' or + (constraint[1] == '{' and constraint[constraint.len - 1] != '}')) + { + return f.fail("CBE: constraint not supported: '{s}'", .{constraint}); } - try writer.writeAll(";\n"); - } - } - for (inputs) |input| { - const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(extra_bytes, 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (constraint.len < 1 or std.mem.indexOfScalar(u8, "=+&%", constraint[0]) != null or - (constraint[0] == '{' and constraint[constraint.len - 1] != '}')) - { - return f.fail("CBE: constraint not supported: '{s}'", .{constraint}); - } - const is_reg = constraint[0] == '{'; - const input_val = try f.resolveInst(input); - if (asmInputNeedsLocal(constraint, input_val)) { - const input_ty = f.air.typeOf(input); - if (is_reg) try writer.writeAll("register "); - _ = try f.allocLocal(input_ty, .Const); + const is_reg = constraint[1] == '{'; if (is_reg) { + const output_ty = if (output == .none) inst_ty else f.air.typeOf(output).childType(); + try writer.writeAll("register "); + const alignment = 0; + const local_value = try f.allocLocalValue(output_ty, alignment); + try f.object.dg.renderTypeAndName( + writer, + output_ty, + local_value, + .Mut, + alignment, + .Complete, + ); try writer.writeAll(" __asm(\""); - try writer.writeAll(constraint["{".len .. constraint.len - "}".len]); + try writer.writeAll(constraint["={".len .. constraint.len - "}".len]); try writer.writeAll("\")"); + if (f.wantSafety()) { + try writer.writeAll(" = "); + try f.writeCValue(writer, .{ .undef = output_ty }, .Initializer); + } + try writer.writeAll(";\n"); } - try writer.writeAll(" = "); - try f.writeCValue(writer, input_val, .Initializer); - try writer.writeAll(";\n"); } - } - { - var clobber_i: u32 = 0; - while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + for (inputs) |input| { + const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. - extra_i += clobber.len / 4 + 1; - } - } - { - const asm_source = mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len]; + extra_i += (constraint.len + name.len + (2 + 3)) / 4; - var stack = std.heap.stackFallback(256, f.object.dg.gpa); - const allocator = stack.get(); - const fixed_asm_source = try allocator.alloc(u8, asm_source.len); - defer allocator.free(fixed_asm_source); + if (constraint.len < 1 or std.mem.indexOfScalar(u8, "=+&%", constraint[0]) != null or + (constraint[0] == '{' and constraint[constraint.len - 1] != '}')) + { + return f.fail("CBE: constraint not supported: '{s}'", .{constraint}); + } - var src_i: usize = 0; - var dst_i: usize = 0; - while (true) { - const literal = mem.sliceTo(asm_source[src_i..], '%'); - src_i += literal.len; + const is_reg = constraint[0] == '{'; + const input_val = try f.resolveInst(input); + if (asmInputNeedsLocal(constraint, input_val)) { + const input_ty = f.air.typeOf(input); + if (is_reg) try writer.writeAll("register "); + const alignment = 0; + const local_value = try f.allocLocalValue(input_ty, alignment); + try f.object.dg.renderTypeAndName( + writer, + input_ty, + local_value, + .Const, + alignment, + .Complete, + ); + if (is_reg) { + try writer.writeAll(" __asm(\""); + try writer.writeAll(constraint["{".len .. constraint.len - "}".len]); + try writer.writeAll("\")"); + } + try writer.writeAll(" = "); + try f.writeCValue(writer, input_val, .Initializer); + try writer.writeAll(";\n"); + } + } + { + var clobber_i: u32 = 0; + while (clobber_i < clobbers_len) : (clobber_i += 1) { + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += clobber.len / 4 + 1; + } + } + + { + const asm_source = mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len]; - mem.copy(u8, fixed_asm_source[dst_i..], literal); - dst_i += literal.len; + var stack = std.heap.stackFallback(256, f.object.dg.gpa); + const allocator = stack.get(); + const fixed_asm_source = try allocator.alloc(u8, asm_source.len); + defer allocator.free(fixed_asm_source); - if (src_i >= asm_source.len) break; + var src_i: usize = 0; + var dst_i: usize = 0; + while (true) { + const literal = mem.sliceTo(asm_source[src_i..], '%'); + src_i += literal.len; - src_i += 1; - if (src_i >= asm_source.len) - return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source}); + mem.copy(u8, fixed_asm_source[dst_i..], literal); + dst_i += literal.len; - fixed_asm_source[dst_i] = '%'; - dst_i += 1; + if (src_i >= asm_source.len) break; - if (asm_source[src_i] != '[') { - // This also handles %% - fixed_asm_source[dst_i] = asm_source[src_i]; src_i += 1; + if (src_i >= asm_source.len) + return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source}); + + fixed_asm_source[dst_i] = '%'; dst_i += 1; - continue; - } - const desc = mem.sliceTo(asm_source[src_i..], ']'); - if (mem.indexOfScalar(u8, desc, ':')) |colon| { - const name = desc[0..colon]; - const modifier = desc[colon + 1 ..]; + if (asm_source[src_i] != '[') { + // This also handles %% + fixed_asm_source[dst_i] = asm_source[src_i]; + src_i += 1; + dst_i += 1; + continue; + } - mem.copy(u8, fixed_asm_source[dst_i..], modifier); - dst_i += modifier.len; - mem.copy(u8, fixed_asm_source[dst_i..], name); - dst_i += name.len; + const desc = mem.sliceTo(asm_source[src_i..], ']'); + if (mem.indexOfScalar(u8, desc, ':')) |colon| { + const name = desc[0..colon]; + const modifier = desc[colon + 1 ..]; - src_i += desc.len; - if (src_i >= asm_source.len) - return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source}); + mem.copy(u8, fixed_asm_source[dst_i..], modifier); + dst_i += modifier.len; + mem.copy(u8, fixed_asm_source[dst_i..], name); + dst_i += name.len; + + src_i += desc.len; + if (src_i >= asm_source.len) + return f.fail("CBE: invalid inline asm string '{s}'", .{asm_source}); + } } + + try writer.writeAll("__asm"); + if (is_volatile) try writer.writeAll(" volatile"); + try writer.print("({s}", .{fmtStringLiteral(fixed_asm_source[0..dst_i])}); } - try writer.writeAll("__asm"); - if (is_volatile) try writer.writeAll(" volatile"); - try writer.print("({s}", .{fmtStringLiteral(fixed_asm_source[0..dst_i])}); - } - - extra_i = constraints_extra_begin; - var locals_index = locals_begin; - try writer.writeByte(':'); - for (outputs) |output, index| { - const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(extra_bytes, 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (index > 0) try writer.writeByte(','); - try writer.writeByte(' '); - if (!std.mem.eql(u8, name, "_")) try writer.print("[{s}]", .{name}); - const is_reg = constraint[1] == '{'; - try writer.print("{s}(", .{fmtStringLiteral(if (is_reg) "=r" else constraint)}); - if (is_reg) { - try f.writeCValue(writer, .{ .local = locals_index }, .Other); - locals_index += 1; - } else if (output == .none) { - try f.writeCValue(writer, local, .FunctionArgument); - } else { - try f.writeCValueDeref(writer, try f.resolveInst(output)); + extra_i = constraints_extra_begin; + var locals_index = locals_begin; + try writer.writeByte(':'); + for (outputs) |output, index| { + const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (index > 0) try writer.writeByte(','); + try writer.writeByte(' '); + if (!std.mem.eql(u8, name, "_")) try writer.print("[{s}]", .{name}); + const is_reg = constraint[1] == '{'; + try writer.print("{s}(", .{fmtStringLiteral(if (is_reg) "=r" else constraint)}); + if (is_reg) { + try f.writeCValue(writer, .{ .local = locals_index }, .Other); + locals_index += 1; + } else if (output == .none) { + try f.writeCValue(writer, local, .FunctionArgument); + } else { + try f.writeCValueDeref(writer, try f.resolveInst(output)); + } + try writer.writeByte(')'); } - try writer.writeByte(')'); - } - try writer.writeByte(':'); - for (inputs) |input, index| { - const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(extra_bytes, 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - if (index > 0) try writer.writeByte(','); - try writer.writeByte(' '); - if (!std.mem.eql(u8, name, "_")) try writer.print("[{s}]", .{name}); - - const is_reg = constraint[0] == '{'; - const input_val = try f.resolveInst(input); - try writer.print("{s}(", .{fmtStringLiteral(if (is_reg) "r" else constraint)}); - try f.writeCValue(writer, if (asmInputNeedsLocal(constraint, input_val)) local: { - const input_local = CValue{ .local = locals_index }; - locals_index += 1; - break :local input_local; - } else input_val, .Other); - try writer.writeByte(')'); - } - try writer.writeByte(':'); - { - var clobber_i: u32 = 0; - while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + try writer.writeByte(':'); + for (inputs) |input, index| { + const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. - extra_i += clobber.len / 4 + 1; + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + if (index > 0) try writer.writeByte(','); + try writer.writeByte(' '); + if (!std.mem.eql(u8, name, "_")) try writer.print("[{s}]", .{name}); + + const is_reg = constraint[0] == '{'; + const input_val = try f.resolveInst(input); + try writer.print("{s}(", .{fmtStringLiteral(if (is_reg) "r" else constraint)}); + try f.writeCValue(writer, if (asmInputNeedsLocal(constraint, input_val)) local: { + const input_local = CValue{ .local = locals_index }; + locals_index += 1; + break :local input_local; + } else input_val, .Other); + try writer.writeByte(')'); + } + try writer.writeByte(':'); + { + var clobber_i: u32 = 0; + while (clobber_i < clobbers_len) : (clobber_i += 1) { + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(f.air.extra[extra_i..]), 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += clobber.len / 4 + 1; - if (clobber.len == 0) continue; + if (clobber.len == 0) continue; - if (clobber_i > 0) try writer.writeByte(','); - try writer.print(" {s}", .{fmtStringLiteral(clobber)}); + if (clobber_i > 0) try writer.writeByte(','); + try writer.print(" {s}", .{fmtStringLiteral(clobber)}); + } } - } - try writer.writeAll(");\n"); + try writer.writeAll(");\n"); - extra_i = constraints_extra_begin; - locals_index = locals_begin; - for (outputs) |output| { - const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); - const constraint = std.mem.sliceTo(extra_bytes, 0); - const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); - // This equation accounts for the fact that even if we have exactly 4 bytes - // for the string, we still use the next u32 for the null terminator. - extra_i += (constraint.len + name.len + (2 + 3)) / 4; - - const is_reg = constraint[1] == '{'; - if (is_reg) { - try f.writeCValueDeref(writer, if (output == .none) - CValue{ .local_ref = local.local } - else - try f.resolveInst(output)); - try writer.writeAll(" = "); - try f.writeCValue(writer, .{ .local = locals_index }, .Other); - locals_index += 1; - try writer.writeAll(";\n"); + extra_i = constraints_extra_begin; + locals_index = locals_begin; + for (outputs) |output| { + const extra_bytes = std.mem.sliceAsBytes(f.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(extra_bytes, 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + // This equation accounts for the fact that even if we have exactly 4 bytes + // for the string, we still use the next u32 for the null terminator. + extra_i += (constraint.len + name.len + (2 + 3)) / 4; + + const is_reg = constraint[1] == '{'; + if (is_reg) { + try f.writeCValueDeref(writer, if (output == .none) + CValue{ .local_ref = local.local } + else + try f.resolveInst(output)); + try writer.writeAll(" = "); + try f.writeCValue(writer, .{ .local = locals_index }, .Other); + locals_index += 1; + try writer.writeAll(";\n"); + } } + + break :r local; + }; + + var bt = iterateBigTomb(f, inst); + for (outputs) |output| { + if (output == .none) continue; + try bt.feed(output); + } + for (inputs) |input| { + try bt.feed(input); } - return local; + return result; } fn airIsNull( @@ -4361,16 +4666,25 @@ fn airIsNull( operator: []const u8, is_ptr: bool, ) !CValue { - if (f.liveness.isUnused(inst)) + const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); return CValue.none; + } - const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); - const local = try f.allocLocal(Type.initTag(.bool), .Const); + const local = try f.allocLocal(inst, Type.bool); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); - try if (is_ptr) f.writeCValueDeref(writer, operand) else f.writeCValue(writer, operand, .Other); + if (is_ptr) { + try f.writeCValueDeref(writer, operand); + } else { + try f.writeCValue(writer, operand, .Other); + } const operand_ty = f.air.typeOf(un_op); const optional_ty = if (is_ptr) operand_ty.childType() else operand_ty; @@ -4402,30 +4716,47 @@ fn airIsNull( } fn airOptionalPayload(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const opt_ty = f.air.typeOf(ty_op.operand); var buf: Type.Payload.ElemType = undefined; const payload_ty = opt_ty.optionalChild(&buf); - if (!payload_ty.hasRuntimeBitsIgnoreComptime()) return CValue.none; - if (opt_ty.optionalReprIsPayload()) return operand; + if (!payload_ty.hasRuntimeBitsIgnoreComptime()) { + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst, inst_ty); + const writer = f.object.writer(); + + if (opt_ty.optionalReprIsPayload()) { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); + return local; + } + const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(inst_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); - const writer = f.object.writer(); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - } else try writer.writeAll(" = "); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + } try f.writeCValueMember(writer, operand, .{ .identifier = "payload" }); if (is_array) { try writer.writeAll(", sizeof("); @@ -4437,11 +4768,16 @@ fn airOptionalPayload(f: *Function, inst: Air.Inst.Index) !CValue { } fn airOptionalPayloadPtr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + const writer = f.object.writer(); const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const ptr_ty = f.air.typeOf(ty_op.operand); const opt_ty = ptr_ty.childType(); const inst_ty = f.air.typeOfIndex(inst); @@ -4456,7 +4792,8 @@ fn airOptionalPayloadPtr(f: *Function, inst: Air.Inst.Index) !CValue { return operand; } - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = &"); try f.writeCValueDerefMember(writer, operand, .{ .identifier = "payload" }); try writer.writeAll(";\n"); @@ -4467,6 +4804,7 @@ fn airOptionalPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { const ty_op = f.air.instructions.items(.data)[inst].ty_op; const writer = f.object.writer(); const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const operand_ty = f.air.typeOf(ty_op.operand); const opt_ty = operand_ty.elemType(); @@ -4483,7 +4821,8 @@ fn airOptionalPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(";\n"); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = &"); try f.writeCValueDeref(writer, operand); @@ -4492,37 +4831,49 @@ fn airOptionalPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { } fn airStructFieldPtr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) + const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; + const extra = f.air.extraData(Air.StructField, ty_pl.payload).data; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{extra.struct_operand}); // TODO this @as is needed because of a stage1 bug return @as(CValue, CValue.none); + } - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; - const extra = f.air.extraData(Air.StructField, ty_pl.payload).data; const struct_ptr = try f.resolveInst(extra.struct_operand); + try reap(f, inst, &.{extra.struct_operand}); const struct_ptr_ty = f.air.typeOf(extra.struct_operand); return structFieldPtr(f, inst, struct_ptr_ty, struct_ptr, extra.field_index); } fn airStructFieldPtrIndex(f: *Function, inst: Air.Inst.Index, index: u8) !CValue { - if (f.liveness.isUnused(inst)) + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); // TODO this @as is needed because of a stage1 bug return @as(CValue, CValue.none); + } - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const struct_ptr = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const struct_ptr_ty = f.air.typeOf(ty_op.operand); return structFieldPtr(f, inst, struct_ptr_ty, struct_ptr, index); } fn airFieldParentPtr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const extra = f.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{extra.field_ptr}); + return CValue.none; + } + const struct_ptr_ty = f.air.typeOfIndex(inst); const field_ptr_ty = f.air.typeOf(extra.field_ptr); const field_ptr_val = try f.resolveInst(extra.field_ptr); + try reap(f, inst, &.{extra.field_ptr}); const target = f.object.dg.module.getTarget(); const struct_ty = struct_ptr_ty.childType(); @@ -4539,7 +4890,8 @@ fn airFieldParentPtr(f: *Function, inst: Air.Inst.Index) !CValue { const u8_ptr_ty = Type.initPayload(&u8_ptr_pl.base); const writer = f.object.writer(); - const local = try f.allocLocal(struct_ptr_ty, .Const); + const local = try f.allocLocal(inst, struct_ptr_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, struct_ptr_ty); try writer.writeAll(")&(("); @@ -4560,7 +4912,8 @@ fn structFieldPtr(f: *Function, inst: Air.Inst.Index, struct_ptr_ty: Type, struc // Ensure complete type definition is visible before accessing fields. try f.renderType(std.io.null_writer, struct_ty); - const local = try f.allocLocal(field_ptr_ty, .Const); + const local = try f.allocLocal(inst, field_ptr_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, field_ptr_ty); try writer.writeByte(')'); @@ -4648,16 +5001,23 @@ fn structFieldPtr(f: *Function, inst: Air.Inst.Index, struct_ptr_ty: Type, struc } fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) + const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; + const extra = f.air.extraData(Air.StructField, ty_pl.payload).data; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{extra.struct_operand}); return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - if (!inst_ty.hasRuntimeBitsIgnoreComptime()) return CValue.none; + if (!inst_ty.hasRuntimeBitsIgnoreComptime()) { + try reap(f, inst, &.{extra.struct_operand}); + return CValue.none; + } - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; - const extra = f.air.extraData(Air.StructField, ty_pl.payload).data; const target = f.object.dg.module.getTarget(); const struct_byval = try f.resolveInst(extra.struct_operand); + try reap(f, inst, &.{extra.struct_operand}); const struct_ty = f.air.typeOf(extra.struct_operand); const writer = f.object.writer(); @@ -4701,7 +5061,8 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { }; const field_int_ty = Type.initPayload(&field_int_pl.base); - const temp_local = try f.allocLocal(field_int_ty, .Const); + const temp_local = try f.allocLocal(inst, field_int_ty); + try f.writeCValue(writer, temp_local, .Other); try writer.writeAll(" = zig_wrap_"); try f.object.dg.renderTypeForBuiltinFnName(writer, field_int_ty); try writer.writeAll("(("); @@ -4717,8 +5078,7 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(");\n"); if (inst_ty.eql(field_int_ty, f.object.dg.module)) return temp_local; - const local = try f.allocLocal(inst_ty, .Mut); - try writer.writeAll(";\n"); + const local = try f.allocLocal(inst, inst_ty); try writer.writeAll("memcpy("); try f.writeCValue(writer, .{ .local_ref = local.local }, .FunctionArgument); try writer.writeAll(", "); @@ -4726,20 +5086,22 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(", sizeof("); try f.renderTypecast(writer, inst_ty); try writer.writeAll("));\n"); + try freeLocal(f, inst, temp_local.local, 0); return local; }, }, .@"union", .union_safety_tagged, .union_tagged => if (struct_ty.containerLayout() == .Packed) { const operand_lval = if (struct_byval == .constant) blk: { - const operand_local = try f.allocLocal(struct_ty, .Const); + const operand_local = try f.allocLocal(inst, struct_ty); + try f.writeCValue(writer, operand_local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, struct_byval, .Initializer); try writer.writeAll(";\n"); break :blk operand_local; } else struct_byval; - const local = try f.allocLocal(inst_ty, .Mut); - try writer.writeAll(";\n"); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll("memcpy(&"); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", &"); @@ -4747,6 +5109,11 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(", sizeof("); try f.renderTypecast(writer, inst_ty); try writer.writeAll("));\n"); + + if (struct_byval == .constant) { + try freeLocal(f, inst, operand_lval.local, 0); + } + return local; } else .{ .identifier = struct_ty.unionFields().keys()[extra.field_index], @@ -4764,13 +5131,15 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { }; const is_array = lowersToArray(inst_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); + const local = try f.allocLocal(inst, inst_ty); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - } else try writer.writeAll(" = "); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + } if (extra_name != .none) { try f.writeCValueMember(writer, struct_byval, extra_name); try writer.writeByte('.'); @@ -4788,9 +5157,13 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { /// *(E!T) -> E /// Note that the result is never a pointer. fn airUnwrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(ty_op.operand); const operand_ty = f.air.typeOf(ty_op.operand); @@ -4800,9 +5173,11 @@ fn airUnwrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { const error_ty = error_union_ty.errorUnionSet(); const payload_ty = error_union_ty.errorUnionPayload(); if (!payload_ty.hasRuntimeBits()) return operand; + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (!error_ty.errorSetIsEmpty()) if (operand_is_ptr) @@ -4816,12 +5191,16 @@ fn airUnwrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { } fn airUnwrapErrUnionPay(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue { - if (f.liveness.isUnused(inst)) + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); return CValue.none; + } - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const operand_ty = f.air.typeOf(ty_op.operand); const operand_is_ptr = operand_ty.zigTypeTag() == .Pointer; const error_union_ty = if (operand_is_ptr) operand_ty.childType() else operand_ty; @@ -4829,8 +5208,9 @@ fn airUnwrapErrUnionPay(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValu if (!error_union_ty.errorUnionPayload().hasRuntimeBits()) { if (!is_ptr) return CValue.none; - const local = try f.allocLocal(inst_ty, .Const); const w = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(w, local, .Other); try w.writeAll(" = ("); try f.renderTypecast(w, inst_ty); try w.writeByte(')'); @@ -4840,7 +5220,8 @@ fn airUnwrapErrUnionPay(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValu } const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (is_ptr) try writer.writeByte('&'); if (operand_is_ptr) @@ -4852,10 +5233,14 @@ fn airUnwrapErrUnionPay(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValu } fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const payload = try f.resolveInst(ty_op.operand); if (inst_ty.optionalReprIsPayload()) return payload; @@ -4863,8 +5248,10 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(payload_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = { .payload = "); try f.writeCValue(writer, if (is_array) CValue{ .undef = payload_ty } else payload, .Initializer); try writer.writeAll(", .is_null = "); @@ -4883,16 +5270,22 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { } fn airWrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const writer = f.object.writer(); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); const error_union_ty = f.air.typeOfIndex(inst); const payload_ty = error_union_ty.errorUnionPayload(); if (!payload_ty.hasRuntimeBits()) return operand; - const local = try f.allocLocal(error_union_ty, .Const); + try reap(f, inst, &.{ty_op.operand}); + + const local = try f.allocLocal(inst, error_union_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = { .payload = "); try f.writeCValue(writer, .{ .undef = payload_ty }, .Initializer); try writer.writeAll(", .error = "); @@ -4919,6 +5312,7 @@ fn airErrUnionPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { return operand; } + try reap(f, inst, &.{ty_op.operand}); try f.writeCValueDeref(writer, operand); try writer.writeAll(".error = "); try f.object.dg.renderValue(writer, error_ty, Value.zero, .Other); @@ -4927,7 +5321,8 @@ fn airErrUnionPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { // Then return the payload pointer (only if it is used) if (f.liveness.isUnused(inst)) return CValue.none; - const local = try f.allocLocal(f.air.typeOfIndex(inst), .Const); + const local = try f.allocLocal(inst, f.air.typeOfIndex(inst)); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = &("); try f.writeCValueDeref(writer, operand); try writer.writeAll(").payload;\n"); @@ -4950,19 +5345,24 @@ fn airSaveErrReturnTraceIndex(f: *Function, inst: Air.Inst.Index) !CValue { } fn airWrapErrUnionPay(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); const error_ty = inst_ty.errorUnionSet(); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const payload_ty = inst_ty.errorUnionPayload(); const payload = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(payload_ty, target); - const local = try f.allocLocal(inst_ty, if (is_array) .Mut else .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = { .payload = "); try f.writeCValue(writer, if (is_array) CValue{ .undef = payload_ty } else payload, .Initializer); try writer.writeAll(", .error = "); @@ -4981,18 +5381,23 @@ fn airWrapErrUnionPay(f: *Function, inst: Air.Inst.Index) !CValue { } fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const u8) !CValue { - if (f.liveness.isUnused(inst)) + const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); return CValue.none; + } - const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const operand_ty = f.air.typeOf(un_op); - const local = try f.allocLocal(Type.initTag(.bool), .Const); + const local = try f.allocLocal(inst, Type.bool); const err_union_ty = if (is_ptr) operand_ty.childType() else operand_ty; const payload_ty = err_union_ty.errorUnionPayload(); const error_ty = err_union_ty.errorUnionSet(); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (!error_ty.errorSetIsEmpty()) @@ -5014,14 +5419,19 @@ fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const } fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); return CValue.none; + } + const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const writer = f.object.writer(); - const operand = try f.resolveInst(ty_op.operand); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); const array_len = f.air.typeOf(ty_op.operand).elemType().arrayLen(); try writer.writeAll(" = { .ptr = "); @@ -5043,11 +5453,16 @@ fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { } fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const operand_ty = f.air.typeOf(ty_op.operand); const target = f.object.dg.module.getTarget(); const operation = if (inst_ty.isRuntimeFloat() and operand_ty.isRuntimeFloat()) @@ -5059,8 +5474,9 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue { else unreachable; - const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (inst_ty.isInt() and operand_ty.isRuntimeFloat()) { @@ -5085,13 +5501,19 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue { } fn airPtrToInt(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - const un_op = f.air.instructions.items(.data)[inst].un_op; const writer = f.object.writer(); - const operand = try f.resolveInst(un_op); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); try f.renderTypecast(writer, inst_ty); @@ -5107,20 +5529,27 @@ fn airUnBuiltinCall( operation: []const u8, info: BuiltinInfo, ) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } + + const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const inst_ty = f.air.typeOfIndex(inst); - const operand = f.air.instructions.items(.data)[inst].ty_op.operand; - const operand_ty = f.air.typeOf(operand); + const operand_ty = f.air.typeOf(ty_op.operand); - const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); try writer.writeByte('('); - try f.writeCValue(writer, try f.resolveInst(operand), .FunctionArgument); + try f.writeCValue(writer, operand, .FunctionArgument); try f.object.dg.renderBuiltinInfo(writer, operand_ty, info); try writer.writeAll(");\n"); return local; @@ -5132,49 +5561,61 @@ fn airBinBuiltinCall( operation: []const u8, info: BuiltinInfo, ) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const bin_op = f.air.instructions.items(.data)[inst].bin_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } + + const lhs = try f.resolveInst(bin_op.lhs); + const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const inst_ty = f.air.typeOfIndex(inst); - const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); - const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); try writer.writeByte('('); - try f.writeCValue(writer, try f.resolveInst(bin_op.lhs), .FunctionArgument); + try f.writeCValue(writer, lhs, .FunctionArgument); try writer.writeAll(", "); - try f.writeCValue(writer, try f.resolveInst(bin_op.rhs), .FunctionArgument); + try f.writeCValue(writer, rhs, .FunctionArgument); try f.object.dg.renderBuiltinInfo(writer, operand_ty, info); try writer.writeAll(");\n"); return local; } -fn airCmpBuiltinCall( +fn cmpBuiltinCall( f: *Function, inst: Air.Inst.Index, operator: []const u8, operation: []const u8, ) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const inst_ty = f.air.typeOfIndex(inst); const bin_op = f.air.instructions.items(.data)[inst].bin_op; const operand_ty = f.air.typeOf(bin_op.lhs); - const local = try f.allocLocal(inst_ty, .Const); + const lhs = try f.resolveInst(bin_op.lhs); + const rhs = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_"); try writer.writeAll(operation); try writer.writeByte('_'); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); try writer.writeByte('('); - try f.writeCValue(writer, try f.resolveInst(bin_op.lhs), .FunctionArgument); + try f.writeCValue(writer, lhs, .FunctionArgument); try writer.writeAll(", "); - try f.writeCValue(writer, try f.resolveInst(bin_op.rhs), .FunctionArgument); + try f.writeCValue(writer, rhs, .FunctionArgument); try writer.print(") {s} {};\n", .{ operator, try f.fmtIntLiteral(Type.initTag(.i32), Value.zero) }); return local; } @@ -5188,9 +5629,11 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue const ptr = try f.resolveInst(extra.ptr); const expected_value = try f.resolveInst(extra.expected_value); const new_value = try f.resolveInst(extra.new_value); + try reap(f, inst, &.{ extra.ptr, extra.expected_value, extra.new_value }); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Mut); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (is_struct) try writer.writeAll("{ .payload = "); try f.writeCValue(writer, expected_value, .Initializer); @@ -5246,8 +5689,10 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue { const ptr_ty = f.air.typeOf(pl_op.operand); const ptr = try f.resolveInst(pl_op.operand); const operand = try f.resolveInst(extra.operand); - const local = try f.allocLocal(inst_ty, .Const); + try reap(f, inst, &.{ pl_op.operand, extra.operand }); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.print(" = zig_atomicrmw_{s}((", .{toAtomicRmwSuffix(extra.op())}); switch (extra.op()) { @@ -5276,13 +5721,16 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue { fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue { const atomic_load = f.air.instructions.items(.data)[inst].atomic_load; const ptr = try f.resolveInst(atomic_load.ptr); + try reap(f, inst, &.{atomic_load.ptr}); const ptr_ty = f.air.typeOf(atomic_load.ptr); - if (!ptr_ty.isVolatilePtr() and f.liveness.isUnused(inst)) + if (!ptr_ty.isVolatilePtr() and f.liveness.isUnused(inst)) { return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_atomic_load((zig_atomic("); try f.renderTypecast(writer, ptr_ty.elemType()); @@ -5302,6 +5750,7 @@ fn airAtomicStore(f: *Function, inst: Air.Inst.Index, order: [*:0]const u8) !CVa const ptr_ty = f.air.typeOf(bin_op.lhs); const ptr = try f.resolveInst(bin_op.lhs); const element = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); try writer.writeAll("zig_atomic_store((zig_atomic("); @@ -5324,6 +5773,7 @@ fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue { const dest_ptr = try f.resolveInst(pl_op.operand); const value = try f.resolveInst(extra.lhs); const len = try f.resolveInst(extra.rhs); + try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs }); const writer = f.object.writer(); if (dest_ty.isVolatilePtr()) { @@ -5332,7 +5782,8 @@ fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue { const u8_ptr_ty = Type.initPayload(&u8_ptr_pl.base); try writer.writeAll("for ("); - const index = try f.allocLocal(Type.usize, .Mut); + const index = try f.allocLocal(inst, Type.usize); + try f.writeCValue(writer, index, .Other); try writer.writeAll(" = "); try f.object.dg.renderValue(writer, Type.usize, Value.zero, .Initializer); try writer.writeAll("; "); @@ -5353,6 +5804,8 @@ fn airMemset(f: *Function, inst: Air.Inst.Index) !CValue { try f.writeCValue(writer, value, .FunctionArgument); try writer.writeAll(";\n"); + try freeLocal(f, inst, index.local, 0); + return CValue.none; } @@ -5373,6 +5826,7 @@ fn airMemcpy(f: *Function, inst: Air.Inst.Index) !CValue { const dest_ptr = try f.resolveInst(pl_op.operand); const src_ptr = try f.resolveInst(extra.lhs); const len = try f.resolveInst(extra.rhs); + try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs }); const writer = f.object.writer(); try writer.writeAll("memcpy("); @@ -5390,6 +5844,7 @@ fn airSetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { const bin_op = f.air.instructions.items(.data)[inst].bin_op; const union_ptr = try f.resolveInst(bin_op.lhs); const new_tag = try f.resolveInst(bin_op.rhs); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); const union_ty = f.air.typeOf(bin_op.lhs).childType(); @@ -5407,20 +5862,27 @@ fn airSetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { } fn airGetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); return CValue.none; + } - const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; - const un_ty = f.air.typeOf(ty_op.operand); - const writer = f.object.writer(); const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); + + const un_ty = f.air.typeOf(ty_op.operand); const target = f.object.dg.module.getTarget(); const layout = un_ty.unionGetLayout(target); if (layout.tag_size == 0) return CValue.none; + const inst_ty = f.air.typeOfIndex(inst); + const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Other); try writer.writeAll(".tag;\n"); @@ -5428,15 +5890,21 @@ fn airGetUnionTag(f: *Function, inst: Air.Inst.Index) !CValue { } fn airTagName(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const inst_ty = f.air.typeOfIndex(inst); const enum_ty = f.air.typeOf(un_op); const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.print(" = {s}(", .{try f.object.dg.getTagNameFn(enum_ty)}); try f.writeCValue(writer, operand, .Other); try writer.writeAll(");\n"); @@ -5445,13 +5913,19 @@ fn airTagName(f: *Function, inst: Air.Inst.Index) !CValue { } fn airErrorName(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const un_op = f.air.instructions.items(.data)[inst].un_op; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(un_op); - const local = try f.allocLocal(inst_ty, .Const); + try reap(f, inst, &.{un_op}); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_errorName["); try f.writeCValue(writer, operand, .Other); @@ -5460,17 +5934,21 @@ fn airErrorName(f: *Function, inst: Air.Inst.Index) !CValue { } fn airSplat(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const ty_op = f.air.instructions.items(.data)[inst].ty_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ty_op.operand}); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const ty_op = f.air.instructions.items(.data)[inst].ty_op; const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); _ = operand; - _ = local; return f.fail("TODO: C backend: implement airSplat", .{}); } @@ -5487,12 +5965,17 @@ fn airShuffle(f: *Function, inst: Air.Inst.Index) !CValue { } fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; + const reduce = f.air.instructions.items(.data)[inst].reduce; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{reduce.operand}); + return CValue.none; + } const target = f.object.dg.module.getTarget(); const scalar_ty = f.air.typeOfIndex(inst); - const reduce = f.air.instructions.items(.data)[inst].reduce; const operand = try f.resolveInst(reduce.operand); + try reap(f, inst, &.{reduce.operand}); const operand_ty = f.air.typeOf(reduce.operand); const vector_len = operand_ty.vectorLen(); const writer = f.object.writer(); @@ -5569,10 +6052,12 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue { // } // break :reduce accum; // } - const it = try f.allocLocal(Type.usize, .Mut); + const it = try f.allocLocal(inst, Type.usize); + try f.writeCValue(writer, it, .Other); try writer.writeAll(" = 0;\n"); - const accum = try f.allocLocal(scalar_ty, .Mut); + const accum = try f.allocLocal(inst, scalar_ty); + try f.writeCValue(writer, accum, .Other); try writer.writeAll(" = "); const init_val = switch (reduce.operation) { @@ -5635,32 +6120,43 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(";\n"); + try freeLocal(f, inst, it.local, 0); + return accum; } fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - - const inst_ty = f.air.typeOfIndex(inst); const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; + const inst_ty = f.air.typeOfIndex(inst); const len = @intCast(usize, inst_ty.arrayLen()); const elements = @ptrCast([]const Air.Inst.Ref, f.air.extra[ty_pl.payload..][0..len]); + const gpa = f.object.dg.gpa; + const resolved_elements = try gpa.alloc(CValue, elements.len); + defer gpa.free(resolved_elements); + { + var bt = iterateBigTomb(f, inst); + for (elements) |element, i| { + resolved_elements[i] = try f.resolveInst(element); + try bt.feed(element); + } + } + + if (f.liveness.isUnused(inst)) return CValue.none; + const target = f.object.dg.module.getTarget(); - const mutability: Mutability = for (elements) |element| { - if (lowersToArray(f.air.typeOf(element), target)) break .Mut; - } else .Const; const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, mutability); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); switch (inst_ty.zigTypeTag()) { .Array, .Vector => { const elem_ty = inst_ty.childType(); try writer.writeByte('{'); var empty = true; - for (elements) |element| { + for (resolved_elements) |element| { if (!empty) try writer.writeAll(", "); - try f.writeCValue(writer, try f.resolveInst(element), .Initializer); + try f.writeCValue(writer, element, .Initializer); empty = false; } if (inst_ty.sentinel()) |sentinel| { @@ -5686,7 +6182,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { const element_ty = f.air.typeOf(element); try f.writeCValue(writer, switch (element_ty.zigTypeTag()) { .Array => CValue{ .undef = element_ty }, - else => try f.resolveInst(element), + else => resolved_elements[index], }, .Initializer); empty = false; } @@ -5709,7 +6205,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll("memcpy("); try f.writeCValueMember(writer, local, field_name); try writer.writeAll(", "); - try f.writeCValue(writer, try f.resolveInst(element), .FunctionArgument); + try f.writeCValue(writer, resolved_elements[index], .FunctionArgument); try writer.writeAll(", sizeof("); try f.renderTypecast(writer, element_ty); try writer.writeAll("));\n"); @@ -5742,7 +6238,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { empty = false; } empty = true; - for (elements) |element, index| { + for (resolved_elements) |element, index| { const field_ty = inst_ty.structFieldType(index); if (!field_ty.hasRuntimeBitsIgnoreComptime()) continue; @@ -5760,7 +6256,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { }); try writer.writeByte(')'); } - try f.writeCValue(writer, try f.resolveInst(element), .Other); + try f.writeCValue(writer, element, .Other); try writer.writeAll(", "); try f.object.dg.renderValue(writer, bit_offset_ty, bit_offset_val, .FunctionArgument); try f.object.dg.renderBuiltinInfo(writer, inst_ty, .Bits); @@ -5781,18 +6277,24 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { } fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const extra = f.air.extraData(Air.UnionInit, ty_pl.payload).data; + + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{extra.init}); + return CValue.none; + } + const union_ty = f.air.typeOfIndex(inst); const target = f.object.dg.module.getTarget(); const union_obj = union_ty.cast(Type.Payload.Union).?.data; const field_name = union_obj.fields.keys()[extra.field_index]; const payload = try f.resolveInst(extra.init); + try reap(f, inst, &.{extra.init}); const writer = f.object.writer(); - const local = try f.allocLocal(union_ty, .Const); + const local = try f.allocLocal(inst, union_ty); + try f.writeCValue(writer, local, .Other); if (union_obj.layout == .Packed) { try writer.writeAll(" = "); try f.writeCValue(writer, payload, .Initializer); @@ -5839,6 +6341,7 @@ fn airPrefetch(f: *Function, inst: Air.Inst.Index) !CValue { .instruction => return CValue.none, } const ptr = try f.resolveInst(prefetch.ptr); + try reap(f, inst, &.{prefetch.ptr}); const writer = f.object.writer(); try writer.writeAll("zig_prefetch("); try f.writeCValue(writer, ptr, .FunctionArgument); @@ -5855,7 +6358,8 @@ fn airWasmMemorySize(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try writer.print("zig_wasm_memory_size({d});\n", .{pl_op.payload}); @@ -5869,7 +6373,9 @@ fn airWasmMemoryGrow(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(pl_op.operand); - const local = try f.allocLocal(inst_ty, .Const); + try reap(f, inst, &.{pl_op.operand}); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try writer.print("zig_wasm_memory_grow({d}, ", .{pl_op.payload}); @@ -5879,15 +6385,19 @@ fn airWasmMemoryGrow(f: *Function, inst: Air.Inst.Index) !CValue { } fn airFloatNeg(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; - const inst_ty = f.air.typeOfIndex(inst); const un_op = f.air.instructions.items(.data)[inst].un_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const operand = try f.resolveInst(un_op); const operand_ty = f.air.typeOf(un_op); - const local = try f.allocLocal(inst_ty, .Const); const writer = f.object.writer(); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_neg_"); try f.object.dg.renderTypeForBuiltinFnName(writer, operand_ty); try writer.writeByte('('); @@ -5897,12 +6407,17 @@ fn airFloatNeg(f: *Function, inst: Air.Inst.Index) !CValue { } fn airUnFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; const un_op = f.air.instructions.items(.data)[inst].un_op; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{un_op}); + return CValue.none; + } + const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); - const operand = try f.resolveInst(un_op); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_libc_name_"); try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty); try writer.writeByte('('); @@ -5914,13 +6429,19 @@ fn airUnFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CVal } fn airBinFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; const bin_op = f.air.instructions.items(.data)[inst].bin_op; - const writer = f.object.writer(); - const inst_ty = f.air.typeOfIndex(inst); + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + return CValue.none; + } const lhs = try f.resolveInst(bin_op.lhs); const rhs = try f.resolveInst(bin_op.rhs); - const local = try f.allocLocal(inst_ty, .Const); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); + + const writer = f.object.writer(); + const inst_ty = f.air.typeOfIndex(inst); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_libc_name_"); try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty); try writer.writeByte('('); @@ -5934,15 +6455,20 @@ fn airBinFloatOp(f: *Function, inst: Air.Inst.Index, operation: []const u8) !CVa } fn airMulAdd(f: *Function, inst: Air.Inst.Index) !CValue { - if (f.liveness.isUnused(inst)) return CValue.none; const pl_op = f.air.instructions.items(.data)[inst].pl_op; - const extra = f.air.extraData(Air.Bin, pl_op.payload).data; + const bin_op = f.air.extraData(Air.Bin, pl_op.payload).data; + if (f.liveness.isUnused(inst)) { + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }); + return CValue.none; + } const inst_ty = f.air.typeOfIndex(inst); - const mulend1 = try f.resolveInst(extra.lhs); - const mulend2 = try f.resolveInst(extra.rhs); + const mulend1 = try f.resolveInst(bin_op.lhs); + const mulend2 = try f.resolveInst(bin_op.rhs); const addend = try f.resolveInst(pl_op.operand); + try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand }); const writer = f.object.writer(); - const local = try f.allocLocal(inst_ty, .Const); + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = zig_libc_name_"); try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty); try writer.writeAll("(fma)("); @@ -6321,3 +6847,63 @@ fn loweredArrayInfo(ty: Type, target: std.Target) ?Type.ArrayInfo { }, } } + +fn reap(f: *Function, inst: Air.Inst.Index, operands: []const Air.Inst.Ref) !void { + assert(operands.len <= Liveness.bpi - 1); + var tomb_bits = f.liveness.getTombBits(inst); + for (operands) |operand| { + const dies = @truncate(u1, tomb_bits) != 0; + tomb_bits >>= 1; + if (!dies) continue; + try die(f, inst, operand); + } +} + +fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void { + const ref_inst = Air.refToIndex(ref) orelse return; + if (f.air.instructions.items(.tag)[ref_inst] == .constant) return; + const c_value = (f.value_map.fetchRemove(ref) orelse return).value; + const local_index = switch (c_value) { + .local => |l| l, + else => return, + }; + try freeLocal(f, inst, local_index, ref_inst); +} + +fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_inst: Air.Inst.Index) !void { + const gpa = f.object.dg.gpa; + const gop = try f.free_locals.getOrPutContext( + gpa, + f.locals.items[local_index].ty, + f.tyHashCtx(), + ); + if (!gop.found_existing) gop.value_ptr.* = .{}; + log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst }); + if (std.debug.runtime_safety) { + // If this trips, it means a local is being inserted into the + // free_locals map while it already exists in the map, which is not + // allowed. + assert(mem.indexOfScalar(LocalIndex, gop.value_ptr.items, local_index) == null); + } + try gop.value_ptr.append(gpa, local_index); +} + +const BigTomb = struct { + f: *Function, + inst: Air.Inst.Index, + lbt: Liveness.BigTomb, + + fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) !void { + const dies = bt.lbt.feed(); + if (!dies) return; + try die(bt.f, bt.inst, op_ref); + } +}; + +fn iterateBigTomb(f: *Function, inst: Air.Inst.Index) BigTomb { + return .{ + .f = f, + .inst = inst, + .lbt = f.liveness.iterateBigTomb(inst), + }; +} diff --git a/src/link/C.zig b/src/link/C.zig index a2d7ee143c..880a3e154e 100644 --- a/src/link/C.zig +++ b/src/link/C.zig @@ -136,16 +136,7 @@ pub fn updateFunc(self: *C, module: *Module, func: *Module.Fn, air: Air, livenes }; function.object.indent_writer = .{ .underlying_writer = function.object.code.writer() }; - defer { - function.blocks.deinit(module.gpa); - function.value_map.deinit(); - function.object.code.deinit(); - for (function.object.dg.typedefs.values()) |typedef| { - module.gpa.free(typedef.rendered); - } - function.object.dg.typedefs.deinit(); - function.object.dg.fwd_decl.deinit(); - } + defer function.deinit(module.gpa); codegen.genFunc(&function) catch |err| switch (err) { error.AnalysisFail => { diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000..d57f2af18f --- /dev/null +++ b/test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +if [[ $1 == --enable-fixed-behavior ]]; then + declare -A offsets + git g -n stage2_c test/behavior | while read -r match; do + printf '\e[36mTrying to enable... %s\e[m\n' "$match" + file=`cut -d: -f1 <<<"$match"` + offset=${offsets[$file]:=0} + let line=`cut -d: -f2 <<<"$match"`-$offset + contents=`cut -d: -f3- <<<"$match"` + sed --in-place "${line}d" "$file" + if zigd test -Itest test/behavior.zig -fno-stage1 -fno-LLVM -ofmt=c; then + printf '\e[32mTest was enabled! :)\e[m\n' + let offsets[$file]+=1 + else + printf '\e[31mTest kept disabled. :(\e[m\n' + sed --in-place "${line}i\\ +$contents" "$file" + fi + done +fi -- cgit v1.2.3 From 73a76b45c5a2c8fc6ff884c4d3c84c29d37c3034 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 2 Dec 2022 23:58:54 -0700 Subject: CBE: take advantage of switch_br and cond_br liveness --- src/codegen/c.zig | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 49d6b9ec18..82ceaacd43 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -4277,6 +4277,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { const extra = f.air.extraData(Air.CondBr, pl_op.payload); const then_body = f.air.extra[extra.end..][0..extra.data.then_body_len]; const else_body = f.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]; + const liveness_condbr = f.liveness.getCondBr(inst); const writer = f.object.writer(); // Keep using the original for the then branch; use a clone of the value @@ -4287,6 +4288,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { var cloned_frees = try f.free_locals.clone(gpa); defer cloned_frees.deinit(gpa); + for (liveness_condbr.then_deaths) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } + try writer.writeAll("if ("); try f.writeCValue(writer, cond, .Other); try writer.writeAll(") "); @@ -4296,6 +4301,9 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { f.value_map = cloned_map.move(); f.free_locals.deinit(gpa); f.free_locals = cloned_frees.move(); + for (liveness_condbr.else_deaths) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } try genBody(f, else_body); try f.object.indent_writer.insertNewline(); @@ -4325,6 +4333,12 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { f.object.indent_writer.pushIndent(); const gpa = f.object.dg.gpa; + const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.data.cases_len + 1); + defer gpa.free(liveness.deaths); + // On the final iteration we do not clone the map. This ensures that + // lowering proceeds after the switch_br taking into account the + // mutations to the liveness information. + const last_case_i = switch_br.data.cases_len - @boolToInt(switch_br.data.else_body_len == 0); var extra_index: usize = switch_br.end; var case_i: u32 = 0; while (case_i < switch_br.data.cases_len) : (case_i += 1) { @@ -4344,15 +4358,16 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { try f.object.dg.renderValue(writer, condition_ty, f.air.value(item).?, .Other); try writer.writeAll(": "); } - // On the final iteration we do not clone the map. This ensures that - // lowering proceeds after the switch_br taking into account the - // mutations to the liveness information. - // The case body must be noreturn so we don't need to insert a break. - if (case_i < switch_br.data.cases_len - 1) { + if (case_i != last_case_i) { const old_value_map = f.value_map; f.value_map = try old_value_map.clone(); const old_free_locals = f.free_locals; f.free_locals = try f.free_locals.clone(gpa); + + for (liveness.deaths[case_i]) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } + defer { f.value_map.deinit(); f.free_locals.deinit(gpa); @@ -4361,14 +4376,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { } try genBody(f, case_body); } else { + for (liveness.deaths[case_i]) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } try genBody(f, case_body); } + // The case body must be noreturn so we don't need to insert a break. } const else_body = f.air.extra[extra_index..][0..switch_br.data.else_body_len]; try f.object.indent_writer.insertNewline(); - try writer.writeAll("default: "); - try genBody(f, else_body); + if (else_body.len > 0) { + for (liveness.deaths[liveness.deaths.len - 1]) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } + try writer.writeAll("default: "); + try genBody(f, else_body); + } else { + try writer.writeAll("default: zig_unreachable();"); + } try f.object.indent_writer.insertNewline(); f.object.indent_writer.popIndent(); -- cgit v1.2.3 From 8bfbfa589c424fc58d9c9a097b1df050738a337c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 00:20:57 -0700 Subject: CBE: fix clone of freed locals not being deep clone --- src/codegen/c.zig | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 82ceaacd43..80047870d1 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -411,7 +411,7 @@ pub const Function = struct { pub fn deinit(f: *Function, gpa: mem.Allocator) void { f.allocs.deinit(gpa); f.locals.deinit(gpa); - f.free_locals.deinit(gpa); + deinitFreeLocalsMap(gpa, &f.free_locals); f.blocks.deinit(gpa); f.value_map.deinit(); f.object.code.deinit(); @@ -4285,8 +4285,8 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { const gpa = f.object.dg.gpa; var cloned_map = try f.value_map.clone(); defer cloned_map.deinit(); - var cloned_frees = try f.free_locals.clone(gpa); - defer cloned_frees.deinit(gpa); + var cloned_frees = try cloneFreeLocalsMap(gpa, &f.free_locals); + defer deinitFreeLocalsMap(gpa, &cloned_frees); for (liveness_condbr.then_deaths) |operand| { try die(f, inst, Air.indexToRef(operand)); @@ -4299,7 +4299,7 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(" else "); f.value_map.deinit(); f.value_map = cloned_map.move(); - f.free_locals.deinit(gpa); + deinitFreeLocalsMap(gpa, &f.free_locals); f.free_locals = cloned_frees.move(); for (liveness_condbr.else_deaths) |operand| { try die(f, inst, Air.indexToRef(operand)); @@ -4362,18 +4362,19 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { const old_value_map = f.value_map; f.value_map = try old_value_map.clone(); const old_free_locals = f.free_locals; - f.free_locals = try f.free_locals.clone(gpa); - - for (liveness.deaths[case_i]) |operand| { - try die(f, inst, Air.indexToRef(operand)); - } + f.free_locals = try cloneFreeLocalsMap(gpa, &f.free_locals); defer { f.value_map.deinit(); - f.free_locals.deinit(gpa); + deinitFreeLocalsMap(gpa, &f.free_locals); f.value_map = old_value_map; f.free_locals = old_free_locals; } + + for (liveness.deaths[case_i]) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } + try genBody(f, case_body); } else { for (liveness.deaths[case_i]) |operand| { @@ -6933,3 +6934,29 @@ fn iterateBigTomb(f: *Function, inst: Air.Inst.Index) BigTomb { .lbt = f.liveness.iterateBigTomb(inst), }; } + +/// A naive clone of this map would create copies of the ArrayList which is +/// stored in the values. This function additionally clones the values. +fn cloneFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) !LocalsMap { + var cloned = try map.clone(gpa); + const values = cloned.values(); + var i: usize = 0; + errdefer { + cloned.deinit(gpa); + while (i > 0) { + i -= 1; + values[i].deinit(gpa); + } + } + while (i < values.len) : (i += 1) { + values[i] = try values[i].clone(gpa); + } + return cloned; +} + +fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void { + for (map.values()) |*value| { + value.deinit(gpa); + } + map.deinit(gpa); +} -- cgit v1.2.3 From 7bd63a602a433044f321534117570b1da338e946 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 02:20:04 -0700 Subject: CBE: fix assignment expr and switch free tracking --- src/codegen/c.zig | 104 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 30 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 80047870d1..f66b7da6eb 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -2488,9 +2488,7 @@ pub fn genFunc(f: *Function) !void { const local = f.locals.items[local_index]; log.debug("inserting local {d} into free_locals", .{local_index}); const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); - if (!gop.found_existing) { - gop.value_ptr.* = .{}; - } + if (!gop.found_existing) gop.value_ptr.* = .{}; try gop.value_ptr.append(gpa, local_index); } @@ -3819,14 +3817,16 @@ fn airSlice(f: *Function, inst: Air.Inst.Index) !CValue { const inst_ty = f.air.typeOfIndex(inst); const local = try f.allocLocal(inst, inst_ty); try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = {("); + try writer.writeAll(".ptr = ("); var buf: Type.SlicePtrFieldTypeBuffer = undefined; try f.renderTypecast(writer, inst_ty.slicePtrFieldType(&buf)); try writer.writeByte(')'); try f.writeCValue(writer, ptr, .Other); - try writer.writeAll(", "); + try writer.writeAll("; "); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(".len = "); try f.writeCValue(writer, len, .Initializer); - try writer.writeAll("};\n"); + try writer.writeAll(";\n"); return local; } @@ -4292,6 +4292,11 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { try die(f, inst, Air.indexToRef(operand)); } + // Remember how many locals there were before entering the then branch so + // that we can notice and use them in the else branch. Any new locals must + // necessarily be free already after the then branch is complete. + const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); + try writer.writeAll("if ("); try f.writeCValue(writer, cond, .Other); try writer.writeAll(") "); @@ -4304,6 +4309,9 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { for (liveness_condbr.else_deaths) |operand| { try die(f, inst, Air.indexToRef(operand)); } + + try noticeBranchFrees(f, pre_locals_len); + try genBody(f, else_body); try f.object.indent_writer.insertNewline(); @@ -4335,10 +4343,12 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { const gpa = f.object.dg.gpa; const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.data.cases_len + 1); defer gpa.free(liveness.deaths); + // On the final iteration we do not clone the map. This ensures that // lowering proceeds after the switch_br taking into account the // mutations to the liveness information. const last_case_i = switch_br.data.cases_len - @boolToInt(switch_br.data.else_body_len == 0); + var extra_index: usize = switch_br.end; var case_i: u32 = 0; while (case_i < switch_br.data.cases_len) : (case_i += 1) { @@ -4358,31 +4368,43 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { try f.object.dg.renderValue(writer, condition_ty, f.air.value(item).?, .Other); try writer.writeAll(": "); } + if (case_i != last_case_i) { const old_value_map = f.value_map; f.value_map = try old_value_map.clone(); const old_free_locals = f.free_locals; f.free_locals = try cloneFreeLocalsMap(gpa, &f.free_locals); - defer { - f.value_map.deinit(); - deinitFreeLocalsMap(gpa, &f.free_locals); - f.value_map = old_value_map; - f.free_locals = old_free_locals; - } + // Remember how many locals there were before entering each branch so that + // we can notice and use them in subsequent branches. Any new locals must + // necessarily be free already after the previous branch is complete. + const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); - for (liveness.deaths[case_i]) |operand| { - try die(f, inst, Air.indexToRef(operand)); + { + defer { + f.value_map.deinit(); + deinitFreeLocalsMap(gpa, &f.free_locals); + f.value_map = old_value_map; + f.free_locals = old_free_locals; + } + + for (liveness.deaths[case_i]) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } + + try genBody(f, case_body); } - try genBody(f, case_body); + try noticeBranchFrees(f, pre_locals_len); } else { for (liveness.deaths[case_i]) |operand| { try die(f, inst, Air.indexToRef(operand)); } try genBody(f, case_body); } + // The case body must be noreturn so we don't need to insert a break. + } const else_body = f.air.extra[extra_index..][0..switch_br.data.else_body_len]; @@ -5278,12 +5300,16 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { try reap(f, inst, &.{ty_op.operand}); const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); + if (!is_array) { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(".payload = "); + try f.writeCValue(writer, payload, .Other); + try writer.writeAll("; "); + } try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = { .payload = "); - try f.writeCValue(writer, if (is_array) CValue{ .undef = payload_ty } else payload, .Initializer); - try writer.writeAll(", .is_null = "); + try writer.writeAll(".is_null = "); try f.object.dg.renderValue(writer, Type.bool, Value.false, .Initializer); - try writer.writeAll(" };\n"); + try writer.writeAll(";\n"); if (is_array) { try writer.writeAll("memcpy("); try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); @@ -5312,12 +5338,14 @@ fn airWrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { try reap(f, inst, &.{ty_op.operand}); const local = try f.allocLocal(inst, error_union_ty); + { + // TODO: set the payload to undefined + //try f.writeCValue(writer, local, .Other); + } try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = { .payload = "); - try f.writeCValue(writer, .{ .undef = payload_ty }, .Initializer); - try writer.writeAll(", .error = "); - try f.writeCValue(writer, operand, .Initializer); - try writer.writeAll(" };\n"); + try writer.writeAll(".error = "); + try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); return local; } @@ -5379,7 +5407,6 @@ fn airWrapErrUnionPay(f: *Function, inst: Air.Inst.Index) !CValue { } const inst_ty = f.air.typeOfIndex(inst); - const error_ty = inst_ty.errorUnionSet(); const payload_ty = inst_ty.errorUnionPayload(); const payload = try f.resolveInst(ty_op.operand); try reap(f, inst, &.{ty_op.operand}); @@ -5389,12 +5416,14 @@ fn airWrapErrUnionPay(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); + if (!is_array) { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(".payload = "); + try f.writeCValue(writer, payload, .Other); + try writer.writeAll("; "); + } try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = { .payload = "); - try f.writeCValue(writer, if (is_array) CValue{ .undef = payload_ty } else payload, .Initializer); - try writer.writeAll(", .error = "); - try f.object.dg.renderValue(writer, error_ty, Value.zero, .Initializer); - try writer.writeAll(" };\n"); + try writer.writeAll(".error = 0;\n"); if (is_array) { try writer.writeAll("memcpy("); try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); @@ -6911,6 +6940,8 @@ fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_in // free_locals map while it already exists in the map, which is not // allowed. assert(mem.indexOfScalar(LocalIndex, gop.value_ptr.items, local_index) == null); + // If this trips, an unfreeable allocation was attempted to be freed. + assert(!f.allocs.contains(local_index)); } try gop.value_ptr.append(gpa, local_index); } @@ -6960,3 +6991,16 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void { } map.deinit(gpa); } + +fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex) !void { + const gpa = f.object.dg.gpa; + var i = pre_locals_len; + while (i < f.locals.items.len) : (i += 1) { + const local = f.locals.items[i]; + const unfreeable = f.allocs.contains(i); + if (unfreeable) continue; + const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.append(gpa, i); + } +} -- cgit v1.2.3 From db1819e8ed02d3bb230b82f875b45fe7a6274c49 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 02:59:02 -0800 Subject: CBE: fix use-after-free of Type keys in free_locals map --- src/codegen/c.zig | 17 ++++++++++++----- src/link/C.zig | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index f66b7da6eb..14da51b276 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -272,6 +272,8 @@ pub const Function = struct { /// variable declarations at the top of a function, sorted descending by /// type alignment. allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, void) = .{}, + /// Needed for memory used by Type objects used as keys in free_locals. + arena: std.heap.ArenaAllocator, fn tyHashCtx(f: Function) Type.HashContext32 { return .{ .mod = f.object.dg.module }; @@ -314,7 +316,7 @@ pub const Function = struct { .ty = ty, .alignment = alignment, }); - return .{ .local = @intCast(LocalIndex, f.locals.items.len - 1) }; + return CValue{ .local = @intCast(LocalIndex, f.locals.items.len - 1) }; } fn allocLocal(f: *Function, inst: Air.Inst.Index, ty: Type) !CValue { @@ -420,6 +422,7 @@ pub const Function = struct { } f.object.dg.typedefs.deinit(); f.object.dg.fwd_decl.deinit(); + f.arena.deinit(); } }; @@ -3228,8 +3231,9 @@ fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue { var deref = is_ptr; const operand = try f.resolveInst(un_op); try reap(f, inst, &.{un_op}); - const ret_val = if (lowersToArray(ret_ty, target)) ret_val: { - const array_local = try f.allocLocal(inst, lowered_ret_ty); + const is_array = lowersToArray(ret_ty, target); + const ret_val = if (is_array) ret_val: { + const array_local = try f.allocLocal(inst, try lowered_ret_ty.copy(f.arena.allocator())); try writer.writeAll("memcpy("); try f.writeCValueMember(writer, array_local, .{ .field = 0 }); try writer.writeAll(", "); @@ -3250,6 +3254,9 @@ fn airRet(f: *Function, inst: Air.Inst.Index, is_ptr: bool) !CValue { else try f.writeCValue(writer, ret_val, .Other); try writer.writeAll(";\n"); + if (is_array) { + try freeLocal(f, inst, ret_val.local, 0); + } } else { try reap(f, inst, &.{un_op}); if (f.object.dg.decl.ty.fnCallingConvention() != .Naked) { @@ -3884,7 +3891,7 @@ fn airCall( try writer.writeByte(')'); break :r .none; } else r: { - const local = try f.allocLocal(inst, lowered_ret_ty); + const local = try f.allocLocal(inst, try lowered_ret_ty.copy(f.arena.allocator())); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); break :r local; @@ -5110,7 +5117,7 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { }; const field_int_ty = Type.initPayload(&field_int_pl.base); - const temp_local = try f.allocLocal(inst, field_int_ty); + const temp_local = try f.allocLocal(inst, try field_int_ty.copy(f.arena.allocator())); try f.writeCValue(writer, temp_local, .Other); try writer.writeAll(" = zig_wrap_"); try f.object.dg.renderTypeForBuiltinFnName(writer, field_int_ty); diff --git a/src/link/C.zig b/src/link/C.zig index 880a3e154e..ad50a20b99 100644 --- a/src/link/C.zig +++ b/src/link/C.zig @@ -133,6 +133,7 @@ pub fn updateFunc(self: *C, module: *Module, func: *Module.Fn, air: Air, livenes .code = code.toManaged(module.gpa), .indent_writer = undefined, // set later so we can get a pointer to object.code }, + .arena = std.heap.ArenaAllocator.init(module.gpa), }; function.object.indent_writer = .{ .underlying_writer = function.object.code.writer() }; -- cgit v1.2.3 From da73410e7ff41fe1f196236ca405f225650a0726 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 14:25:42 -0700 Subject: CBE: avoid curly inits because they don't work in assignments --- src/codegen/c.zig | 107 +++++++++++++++++++++++++++++------------------------- 1 file changed, 57 insertions(+), 50 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 14da51b276..ff8425b433 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -3843,6 +3843,8 @@ fn airCall( inst: Air.Inst.Index, modifier: std.builtin.CallOptions.Modifier, ) !CValue { + // Not even allowed to call panic in a naked function. + if (f.object.dg.decl.ty.fnCallingConvention() == .Naked) return .none; const gpa = f.object.dg.gpa; switch (modifier) { @@ -5497,7 +5499,7 @@ fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { try f.writeCValue(writer, local, .Other); const array_len = f.air.typeOf(ty_op.operand).elemType().arrayLen(); - try writer.writeAll(" = { .ptr = "); + try writer.writeAll(".ptr = "); if (operand == .undef) { // Unfortunately, C does not support any equivalent to // &(*(void *)p)[0], although LLVM does via GetElementPtr @@ -5511,7 +5513,9 @@ fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { var len_pl: Value.Payload.U64 = .{ .base = .{ .tag = .int_u64 }, .data = array_len }; const len_val = Value.initPayload(&len_pl.base); - try writer.print(", .len = {} }};\n", .{try f.fmtIntLiteral(Type.usize, len_val)}); + try writer.writeAll("; "); + try f.writeCValue(writer, local, .Other); + try writer.print(".len = {};\n", .{try f.fmtIntLiteral(Type.usize, len_val)}); return local; } @@ -5687,59 +5691,62 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue const ty_pl = f.air.instructions.items(.data)[inst].ty_pl; const extra = f.air.extraData(Air.Cmpxchg, ty_pl.payload).data; const inst_ty = f.air.typeOfIndex(inst); - const is_struct = !inst_ty.isPtrLikeOptional(); - const ptr_ty = f.air.typeOf(extra.ptr); const ptr = try f.resolveInst(extra.ptr); const expected_value = try f.resolveInst(extra.expected_value); const new_value = try f.resolveInst(extra.new_value); try reap(f, inst, &.{ extra.ptr, extra.expected_value, extra.new_value }); const writer = f.object.writer(); - + const ptr_ty = f.air.typeOf(extra.ptr); const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = "); - if (is_struct) try writer.writeAll("{ .payload = "); - try f.writeCValue(writer, expected_value, .Initializer); - if (is_struct) { - try writer.writeAll(", .is_null = "); - try f.object.dg.renderValue(writer, Type.bool, Value.false, .Initializer); - try writer.writeAll(" }"); - } - try writer.writeAll(";\n"); - - if (is_struct) { + if (inst_ty.isPtrLikeOptional()) { try f.writeCValue(writer, local, .Other); - try writer.writeAll(".is_null = "); - } else { + try writer.writeAll(" = "); + try f.writeCValue(writer, expected_value, .Initializer); + try writer.writeAll(";\n"); try writer.writeAll("if ("); - } - try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor}); - try f.renderTypecast(writer, ptr_ty.elemType()); - try writer.writeByte(')'); - if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); - try writer.writeAll(" *)"); - try f.writeCValue(writer, ptr, .Other); - try writer.writeAll(", "); - if (is_struct) - try f.writeCValueMember(writer, local, .{ .identifier = "payload" }) - else + try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor}); + try f.renderTypecast(writer, ptr_ty.elemType()); + try writer.writeByte(')'); + if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); + try writer.writeAll(" *)"); + try f.writeCValue(writer, ptr, .Other); + try writer.writeAll(", "); try f.writeCValue(writer, local, .FunctionArgument); - try writer.writeAll(", "); - try f.writeCValue(writer, new_value, .FunctionArgument); - try writer.writeAll(", "); - try writeMemoryOrder(writer, extra.successOrder()); - try writer.writeAll(", "); - try writeMemoryOrder(writer, extra.failureOrder()); - try writer.writeByte(')'); - if (is_struct) { - try writer.writeAll(";\n"); - } else { + try writer.writeAll(", "); + try f.writeCValue(writer, new_value, .FunctionArgument); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.successOrder()); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.failureOrder()); + try writer.writeByte(')'); try writer.writeAll(") {\n"); f.object.indent_writer.pushIndent(); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = NULL;\n"); f.object.indent_writer.popIndent(); try writer.writeAll("}\n"); + } else { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(".payload = "); + try f.writeCValue(writer, expected_value, .Other); + try writer.writeAll(";\n"); + try f.writeCValue(writer, local, .Other); + try writer.print(".is_null = zig_cmpxchg_{s}((zig_atomic(", .{flavor}); + try f.renderTypecast(writer, ptr_ty.elemType()); + try writer.writeByte(')'); + if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); + try writer.writeAll(" *)"); + try f.writeCValue(writer, ptr, .Other); + try writer.writeAll(", "); + try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); + try writer.writeAll(", "); + try f.writeCValue(writer, new_value, .FunctionArgument); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.successOrder()); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.failureOrder()); + try writer.writeByte(')'); + try writer.writeAll(";\n"); } return local; @@ -6211,7 +6218,9 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = "); + try writer.writeAll(" = ("); + try f.renderTypecast(writer, inst_ty); + try writer.writeAll(")"); switch (inst_ty.zigTypeTag()) { .Array, .Vector => { const elem_ty = inst_ty.childType(); @@ -6357,15 +6366,14 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const local = try f.allocLocal(inst, union_ty); - try f.writeCValue(writer, local, .Other); if (union_obj.layout == .Packed) { + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, payload, .Initializer); try writer.writeAll(";\n"); return local; } - try writer.writeAll(" = {"); if (union_ty.unionTagTypeSafety()) |tag_ty| { const layout = union_ty.unionGetLayout(target); if (layout.tag_size != 0) { @@ -6380,16 +6388,15 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { var int_pl: Value.Payload.U64 = undefined; const int_val = tag_val.enumToInt(tag_ty, &int_pl); - try writer.print(".tag = {}, ", .{try f.fmtIntLiteral(tag_ty, int_val)}); + try f.writeCValue(writer, local, .Other); + try writer.print(".tag = {}; ", .{try f.fmtIntLiteral(tag_ty, int_val)}); } - try writer.writeAll(".payload = {"); } - try writer.print(".{ } = ", .{fmtIdent(field_name)}); - try f.writeCValue(writer, payload, .Initializer); - - if (union_ty.unionTagTypeSafety()) |_| try writer.writeByte('}'); - try writer.writeAll("};\n"); + try f.writeCValue(writer, local, .Other); + try writer.print(".payload.{ } = ", .{fmtIdent(field_name)}); + try f.writeCValue(writer, payload, .Other); + try writer.writeAll(";\n"); return local; } -- cgit v1.2.3 From 6c0a1417c6edb40cfc86e546c5c853a2d19c22f0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 15:35:18 -0700 Subject: CBE: fix static allocs being double allocated --- src/codegen/c.zig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index ff8425b433..99ecd3d32f 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -271,7 +271,8 @@ pub const Function = struct { /// the locals within so that it can be used to render the block of /// variable declarations at the top of a function, sorted descending by /// type alignment. - allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, void) = .{}, + /// The value is whether the alloc is static or not. + allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{}, /// Needed for memory used by Type objects used as keys in free_locals. arena: std.heap.ArenaAllocator, @@ -290,6 +291,8 @@ pub const Function = struct { const writer = f.object.code_header.writer(); const alignment = 0; const decl_c_value = try f.allocLocalValue(ty, alignment); + const gpa = f.object.dg.gpa; + try f.allocs.put(gpa, decl_c_value.local, true); try writer.writeAll("static "); try f.object.dg.renderTypeAndName(writer, ty, decl_c_value, .Const, alignment, .Complete); try writer.writeAll(" = "); @@ -2487,7 +2490,9 @@ pub fn genFunc(f: *Function) !void { // Liveness analysis, however, locals from alloc instructions will be // missing. These are added now to complete the map. Then we can sort by // alignment, descending. - for (f.allocs.keys()) |local_index| { + const values = f.allocs.values(); + for (f.allocs.keys()) |local_index, i| { + if (values[i]) continue; // static const local = f.locals.items[local_index]; log.debug("inserting local {d} into free_locals", .{local_index}); const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); @@ -3110,7 +3115,7 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue { const local = try f.allocAlignedLocal(elem_type, mutability, inst_ty.ptrAlignment(target)); log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.local }); const gpa = f.object.dg.module.gpa; - try f.allocs.put(gpa, local.local, {}); + try f.allocs.put(gpa, local.local, false); return CValue{ .local_ref = local.local }; } @@ -3127,7 +3132,7 @@ fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue { const local = try f.allocAlignedLocal(elem_ty, mutability, inst_ty.ptrAlignment(target)); log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.local }); const gpa = f.object.dg.module.gpa; - try f.allocs.put(gpa, local.local, {}); + try f.allocs.put(gpa, local.local, false); return CValue{ .local_ref = local.local }; } -- cgit v1.2.3 From f2e59e41c1ea3884a50e45f54a74c29f52954b24 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 19:20:15 -0700 Subject: CBE: fix various regressions caught by behavior tests --- src/codegen/c.zig | 136 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 51 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 99ecd3d32f..372e89bac7 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -4096,9 +4096,7 @@ fn lowerTry( const is_array = lowersToArray(payload_ty, target); try reap(f, inst, &.{operand}); const local = try f.allocLocal(inst, result_ty); - try f.writeCValue(writer, local, .Other); if (is_array) { - try writer.writeAll(";\n"); try writer.writeAll("memcpy("); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); @@ -4107,6 +4105,7 @@ fn lowerTry( try f.renderTypecast(writer, payload_ty); try writer.writeAll("));\n"); } else { + try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); if (operand_is_ptr or isByRef(payload_ty)) { try writer.writeByte('&'); @@ -4849,16 +4848,18 @@ fn airOptionalPayloadPtr(f: *Function, inst: Air.Inst.Index) !CValue { return CValue{ .undef = inst_ty }; } + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); + if (opt_ty.optionalReprIsPayload()) { // the operand is just a regular pointer, no need to do anything special. // *?*T -> **T and ?*T -> *T are **T -> **T and *T -> *T in C - return operand; + try writer.writeAll(" = "); + try f.writeCValue(writer, operand, .Other); + } else { + try writer.writeAll(" = &"); + try f.writeCValueDerefMember(writer, operand, .{ .identifier = "payload" }); } - - const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = &"); - try f.writeCValueDerefMember(writer, operand, .{ .identifier = "payload" }); try writer.writeAll(";\n"); return local; } @@ -4872,25 +4873,37 @@ fn airOptionalPayloadPtrSet(f: *Function, inst: Air.Inst.Index) !CValue { const opt_ty = operand_ty.elemType(); + const inst_ty = f.air.typeOfIndex(inst); + if (opt_ty.optionalReprIsPayload()) { + if (f.liveness.isUnused(inst)) { + return CValue.none; + } + const local = try f.allocLocal(inst, inst_ty); // The payload and the optional are the same value. // Setting to non-null will be done when the payload is set. - return operand; - } - - try f.writeCValueDeref(writer, operand); - try writer.writeAll(".is_null = "); - try f.object.dg.renderValue(writer, Type.bool, Value.false, .Initializer); - try writer.writeAll(";\n"); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); + return local; + } else { + try f.writeCValueDeref(writer, operand); + try writer.writeAll(".is_null = "); + try f.object.dg.renderValue(writer, Type.bool, Value.false, .Initializer); + try writer.writeAll(";\n"); - const inst_ty = f.air.typeOfIndex(inst); - const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = &"); - try f.writeCValueDeref(writer, operand); + if (f.liveness.isUnused(inst)) { + return CValue.none; + } - try writer.writeAll(".payload;\n"); - return local; + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = &"); + try f.writeCValueDeref(writer, operand); + try writer.writeAll(".payload;\n"); + return local; + } } fn airStructFieldPtr(f: *Function, inst: Air.Inst.Index) !CValue { @@ -5164,7 +5177,6 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { } else struct_byval; const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); try writer.writeAll("memcpy(&"); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", &"); @@ -5230,25 +5242,28 @@ fn airUnwrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { const inst_ty = f.air.typeOfIndex(inst); const operand = try f.resolveInst(ty_op.operand); const operand_ty = f.air.typeOf(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const operand_is_ptr = operand_ty.zigTypeTag() == .Pointer; const error_union_ty = if (operand_is_ptr) operand_ty.childType() else operand_ty; const error_ty = error_union_ty.errorUnionSet(); const payload_ty = error_union_ty.errorUnionPayload(); - if (!payload_ty.hasRuntimeBits()) return operand; - try reap(f, inst, &.{ty_op.operand}); - - const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); + const writer = f.object.writer(); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); - if (!error_ty.errorSetIsEmpty()) - if (operand_is_ptr) - try f.writeCValueDerefMember(writer, operand, .{ .identifier = "error" }) + + if (!payload_ty.hasRuntimeBits()) { + try f.writeCValue(writer, operand, .Other); + } else { + if (!error_ty.errorSetIsEmpty()) + if (operand_is_ptr) + try f.writeCValueDerefMember(writer, operand, .{ .identifier = "error" }) + else + try f.writeCValueMember(writer, operand, .{ .identifier = "error" }) else - try f.writeCValueMember(writer, operand, .{ .identifier = "error" }) - else - try f.object.dg.renderValue(writer, error_ty, Value.zero, .Initializer); + try f.object.dg.renderValue(writer, error_ty, Value.zero, .Initializer); + } try writer.writeAll(";\n"); return local; } @@ -5345,13 +5360,19 @@ fn airWrapErrUnionErr(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const operand = try f.resolveInst(ty_op.operand); + try reap(f, inst, &.{ty_op.operand}); const error_union_ty = f.air.typeOfIndex(inst); const payload_ty = error_union_ty.errorUnionPayload(); - if (!payload_ty.hasRuntimeBits()) return operand; + const local = try f.allocLocal(inst, error_union_ty); - try reap(f, inst, &.{ty_op.operand}); + if (!payload_ty.hasRuntimeBits()) { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + try f.writeCValue(writer, operand, .Other); + try writer.writeAll(";\n"); + return local; + } - const local = try f.allocLocal(inst, error_union_ty); { // TODO: set the payload to undefined //try f.writeCValue(writer, local, .Other); @@ -5754,6 +5775,11 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue try writer.writeAll(";\n"); } + if (f.liveness.isUnused(inst)) { + try freeLocal(f, inst, local.local, 0); + return CValue.none; + } + return local; } @@ -5790,6 +5816,11 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue { try writeMemoryOrder(writer, extra.ordering()); try writer.writeAll(");\n"); + if (f.liveness.isUnused(inst)) { + try freeLocal(f, inst, local.local, 0); + return CValue.none; + } + return local; } @@ -6222,30 +6253,28 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = ("); - try f.renderTypecast(writer, inst_ty); - try writer.writeAll(")"); switch (inst_ty.zigTypeTag()) { .Array, .Vector => { const elem_ty = inst_ty.childType(); - try writer.writeByte('{'); - var empty = true; - for (resolved_elements) |element| { - if (!empty) try writer.writeAll(", "); - try f.writeCValue(writer, element, .Initializer); - empty = false; + for (resolved_elements) |element, i| { + try f.writeCValue(writer, local, .Other); + try writer.print("[{d}] = ", .{i}); + try f.writeCValue(writer, element, .Other); + try writer.writeAll(";\n"); } if (inst_ty.sentinel()) |sentinel| { - if (!empty) try writer.writeAll(", "); - try f.object.dg.renderValue(writer, elem_ty, sentinel, .Initializer); - empty = false; + try f.writeCValue(writer, local, .Other); + try writer.print("[{d}] = ", .{resolved_elements.len}); + try f.object.dg.renderValue(writer, elem_ty, sentinel, .Other); + try writer.writeAll(";\n"); } - if (empty) try writer.print("{}", .{try f.fmtIntLiteral(Type.u8, Value.zero)}); - try writer.writeAll("};\n"); }, .Struct => switch (inst_ty.containerLayout()) { .Auto, .Extern => { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = ("); + try f.renderTypecast(writer, inst_ty); + try writer.writeAll(")"); try writer.writeByte('{'); var empty = true; for (elements) |element, index| { @@ -6291,6 +6320,10 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { } }, .Packed => { + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = ("); + try f.renderTypecast(writer, inst_ty); + try writer.writeAll(")"); const int_info = inst_ty.intInfo(target); var bit_offset_ty_pl = Type.Payload.Bits{ @@ -6468,6 +6501,7 @@ fn airFloatNeg(f: *Function, inst: Air.Inst.Index) !CValue { } const operand = try f.resolveInst(un_op); + try reap(f, inst, &.{un_op}); const operand_ty = f.air.typeOf(un_op); const writer = f.object.writer(); -- cgit v1.2.3 From 4aae0b09cfdc444a0b4e98d3ed7f83bfe421c06f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 19:30:08 -0700 Subject: CBE and LLVM: handle unused try instructions In both backends they did not observe the Liveness information for try instructions. Now they do. For the C backend this is necessary for correctness; for the LLVM backend, it improves code generation. --- src/codegen/c.zig | 7 ++++++- src/codegen/llvm.zig | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 372e89bac7..ac22ac3fa6 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -4092,9 +4092,14 @@ fn lowerTry( } } + try reap(f, inst, &.{operand}); + + if (f.liveness.isUnused(inst)) { + return CValue.none; + } + const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(payload_ty, target); - try reap(f, inst, &.{operand}); const local = try f.allocLocal(inst, result_ty); if (is_array) { try writer.writeAll("memcpy("); diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 3fd1effc21..29d074ea1c 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -5346,7 +5346,8 @@ pub const FuncGen = struct { const err_union_ty = self.air.typeOf(pl_op.operand); const payload_ty = self.air.typeOfIndex(inst); const can_elide_load = if (isByRef(payload_ty)) self.canElideLoad(body_tail) else false; - return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, payload_ty); + const is_unused = self.liveness.isUnused(inst); + return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, payload_ty); } fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !?*llvm.Value { @@ -5356,7 +5357,8 @@ pub const FuncGen = struct { const body = self.air.extra[extra.end..][0..extra.data.body_len]; const err_union_ty = self.air.typeOf(extra.data.ptr).childType(); const payload_ty = self.air.typeOfIndex(inst); - return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, payload_ty); + const is_unused = self.liveness.isUnused(inst); + return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, payload_ty); } fn lowerTry( @@ -5366,6 +5368,7 @@ pub const FuncGen = struct { err_union_ty: Type, operand_is_ptr: bool, can_elide_load: bool, + is_unused: bool, result_ty: Type, ) !?*llvm.Value { const payload_ty = err_union_ty.errorUnionPayload(); @@ -5405,6 +5408,9 @@ pub const FuncGen = struct { fg.builder.positionBuilderAtEnd(continue_block); } + if (is_unused) { + return null; + } if (!payload_has_bits) { if (!operand_is_ptr) return null; -- cgit v1.2.3 From aa98517b3e4df84eb28fe1f3352e34407ef51343 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 19:36:26 -0700 Subject: CBE: aggregate_init: resolve all operands before processing Liveness --- src/codegen/c.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index ac22ac3fa6..d2ea876065 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -6244,10 +6244,12 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { const gpa = f.object.dg.gpa; const resolved_elements = try gpa.alloc(CValue, elements.len); defer gpa.free(resolved_elements); + for (elements) |element, i| { + resolved_elements[i] = try f.resolveInst(element); + } { var bt = iterateBigTomb(f, inst); - for (elements) |element, i| { - resolved_elements[i] = try f.resolveInst(element); + for (elements) |element| { try bt.feed(element); } } -- cgit v1.2.3 From 701cebeb30252abb59505e8c03a8b762f16c452f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 19:41:35 -0700 Subject: CBE: fix union init wrong field name --- src/codegen/c.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index d2ea876065..adc4035482 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -6436,10 +6436,15 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { try f.writeCValue(writer, local, .Other); try writer.print(".tag = {}; ", .{try f.fmtIntLiteral(tag_ty, int_val)}); } + try f.writeCValue(writer, local, .Other); + try writer.print(".payload.{ } = ", .{fmtIdent(field_name)}); + try f.writeCValue(writer, payload, .Other); + try writer.writeAll(";\n"); + return local; } try f.writeCValue(writer, local, .Other); - try writer.print(".payload.{ } = ", .{fmtIdent(field_name)}); + try writer.print(".{ } = ", .{fmtIdent(field_name)}); try f.writeCValue(writer, payload, .Other); try writer.writeAll(";\n"); -- cgit v1.2.3 From 4a6deaff5f4d57641b794071eb60629faa9f393a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 19:49:17 -0700 Subject: CBE: remove stray comment --- src/codegen/c.zig | 1 - 1 file changed, 1 deletion(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index adc4035482..805b60b096 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -4468,7 +4468,6 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const inst_ty = f.air.typeOfIndex(inst); const local = if (inst_ty.hasRuntimeBitsIgnoreComptime()) local: { - // TODO free this after using it const local = try f.allocLocal(inst, inst_ty); if (f.wantSafety()) { try f.writeCValue(writer, local, .Other); -- cgit v1.2.3 From cc8bcae8c0e7883423a9fec2c9bc19703538cb9f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 3 Dec 2022 21:03:10 -0700 Subject: CBE: fix liveness issue with wrapping optionals --- src/codegen/c.zig | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 805b60b096..70780cf468 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -5324,14 +5324,22 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { const inst_ty = f.air.typeOfIndex(inst); const payload = try f.resolveInst(ty_op.operand); - if (inst_ty.optionalReprIsPayload()) return payload; + try reap(f, inst, &.{ty_op.operand}); + const writer = f.object.writer(); + + if (inst_ty.optionalReprIsPayload()) { + const local = try f.allocLocal(inst, inst_ty); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(" = "); + try f.writeCValue(writer, payload, .Other); + try writer.writeAll(";\n"); + return local; + } const payload_ty = f.air.typeOf(ty_op.operand); const target = f.object.dg.module.getTarget(); const is_array = lowersToArray(payload_ty, target); - try reap(f, inst, &.{ty_op.operand}); - const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); if (!is_array) { try f.writeCValue(writer, local, .Other); @@ -5340,9 +5348,7 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll("; "); } try f.writeCValue(writer, local, .Other); - try writer.writeAll(".is_null = "); - try f.object.dg.renderValue(writer, Type.bool, Value.false, .Initializer); - try writer.writeAll(";\n"); + try writer.writeAll(".is_null = false;\n"); if (is_array) { try writer.writeAll("memcpy("); try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); -- cgit v1.2.3 From 7d3cc3bc8d772519d390b7f13346eeab73bc0c21 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sun, 4 Dec 2022 07:58:59 -0500 Subject: CBE: defer invariant local reuse in loops When a local defined outside a loop dies inside the loop, it can still be needed on subsequent loop iterations, so reuse of the local must be deferred until after the loop ends. This causes behavior tests to pass. --- src/codegen/c.zig | 129 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 39 deletions(-) (limited to 'src/codegen') diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 70780cf468..bfd2ff7a3a 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -69,14 +69,18 @@ pub const TypedefMap = std.ArrayHashMap( true, ); +const LoopDepth = u16; const Local = struct { ty: Type, alignment: u32, + /// How many loops the last definition was nested in. + loop_depth: LoopDepth, }; const LocalIndex = u16; const LocalsList = std.ArrayListUnmanaged(LocalIndex); const LocalsMap = std.ArrayHashMapUnmanaged(Type, LocalsList, Type.HashContext32, true); +const LocalsStack = std.ArrayListUnmanaged(LocalsMap); const FormatTypeAsCIdentContext = struct { ty: Type, @@ -265,15 +269,18 @@ pub const Function = struct { /// All the locals, to be emitted at the top of the function. locals: std.ArrayListUnmanaged(Local) = .{}, /// Which locals are available for reuse, based on Type. - free_locals: LocalsMap = .{}, + /// Only locals in the last stack entry are available for reuse, + /// other entries will become available on loop exit. + free_locals_stack: LocalsStack = .{}, + free_locals_clone_depth: LoopDepth = 0, /// Locals which will not be freed by Liveness. This is used after a - /// Function body is lowered in order to make `free_locals` have 100% of - /// the locals within so that it can be used to render the block of - /// variable declarations at the top of a function, sorted descending by - /// type alignment. + /// Function body is lowered in order to make `free_locals_stack` have + /// 100% of the locals within so that it can be used to render the block + /// of variable declarations at the top of a function, sorted descending + /// by type alignment. /// The value is whether the alloc is static or not. allocs: std.AutoArrayHashMapUnmanaged(LocalIndex, bool) = .{}, - /// Needed for memory used by Type objects used as keys in free_locals. + /// Needed for memory used by the keys of free_locals_stack entries. arena: std.heap.ArenaAllocator, fn tyHashCtx(f: Function) Type.HashContext32 { @@ -312,12 +319,17 @@ pub const Function = struct { }; } + fn getFreeLocals(f: *Function) *LocalsMap { + return &f.free_locals_stack.items[f.free_locals_stack.items.len - 1]; + } + /// Skips the reuse logic. fn allocLocalValue(f: *Function, ty: Type, alignment: u32) !CValue { const gpa = f.object.dg.gpa; try f.locals.append(gpa, .{ .ty = ty, .alignment = alignment, + .loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1), }); return CValue{ .local = @intCast(LocalIndex, f.locals.items.len - 1) }; } @@ -332,10 +344,11 @@ pub const Function = struct { fn allocAlignedLocal(f: *Function, ty: Type, mutability: Mutability, alignment: u32) !CValue { _ = mutability; - if (f.free_locals.getPtrContext(ty, f.tyHashCtx())) |locals_list| { + if (f.getFreeLocals().getPtrContext(ty, f.tyHashCtx())) |locals_list| { for (locals_list.items) |local_index, i| { - const local = f.locals.items[local_index]; + const local = &f.locals.items[local_index]; if (local.alignment >= alignment) { + local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1); _ = locals_list.swapRemove(i); return CValue{ .local = local_index }; } @@ -416,7 +429,10 @@ pub const Function = struct { pub fn deinit(f: *Function, gpa: mem.Allocator) void { f.allocs.deinit(gpa); f.locals.deinit(gpa); - deinitFreeLocalsMap(gpa, &f.free_locals); + for (f.free_locals_stack.items) |*free_locals| { + deinitFreeLocalsMap(gpa, free_locals); + } + f.free_locals_stack.deinit(gpa); f.blocks.deinit(gpa); f.value_map.deinit(); f.object.code.deinit(); @@ -2480,6 +2496,9 @@ pub fn genFunc(f: *Function) !void { o.code_header.appendSliceAssumeCapacity("{\n "); const empty_header_len = o.code_header.items.len; + f.free_locals_stack.clearRetainingCapacity(); + try f.free_locals_stack.append(gpa, .{}); + const main_body = f.air.getMainBody(); try genBody(f, main_body); @@ -2490,12 +2509,13 @@ pub fn genFunc(f: *Function) !void { // Liveness analysis, however, locals from alloc instructions will be // missing. These are added now to complete the map. Then we can sort by // alignment, descending. + const free_locals = f.getFreeLocals(); const values = f.allocs.values(); for (f.allocs.keys()) |local_index, i| { if (values[i]) continue; // static const local = f.locals.items[local_index]; log.debug("inserting local {d} into free_locals", .{local_index}); - const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); + const gop = try free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); if (!gop.found_existing) gop.value_ptr.* = .{}; try gop.value_ptr.append(gpa, local_index); } @@ -2511,10 +2531,10 @@ pub fn genFunc(f: *Function) !void { } }; const target = o.dg.module.getTarget(); - f.free_locals.sort(SortContext{ .target = target, .keys = f.free_locals.keys() }); + free_locals.sort(SortContext{ .target = target, .keys = free_locals.keys() }); const w = o.code_header.writer(); - for (f.free_locals.values()) |list| { + for (free_locals.values()) |list| { for (list.items) |local_index| { const local = f.locals.items[local_index]; try o.dg.renderTypeAndName( @@ -4282,9 +4302,30 @@ fn airLoop(f: *Function, inst: Air.Inst.Index) !CValue { const loop = f.air.extraData(Air.Block, ty_pl.payload); const body = f.air.extra[loop.end..][0..loop.data.body_len]; const writer = f.object.writer(); + + const gpa = f.object.dg.gpa; + try f.free_locals_stack.insert(gpa, f.free_locals_stack.items.len - 1, .{}); + try writer.writeAll("for (;;) "); try genBody(f, body); try writer.writeByte('\n'); + + var old_free_locals = f.free_locals_stack.pop(); + defer deinitFreeLocalsMap(gpa, &old_free_locals); + const new_free_locals = f.getFreeLocals(); + var it = new_free_locals.iterator(); + while (it.next()) |entry| { + const gop = try old_free_locals.getOrPutContext(gpa, entry.key_ptr.*, f.tyHashCtx()); + if (gop.found_existing) { + try gop.value_ptr.appendSlice(gpa, entry.value_ptr.items); + } else { + gop.value_ptr.* = entry.value_ptr.*; + entry.value_ptr.* = .{}; + } + } + deinitFreeLocalsMap(gpa, new_free_locals); + new_free_locals.* = old_free_locals.move(); + return CValue.none; } @@ -4303,17 +4344,19 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { const gpa = f.object.dg.gpa; var cloned_map = try f.value_map.clone(); defer cloned_map.deinit(); - var cloned_frees = try cloneFreeLocalsMap(gpa, &f.free_locals); + var cloned_frees = try cloneFreeLocalsMap(gpa, f.getFreeLocals()); defer deinitFreeLocalsMap(gpa, &cloned_frees); - for (liveness_condbr.then_deaths) |operand| { - try die(f, inst, Air.indexToRef(operand)); - } - // Remember how many locals there were before entering the then branch so // that we can notice and use them in the else branch. Any new locals must // necessarily be free already after the then branch is complete. const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); + const pre_clone_depth = f.free_locals_clone_depth; + f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len); + + for (liveness_condbr.then_deaths) |operand| { + try die(f, inst, Air.indexToRef(operand)); + } try writer.writeAll("if ("); try f.writeCValue(writer, cond, .Other); @@ -4322,13 +4365,15 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(" else "); f.value_map.deinit(); f.value_map = cloned_map.move(); - deinitFreeLocalsMap(gpa, &f.free_locals); - f.free_locals = cloned_frees.move(); + const free_locals = f.getFreeLocals(); + deinitFreeLocalsMap(gpa, free_locals); + free_locals.* = cloned_frees.move(); + f.free_locals_clone_depth = pre_clone_depth; for (liveness_condbr.else_deaths) |operand| { try die(f, inst, Air.indexToRef(operand)); } - try noticeBranchFrees(f, pre_locals_len); + try noticeBranchFrees(f, pre_locals_len, inst); try genBody(f, else_body); try f.object.indent_writer.insertNewline(); @@ -4390,20 +4435,25 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { if (case_i != last_case_i) { const old_value_map = f.value_map; f.value_map = try old_value_map.clone(); - const old_free_locals = f.free_locals; - f.free_locals = try cloneFreeLocalsMap(gpa, &f.free_locals); + var free_locals = f.getFreeLocals(); + const old_free_locals = free_locals.*; + free_locals.* = try cloneFreeLocalsMap(gpa, free_locals); // Remember how many locals there were before entering each branch so that // we can notice and use them in subsequent branches. Any new locals must // necessarily be free already after the previous branch is complete. const pre_locals_len = @intCast(LocalIndex, f.locals.items.len); + const pre_clone_depth = f.free_locals_clone_depth; + f.free_locals_clone_depth = @intCast(LoopDepth, f.free_locals_stack.items.len); { defer { + f.free_locals_clone_depth = pre_clone_depth; f.value_map.deinit(); - deinitFreeLocalsMap(gpa, &f.free_locals); + free_locals = f.getFreeLocals(); + deinitFreeLocalsMap(gpa, free_locals); f.value_map = old_value_map; - f.free_locals = old_free_locals; + free_locals.* = old_free_locals; } for (liveness.deaths[case_i]) |operand| { @@ -4413,7 +4463,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue { try genBody(f, case_body); } - try noticeBranchFrees(f, pre_locals_len); + try noticeBranchFrees(f, pre_locals_len, inst); } else { for (liveness.deaths[case_i]) |operand| { try die(f, inst, Air.indexToRef(operand)); @@ -6193,7 +6243,7 @@ fn airReduce(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(init_val); try writer.writeAll(";"); try f.object.indent_writer.insertNewline(); - try writer.writeAll("for(;"); + try writer.writeAll("for (;"); try f.writeCValue(writer, it, .Other); try writer.print("<{d};++", .{vector_len}); try f.writeCValue(writer, it, .Other); @@ -6998,13 +7048,15 @@ fn die(f: *Function, inst: Air.Inst.Index, ref: Air.Inst.Ref) !void { fn freeLocal(f: *Function, inst: Air.Inst.Index, local_index: LocalIndex, ref_inst: Air.Inst.Index) !void { const gpa = f.object.dg.gpa; - const gop = try f.free_locals.getOrPutContext( + const local = &f.locals.items[local_index]; + log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst }); + if (local.loop_depth < f.free_locals_clone_depth) return; + const gop = try f.free_locals_stack.items[local.loop_depth].getOrPutContext( gpa, - f.locals.items[local_index].ty, + local.ty, f.tyHashCtx(), ); if (!gop.found_existing) gop.value_ptr.* = .{}; - log.debug("%{d}: freeing t{d} (operand %{d})", .{ inst, local_index, ref_inst }); if (std.debug.runtime_safety) { // If this trips, it means a local is being inserted into the // free_locals map while it already exists in the map, which is not @@ -7062,15 +7114,14 @@ fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void { map.deinit(gpa); } -fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex) !void { - const gpa = f.object.dg.gpa; - var i = pre_locals_len; - while (i < f.locals.items.len) : (i += 1) { - const local = f.locals.items[i]; - const unfreeable = f.allocs.contains(i); - if (unfreeable) continue; - const gop = try f.free_locals.getOrPutContext(gpa, local.ty, f.tyHashCtx()); - if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.append(gpa, i); +fn noticeBranchFrees(f: *Function, pre_locals_len: LocalIndex, inst: Air.Inst.Index) !void { + for (f.locals.items[pre_locals_len..]) |*local, local_offset| { + const local_index = pre_locals_len + @intCast(LocalIndex, local_offset); + if (f.allocs.contains(local_index)) continue; // allocs are not freeable + + // free more deeply nested locals from other branches at current depth + assert(local.loop_depth >= f.free_locals_stack.items.len - 1); + local.loop_depth = @intCast(LoopDepth, f.free_locals_stack.items.len - 1); + try freeLocal(f, inst, local_index, 0); } } -- cgit v1.2.3