diff options
Diffstat (limited to 'src/codegen.cpp')
| -rw-r--r-- | src/codegen.cpp | 589 |
1 files changed, 572 insertions, 17 deletions
diff --git a/src/codegen.cpp b/src/codegen.cpp index 15648cbdec..0cfd27322f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -381,6 +381,8 @@ static LLVMCallConv get_llvm_cc(CodeGen *g, CallingConvention cc) { } else { return LLVMCCallConv; } + case CallingConventionAsync: + return LLVMFastCallConv; } zig_unreachable(); } @@ -410,10 +412,10 @@ static uint32_t get_err_ret_trace_arg_index(CodeGen *g, FnTableEntry *fn_table_e return UINT32_MAX; } TypeTableEntry *fn_type = fn_table_entry->type_entry; - TypeTableEntry *return_type = fn_type->data.fn.fn_type_id.return_type; - if (return_type->id != TypeTableEntryIdErrorUnion && return_type->id != TypeTableEntryIdErrorSet) { + if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) { return UINT32_MAX; } + TypeTableEntry *return_type = fn_type->data.fn.fn_type_id.return_type; bool first_arg_ret = type_has_bits(return_type) && handle_is_ptr(return_type); return first_arg_ret ? 1 : 0; } @@ -540,7 +542,7 @@ static LLVMValueRef fn_llvm_value(CodeGen *g, FnTableEntry *fn_table_entry) { if (!type_has_bits(return_type)) { // nothing to do - } else if (return_type->id == TypeTableEntryIdPointer || return_type->id == TypeTableEntryIdFn) { + } else if (type_is_codegen_pointer(return_type)) { addLLVMAttr(fn_table_entry->llvm_value, 0, "nonnull"); } else if (handle_is_ptr(return_type) && calling_convention_does_first_arg_return(fn_type->data.fn.fn_type_id.cc)) @@ -925,6 +927,177 @@ static LLVMValueRef get_memcpy_fn_val(CodeGen *g) { return g->memcpy_fn_val; } +static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) { + if (g->coro_destroy_fn_val) + return g->coro_destroy_fn_val; + + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false); + Buf *name = buf_sprintf("llvm.coro.destroy"); + g->coro_destroy_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_destroy_fn_val)); + + return g->coro_destroy_fn_val; +} + +static LLVMValueRef get_coro_id_fn_val(CodeGen *g) { + if (g->coro_id_fn_val) + return g->coro_id_fn_val; + + LLVMTypeRef param_types[] = { + LLVMInt32Type(), + LLVMPointerType(LLVMInt8Type(), 0), + LLVMPointerType(LLVMInt8Type(), 0), + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 4, false); + Buf *name = buf_sprintf("llvm.coro.id"); + g->coro_id_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_id_fn_val)); + + return g->coro_id_fn_val; +} + +static LLVMValueRef get_coro_alloc_fn_val(CodeGen *g) { + if (g->coro_alloc_fn_val) + return g->coro_alloc_fn_val; + + LLVMTypeRef param_types[] = { + ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 1, false); + Buf *name = buf_sprintf("llvm.coro.alloc"); + g->coro_alloc_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_alloc_fn_val)); + + return g->coro_alloc_fn_val; +} + +static LLVMValueRef get_coro_size_fn_val(CodeGen *g) { + if (g->coro_size_fn_val) + return g->coro_size_fn_val; + + LLVMTypeRef fn_type = LLVMFunctionType(g->builtin_types.entry_usize->type_ref, nullptr, 0, false); + Buf *name = buf_sprintf("llvm.coro.size.i%d", g->pointer_size_bytes * 8); + g->coro_size_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_size_fn_val)); + + return g->coro_size_fn_val; +} + +static LLVMValueRef get_coro_begin_fn_val(CodeGen *g) { + if (g->coro_begin_fn_val) + return g->coro_begin_fn_val; + + LLVMTypeRef param_types[] = { + ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false); + Buf *name = buf_sprintf("llvm.coro.begin"); + g->coro_begin_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_begin_fn_val)); + + return g->coro_begin_fn_val; +} + +static LLVMValueRef get_coro_suspend_fn_val(CodeGen *g) { + if (g->coro_suspend_fn_val) + return g->coro_suspend_fn_val; + + LLVMTypeRef param_types[] = { + ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), + LLVMInt1Type(), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt8Type(), param_types, 2, false); + Buf *name = buf_sprintf("llvm.coro.suspend"); + g->coro_suspend_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_suspend_fn_val)); + + return g->coro_suspend_fn_val; +} + +static LLVMValueRef get_coro_end_fn_val(CodeGen *g) { + if (g->coro_end_fn_val) + return g->coro_end_fn_val; + + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + LLVMInt1Type(), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 2, false); + Buf *name = buf_sprintf("llvm.coro.end"); + g->coro_end_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_end_fn_val)); + + return g->coro_end_fn_val; +} + +static LLVMValueRef get_coro_free_fn_val(CodeGen *g) { + if (g->coro_free_fn_val) + return g->coro_free_fn_val; + + LLVMTypeRef param_types[] = { + ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false); + Buf *name = buf_sprintf("llvm.coro.free"); + g->coro_free_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_free_fn_val)); + + return g->coro_free_fn_val; +} + +static LLVMValueRef get_coro_resume_fn_val(CodeGen *g) { + if (g->coro_resume_fn_val) + return g->coro_resume_fn_val; + + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false); + Buf *name = buf_sprintf("llvm.coro.resume"); + g->coro_resume_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_resume_fn_val)); + + return g->coro_resume_fn_val; +} + +static LLVMValueRef get_coro_save_fn_val(CodeGen *g) { + if (g->coro_save_fn_val) + return g->coro_save_fn_val; + + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + }; + LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 1, false); + Buf *name = buf_sprintf("llvm.coro.save"); + g->coro_save_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_save_fn_val)); + + return g->coro_save_fn_val; +} + +static LLVMValueRef get_coro_promise_fn_val(CodeGen *g) { + if (g->coro_promise_fn_val) + return g->coro_promise_fn_val; + + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + LLVMInt32Type(), + LLVMInt1Type(), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 3, false); + Buf *name = buf_sprintf("llvm.coro.promise"); + g->coro_promise_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); + assert(LLVMGetIntrinsicID(g->coro_promise_fn_val)); + + return g->coro_promise_fn_val; +} + static LLVMValueRef get_return_address_fn_val(CodeGen *g) { if (g->return_address_fn_val) return g->return_address_fn_val; @@ -2506,6 +2679,25 @@ static LLVMValueRef ir_render_elem_ptr(CodeGen *g, IrExecutable *executable, IrI } } +static bool get_prefix_arg_err_ret_stack(CodeGen *g, FnTypeId *fn_type_id) { + return g->have_err_ret_tracing && + (fn_type_id->return_type->id == TypeTableEntryIdErrorUnion || + fn_type_id->return_type->id == TypeTableEntryIdErrorSet || + fn_type_id->cc == CallingConventionAsync); +} + +static size_t get_async_allocator_arg_index(CodeGen *g, FnTypeId *fn_type_id) { + // 0 1 2 3 + // err_ret_stack allocator_ptr err_code other_args... + return get_prefix_arg_err_ret_stack(g, fn_type_id) ? 1 : 0; +} + +static size_t get_async_err_code_arg_index(CodeGen *g, FnTypeId *fn_type_id) { + // 0 1 2 3 + // err_ret_stack allocator_ptr err_code other_args... + return 1 + get_async_allocator_arg_index(g, fn_type_id); +} + static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstructionCall *instruction) { LLVMValueRef fn_val; TypeTableEntry *fn_type; @@ -2519,11 +2711,15 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr } FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id; + TypeTableEntry *src_return_type = fn_type_id->return_type; bool ret_has_bits = type_has_bits(src_return_type); - bool first_arg_ret = ret_has_bits && handle_is_ptr(src_return_type); - bool prefix_arg_err_ret_stack = g->have_err_ret_tracing && (src_return_type->id == TypeTableEntryIdErrorUnion || src_return_type->id == TypeTableEntryIdErrorSet); - size_t actual_param_count = instruction->arg_count + (first_arg_ret ? 1 : 0) + (prefix_arg_err_ret_stack ? 1 : 0); + + bool first_arg_ret = ret_has_bits && handle_is_ptr(src_return_type) && + calling_convention_does_first_arg_return(fn_type->data.fn.fn_type_id.cc); + bool prefix_arg_err_ret_stack = get_prefix_arg_err_ret_stack(g, fn_type_id); + // +2 for the async args + size_t actual_param_count = instruction->arg_count + (first_arg_ret ? 1 : 0) + (prefix_arg_err_ret_stack ? 1 : 0) + 2; bool is_var_args = fn_type_id->is_var_args; LLVMValueRef *gen_param_values = allocate<LLVMValueRef>(actual_param_count); size_t gen_param_index = 0; @@ -2535,6 +2731,14 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr gen_param_values[gen_param_index] = g->cur_err_ret_trace_val; gen_param_index += 1; } + if (instruction->is_async) { + gen_param_values[gen_param_index] = ir_llvm_value(g, instruction->async_allocator); + gen_param_index += 1; + + LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, instruction->tmp_ptr, err_union_err_index, ""); + gen_param_values[gen_param_index] = err_val_ptr; + gen_param_index += 1; + } for (size_t call_i = 0; call_i < instruction->arg_count; call_i += 1) { IrInstruction *param_instruction = instruction->args[call_i]; TypeTableEntry *param_type = param_instruction->value.type; @@ -2572,6 +2776,12 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr } } + if (instruction->is_async) { + LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, instruction->tmp_ptr, err_union_payload_index, ""); + LLVMBuildStore(g->builder, result, payload_ptr); + return instruction->tmp_ptr; + } + if (src_return_type->id == TypeTableEntryIdUnreachable) { return LLVMBuildUnreachable(g->builder); } else if (!ret_has_bits) { @@ -2783,7 +2993,7 @@ static LLVMValueRef gen_non_null_bit(CodeGen *g, TypeTableEntry *maybe_type, LLV if (child_type->zero_bits) { return maybe_handle; } else { - bool maybe_is_ptr = (child_type->id == TypeTableEntryIdPointer || child_type->id == TypeTableEntryIdFn); + bool maybe_is_ptr = type_is_codegen_pointer(child_type); if (maybe_is_ptr) { return LLVMBuildICmp(g->builder, LLVMIntNE, maybe_handle, LLVMConstNull(maybe_type->type_ref), ""); } else { @@ -2823,7 +3033,7 @@ static LLVMValueRef ir_render_unwrap_maybe(CodeGen *g, IrExecutable *executable, if (child_type->zero_bits) { return nullptr; } else { - bool maybe_is_ptr = (child_type->id == TypeTableEntryIdPointer || child_type->id == TypeTableEntryIdFn); + bool maybe_is_ptr = type_is_codegen_pointer(child_type); if (maybe_is_ptr) { return maybe_ptr; } else { @@ -3046,6 +3256,10 @@ static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, I { align_bytes = target_type->data.maybe.child_type->data.fn.fn_type_id.alignment; ptr_val = target_val; + } else if (target_type->id == TypeTableEntryIdMaybe && + target_type->data.maybe.child_type->id == TypeTableEntryIdPromise) + { + zig_panic("TODO audit this function"); } else if (target_type->id == TypeTableEntryIdStruct && target_type->data.structure.is_slice) { TypeTableEntry *slice_ptr_type = target_type->data.structure.fields[slice_ptr_index].type_entry; align_bytes = slice_ptr_type->data.pointer.alignment; @@ -3088,6 +3302,20 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *execu return g->cur_err_ret_trace_val; } +static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { + LLVMValueRef target_handle = ir_llvm_value(g, instruction->target); + LLVMBuildCall(g->builder, get_coro_destroy_fn_val(g), &target_handle, 1, ""); + return nullptr; +} + +static LLVMValueRef ir_render_get_implicit_allocator(CodeGen *g, IrExecutable *executable, + IrInstructionGetImplicitAllocator *instruction) +{ + assert(instruction->id == ImplicitAllocatorIdArg); + size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); + return LLVMGetParam(g->cur_fn_val, allocator_arg_index); +} + static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { switch (atomic_order) { case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered; @@ -3100,6 +3328,23 @@ static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { zig_unreachable(); } +static LLVMAtomicRMWBinOp to_LLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed) { + switch (op) { + case AtomicRmwOp_xchg: return LLVMAtomicRMWBinOpXchg; + case AtomicRmwOp_add: return LLVMAtomicRMWBinOpAdd; + case AtomicRmwOp_sub: return LLVMAtomicRMWBinOpSub; + case AtomicRmwOp_and: return LLVMAtomicRMWBinOpAnd; + case AtomicRmwOp_nand: return LLVMAtomicRMWBinOpNand; + case AtomicRmwOp_or: return LLVMAtomicRMWBinOpOr; + case AtomicRmwOp_xor: return LLVMAtomicRMWBinOpXor; + case AtomicRmwOp_max: + return is_signed ? LLVMAtomicRMWBinOpMax : LLVMAtomicRMWBinOpUMax; + case AtomicRmwOp_min: + return is_signed ? LLVMAtomicRMWBinOpMin : LLVMAtomicRMWBinOpUMin; + } + zig_unreachable(); +} + static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutable *executable, IrInstructionCmpxchg *instruction) { LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr); LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value); @@ -3508,9 +3753,7 @@ static LLVMValueRef ir_render_maybe_wrap(CodeGen *g, IrExecutable *executable, I } LLVMValueRef payload_val = ir_llvm_value(g, instruction->value); - if (child_type->id == TypeTableEntryIdPointer || - child_type->id == TypeTableEntryIdFn) - { + if (type_is_codegen_pointer(child_type)) { return payload_val; } @@ -3682,6 +3925,264 @@ static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInst return nullptr; } +static LLVMValueRef ir_render_coro_id(CodeGen *g, IrExecutable *executable, IrInstructionCoroId *instruction) { + LLVMValueRef promise_ptr = ir_llvm_value(g, instruction->promise_ptr); + LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false); + LLVMValueRef null = LLVMConstIntToPtr(LLVMConstNull(g->builtin_types.entry_usize->type_ref), + LLVMPointerType(LLVMInt8Type(), 0)); + LLVMValueRef params[] = { + align_val, + promise_ptr, + null, + null, + }; + return LLVMBuildCall(g->builder, get_coro_id_fn_val(g), params, 4, ""); +} + +static LLVMValueRef ir_render_coro_alloc(CodeGen *g, IrExecutable *executable, IrInstructionCoroAlloc *instruction) { + LLVMValueRef token = ir_llvm_value(g, instruction->coro_id); + return LLVMBuildCall(g->builder, get_coro_alloc_fn_val(g), &token, 1, ""); +} + +static LLVMValueRef ir_render_coro_size(CodeGen *g, IrExecutable *executable, IrInstructionCoroSize *instruction) { + return LLVMBuildCall(g->builder, get_coro_size_fn_val(g), nullptr, 0, ""); +} + +static LLVMValueRef ir_render_coro_begin(CodeGen *g, IrExecutable *executable, IrInstructionCoroBegin *instruction) { + LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id); + LLVMValueRef coro_mem_ptr = ir_llvm_value(g, instruction->coro_mem_ptr); + LLVMValueRef params[] = { + coro_id, + coro_mem_ptr, + }; + return LLVMBuildCall(g->builder, get_coro_begin_fn_val(g), params, 2, ""); +} + +static LLVMValueRef ir_render_coro_alloc_fail(CodeGen *g, IrExecutable *executable, + IrInstructionCoroAllocFail *instruction) +{ + size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); + LLVMValueRef err_code_ptr_val = LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index); + LLVMValueRef err_code = ir_llvm_value(g, instruction->err_val); + LLVMBuildStore(g->builder, err_code, err_code_ptr_val); + + LLVMValueRef return_value; + if (ir_want_runtime_safety(g, &instruction->base)) { + return_value = LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)); + } else { + return_value = LLVMGetUndef(LLVMPointerType(LLVMInt8Type(), 0)); + } + LLVMBuildRet(g->builder, return_value); + return nullptr; +} + +static LLVMValueRef ir_render_coro_suspend(CodeGen *g, IrExecutable *executable, IrInstructionCoroSuspend *instruction) { + LLVMValueRef save_point; + if (instruction->save_point == nullptr) { + save_point = LLVMConstNull(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext())); + } else { + save_point = ir_llvm_value(g, instruction->save_point); + } + LLVMValueRef is_final = ir_llvm_value(g, instruction->is_final); + LLVMValueRef params[] = { + save_point, + is_final, + }; + return LLVMBuildCall(g->builder, get_coro_suspend_fn_val(g), params, 2, ""); +} + +static LLVMValueRef ir_render_coro_end(CodeGen *g, IrExecutable *executable, IrInstructionCoroEnd *instruction) { + LLVMValueRef params[] = { + LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)), + LLVMConstNull(LLVMInt1Type()), + }; + return LLVMBuildCall(g->builder, get_coro_end_fn_val(g), params, 2, ""); +} + +static LLVMValueRef ir_render_coro_free(CodeGen *g, IrExecutable *executable, IrInstructionCoroFree *instruction) { + LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id); + LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); + LLVMValueRef params[] = { + coro_id, + coro_handle, + }; + return LLVMBuildCall(g->builder, get_coro_free_fn_val(g), params, 2, ""); +} + +static LLVMValueRef ir_render_coro_resume(CodeGen *g, IrExecutable *executable, IrInstructionCoroResume *instruction) { + LLVMValueRef awaiter_handle = ir_llvm_value(g, instruction->awaiter_handle); + return LLVMBuildCall(g->builder, get_coro_resume_fn_val(g), &awaiter_handle, 1, ""); +} + +static LLVMValueRef ir_render_coro_save(CodeGen *g, IrExecutable *executable, IrInstructionCoroSave *instruction) { + LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); + return LLVMBuildCall(g->builder, get_coro_save_fn_val(g), &coro_handle, 1, ""); +} + +static LLVMValueRef ir_render_coro_promise(CodeGen *g, IrExecutable *executable, IrInstructionCoroPromise *instruction) { + LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); + LLVMValueRef params[] = { + coro_handle, + LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false), + LLVMConstNull(LLVMInt1Type()), + }; + LLVMValueRef uncasted_result = LLVMBuildCall(g->builder, get_coro_promise_fn_val(g), params, 3, ""); + return LLVMBuildBitCast(g->builder, uncasted_result, instruction->base.value.type->type_ref, ""); +} + +static LLVMValueRef get_coro_alloc_helper_fn_val(CodeGen *g, LLVMTypeRef alloc_fn_type_ref, TypeTableEntry *fn_type) { + if (g->coro_alloc_helper_fn_val != nullptr) + return g->coro_alloc_helper_fn_val; + + assert(fn_type->id == TypeTableEntryIdFn); + + TypeTableEntry *ptr_to_err_code_type = get_pointer_to_type(g, g->builtin_types.entry_global_error_set, false); + + LLVMTypeRef alloc_raw_fn_type_ref = LLVMGetElementType(alloc_fn_type_ref); + LLVMTypeRef *alloc_fn_arg_types = allocate<LLVMTypeRef>(LLVMCountParamTypes(alloc_raw_fn_type_ref)); + LLVMGetParamTypes(alloc_raw_fn_type_ref, alloc_fn_arg_types); + + ZigList<LLVMTypeRef> arg_types = {}; + arg_types.append(alloc_fn_type_ref); + if (g->have_err_ret_tracing) { + arg_types.append(alloc_fn_arg_types[1]); + } + arg_types.append(alloc_fn_arg_types[g->have_err_ret_tracing ? 2 : 1]); + arg_types.append(ptr_to_err_code_type->type_ref); + arg_types.append(g->builtin_types.entry_usize->type_ref); + + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), + arg_types.items, arg_types.length, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_coro_alloc_helper"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); + addLLVMArgAttr(fn_val, (unsigned)1, "nonnull"); + + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + FnTableEntry *prev_cur_fn = g->cur_fn; + LLVMValueRef prev_cur_fn_val = g->cur_fn_val; + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + g->cur_fn = nullptr; + g->cur_fn_val = fn_val; + + LLVMValueRef sret_ptr = LLVMBuildAlloca(g->builder, LLVMGetElementType(alloc_fn_arg_types[0]), ""); + + size_t next_arg = 0; + LLVMValueRef alloc_fn_val = LLVMGetParam(fn_val, next_arg); + next_arg += 1; + + LLVMValueRef stack_trace_val; + if (g->have_err_ret_tracing) { + stack_trace_val = LLVMGetParam(fn_val, next_arg); + next_arg += 1; + } + + LLVMValueRef allocator_val = LLVMGetParam(fn_val, next_arg); + next_arg += 1; + LLVMValueRef err_code_ptr = LLVMGetParam(fn_val, next_arg); + next_arg += 1; + LLVMValueRef coro_size = LLVMGetParam(fn_val, next_arg); + next_arg += 1; + LLVMValueRef alignment_val = LLVMConstInt(g->builtin_types.entry_u29->type_ref, + get_coro_frame_align_bytes(g), false); + + ZigList<LLVMValueRef> args = {}; + args.append(sret_ptr); + if (g->have_err_ret_tracing) { + args.append(stack_trace_val); + } + args.append(allocator_val); + args.append(coro_size); + args.append(alignment_val); + ZigLLVMBuildCall(g->builder, alloc_fn_val, args.items, args.length, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_err_index, ""); + LLVMValueRef err_val = LLVMBuildLoad(g->builder, err_val_ptr, ""); + LLVMBuildStore(g->builder, err_val, err_code_ptr); + LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_val, LLVMConstNull(LLVMTypeOf(err_val)), ""); + LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(fn_val, "AllocOk"); + LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(fn_val, "AllocFail"); + LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block); + + LLVMPositionBuilderAtEnd(g->builder, ok_block); + LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_payload_index, ""); + TypeTableEntry *u8_ptr_type = get_pointer_to_type(g, g->builtin_types.entry_u8, false); + TypeTableEntry *slice_type = get_slice_type(g, u8_ptr_type); + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, payload_ptr, ptr_field_index, ""); + LLVMValueRef ptr_val = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); + LLVMBuildRet(g->builder, ptr_val); + + LLVMPositionBuilderAtEnd(g->builder, fail_block); + LLVMBuildRet(g->builder, LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0))); + + g->cur_fn = prev_cur_fn; + g->cur_fn_val = prev_cur_fn_val; + LLVMPositionBuilderAtEnd(g->builder, prev_block); + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + + g->coro_alloc_helper_fn_val = fn_val; + return fn_val; +} + +static LLVMValueRef ir_render_coro_alloc_helper(CodeGen *g, IrExecutable *executable, + IrInstructionCoroAllocHelper *instruction) +{ + LLVMValueRef alloc_fn = ir_llvm_value(g, instruction->alloc_fn); + LLVMValueRef coro_size = ir_llvm_value(g, instruction->coro_size); + LLVMValueRef fn_val = get_coro_alloc_helper_fn_val(g, LLVMTypeOf(alloc_fn), instruction->alloc_fn->value.type); + size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); + size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); + + ZigList<LLVMValueRef> params = {}; + params.append(alloc_fn); + uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, g->cur_fn); + if (err_ret_trace_arg_index != UINT32_MAX) { + params.append(LLVMGetParam(g->cur_fn_val, err_ret_trace_arg_index)); + } + params.append(LLVMGetParam(g->cur_fn_val, allocator_arg_index)); + params.append(LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index)); + params.append(coro_size); + + return ZigLLVMBuildCall(g->builder, fn_val, params.items, params.length, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); +} + +static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable, + IrInstructionAtomicRmw *instruction) +{ + bool is_signed; + TypeTableEntry *operand_type = instruction->operand->value.type; + if (operand_type->id == TypeTableEntryIdInt) { + is_signed = operand_type->data.integral.is_signed; + } else { + is_signed = false; + } + LLVMAtomicRMWBinOp op = to_LLVMAtomicRMWBinOp(instruction->resolved_op, is_signed); + LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering); + LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr); + LLVMValueRef operand = ir_llvm_value(g, instruction->operand); + + if (get_codegen_ptr_type(operand_type) == nullptr) { + return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, false); + } + + // it's a pointer but we need to treat it as an int + LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr, + LLVMPointerType(g->builtin_types.entry_usize->type_ref, 0), ""); + LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->type_ref, ""); + LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering, false); + return LLVMBuildIntToPtr(g->builder, uncasted_result, operand_type->type_ref, ""); +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -3745,7 +4246,9 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdTagType: case IrInstructionIdExport: case IrInstructionIdErrorUnion: + case IrInstructionIdPromiseResultType: zig_unreachable(); + case IrInstructionIdReturn: return ir_render_return(g, executable, (IrInstructionReturn *)instruction); case IrInstructionIdDeclVar: @@ -3862,12 +4365,43 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_align_cast(g, executable, (IrInstructionAlignCast *)instruction); case IrInstructionIdErrorReturnTrace: return ir_render_error_return_trace(g, executable, (IrInstructionErrorReturnTrace *)instruction); + case IrInstructionIdCancel: + return ir_render_cancel(g, executable, (IrInstructionCancel *)instruction); + case IrInstructionIdGetImplicitAllocator: + return ir_render_get_implicit_allocator(g, executable, (IrInstructionGetImplicitAllocator *)instruction); + case IrInstructionIdCoroId: + return ir_render_coro_id(g, executable, (IrInstructionCoroId *)instruction); + case IrInstructionIdCoroAlloc: + return ir_render_coro_alloc(g, executable, (IrInstructionCoroAlloc *)instruction); + case IrInstructionIdCoroSize: + return ir_render_coro_size(g, executable, (IrInstructionCoroSize *)instruction); + case IrInstructionIdCoroBegin: + return ir_render_coro_begin(g, executable, (IrInstructionCoroBegin *)instruction); + case IrInstructionIdCoroAllocFail: + return ir_render_coro_alloc_fail(g, executable, (IrInstructionCoroAllocFail *)instruction); + case IrInstructionIdCoroSuspend: + return ir_render_coro_suspend(g, executable, (IrInstructionCoroSuspend *)instruction); + case IrInstructionIdCoroEnd: + return ir_render_coro_end(g, executable, (IrInstructionCoroEnd *)instruction); + case IrInstructionIdCoroFree: + return ir_render_coro_free(g, executable, (IrInstructionCoroFree *)instruction); + case IrInstructionIdCoroResume: + return ir_render_coro_resume(g, executable, (IrInstructionCoroResume *)instruction); + case IrInstructionIdCoroSave: + return ir_render_coro_save(g, executable, (IrInstructionCoroSave *)instruction); + case IrInstructionIdCoroPromise: + return ir_render_coro_promise(g, executable, (IrInstructionCoroPromise *)instruction); + case IrInstructionIdCoroAllocHelper: + return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction); + case IrInstructionIdAtomicRmw: + return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); } zig_unreachable(); } static void ir_render(CodeGen *g, FnTableEntry *fn_entry) { assert(fn_entry); + IrExecutable *executable = &fn_entry->analyzed_executable; assert(executable->basic_block_list.length > 0); for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) { @@ -4009,6 +4543,7 @@ static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, Con case TypeTableEntryIdPointer: case TypeTableEntryIdFn: case TypeTableEntryIdMaybe: + case TypeTableEntryIdPromise: { LLVMValueRef ptr_val = gen_const_val(g, const_val, ""); LLVMValueRef ptr_size_int_val = LLVMConstPtrToInt(ptr_val, g->builtin_types.entry_usize->type_ref); @@ -4104,9 +4639,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c TypeTableEntry *child_type = type_entry->data.maybe.child_type; if (child_type->zero_bits) { return LLVMConstInt(LLVMInt1Type(), const_val->data.x_maybe ? 1 : 0, false); - } else if (child_type->id == TypeTableEntryIdPointer || - child_type->id == TypeTableEntryIdFn) - { + } else if (type_is_codegen_pointer(child_type)) { if (const_val->data.x_maybe) { return gen_const_val(g, const_val->data.x_maybe, ""); } else { @@ -4426,6 +4959,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c case TypeTableEntryIdVar: case TypeTableEntryIdArgTuple: case TypeTableEntryIdOpaque: + case TypeTableEntryIdPromise: zig_unreachable(); } @@ -5235,6 +5769,7 @@ static void define_builtin_types(CodeGen *g) { g->builtin_types.entry_u8 = get_int_type(g, false, 8); g->builtin_types.entry_u16 = get_int_type(g, false, 16); + g->builtin_types.entry_u29 = get_int_type(g, false, 29); g->builtin_types.entry_u32 = get_int_type(g, false, 32); g->builtin_types.entry_u64 = get_int_type(g, false, 64); g->builtin_types.entry_u128 = get_int_type(g, false, 128); @@ -5271,6 +5806,10 @@ static void define_builtin_types(CodeGen *g) { g->primitive_type_table.put(&entry->name, entry); } + { + TypeTableEntry *entry = get_promise_type(g, nullptr); + g->primitive_type_table.put(&entry->name, entry); + } } @@ -5348,6 +5887,7 @@ static void define_builtin_fns(CodeGen *g) { create_builtin_fn(g, BuiltinFnIdArgType, "ArgType", 2); create_builtin_fn(g, BuiltinFnIdExport, "export", 3); create_builtin_fn(g, BuiltinFnIdErrorReturnTrace, "errorReturnTrace", 0); + create_builtin_fn(g, BuiltinFnIdAtomicRmw, "atomicRmw", 5); } static const char *bool_to_str(bool b) { @@ -5479,6 +6019,20 @@ static void define_builtin_compile_vars(CodeGen *g) { } { buf_appendf(contents, + "pub const AtomicRmwOp = enum {\n" + " Xchg,\n" + " Add,\n" + " Sub,\n" + " And,\n" + " Nand,\n" + " Or,\n" + " Xor,\n" + " Max,\n" + " Min,\n" + "};\n\n"); + } + { + buf_appendf(contents, "pub const Mode = enum {\n" " Debug,\n" " ReleaseSafe,\n" @@ -5898,6 +6452,7 @@ static void prepend_c_type_to_decl_list(CodeGen *g, GenH *gen_h, TypeTableEntry case TypeTableEntryIdArgTuple: case TypeTableEntryIdErrorUnion: case TypeTableEntryIdErrorSet: + case TypeTableEntryIdPromise: zig_unreachable(); case TypeTableEntryIdVoid: case TypeTableEntryIdUnreachable: @@ -6027,9 +6582,7 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf if (child_type->zero_bits) { buf_init_from_str(out_buf, "bool"); return; - } else if (child_type->id == TypeTableEntryIdPointer || - child_type->id == TypeTableEntryIdFn) - { + } else if (type_is_codegen_pointer(child_type)) { return get_c_type(g, gen_h, child_type, out_buf); } else { zig_unreachable(); @@ -6084,6 +6637,7 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf case TypeTableEntryIdNullLit: case TypeTableEntryIdVar: case TypeTableEntryIdArgTuple: + case TypeTableEntryIdPromise: zig_unreachable(); } } @@ -6244,6 +6798,7 @@ static void gen_h_file(CodeGen *g) { case TypeTableEntryIdArgTuple: case TypeTableEntryIdMaybe: case TypeTableEntryIdFn: + case TypeTableEntryIdPromise: zig_unreachable(); case TypeTableEntryIdEnum: assert(type_entry->data.enumeration.layout == ContainerLayoutExtern); |