diff options
Diffstat (limited to 'src/codegen.cpp')
| -rw-r--r-- | src/codegen.cpp | 2148 |
1 files changed, 1230 insertions, 918 deletions
diff --git a/src/codegen.cpp b/src/codegen.cpp index 32e6d2fbee..45e2e4122f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -24,6 +24,12 @@ #include <stdio.h> #include <errno.h> +enum ResumeId { + ResumeIdManual, + ResumeIdReturn, + ResumeIdCall, +}; + static void init_darwin_native(CodeGen *g) { char *osx_target = getenv("MACOSX_DEPLOYMENT_TARGET"); char *ios_target = getenv("IPHONEOS_DEPLOYMENT_TARGET"); @@ -297,12 +303,42 @@ static LLVMLinkage to_llvm_linkage(GlobalLinkageId id) { zig_unreachable(); } +// label (grep this): [fn_frame_struct_layout] +static uint32_t frame_index_trace_arg(CodeGen *g, ZigType *return_type) { + // [0] *ReturnType (callee's) + // [1] *ReturnType (awaiter's) + // [2] ReturnType + uint32_t return_field_count = type_has_bits(return_type) ? 3 : 0; + return frame_ret_start + return_field_count; +} + +// label (grep this): [fn_frame_struct_layout] +static uint32_t frame_index_arg(CodeGen *g, ZigType *return_type) { + bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, return_type); + // [0] *StackTrace (callee's) + // [1] *StackTrace (awaiter's) + uint32_t trace_field_count = have_stack_trace ? 2 : 0; + return frame_index_trace_arg(g, return_type) + trace_field_count; +} + +// label (grep this): [fn_frame_struct_layout] +static uint32_t frame_index_trace_stack(CodeGen *g, FnTypeId *fn_type_id) { + uint32_t result = frame_index_arg(g, fn_type_id->return_type); + for (size_t i = 0; i < fn_type_id->param_count; i += 1) { + if (type_has_bits(fn_type_id->param_info->type)) { + result += 1; + } + } + return result; +} + + static uint32_t get_err_ret_trace_arg_index(CodeGen *g, ZigFn *fn_table_entry) { if (!g->have_err_ret_tracing) { return UINT32_MAX; } - if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { - return 0; + if (fn_is_async(fn_table_entry)) { + return UINT32_MAX; } ZigType *fn_type = fn_table_entry->type_entry; if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) { @@ -343,27 +379,28 @@ static bool cc_want_sret_attr(CallingConvention cc) { zig_unreachable(); } -static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn_table_entry) { - if (fn_table_entry->llvm_value) - return fn_table_entry->llvm_value; +static bool codegen_have_frame_pointer(CodeGen *g) { + return g->build_mode == BuildModeDebug; +} - Buf *unmangled_name = &fn_table_entry->symbol_name; +static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) { + Buf *unmangled_name = &fn->symbol_name; Buf *symbol_name; GlobalLinkageId linkage; - if (fn_table_entry->body_node == nullptr) { + if (fn->body_node == nullptr) { symbol_name = unmangled_name; linkage = GlobalLinkageIdStrong; - } else if (fn_table_entry->export_list.length == 0) { + } else if (fn->export_list.length == 0) { symbol_name = get_mangled_name(g, unmangled_name, false); linkage = GlobalLinkageIdInternal; } else { - GlobalExport *fn_export = &fn_table_entry->export_list.items[0]; + GlobalExport *fn_export = &fn->export_list.items[0]; symbol_name = &fn_export->name; linkage = fn_export->linkage; } bool external_linkage = linkage != GlobalLinkageIdInternal; - CallingConvention cc = fn_table_entry->type_entry->data.fn.fn_type_id.cc; + CallingConvention cc = fn->type_entry->data.fn.fn_type_id.cc; if (cc == CallingConventionStdcall && external_linkage && g->zig_target->arch == ZigLLVM_x86) { @@ -371,130 +408,125 @@ static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn_table_entry) { symbol_name = buf_sprintf("\x01_%s", buf_ptr(symbol_name)); } + bool is_async = fn_is_async(fn); - ZigType *fn_type = fn_table_entry->type_entry; + + ZigType *fn_type = fn->type_entry; // Make the raw_type_ref populated - (void)get_llvm_type(g, fn_type); - LLVMTypeRef fn_llvm_type = fn_type->data.fn.raw_type_ref; - if (fn_table_entry->body_node == nullptr) { + resolve_llvm_types_fn(g, fn); + LLVMTypeRef fn_llvm_type = fn->raw_type_ref; + LLVMValueRef llvm_fn = nullptr; + if (fn->body_node == nullptr) { LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, buf_ptr(symbol_name)); if (existing_llvm_fn) { - fn_table_entry->llvm_value = LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, 0)); - return fn_table_entry->llvm_value; + return LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, 0)); } else { auto entry = g->exported_symbol_names.maybe_get(symbol_name); if (entry == nullptr) { - fn_table_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type); + llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type); if (target_is_wasm(g->zig_target)) { - assert(fn_table_entry->proto_node->type == NodeTypeFnProto); - AstNodeFnProto *fn_proto = &fn_table_entry->proto_node->data.fn_proto; + assert(fn->proto_node->type == NodeTypeFnProto); + AstNodeFnProto *fn_proto = &fn->proto_node->data.fn_proto; if (fn_proto-> is_extern && fn_proto->lib_name != nullptr ) { - addLLVMFnAttrStr(fn_table_entry->llvm_value, "wasm-import-module", buf_ptr(fn_proto->lib_name)); + addLLVMFnAttrStr(llvm_fn, "wasm-import-module", buf_ptr(fn_proto->lib_name)); } } } else { assert(entry->value->id == TldIdFn); TldFn *tld_fn = reinterpret_cast<TldFn *>(entry->value); // Make the raw_type_ref populated - (void)get_llvm_type(g, tld_fn->fn_entry->type_entry); + resolve_llvm_types_fn(g, tld_fn->fn_entry); tld_fn->fn_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name), - tld_fn->fn_entry->type_entry->data.fn.raw_type_ref); - fn_table_entry->llvm_value = LLVMConstBitCast(tld_fn->fn_entry->llvm_value, - LLVMPointerType(fn_llvm_type, 0)); - return fn_table_entry->llvm_value; + tld_fn->fn_entry->raw_type_ref); + llvm_fn = LLVMConstBitCast(tld_fn->fn_entry->llvm_value, LLVMPointerType(fn_llvm_type, 0)); + return llvm_fn; } } } else { - if (fn_table_entry->llvm_value == nullptr) { - fn_table_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type); + if (llvm_fn == nullptr) { + llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type); } - for (size_t i = 1; i < fn_table_entry->export_list.length; i += 1) { - GlobalExport *fn_export = &fn_table_entry->export_list.items[i]; - LLVMAddAlias(g->module, LLVMTypeOf(fn_table_entry->llvm_value), - fn_table_entry->llvm_value, buf_ptr(&fn_export->name)); + for (size_t i = 1; i < fn->export_list.length; i += 1) { + GlobalExport *fn_export = &fn->export_list.items[i]; + LLVMAddAlias(g->module, LLVMTypeOf(llvm_fn), llvm_fn, buf_ptr(&fn_export->name)); } } - fn_table_entry->llvm_name = strdup(LLVMGetValueName(fn_table_entry->llvm_value)); - switch (fn_table_entry->fn_inline) { + switch (fn->fn_inline) { case FnInlineAlways: - addLLVMFnAttr(fn_table_entry->llvm_value, "alwaysinline"); - g->inline_fns.append(fn_table_entry); + addLLVMFnAttr(llvm_fn, "alwaysinline"); + g->inline_fns.append(fn); break; case FnInlineNever: - addLLVMFnAttr(fn_table_entry->llvm_value, "noinline"); + addLLVMFnAttr(llvm_fn, "noinline"); break; case FnInlineAuto: - if (fn_table_entry->alignstack_value != 0) { - addLLVMFnAttr(fn_table_entry->llvm_value, "noinline"); + if (fn->alignstack_value != 0) { + addLLVMFnAttr(llvm_fn, "noinline"); } break; } if (cc == CallingConventionNaked) { - addLLVMFnAttr(fn_table_entry->llvm_value, "naked"); + addLLVMFnAttr(llvm_fn, "naked"); } else { - LLVMSetFunctionCallConv(fn_table_entry->llvm_value, get_llvm_cc(g, fn_type->data.fn.fn_type_id.cc)); - } - if (cc == CallingConventionAsync) { - addLLVMFnAttr(fn_table_entry->llvm_value, "optnone"); - addLLVMFnAttr(fn_table_entry->llvm_value, "noinline"); + LLVMSetFunctionCallConv(llvm_fn, get_llvm_cc(g, fn_type->data.fn.fn_type_id.cc)); } - bool want_cold = fn_table_entry->is_cold || cc == CallingConventionCold; + bool want_cold = fn->is_cold || cc == CallingConventionCold; if (want_cold) { - ZigLLVMAddFunctionAttrCold(fn_table_entry->llvm_value); + ZigLLVMAddFunctionAttrCold(llvm_fn); } - LLVMSetLinkage(fn_table_entry->llvm_value, to_llvm_linkage(linkage)); + LLVMSetLinkage(llvm_fn, to_llvm_linkage(linkage)); if (linkage == GlobalLinkageIdInternal) { - LLVMSetUnnamedAddr(fn_table_entry->llvm_value, true); + LLVMSetUnnamedAddr(llvm_fn, true); } ZigType *return_type = fn_type->data.fn.fn_type_id.return_type; if (return_type->id == ZigTypeIdUnreachable) { - addLLVMFnAttr(fn_table_entry->llvm_value, "noreturn"); + addLLVMFnAttr(llvm_fn, "noreturn"); } - if (fn_table_entry->body_node != nullptr) { - maybe_export_dll(g, fn_table_entry->llvm_value, linkage); + if (fn->body_node != nullptr) { + maybe_export_dll(g, llvm_fn, linkage); bool want_fn_safety = g->build_mode != BuildModeFastRelease && g->build_mode != BuildModeSmallRelease && - !fn_table_entry->def_scope->safety_off; + !fn->def_scope->safety_off; if (want_fn_safety) { if (g->libc_link_lib != nullptr) { - addLLVMFnAttr(fn_table_entry->llvm_value, "sspstrong"); - addLLVMFnAttrStr(fn_table_entry->llvm_value, "stack-protector-buffer-size", "4"); + addLLVMFnAttr(llvm_fn, "sspstrong"); + addLLVMFnAttrStr(llvm_fn, "stack-protector-buffer-size", "4"); } } - if (g->have_stack_probing && !fn_table_entry->def_scope->safety_off) { - addLLVMFnAttrStr(fn_table_entry->llvm_value, "probe-stack", "__zig_probe_stack"); + if (g->have_stack_probing && !fn->def_scope->safety_off) { + addLLVMFnAttrStr(llvm_fn, "probe-stack", "__zig_probe_stack"); } } else { - maybe_import_dll(g, fn_table_entry->llvm_value, linkage); + maybe_import_dll(g, llvm_fn, linkage); } - if (fn_table_entry->alignstack_value != 0) { - addLLVMFnAttrInt(fn_table_entry->llvm_value, "alignstack", fn_table_entry->alignstack_value); + if (fn->alignstack_value != 0) { + addLLVMFnAttrInt(llvm_fn, "alignstack", fn->alignstack_value); } - addLLVMFnAttr(fn_table_entry->llvm_value, "nounwind"); - add_uwtable_attr(g, fn_table_entry->llvm_value); - addLLVMFnAttr(fn_table_entry->llvm_value, "nobuiltin"); - if (g->build_mode == BuildModeDebug && fn_table_entry->fn_inline != FnInlineAlways) { - ZigLLVMAddFunctionAttr(fn_table_entry->llvm_value, "no-frame-pointer-elim", "true"); - ZigLLVMAddFunctionAttr(fn_table_entry->llvm_value, "no-frame-pointer-elim-non-leaf", nullptr); + addLLVMFnAttr(llvm_fn, "nounwind"); + add_uwtable_attr(g, llvm_fn); + addLLVMFnAttr(llvm_fn, "nobuiltin"); + if (codegen_have_frame_pointer(g) && fn->fn_inline != FnInlineAlways) { + ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim-non-leaf", nullptr); } - if (fn_table_entry->section_name) { - LLVMSetSection(fn_table_entry->llvm_value, buf_ptr(fn_table_entry->section_name)); + if (fn->section_name) { + LLVMSetSection(llvm_fn, buf_ptr(fn->section_name)); } - if (fn_table_entry->align_bytes > 0) { - LLVMSetAlignment(fn_table_entry->llvm_value, (unsigned)fn_table_entry->align_bytes); + if (fn->align_bytes > 0) { + LLVMSetAlignment(llvm_fn, (unsigned)fn->align_bytes); } else { // We'd like to set the best alignment for the function here, but on Darwin LLVM gives // "Cannot getTypeInfo() on a type that is unsized!" assertion failure when calling @@ -502,36 +534,50 @@ static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn_table_entry) { // use the ABI alignment, which is fine. } - unsigned init_gen_i = 0; - if (!type_has_bits(return_type)) { - // nothing to do - } else if (type_is_nonnull_ptr(return_type)) { - addLLVMAttr(fn_table_entry->llvm_value, 0, "nonnull"); - } else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) { - // Sret pointers must not be address 0 - addLLVMArgAttr(fn_table_entry->llvm_value, 0, "nonnull"); - addLLVMArgAttr(fn_table_entry->llvm_value, 0, "sret"); - if (cc_want_sret_attr(cc)) { - addLLVMArgAttr(fn_table_entry->llvm_value, 0, "noalias"); + if (is_async) { + addLLVMArgAttr(llvm_fn, 0, "nonnull"); + } else { + unsigned init_gen_i = 0; + if (!type_has_bits(return_type)) { + // nothing to do + } else if (type_is_nonnull_ptr(return_type)) { + addLLVMAttr(llvm_fn, 0, "nonnull"); + } else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) { + // Sret pointers must not be address 0 + addLLVMArgAttr(llvm_fn, 0, "nonnull"); + addLLVMArgAttr(llvm_fn, 0, "sret"); + if (cc_want_sret_attr(cc)) { + addLLVMArgAttr(llvm_fn, 0, "noalias"); + } + init_gen_i = 1; } - init_gen_i = 1; - } - // set parameter attributes - FnWalk fn_walk = {}; - fn_walk.id = FnWalkIdAttrs; - fn_walk.data.attrs.fn = fn_table_entry; - fn_walk.data.attrs.gen_i = init_gen_i; - walk_function_params(g, fn_type, &fn_walk); + // set parameter attributes + FnWalk fn_walk = {}; + fn_walk.id = FnWalkIdAttrs; + fn_walk.data.attrs.fn = fn; + fn_walk.data.attrs.llvm_fn = llvm_fn; + fn_walk.data.attrs.gen_i = init_gen_i; + walk_function_params(g, fn_type, &fn_walk); - uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry); - if (err_ret_trace_arg_index != UINT32_MAX) { - // Error return trace memory is in the stack, which is impossible to be at address 0 - // on any architecture. - addLLVMArgAttr(fn_table_entry->llvm_value, (unsigned)err_ret_trace_arg_index, "nonnull"); + uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn); + if (err_ret_trace_arg_index != UINT32_MAX) { + // Error return trace memory is in the stack, which is impossible to be at address 0 + // on any architecture. + addLLVMArgAttr(llvm_fn, (unsigned)err_ret_trace_arg_index, "nonnull"); + } } - return fn_table_entry->llvm_value; + return llvm_fn; +} + +static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn) { + if (fn->llvm_value) + return fn->llvm_value; + + fn->llvm_value = make_fn_llvm_value(g, fn); + fn->llvm_name = strdup(LLVMGetValueName(fn->llvm_value)); + return fn->llvm_value; } static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) { @@ -559,10 +605,11 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) { unsigned flags = ZigLLVM_DIFlags_StaticMember; ZigLLVMDIScope *fn_di_scope = get_di_scope(g, scope->parent); assert(fn_di_scope != nullptr); + assert(fn_table_entry->raw_di_type != nullptr); ZigLLVMDISubprogram *subprogram = ZigLLVMCreateFunction(g->dbuilder, fn_di_scope, buf_ptr(&fn_table_entry->symbol_name), "", import->data.structure.root_struct->di_file, line_number, - fn_table_entry->type_entry->data.fn.raw_di_type, is_internal_linkage, + fn_table_entry->raw_di_type, is_internal_linkage, is_definition, scope_line, flags, is_optimized, nullptr); scope->di_scope = ZigLLVMSubprogramToScope(subprogram); @@ -597,7 +644,6 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) { case ScopeIdLoop: case ScopeIdSuspend: case ScopeIdCompTime: - case ScopeIdCoroPrelude: case ScopeIdRuntime: return get_di_scope(g, scope->parent); } @@ -798,9 +844,8 @@ static bool ir_want_fast_math(CodeGen *g, IrInstruction *instruction) { return false; } -static bool ir_want_runtime_safety(CodeGen *g, IrInstruction *instruction) { +static bool ir_want_runtime_safety_scope(CodeGen *g, Scope *scope) { // TODO memoize - Scope *scope = instruction->scope; while (scope) { if (scope->id == ScopeIdBlock) { ScopeBlock *block_scope = (ScopeBlock *)scope; @@ -818,6 +863,10 @@ static bool ir_want_runtime_safety(CodeGen *g, IrInstruction *instruction) { g->build_mode != BuildModeSmallRelease); } +static bool ir_want_runtime_safety(CodeGen *g, IrInstruction *instruction) { + return ir_want_runtime_safety_scope(g, instruction->scope); +} + static Buf *panic_msg_buf(PanicMsgId msg_id) { switch (msg_id) { case PanicMsgIdCount: @@ -858,6 +907,18 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) { return buf_create_from_str("integer part of floating point value out of bounds"); case PanicMsgIdPtrCastNull: return buf_create_from_str("cast causes pointer to be null"); + case PanicMsgIdBadResume: + return buf_create_from_str("resumed an async function which already returned"); + case PanicMsgIdBadAwait: + return buf_create_from_str("async function awaited twice"); + case PanicMsgIdBadReturn: + return buf_create_from_str("async function returned twice"); + case PanicMsgIdResumedAnAwaitingFn: + return buf_create_from_str("awaiting function resumed"); + case PanicMsgIdFrameTooSmall: + return buf_create_from_str("frame too small"); + case PanicMsgIdResumedFnPendingAwait: + return buf_create_from_str("resumed an async function which can only be awaited"); } zig_unreachable(); } @@ -882,13 +943,16 @@ static LLVMValueRef get_panic_msg_ptr_val(CodeGen *g, PanicMsgId msg_id) { return LLVMConstBitCast(val->global_refs->llvm_global, LLVMPointerType(get_llvm_type(g, str_type), 0)); } +static ZigType *ptr_to_stack_trace_type(CodeGen *g) { + return get_pointer_to_type(g, get_stack_trace_type(g), false); +} + static void gen_panic(CodeGen *g, LLVMValueRef msg_arg, LLVMValueRef stack_trace_arg) { assert(g->panic_fn != nullptr); LLVMValueRef fn_val = fn_llvm_value(g, g->panic_fn); LLVMCallConv llvm_cc = get_llvm_cc(g, g->panic_fn->type_entry->data.fn.fn_type_id.cc); if (stack_trace_arg == nullptr) { - ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); - stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type)); + stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g))); } LLVMValueRef args[] = { msg_arg, @@ -904,14 +968,18 @@ static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) { gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr); } -static void gen_assertion(CodeGen *g, PanicMsgId msg_id, IrInstruction *source_instruction) { - if (ir_want_runtime_safety(g, source_instruction)) { +static void gen_assertion_scope(CodeGen *g, PanicMsgId msg_id, Scope *source_scope) { + if (ir_want_runtime_safety_scope(g, source_scope)) { gen_safety_crash(g, msg_id); } else { LLVMBuildUnreachable(g->builder); } } +static void gen_assertion(CodeGen *g, PanicMsgId msg_id, IrInstruction *source_instruction) { + return gen_assertion_scope(g, msg_id, source_instruction->scope); +} + static LLVMValueRef get_stacksave_fn_val(CodeGen *g) { if (g->stacksave_fn_val) return g->stacksave_fn_val; @@ -959,177 +1027,6 @@ static LLVMValueRef get_write_register_fn_val(CodeGen *g) { return g->write_register_fn_val; } -static LLVMValueRef get_coro_destroy_fn_val(CodeGen *g) { - if (g->coro_destroy_fn_val) - return g->coro_destroy_fn_val; - - LLVMTypeRef param_types[] = { - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false); - Buf *name = buf_sprintf("llvm.coro.destroy"); - g->coro_destroy_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_destroy_fn_val)); - - return g->coro_destroy_fn_val; -} - -static LLVMValueRef get_coro_id_fn_val(CodeGen *g) { - if (g->coro_id_fn_val) - return g->coro_id_fn_val; - - LLVMTypeRef param_types[] = { - LLVMInt32Type(), - LLVMPointerType(LLVMInt8Type(), 0), - LLVMPointerType(LLVMInt8Type(), 0), - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 4, false); - Buf *name = buf_sprintf("llvm.coro.id"); - g->coro_id_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_id_fn_val)); - - return g->coro_id_fn_val; -} - -static LLVMValueRef get_coro_alloc_fn_val(CodeGen *g) { - if (g->coro_alloc_fn_val) - return g->coro_alloc_fn_val; - - LLVMTypeRef param_types[] = { - ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 1, false); - Buf *name = buf_sprintf("llvm.coro.alloc"); - g->coro_alloc_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_alloc_fn_val)); - - return g->coro_alloc_fn_val; -} - -static LLVMValueRef get_coro_size_fn_val(CodeGen *g) { - if (g->coro_size_fn_val) - return g->coro_size_fn_val; - - LLVMTypeRef fn_type = LLVMFunctionType(g->builtin_types.entry_usize->llvm_type, nullptr, 0, false); - Buf *name = buf_sprintf("llvm.coro.size.i%d", g->pointer_size_bytes * 8); - g->coro_size_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_size_fn_val)); - - return g->coro_size_fn_val; -} - -static LLVMValueRef get_coro_begin_fn_val(CodeGen *g) { - if (g->coro_begin_fn_val) - return g->coro_begin_fn_val; - - LLVMTypeRef param_types[] = { - ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false); - Buf *name = buf_sprintf("llvm.coro.begin"); - g->coro_begin_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_begin_fn_val)); - - return g->coro_begin_fn_val; -} - -static LLVMValueRef get_coro_suspend_fn_val(CodeGen *g) { - if (g->coro_suspend_fn_val) - return g->coro_suspend_fn_val; - - LLVMTypeRef param_types[] = { - ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), - LLVMInt1Type(), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt8Type(), param_types, 2, false); - Buf *name = buf_sprintf("llvm.coro.suspend"); - g->coro_suspend_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_suspend_fn_val)); - - return g->coro_suspend_fn_val; -} - -static LLVMValueRef get_coro_end_fn_val(CodeGen *g) { - if (g->coro_end_fn_val) - return g->coro_end_fn_val; - - LLVMTypeRef param_types[] = { - LLVMPointerType(LLVMInt8Type(), 0), - LLVMInt1Type(), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMInt1Type(), param_types, 2, false); - Buf *name = buf_sprintf("llvm.coro.end"); - g->coro_end_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_end_fn_val)); - - return g->coro_end_fn_val; -} - -static LLVMValueRef get_coro_free_fn_val(CodeGen *g) { - if (g->coro_free_fn_val) - return g->coro_free_fn_val; - - LLVMTypeRef param_types[] = { - ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 2, false); - Buf *name = buf_sprintf("llvm.coro.free"); - g->coro_free_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_free_fn_val)); - - return g->coro_free_fn_val; -} - -static LLVMValueRef get_coro_resume_fn_val(CodeGen *g) { - if (g->coro_resume_fn_val) - return g->coro_resume_fn_val; - - LLVMTypeRef param_types[] = { - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 1, false); - Buf *name = buf_sprintf("llvm.coro.resume"); - g->coro_resume_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_resume_fn_val)); - - return g->coro_resume_fn_val; -} - -static LLVMValueRef get_coro_save_fn_val(CodeGen *g) { - if (g->coro_save_fn_val) - return g->coro_save_fn_val; - - LLVMTypeRef param_types[] = { - LLVMPointerType(LLVMInt8Type(), 0), - }; - LLVMTypeRef fn_type = LLVMFunctionType(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext()), param_types, 1, false); - Buf *name = buf_sprintf("llvm.coro.save"); - g->coro_save_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_save_fn_val)); - - return g->coro_save_fn_val; -} - -static LLVMValueRef get_coro_promise_fn_val(CodeGen *g) { - if (g->coro_promise_fn_val) - return g->coro_promise_fn_val; - - LLVMTypeRef param_types[] = { - LLVMPointerType(LLVMInt8Type(), 0), - LLVMInt32Type(), - LLVMInt1Type(), - }; - LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), param_types, 3, false); - Buf *name = buf_sprintf("llvm.coro.promise"); - g->coro_promise_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_promise_fn_val)); - - return g->coro_promise_fn_val; -} - static LLVMValueRef get_return_address_fn_val(CodeGen *g) { if (g->return_address_fn_val) return g->return_address_fn_val; @@ -1149,7 +1046,7 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) { return g->add_error_return_trace_addr_fn_val; LLVMTypeRef arg_types[] = { - get_llvm_type(g, get_ptr_to_stack_trace_type(g)), + get_llvm_type(g, ptr_to_stack_trace_type(g)), g->builtin_types.entry_usize->llvm_type, }; LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false); @@ -1164,7 +1061,7 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) { // Error return trace memory is in the stack, which is impossible to be at address 0 // on any architecture. addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); - if (g->build_mode == BuildModeDebug) { + if (codegen_have_frame_pointer(g)) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } @@ -1222,140 +1119,6 @@ static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) { return fn_val; } -static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { - if (g->merge_err_ret_traces_fn_val) - return g->merge_err_ret_traces_fn_val; - - assert(g->stack_trace_type != nullptr); - - LLVMTypeRef param_types[] = { - get_llvm_type(g, get_ptr_to_stack_trace_type(g)), - get_llvm_type(g, get_ptr_to_stack_trace_type(g)), - }; - LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false); - - Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false); - LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); - LLVMSetLinkage(fn_val, LLVMInternalLinkage); - LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); - addLLVMFnAttr(fn_val, "nounwind"); - add_uwtable_attr(g, fn_val); - // Error return trace memory is in the stack, which is impossible to be at address 0 - // on any architecture. - addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); - addLLVMArgAttr(fn_val, (unsigned)0, "noalias"); - addLLVMArgAttr(fn_val, (unsigned)0, "writeonly"); - // Error return trace memory is in the stack, which is impossible to be at address 0 - // on any architecture. - addLLVMArgAttr(fn_val, (unsigned)1, "nonnull"); - addLLVMArgAttr(fn_val, (unsigned)1, "noalias"); - addLLVMArgAttr(fn_val, (unsigned)1, "readonly"); - if (g->build_mode == BuildModeDebug) { - ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); - ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); - } - - // this is above the ZigLLVMClearCurrentDebugLocation - LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); - - LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); - LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); - LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); - LLVMPositionBuilderAtEnd(g->builder, entry_block); - ZigLLVMClearCurrentDebugLocation(g->builder); - - // var frame_index: usize = undefined; - // var frames_left: usize = undefined; - // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) { - // frame_index = 0; - // frames_left = src_stack_trace.index; - // if (frames_left == 0) return; - // } else { - // frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len; - // frames_left = src_stack_trace.instruction_addresses.len; - // } - // while (true) { - // __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]); - // frames_left -= 1; - // if (frames_left == 0) return; - // frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len; - // } - LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return"); - - LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index"); - LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left"); - - LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0); - LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1); - - size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, - (unsigned)src_index_field_index, ""); - LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, - (unsigned)src_addresses_field_index, ""); - ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, ""); - size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; - LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, ""); - LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, ""); - LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, ""); - LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, ""); - LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, ""); - LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap"); - LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap"); - LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop"); - LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block); - - LLVMPositionBuilderAtEnd(g->builder, no_wrap_block); - LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type); - LLVMBuildStore(g->builder, usize_zero, frame_index_ptr); - LLVMBuildStore(g->builder, src_index_val, frames_left_ptr); - LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, ""); - LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block); - - LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block); - LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false); - LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, ""); - LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, ""); - LLVMBuildStore(g->builder, mod_len, frame_index_ptr); - LLVMBuildStore(g->builder, src_len_val, frames_left_ptr); - LLVMBuildBr(g->builder, loop_block); - - LLVMPositionBuilderAtEnd(g->builder, loop_block); - LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); - LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); - LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); - LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; - ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); - LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); - LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); - LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); - LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue"); - LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block); - - LLVMPositionBuilderAtEnd(g->builder, return_block); - LLVMBuildRetVoid(g->builder); - - LLVMPositionBuilderAtEnd(g->builder, continue_block); - LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr); - LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); - LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, ""); - LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, ""); - LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr); - LLVMBuildBr(g->builder, loop_block); - - LLVMPositionBuilderAtEnd(g->builder, prev_block); - if (!g->strip_debug_symbols) { - LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); - } - - g->merge_err_ret_traces_fn_val = fn_val; - return fn_val; - -} - static LLVMValueRef get_return_err_fn(CodeGen *g) { if (g->return_err_fn != nullptr) return g->return_err_fn; @@ -1364,7 +1127,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { LLVMTypeRef arg_types[] = { // error return trace pointer - get_llvm_type(g, get_ptr_to_stack_trace_type(g)), + get_llvm_type(g, ptr_to_stack_trace_type(g)), }; LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false); @@ -1376,10 +1139,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); addLLVMFnAttr(fn_val, "nounwind"); add_uwtable_attr(g, fn_val); - // Error return trace memory is in the stack, which is impossible to be at address 0 - // on any architecture. - addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); - if (g->build_mode == BuildModeDebug) { + if (codegen_have_frame_pointer(g)) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } @@ -1400,6 +1160,17 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, ""); LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, ""); + LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return"); + LLVMBasicBlockRef dest_non_null_block = LLVMAppendBasicBlock(fn_val, "DestNonNull"); + + LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_ret_trace_ptr, + LLVMConstNull(LLVMTypeOf(err_ret_trace_ptr)), ""); + LLVMBuildCondBr(g->builder, null_dest_bit, return_block, dest_non_null_block); + + LLVMPositionBuilderAtEnd(g->builder, return_block); + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, dest_non_null_block); LLVMValueRef args[] = { err_ret_trace_ptr, return_address }; ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); LLVMBuildRetVoid(g->builder); @@ -1434,7 +1205,7 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) { LLVMTypeRef fn_type_ref; if (g->have_err_ret_tracing) { LLVMTypeRef arg_types[] = { - get_llvm_type(g, g->ptr_to_stack_trace_type), + get_llvm_type(g, get_pointer_to_type(g, get_stack_trace_type(g), false)), get_llvm_type(g, g->err_tag_type), }; fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false); @@ -1451,7 +1222,7 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) { LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); addLLVMFnAttr(fn_val, "nounwind"); add_uwtable_attr(g, fn_val); - if (g->build_mode == BuildModeDebug) { + if (codegen_have_frame_pointer(g)) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } @@ -1543,25 +1314,10 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) { return fn_val; } -static bool is_coro_prelude_scope(Scope *scope) { - while (scope != nullptr) { - if (scope->id == ScopeIdCoroPrelude) { - return true; - } else if (scope->id == ScopeIdFnDef) { - break; - } - scope = scope->parent; - } - return false; -} - static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) { if (!g->have_err_ret_tracing) { return nullptr; } - if (g->cur_fn->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { - return is_coro_prelude_scope(scope) ? g->cur_err_ret_trace_val_arg : g->cur_err_ret_trace_val_stack; - } if (g->cur_err_ret_trace_val_stack != nullptr) { return g->cur_err_ret_trace_val_stack; } @@ -1574,8 +1330,7 @@ static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *sc if (g->have_err_ret_tracing) { LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope); if (err_ret_trace_val == nullptr) { - ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); - err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type)); + err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g))); } LLVMValueRef args[] = { err_ret_trace_val, @@ -1820,14 +1575,14 @@ static LLVMRealPredicate cmp_op_to_real_predicate(IrBinOp cmp_op) { } } -static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type, +static void gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type, LLVMValueRef value) { assert(ptr_type->id == ZigTypeIdPointer); ZigType *child_type = ptr_type->data.pointer.child_type; if (!type_has_bits(child_type)) - return nullptr; + return; if (handle_is_ptr(child_type)) { assert(LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMPointerTypeKind); @@ -1847,13 +1602,13 @@ static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_ty ZigLLVMBuildMemCpy(g->builder, dest_ptr, align_bytes, src_ptr, align_bytes, LLVMConstInt(usize->llvm_type, size_bytes, false), ptr_type->data.pointer.is_volatile); - return nullptr; + return; } uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes; if (host_int_bytes == 0) { gen_store(g, value, ptr, ptr_type); - return nullptr; + return; } bool big_endian = g->is_big_endian; @@ -1883,7 +1638,7 @@ static LLVMValueRef gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_ty LLVMValueRef ored_value = LLVMBuildOr(g->builder, shifted_value, anded_containing_int, ""); gen_store(g, ored_value, ptr, ptr_type); - return nullptr; + return; } static void gen_var_debug_decl(CodeGen *g, ZigVar *var) { @@ -1967,7 +1722,7 @@ static bool iter_function_params_c_abi(CodeGen *g, ZigType *fn_type, FnWalk *fn_ param_info = &fn_type->data.fn.fn_type_id.param_info[src_i]; ty = param_info->type; source_node = fn_walk->data.attrs.fn->proto_node; - llvm_fn = fn_walk->data.attrs.fn->llvm_value; + llvm_fn = fn_walk->data.attrs.llvm_fn; break; case FnWalkIdCall: { if (src_i >= fn_walk->data.call.inst->arg_count) @@ -2149,10 +1904,12 @@ static bool iter_function_params_c_abi(CodeGen *g, ZigType *fn_type, FnWalk *fn_ } case FnWalkIdInits: { clear_debug_source_node(g); - LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i); - LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0); - LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, ""); - gen_store_untyped(g, arg, bitcasted, var->align_bytes, false); + if (!fn_is_async(fn_walk->data.inits.fn)) { + LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i); + LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0); + LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, ""); + gen_store_untyped(g, arg, bitcasted, var->align_bytes, false); + } if (var->decl_node) { gen_var_debug_decl(g, var); } @@ -2201,6 +1958,7 @@ void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) { LLVMValueRef param_value = ir_llvm_value(g, param_instruction); assert(param_value); fn_walk->data.call.gen_param_values->append(param_value); + fn_walk->data.call.gen_param_types->append(param_type); } } return; @@ -2216,7 +1974,7 @@ void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) { switch (fn_walk->id) { case FnWalkIdAttrs: { - LLVMValueRef llvm_fn = fn_walk->data.attrs.fn->llvm_value; + LLVMValueRef llvm_fn = fn_walk->data.attrs.llvm_fn; bool is_byval = gen_info->is_byval; FnTypeParamInfo *param_info = &fn_type->data.fn.fn_type_id.param_info[param_i]; @@ -2245,7 +2003,7 @@ void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) { assert(variable); assert(variable->value_ref); - if (!handle_is_ptr(variable->var_type)) { + if (!handle_is_ptr(variable->var_type) && !fn_is_async(fn_walk->data.inits.fn)) { clear_debug_source_node(g); ZigType *fn_type = fn_table_entry->type_entry; unsigned gen_arg_index = fn_type->data.fn.gen_param_info[variable->src_arg_index].gen_index; @@ -2271,48 +2029,357 @@ void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) { } } +static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { + if (g->merge_err_ret_traces_fn_val) + return g->merge_err_ret_traces_fn_val; + + assert(g->stack_trace_type != nullptr); + + LLVMTypeRef param_types[] = { + get_llvm_type(g, ptr_to_stack_trace_type(g)), + get_llvm_type(g, ptr_to_stack_trace_type(g)), + }; + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + add_uwtable_attr(g, fn_val); + addLLVMArgAttr(fn_val, (unsigned)0, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)0, "writeonly"); + + addLLVMArgAttr(fn_val, (unsigned)1, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)1, "readonly"); + if (g->build_mode == BuildModeDebug) { + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); + } + + // this is above the ZigLLVMClearCurrentDebugLocation + LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + + // if (dest_stack_trace == null or src_stack_trace == null) return; + // var frame_index: usize = undefined; + // var frames_left: usize = undefined; + // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) { + // frame_index = 0; + // frames_left = src_stack_trace.index; + // if (frames_left == 0) return; + // } else { + // frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len; + // frames_left = src_stack_trace.instruction_addresses.len; + // } + // while (true) { + // __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]); + // frames_left -= 1; + // if (frames_left == 0) return; + // frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len; + // } + LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return"); + LLVMBasicBlockRef non_null_block = LLVMAppendBasicBlock(fn_val, "NonNull"); + + LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index"); + LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left"); + + LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0); + LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1); + + LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, dest_stack_trace_ptr, + LLVMConstNull(LLVMTypeOf(dest_stack_trace_ptr)), ""); + LLVMValueRef null_src_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_stack_trace_ptr, + LLVMConstNull(LLVMTypeOf(src_stack_trace_ptr)), ""); + LLVMValueRef null_bit = LLVMBuildOr(g->builder, null_dest_bit, null_src_bit, ""); + LLVMBuildCondBr(g->builder, null_bit, return_block, non_null_block); + + LLVMPositionBuilderAtEnd(g->builder, non_null_block); + size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_index_field_index, ""); + LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_addresses_field_index, ""); + ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, ""); + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, ""); + LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, ""); + LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, ""); + LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, ""); + LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, ""); + LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap"); + LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap"); + LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop"); + LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block); + + LLVMPositionBuilderAtEnd(g->builder, no_wrap_block); + LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type); + LLVMBuildStore(g->builder, usize_zero, frame_index_ptr); + LLVMBuildStore(g->builder, src_index_val, frames_left_ptr); + LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, ""); + LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block); + LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false); + LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, ""); + LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, mod_len, frame_index_ptr); + LLVMBuildStore(g->builder, src_len_val, frames_left_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, loop_block); + LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); + LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); + LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; + ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); + LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); + LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); + LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); + LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue"); + LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block); + + LLVMPositionBuilderAtEnd(g->builder, return_block); + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, continue_block); + LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr); + LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, ""); + LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, prev_block); + if (!g->strip_debug_symbols) { + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + } + + g->merge_err_ret_traces_fn_val = fn_val; + return fn_val; + +} static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *executable, IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction) { assert(g->have_err_ret_tracing); LLVMValueRef return_err_fn = get_return_err_fn(g); - LLVMValueRef args[] = { - get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope), - }; - LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, + LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope); + ZigLLVMBuildCall(g->builder, return_err_fn, &my_err_trace_val, 1, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - return call_instruction; + + ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type; + if (fn_is_async(g->cur_fn) && codegen_fn_has_err_ret_tracing_arg(g, ret_type)) { + LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + frame_index_trace_arg(g, ret_type), ""); + LLVMBuildStore(g->builder, my_err_trace_val, trace_ptr_ptr); + } + + return nullptr; } -static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *return_instruction) { +static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, ResumeId resume_id, PanicMsgId msg_id, + LLVMBasicBlockRef end_bb) +{ + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume"); + if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume"); + LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref), + LLVMConstInt(usize_type_ref, resume_id, false)); + LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, ""); + LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block); + + LLVMPositionBuilderAtEnd(g->builder, bad_resume_block); + gen_assertion(g, msg_id, source_instr); + + LLVMPositionBuilderAtEnd(g->builder, end_bb); +} + +static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef target_frame_ptr, ResumeId resume_id) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + if (fn_val == nullptr) { + LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_fn_ptr_index, ""); + fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, ""); + } + LLVMValueRef arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref), + LLVMConstInt(usize_type_ref, resume_id, false), ""); + LLVMValueRef args[] = {target_frame_ptr, arg_val}; + return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, ""); +} + +static LLVMBasicBlockRef gen_suspend_begin(CodeGen *g, const char *name_hint) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint); + size_t new_block_index = g->cur_resume_block_count; + g->cur_resume_block_count += 1; + LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false); + LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb); + LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr); + return resume_bb; +} + +static void set_tail_call_if_appropriate(CodeGen *g, LLVMValueRef call_inst) { + LLVMSetTailCall(call_inst, true); +} + +static LLVMValueRef gen_maybe_atomic_op(CodeGen *g, LLVMAtomicRMWBinOp op, LLVMValueRef ptr, LLVMValueRef val, + LLVMAtomicOrdering order) +{ + if (g->is_single_threaded) { + LLVMValueRef loaded = LLVMBuildLoad(g->builder, ptr, ""); + LLVMValueRef modified; + switch (op) { + case LLVMAtomicRMWBinOpXchg: + modified = val; + break; + case LLVMAtomicRMWBinOpXor: + modified = LLVMBuildXor(g->builder, loaded, val, ""); + break; + default: + zig_unreachable(); + } + LLVMBuildStore(g->builder, modified, ptr); + return loaded; + } else { + return LLVMBuildAtomicRMW(g->builder, op, ptr, val, order, false); + } +} + +static void gen_async_return(CodeGen *g, IrInstructionReturn *instruction) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + + ZigType *operand_type = (instruction->operand != nullptr) ? instruction->operand->value.type : nullptr; + bool operand_has_bits = (operand_type != nullptr) && type_has_bits(operand_type); + ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type; + bool ret_type_has_bits = type_has_bits(ret_type); + + if (operand_has_bits && instruction->operand != nullptr) { + bool need_store = instruction->operand->value.special != ConstValSpecialRuntime || !handle_is_ptr(ret_type); + if (need_store) { + // It didn't get written to the result ptr. We do that now. + ZigType *ret_ptr_type = get_pointer_to_type(g, ret_type, true); + gen_assign_raw(g, g->cur_ret_ptr, ret_ptr_type, ir_llvm_value(g, instruction->operand)); + } + } + + // Whether we tail resume the awaiter, or do an early return, we are done and will not be resumed. + if (ir_want_runtime_safety(g, &instruction->base)) { + LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref); + LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr); + } + + LLVMValueRef zero = LLVMConstNull(usize_type_ref); + LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); + + LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXor, g->cur_async_awaiter_ptr, + all_ones, LLVMAtomicOrderingAcquire); + + LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn"); + LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn"); + LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem"); + + LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2); + + LLVMAddCase(switch_instr, zero, early_return_block); + LLVMAddCase(switch_instr, all_ones, bad_return_block); + + // Something has gone horribly wrong, and this is an invalid second return. + LLVMPositionBuilderAtEnd(g->builder, bad_return_block); + gen_assertion(g, PanicMsgIdBadReturn, &instruction->base); + + // There is no awaiter yet, but we're completely done. + LLVMPositionBuilderAtEnd(g->builder, early_return_block); + LLVMBuildRetVoid(g->builder); + + // We need to resume the caller by tail calling them, + // but first write through the result pointer and possibly + // error return trace pointer. + LLVMPositionBuilderAtEnd(g->builder, resume_them_block); + + if (ret_type_has_bits) { + // If the awaiter result pointer is non-null, we need to copy the result to there. + LLVMBasicBlockRef copy_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResult"); + LLVMBasicBlockRef copy_end_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResultEnd"); + LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start + 1, ""); + LLVMValueRef awaiter_ret_ptr = LLVMBuildLoad(g->builder, awaiter_ret_ptr_ptr, ""); + LLVMValueRef zero_ptr = LLVMConstNull(LLVMTypeOf(awaiter_ret_ptr)); + LLVMValueRef need_copy_bit = LLVMBuildICmp(g->builder, LLVMIntNE, awaiter_ret_ptr, zero_ptr, ""); + LLVMBuildCondBr(g->builder, need_copy_bit, copy_block, copy_end_block); + + LLVMPositionBuilderAtEnd(g->builder, copy_block); + LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0); + LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, awaiter_ret_ptr, ptr_u8, ""); + LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, g->cur_ret_ptr, ptr_u8, ""); + bool is_volatile = false; + uint32_t abi_align = get_abi_alignment(g, ret_type); + LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, ret_type), false); + ZigLLVMBuildMemCpy(g->builder, + dest_ptr_casted, abi_align, + src_ptr_casted, abi_align, byte_count_val, is_volatile); + LLVMBuildBr(g->builder, copy_end_block); + + LLVMPositionBuilderAtEnd(g->builder, copy_end_block); + if (codegen_fn_has_err_ret_tracing_arg(g, ret_type)) { + LLVMValueRef awaiter_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + frame_index_trace_arg(g, ret_type) + 1, ""); + LLVMValueRef dest_trace_ptr = LLVMBuildLoad(g->builder, awaiter_trace_ptr_ptr, ""); + LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); + LLVMValueRef args[] = { dest_trace_ptr, my_err_trace_val }; + ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + } + } + + // Resume the caller by tail calling them. + ZigType *any_frame_type = get_any_frame_type(g, ret_type); + LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, prev_val, get_llvm_type(g, any_frame_type), ""); + LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn); + set_tail_call_if_appropriate(g, call_inst); + LLVMBuildRetVoid(g->builder); +} + +static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *instruction) { + if (fn_is_async(g->cur_fn)) { + gen_async_return(g, instruction); + return nullptr; + } + if (want_first_arg_sret(g, &g->cur_fn->type_entry->data.fn.fn_type_id)) { - if (return_instruction->value == nullptr) { + if (instruction->operand == nullptr) { LLVMBuildRetVoid(g->builder); return nullptr; } assert(g->cur_ret_ptr); - src_assert(return_instruction->value->value.special != ConstValSpecialRuntime, - return_instruction->base.source_node); - LLVMValueRef value = ir_llvm_value(g, return_instruction->value); - ZigType *return_type = return_instruction->value->value.type; + src_assert(instruction->operand->value.special != ConstValSpecialRuntime, + instruction->base.source_node); + LLVMValueRef value = ir_llvm_value(g, instruction->operand); + ZigType *return_type = instruction->operand->value.type; gen_assign_raw(g, g->cur_ret_ptr, get_pointer_to_type(g, return_type, false), value); LLVMBuildRetVoid(g->builder); } else if (g->cur_fn->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync && handle_is_ptr(g->cur_fn->type_entry->data.fn.fn_type_id.return_type)) { - if (return_instruction->value == nullptr) { + if (instruction->operand == nullptr) { LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, ""); LLVMBuildRet(g->builder, by_val_value); } else { - LLVMValueRef value = ir_llvm_value(g, return_instruction->value); + LLVMValueRef value = ir_llvm_value(g, instruction->operand); LLVMValueRef by_val_value = gen_load_untyped(g, value, 0, false, ""); LLVMBuildRet(g->builder, by_val_value); } - } else if (return_instruction->value == nullptr) { + } else if (instruction->operand == nullptr) { LLVMBuildRetVoid(g->builder); } else { - LLVMValueRef value = ir_llvm_value(g, return_instruction->value); + LLVMValueRef value = ir_llvm_value(g, instruction->operand); LLVMBuildRet(g->builder, value); } return nullptr; @@ -3242,14 +3309,17 @@ static LLVMValueRef ir_render_bool_not(CodeGen *g, IrExecutable *executable, IrI return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, ""); } -static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable, IrInstructionDeclVarGen *instruction) { - ZigVar *var = instruction->var; - +static void render_decl_var(CodeGen *g, ZigVar *var) { if (!type_has_bits(var->var_type)) - return nullptr; + return; - var->value_ref = ir_llvm_value(g, instruction->var_ptr); + var->value_ref = ir_llvm_value(g, var->ptr_instruction); gen_var_debug_decl(g, var); +} + +static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable, IrInstructionDeclVarGen *instruction) { + instruction->var->ptr_instruction = instruction->var_ptr; + render_decl_var(g, instruction->var); return nullptr; } @@ -3467,8 +3537,9 @@ static LLVMValueRef ir_render_var_ptr(CodeGen *g, IrExecutable *executable, IrIn static LLVMValueRef ir_render_return_ptr(CodeGen *g, IrExecutable *executable, IrInstructionReturnPtr *instruction) { - src_assert(g->cur_ret_ptr != nullptr || !type_has_bits(instruction->base.value.type), - instruction->base.source_node); + if (!type_has_bits(instruction->base.value.type)) + return nullptr; + src_assert(g->cur_ret_ptr != nullptr, instruction->base.source_node); return g->cur_ret_ptr; } @@ -3566,26 +3637,6 @@ static LLVMValueRef ir_render_elem_ptr(CodeGen *g, IrExecutable *executable, IrI } } -static bool get_prefix_arg_err_ret_stack(CodeGen *g, FnTypeId *fn_type_id) { - return g->have_err_ret_tracing && - (fn_type_id->return_type->id == ZigTypeIdErrorUnion || - fn_type_id->return_type->id == ZigTypeIdErrorSet || - fn_type_id->cc == CallingConventionAsync); -} - -static size_t get_async_allocator_arg_index(CodeGen *g, FnTypeId *fn_type_id) { - // 0 1 2 3 - // err_ret_stack allocator_ptr err_code other_args... - return get_prefix_arg_err_ret_stack(g, fn_type_id) ? 1 : 0; -} - -static size_t get_async_err_code_arg_index(CodeGen *g, FnTypeId *fn_type_id) { - // 0 1 2 3 - // err_ret_stack allocator_ptr err_code other_args... - return 1 + get_async_allocator_arg_index(g, fn_type_id); -} - - static LLVMValueRef get_new_stack_addr(CodeGen *g, LLVMValueRef new_stack) { LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_ptr_index, ""); LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_len_index, ""); @@ -3623,16 +3674,124 @@ static void set_call_instr_sret(CodeGen *g, LLVMValueRef call_instr) { LLVMAddCallSiteAttribute(call_instr, 1, sret_attr); } +static void render_async_spills(CodeGen *g) { + ZigType *fn_type = g->cur_fn->type_entry; + ZigType *import = get_scope_import(&g->cur_fn->fndef_scope->base); + uint32_t async_var_index = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type); + for (size_t var_i = 0; var_i < g->cur_fn->variable_list.length; var_i += 1) { + ZigVar *var = g->cur_fn->variable_list.at(var_i); + + if (!type_has_bits(var->var_type)) { + continue; + } + if (ir_get_var_is_comptime(var)) + continue; + switch (type_requires_comptime(g, var->var_type)) { + case ReqCompTimeInvalid: + zig_unreachable(); + case ReqCompTimeYes: + continue; + case ReqCompTimeNo: + break; + } + if (var->src_arg_index == SIZE_MAX) { + continue; + } + + var->value_ref = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, async_var_index, + buf_ptr(&var->name)); + async_var_index += 1; + if (var->decl_node) { + var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope), + buf_ptr(&var->name), import->data.structure.root_struct->di_file, + (unsigned)(var->decl_node->line + 1), + get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0); + gen_var_debug_decl(g, var); + } + } + + ZigType *frame_type = g->cur_fn->frame_type->data.frame.locals_struct; + + for (size_t alloca_i = 0; alloca_i < g->cur_fn->alloca_gen_list.length; alloca_i += 1) { + IrInstructionAllocaGen *instruction = g->cur_fn->alloca_gen_list.at(alloca_i); + if (instruction->field_index == SIZE_MAX) + continue; + + size_t gen_index = frame_type->data.structure.fields[instruction->field_index].gen_index; + instruction->base.llvm_value = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, gen_index, + instruction->name_hint); + } +} + +static void render_async_var_decls(CodeGen *g, Scope *scope) { + for (;;) { + switch (scope->id) { + case ScopeIdCImport: + zig_unreachable(); + case ScopeIdFnDef: + return; + case ScopeIdVarDecl: { + ZigVar *var = reinterpret_cast<ScopeVarDecl *>(scope)->var; + if (var->ptr_instruction != nullptr) { + render_decl_var(g, var); + } + // fallthrough + } + case ScopeIdDecls: + case ScopeIdBlock: + case ScopeIdDefer: + case ScopeIdDeferExpr: + case ScopeIdLoop: + case ScopeIdSuspend: + case ScopeIdCompTime: + case ScopeIdRuntime: + scope = scope->parent; + continue; + } + } +} + +static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) { + LLVMTypeRef usize_llvm_type = g->builtin_types.entry_usize->llvm_type; + LLVMTypeRef ptr_usize_llvm_type = LLVMPointerType(usize_llvm_type, 0); + LLVMValueRef casted_fn_val = LLVMBuildBitCast(g->builder, fn_val, ptr_usize_llvm_type, ""); + LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true); + LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, ""); + return LLVMBuildLoad(g->builder, prefix_ptr, ""); +} + +static void gen_init_stack_trace(CodeGen *g, LLVMValueRef trace_field_ptr, LLVMValueRef addrs_field_ptr) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMValueRef zero = LLVMConstNull(usize_type_ref); + + LLVMValueRef index_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 0, ""); + LLVMBuildStore(g->builder, zero, index_ptr); + + LLVMValueRef addrs_slice_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 1, ""); + LLVMValueRef addrs_ptr_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_ptr_index, ""); + LLVMValueRef indices[] = { LLVMConstNull(usize_type_ref), LLVMConstNull(usize_type_ref) }; + LLVMValueRef trace_field_addrs_as_ptr = LLVMBuildInBoundsGEP(g->builder, addrs_field_ptr, indices, 2, ""); + LLVMBuildStore(g->builder, trace_field_addrs_as_ptr, addrs_ptr_ptr); + + LLVMValueRef addrs_len_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_len_index, ""); + LLVMBuildStore(g->builder, LLVMConstInt(usize_type_ref, stack_trace_ptr_count, false), addrs_len_ptr); +} + static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstructionCallGen *instruction) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMValueRef fn_val; ZigType *fn_type; + bool callee_is_async; if (instruction->fn_entry) { fn_val = fn_llvm_value(g, instruction->fn_entry); fn_type = instruction->fn_entry->type_entry; + callee_is_async = fn_is_async(instruction->fn_entry); } else { assert(instruction->fn_ref); fn_val = ir_llvm_value(g, instruction->fn_ref); fn_type = instruction->fn_ref->value.type; + callee_is_async = fn_type->data.fn.fn_type_id.cc == CallingConventionAsync; } FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id; @@ -3643,27 +3802,154 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr CallingConvention cc = fn_type->data.fn.fn_type_id.cc; bool first_arg_ret = ret_has_bits && want_first_arg_sret(g, fn_type_id); - bool prefix_arg_err_ret_stack = get_prefix_arg_err_ret_stack(g, fn_type_id); + bool prefix_arg_err_ret_stack = codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type); bool is_var_args = fn_type_id->is_var_args; ZigList<LLVMValueRef> gen_param_values = {}; + ZigList<ZigType *> gen_param_types = {}; LLVMValueRef result_loc = instruction->result_loc ? ir_llvm_value(g, instruction->result_loc) : nullptr; - if (first_arg_ret) { - gen_param_values.append(result_loc); - } - if (prefix_arg_err_ret_stack) { - gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope)); - } - if (instruction->is_async) { - gen_param_values.append(ir_llvm_value(g, instruction->async_allocator)); + LLVMValueRef zero = LLVMConstNull(usize_type_ref); + LLVMValueRef frame_result_loc; + LLVMValueRef awaiter_init_val; + LLVMValueRef ret_ptr; + if (callee_is_async) { + if (instruction->is_async) { + if (instruction->new_stack == nullptr) { + awaiter_init_val = zero; + frame_result_loc = result_loc; + + if (ret_has_bits) { + // Use the result location which is inside the frame if this is an async call. + ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, ""); + } + } else if (cc == CallingConventionAsync) { + awaiter_init_val = zero; + LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack); + if (ir_want_runtime_safety(g, &instruction->base)) { + LLVMValueRef given_len_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_len_index, ""); + LLVMValueRef given_frame_len = LLVMBuildLoad(g->builder, given_len_ptr, ""); + LLVMValueRef actual_frame_len = gen_frame_size(g, fn_val); + + LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckFail"); + LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckOk"); + + LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntUGE, given_frame_len, actual_frame_len, ""); + LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block); + + LLVMPositionBuilderAtEnd(g->builder, fail_block); + gen_safety_crash(g, PanicMsgIdFrameTooSmall); + + LLVMPositionBuilderAtEnd(g->builder, ok_block); + } + LLVMValueRef frame_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_ptr_index, ""); + LLVMValueRef frame_ptr = LLVMBuildLoad(g->builder, frame_ptr_ptr, ""); + frame_result_loc = LLVMBuildBitCast(g->builder, frame_ptr, + get_llvm_type(g, instruction->base.value.type), ""); + + if (ret_has_bits) { + // Use the result location provided to the @asyncCall builtin + ret_ptr = result_loc; + } + } + + // even if prefix_arg_err_ret_stack is true, let the async function do its own + // initialization. + } else { + // async function called as a normal function + + frame_result_loc = ir_llvm_value(g, instruction->frame_result_loc); + awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); // caller's own frame pointer + if (ret_has_bits) { + if (result_loc == nullptr) { + // return type is a scalar, but we still need a pointer to it. Use the async fn frame. + ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, ""); + } else { + // Use the call instruction's result location. + ret_ptr = result_loc; + } + + // Store a zero in the awaiter's result ptr to indicate we do not need a copy made. + LLVMValueRef awaiter_ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 1, ""); + LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr))); + LLVMBuildStore(g->builder, zero_ptr, awaiter_ret_ptr); + } + + if (prefix_arg_err_ret_stack) { + LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, + frame_index_trace_arg(g, src_return_type) + 1, ""); + LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); + LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr); + } + } + + assert(frame_result_loc != nullptr); + + LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_fn_ptr_index, ""); + LLVMValueRef bitcasted_fn_val = LLVMBuildBitCast(g->builder, fn_val, + LLVMGetElementType(LLVMTypeOf(fn_ptr_ptr)), ""); + LLVMBuildStore(g->builder, bitcasted_fn_val, fn_ptr_ptr); + + LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_resume_index, ""); + LLVMBuildStore(g->builder, zero, resume_index_ptr); - LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, ""); - gen_param_values.append(err_val_ptr); + LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, ""); + LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr); + + if (ret_has_bits) { + LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, ""); + LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr); + } + } else if (instruction->is_async) { + // Async call of blocking function + if (instruction->new_stack != nullptr) { + zig_panic("TODO @asyncCall of non-async function"); + } + frame_result_loc = result_loc; + awaiter_init_val = LLVMConstAllOnes(usize_type_ref); + + LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, ""); + LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr); + + if (ret_has_bits) { + LLVMValueRef ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, ""); + LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, ""); + LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr); + + if (first_arg_ret) { + gen_param_values.append(ret_ptr); + } + if (prefix_arg_err_ret_stack) { + // Set up the callee stack trace pointer pointing into the frame. + // Then we have to wire up the StackTrace pointers. + // Await is responsible for merging error return traces. + uint32_t trace_field_index_start = frame_index_trace_arg(g, src_return_type); + LLVMValueRef callee_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, + trace_field_index_start, ""); + LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, + trace_field_index_start + 2, ""); + LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, + trace_field_index_start + 3, ""); + + LLVMBuildStore(g->builder, trace_field_ptr, callee_trace_ptr_ptr); + + gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr); + + gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope)); + } + } + } else { + if (first_arg_ret) { + gen_param_values.append(result_loc); + } + if (prefix_arg_err_ret_stack) { + gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope)); + } } FnWalk fn_walk = {}; fn_walk.id = FnWalkIdCall; fn_walk.data.call.inst = instruction; fn_walk.data.call.is_var_args = is_var_args; fn_walk.data.call.gen_param_values = &gen_param_values; + fn_walk.data.call.gen_param_types = &gen_param_types; walk_function_params(g, fn_type, &fn_walk); ZigLLVM_FnInline fn_inline; @@ -3682,9 +3968,68 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr LLVMCallConv llvm_cc = get_llvm_cc(g, cc); LLVMValueRef result; + if (callee_is_async) { + uint32_t arg_start_i = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type); + + LLVMValueRef casted_frame; + if (instruction->new_stack != nullptr) { + // We need the frame type to be a pointer to a struct that includes the args + size_t field_count = arg_start_i + gen_param_values.length; + LLVMTypeRef *field_types = allocate_nonzero<LLVMTypeRef>(field_count); + LLVMGetStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc)), field_types); + assert(LLVMCountStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc))) == arg_start_i); + for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) { + field_types[arg_start_i + arg_i] = LLVMTypeOf(gen_param_values.at(arg_i)); + } + LLVMTypeRef frame_with_args_type = LLVMStructType(field_types, field_count, false); + LLVMTypeRef ptr_frame_with_args_type = LLVMPointerType(frame_with_args_type, 0); + + casted_frame = LLVMBuildBitCast(g->builder, frame_result_loc, ptr_frame_with_args_type, ""); + } else { + casted_frame = frame_result_loc; + } + + for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) { + LLVMValueRef arg_ptr = LLVMBuildStructGEP(g->builder, casted_frame, arg_start_i + arg_i, ""); + gen_assign_raw(g, arg_ptr, get_pointer_to_type(g, gen_param_types.at(arg_i), true), + gen_param_values.at(arg_i)); + } + + if (instruction->is_async) { + gen_resume(g, fn_val, frame_result_loc, ResumeIdCall); + if (instruction->new_stack != nullptr) { + return frame_result_loc; + } + return nullptr; + } else { + ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true); + + LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume"); + + LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall); + set_tail_call_if_appropriate(g, call_inst); + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, call_bb); + gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr); + render_async_var_decls(g, instruction->base.scope); + + if (!type_has_bits(src_return_type)) + return nullptr; + + if (result_loc != nullptr) + return get_handle_value(g, result_loc, src_return_type, ptr_result_type); + + LLVMValueRef result_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, ""); + return LLVMBuildLoad(g->builder, result_ptr, ""); + } + } + if (instruction->new_stack == nullptr) { result = ZigLLVMBuildCall(g->builder, fn_val, gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, fn_inline, ""); + } else if (instruction->is_async) { + zig_panic("TODO @asyncCall of non-async function"); } else { LLVMValueRef stacksave_fn_val = get_stacksave_fn_val(g); LLVMValueRef stackrestore_fn_val = get_stackrestore_fn_val(g); @@ -3697,13 +4042,6 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr LLVMBuildCall(g->builder, stackrestore_fn_val, &old_stack_ref, 1, ""); } - - if (instruction->is_async) { - LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_payload_index, ""); - LLVMBuildStore(g->builder, result, payload_ptr); - return result_loc; - } - if (src_return_type->id == ZigTypeIdUnreachable) { return LLVMBuildUnreachable(g->builder); } else if (!ret_has_bits) { @@ -4200,7 +4538,7 @@ static LLVMValueRef get_enum_tag_name_function(CodeGen *g, ZigType *enum_type) { LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); addLLVMFnAttr(fn_val, "nounwind"); add_uwtable_attr(g, fn_val); - if (g->build_mode == BuildModeDebug) { + if (codegen_have_frame_pointer(g)) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } @@ -4347,10 +4685,6 @@ static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, I { align_bytes = target_type->data.maybe.child_type->data.fn.fn_type_id.alignment; ptr_val = target_val; - } else if (target_type->id == ZigTypeIdOptional && - target_type->data.maybe.child_type->id == ZigTypeIdPromise) - { - zig_panic("TODO audit this function"); } else if (target_type->id == ZigTypeIdStruct && target_type->data.structure.is_slice) { ZigType *slice_ptr_type = target_type->data.structure.fields[slice_ptr_index].type_entry; align_bytes = get_ptr_align(g, slice_ptr_type); @@ -4388,26 +4722,11 @@ static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *execu { LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); if (cur_err_ret_trace_val == nullptr) { - ZigType *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); - return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type)); + return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g))); } return cur_err_ret_trace_val; } -static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { - LLVMValueRef target_handle = ir_llvm_value(g, instruction->target); - LLVMBuildCall(g->builder, get_coro_destroy_fn_val(g), &target_handle, 1, ""); - return nullptr; -} - -static LLVMValueRef ir_render_get_implicit_allocator(CodeGen *g, IrExecutable *executable, - IrInstructionGetImplicitAllocator *instruction) -{ - assert(instruction->id == ImplicitAllocatorIdArg); - size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); - return LLVMGetParam(g->cur_fn_val, allocator_arg_index); -} - static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) { switch (atomic_order) { case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered; @@ -4722,24 +5041,8 @@ static LLVMValueRef ir_render_frame_address(CodeGen *g, IrExecutable *executable return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, ""); } -static LLVMValueRef get_handle_fn_val(CodeGen *g) { - if (g->coro_frame_fn_val) - return g->coro_frame_fn_val; - - LLVMTypeRef fn_type = LLVMFunctionType( LLVMPointerType(LLVMInt8Type(), 0) - , nullptr, 0, false); - Buf *name = buf_sprintf("llvm.coro.frame"); - g->coro_frame_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type); - assert(LLVMGetIntrinsicID(g->coro_frame_fn_val)); - - return g->coro_frame_fn_val; -} - -static LLVMValueRef ir_render_handle(CodeGen *g, IrExecutable *executable, - IrInstructionHandle *instruction) -{ - LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->builtin_types.entry_promise)); - return LLVMBuildCall(g->builder, get_handle_fn_val(g), &zero, 0, ""); +static LLVMValueRef ir_render_handle(CodeGen *g, IrExecutable *executable, IrInstructionFrameHandle *instruction) { + return g->cur_frame_ptr; } static LLVMValueRef render_shl_with_overflow(CodeGen *g, IrInstructionOverflowOp *instruction) { @@ -5005,248 +5308,6 @@ static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInst return nullptr; } -static LLVMValueRef ir_render_coro_id(CodeGen *g, IrExecutable *executable, IrInstructionCoroId *instruction) { - LLVMValueRef promise_ptr = ir_llvm_value(g, instruction->promise_ptr); - LLVMValueRef align_val = LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false); - LLVMValueRef null = LLVMConstIntToPtr(LLVMConstNull(g->builtin_types.entry_usize->llvm_type), - LLVMPointerType(LLVMInt8Type(), 0)); - LLVMValueRef params[] = { - align_val, - promise_ptr, - null, - null, - }; - return LLVMBuildCall(g->builder, get_coro_id_fn_val(g), params, 4, ""); -} - -static LLVMValueRef ir_render_coro_alloc(CodeGen *g, IrExecutable *executable, IrInstructionCoroAlloc *instruction) { - LLVMValueRef token = ir_llvm_value(g, instruction->coro_id); - return LLVMBuildCall(g->builder, get_coro_alloc_fn_val(g), &token, 1, ""); -} - -static LLVMValueRef ir_render_coro_size(CodeGen *g, IrExecutable *executable, IrInstructionCoroSize *instruction) { - return LLVMBuildCall(g->builder, get_coro_size_fn_val(g), nullptr, 0, ""); -} - -static LLVMValueRef ir_render_coro_begin(CodeGen *g, IrExecutable *executable, IrInstructionCoroBegin *instruction) { - LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id); - LLVMValueRef coro_mem_ptr = ir_llvm_value(g, instruction->coro_mem_ptr); - LLVMValueRef params[] = { - coro_id, - coro_mem_ptr, - }; - return LLVMBuildCall(g->builder, get_coro_begin_fn_val(g), params, 2, ""); -} - -static LLVMValueRef ir_render_coro_alloc_fail(CodeGen *g, IrExecutable *executable, - IrInstructionCoroAllocFail *instruction) -{ - size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); - LLVMValueRef err_code_ptr_val = LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index); - LLVMValueRef err_code = ir_llvm_value(g, instruction->err_val); - LLVMBuildStore(g->builder, err_code, err_code_ptr_val); - - LLVMValueRef return_value; - if (ir_want_runtime_safety(g, &instruction->base)) { - return_value = LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)); - } else { - return_value = LLVMGetUndef(LLVMPointerType(LLVMInt8Type(), 0)); - } - LLVMBuildRet(g->builder, return_value); - return nullptr; -} - -static LLVMValueRef ir_render_coro_suspend(CodeGen *g, IrExecutable *executable, IrInstructionCoroSuspend *instruction) { - LLVMValueRef save_point; - if (instruction->save_point == nullptr) { - save_point = LLVMConstNull(ZigLLVMTokenTypeInContext(LLVMGetGlobalContext())); - } else { - save_point = ir_llvm_value(g, instruction->save_point); - } - LLVMValueRef is_final = ir_llvm_value(g, instruction->is_final); - LLVMValueRef params[] = { - save_point, - is_final, - }; - return LLVMBuildCall(g->builder, get_coro_suspend_fn_val(g), params, 2, ""); -} - -static LLVMValueRef ir_render_coro_end(CodeGen *g, IrExecutable *executable, IrInstructionCoroEnd *instruction) { - LLVMValueRef params[] = { - LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0)), - LLVMConstNull(LLVMInt1Type()), - }; - return LLVMBuildCall(g->builder, get_coro_end_fn_val(g), params, 2, ""); -} - -static LLVMValueRef ir_render_coro_free(CodeGen *g, IrExecutable *executable, IrInstructionCoroFree *instruction) { - LLVMValueRef coro_id = ir_llvm_value(g, instruction->coro_id); - LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); - LLVMValueRef params[] = { - coro_id, - coro_handle, - }; - return LLVMBuildCall(g->builder, get_coro_free_fn_val(g), params, 2, ""); -} - -static LLVMValueRef ir_render_coro_resume(CodeGen *g, IrExecutable *executable, IrInstructionCoroResume *instruction) { - LLVMValueRef awaiter_handle = ir_llvm_value(g, instruction->awaiter_handle); - return LLVMBuildCall(g->builder, get_coro_resume_fn_val(g), &awaiter_handle, 1, ""); -} - -static LLVMValueRef ir_render_coro_save(CodeGen *g, IrExecutable *executable, IrInstructionCoroSave *instruction) { - LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); - return LLVMBuildCall(g->builder, get_coro_save_fn_val(g), &coro_handle, 1, ""); -} - -static LLVMValueRef ir_render_coro_promise(CodeGen *g, IrExecutable *executable, IrInstructionCoroPromise *instruction) { - LLVMValueRef coro_handle = ir_llvm_value(g, instruction->coro_handle); - LLVMValueRef params[] = { - coro_handle, - LLVMConstInt(LLVMInt32Type(), get_coro_frame_align_bytes(g), false), - LLVMConstNull(LLVMInt1Type()), - }; - LLVMValueRef uncasted_result = LLVMBuildCall(g->builder, get_coro_promise_fn_val(g), params, 3, ""); - return LLVMBuildBitCast(g->builder, uncasted_result, get_llvm_type(g, instruction->base.value.type), ""); -} - -static LLVMValueRef get_coro_alloc_helper_fn_val(CodeGen *g, LLVMTypeRef alloc_fn_type_ref, ZigType *fn_type) { - if (g->coro_alloc_helper_fn_val != nullptr) - return g->coro_alloc_helper_fn_val; - - assert(fn_type->id == ZigTypeIdFn); - - ZigType *ptr_to_err_code_type = get_pointer_to_type(g, g->builtin_types.entry_global_error_set, false); - - LLVMTypeRef alloc_raw_fn_type_ref = LLVMGetElementType(alloc_fn_type_ref); - LLVMTypeRef *alloc_fn_arg_types = allocate<LLVMTypeRef>(LLVMCountParamTypes(alloc_raw_fn_type_ref)); - LLVMGetParamTypes(alloc_raw_fn_type_ref, alloc_fn_arg_types); - - ZigList<LLVMTypeRef> arg_types = {}; - arg_types.append(alloc_fn_type_ref); - if (g->have_err_ret_tracing) { - arg_types.append(alloc_fn_arg_types[1]); - } - arg_types.append(alloc_fn_arg_types[g->have_err_ret_tracing ? 2 : 1]); - arg_types.append(get_llvm_type(g, ptr_to_err_code_type)); - arg_types.append(g->builtin_types.entry_usize->llvm_type); - - LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), - arg_types.items, arg_types.length, false); - - Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_coro_alloc_helper"), false); - LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); - LLVMSetLinkage(fn_val, LLVMInternalLinkage); - LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); - addLLVMFnAttr(fn_val, "nounwind"); - addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); - addLLVMArgAttr(fn_val, (unsigned)1, "nonnull"); - - LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); - LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); - ZigFn *prev_cur_fn = g->cur_fn; - LLVMValueRef prev_cur_fn_val = g->cur_fn_val; - - LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); - LLVMPositionBuilderAtEnd(g->builder, entry_block); - ZigLLVMClearCurrentDebugLocation(g->builder); - g->cur_fn = nullptr; - g->cur_fn_val = fn_val; - - LLVMValueRef sret_ptr = LLVMBuildAlloca(g->builder, LLVMGetElementType(alloc_fn_arg_types[0]), ""); - - size_t next_arg = 0; - LLVMValueRef realloc_fn_val = LLVMGetParam(fn_val, next_arg); - next_arg += 1; - - LLVMValueRef stack_trace_val; - if (g->have_err_ret_tracing) { - stack_trace_val = LLVMGetParam(fn_val, next_arg); - next_arg += 1; - } - - LLVMValueRef allocator_val = LLVMGetParam(fn_val, next_arg); - next_arg += 1; - LLVMValueRef err_code_ptr = LLVMGetParam(fn_val, next_arg); - next_arg += 1; - LLVMValueRef coro_size = LLVMGetParam(fn_val, next_arg); - next_arg += 1; - LLVMValueRef alignment_val = LLVMConstInt(g->builtin_types.entry_u29->llvm_type, - get_coro_frame_align_bytes(g), false); - - ConstExprValue *zero_array = create_const_str_lit(g, buf_create_from_str("")); - ConstExprValue *undef_slice_zero = create_const_slice(g, zero_array, 0, 0, false); - render_const_val(g, undef_slice_zero, ""); - render_const_val_global(g, undef_slice_zero, ""); - - ZigList<LLVMValueRef> args = {}; - args.append(sret_ptr); - if (g->have_err_ret_tracing) { - args.append(stack_trace_val); - } - args.append(allocator_val); - args.append(undef_slice_zero->global_refs->llvm_global); - args.append(LLVMGetUndef(g->builtin_types.entry_u29->llvm_type)); - args.append(coro_size); - args.append(alignment_val); - LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, realloc_fn_val, args.items, args.length, - get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - set_call_instr_sret(g, call_instruction); - LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_err_index, ""); - LLVMValueRef err_val = LLVMBuildLoad(g->builder, err_val_ptr, ""); - LLVMBuildStore(g->builder, err_val, err_code_ptr); - LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_val, LLVMConstNull(LLVMTypeOf(err_val)), ""); - LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(fn_val, "AllocOk"); - LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(fn_val, "AllocFail"); - LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block); - - LLVMPositionBuilderAtEnd(g->builder, ok_block); - LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, sret_ptr, err_union_payload_index, ""); - ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, false, false, - PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false); - ZigType *slice_type = get_slice_type(g, u8_ptr_type); - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, payload_ptr, ptr_field_index, ""); - LLVMValueRef ptr_val = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); - LLVMBuildRet(g->builder, ptr_val); - - LLVMPositionBuilderAtEnd(g->builder, fail_block); - LLVMBuildRet(g->builder, LLVMConstNull(LLVMPointerType(LLVMInt8Type(), 0))); - - g->cur_fn = prev_cur_fn; - g->cur_fn_val = prev_cur_fn_val; - LLVMPositionBuilderAtEnd(g->builder, prev_block); - if (!g->strip_debug_symbols) { - LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); - } - - g->coro_alloc_helper_fn_val = fn_val; - return fn_val; -} - -static LLVMValueRef ir_render_coro_alloc_helper(CodeGen *g, IrExecutable *executable, - IrInstructionCoroAllocHelper *instruction) -{ - LLVMValueRef realloc_fn = ir_llvm_value(g, instruction->realloc_fn); - LLVMValueRef coro_size = ir_llvm_value(g, instruction->coro_size); - LLVMValueRef fn_val = get_coro_alloc_helper_fn_val(g, LLVMTypeOf(realloc_fn), instruction->realloc_fn->value.type); - size_t err_code_ptr_arg_index = get_async_err_code_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); - size_t allocator_arg_index = get_async_allocator_arg_index(g, &g->cur_fn->type_entry->data.fn.fn_type_id); - - ZigList<LLVMValueRef> params = {}; - params.append(realloc_fn); - uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, g->cur_fn); - if (err_ret_trace_arg_index != UINT32_MAX) { - params.append(LLVMGetParam(g->cur_fn_val, err_ret_trace_arg_index)); - } - params.append(LLVMGetParam(g->cur_fn_val, allocator_arg_index)); - params.append(LLVMGetParam(g->cur_fn_val, err_code_ptr_arg_index)); - params.append(coro_size); - - return ZigLLVMBuildCall(g->builder, fn_val, params.items, params.length, - get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); -} - static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable, IrInstructionAtomicRmw *instruction) { @@ -5263,14 +5324,15 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable, LLVMValueRef operand = ir_llvm_value(g, instruction->operand); if (get_codegen_ptr_type(operand_type) == nullptr) { - return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, false); + return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded); } // it's a pointer but we need to treat it as an int LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr, LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), ""); LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, ""); - LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering, false); + LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering, + g->is_single_threaded); return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), ""); } @@ -5284,27 +5346,6 @@ static LLVMValueRef ir_render_atomic_load(CodeGen *g, IrExecutable *executable, return load_inst; } -static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *executable, - IrInstructionMergeErrRetTraces *instruction) -{ - assert(g->have_err_ret_tracing); - - LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->src_err_ret_trace_ptr); - LLVMValueRef dest_trace_ptr = ir_llvm_value(g, instruction->dest_err_ret_trace_ptr); - - LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; - ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - return nullptr; -} - -static LLVMValueRef ir_render_mark_err_ret_trace_ptr(CodeGen *g, IrExecutable *executable, - IrInstructionMarkErrRetTracePtr *instruction) -{ - assert(g->have_err_ret_tracing); - g->cur_err_ret_trace_val_stack = ir_llvm_value(g, instruction->err_ret_trace_ptr); - return nullptr; -} - static LLVMValueRef ir_render_float_op(CodeGen *g, IrExecutable *executable, IrInstructionFloatOp *instruction) { LLVMValueRef op = ir_llvm_value(g, instruction->op1); assert(instruction->base.value.type->id == ZigTypeIdFloat); @@ -5424,6 +5465,174 @@ static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executab return nullptr; } +static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable, + IrInstructionSuspendBegin *instruction) +{ + instruction->resume_bb = gen_suspend_begin(g, "SuspendResume"); + return nullptr; +} + +static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executable, + IrInstructionSuspendFinish *instruction) +{ + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, instruction->begin->resume_bb); + render_async_var_decls(g, instruction->base.scope); + return nullptr; +} + +static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) { + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMValueRef zero = LLVMConstNull(usize_type_ref); + LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame); + ZigType *result_type = instruction->base.value.type; + ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true); + + // Prepare to be suspended + LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume"); + LLVMBasicBlockRef end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitEnd"); + + // At this point resuming the function will continue from resume_bb. + // This code is as if it is running inside the suspend block. + + // supply the awaiter return pointer + LLVMValueRef result_loc = (instruction->result_loc == nullptr) ? + nullptr : ir_llvm_value(g, instruction->result_loc); + if (type_has_bits(result_type)) { + LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start + 1, ""); + if (result_loc == nullptr) { + // no copy needed + LLVMBuildStore(g->builder, LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr_ptr))), + awaiter_ret_ptr_ptr); + } else { + LLVMBuildStore(g->builder, result_loc, awaiter_ret_ptr_ptr); + } + } + + // supply the error return trace pointer + if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) { + LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); + assert(my_err_ret_trace_val != nullptr); + LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, + frame_index_trace_arg(g, result_type) + 1, ""); + LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr); + } + + // caller's own frame pointer + LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); + LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_awaiter_index, ""); + LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val, + LLVMAtomicOrderingRelease); + + LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait"); + LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend"); + LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn"); + + LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref); + LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2); + + LLVMAddCase(switch_instr, zero, complete_suspend_block); + LLVMAddCase(switch_instr, all_ones, early_return_block); + + // We discovered that another awaiter was already here. + LLVMPositionBuilderAtEnd(g->builder, bad_await_block); + gen_assertion(g, PanicMsgIdBadAwait, &instruction->base); + + // Rely on the target to resume us from suspension. + LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block); + LLVMBuildRetVoid(g->builder); + + // Early return: The async function has already completed. We must copy the result and + // the error return trace if applicable. + LLVMPositionBuilderAtEnd(g->builder, early_return_block); + if (type_has_bits(result_type) && result_loc != nullptr) { + LLVMValueRef their_result_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start, ""); + LLVMValueRef their_result_ptr = LLVMBuildLoad(g->builder, their_result_ptr_ptr, ""); + LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0); + LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, result_loc, ptr_u8, ""); + LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, their_result_ptr, ptr_u8, ""); + bool is_volatile = false; + uint32_t abi_align = get_abi_alignment(g, result_type); + LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, result_type), false); + ZigLLVMBuildMemCpy(g->builder, + dest_ptr_casted, abi_align, + src_ptr_casted, abi_align, byte_count_val, is_volatile); + } + if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) { + LLVMValueRef their_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, + frame_index_trace_arg(g, result_type), ""); + LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, their_trace_ptr_ptr, ""); + LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); + LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; + ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + } + LLVMBuildBr(g->builder, end_bb); + + LLVMPositionBuilderAtEnd(g->builder, resume_bb); + gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr); + LLVMBuildBr(g->builder, end_bb); + + LLVMPositionBuilderAtEnd(g->builder, end_bb); + if (type_has_bits(result_type) && result_loc != nullptr) { + return get_handle_value(g, result_loc, result_type, ptr_result_type); + } + return nullptr; +} + +static LLVMValueRef ir_render_resume(CodeGen *g, IrExecutable *executable, IrInstructionResume *instruction) { + LLVMValueRef frame = ir_llvm_value(g, instruction->frame); + ZigType *frame_type = instruction->frame->value.type; + assert(frame_type->id == ZigTypeIdAnyFrame); + + gen_resume(g, nullptr, frame, ResumeIdManual); + return nullptr; +} + +static LLVMValueRef ir_render_frame_size(CodeGen *g, IrExecutable *executable, + IrInstructionFrameSizeGen *instruction) +{ + LLVMValueRef fn_val = ir_llvm_value(g, instruction->fn); + return gen_frame_size(g, fn_val); +} + +static LLVMValueRef ir_render_spill_begin(CodeGen *g, IrExecutable *executable, + IrInstructionSpillBegin *instruction) +{ + if (!fn_is_async(g->cur_fn)) + return nullptr; + + switch (instruction->spill_id) { + case SpillIdInvalid: + zig_unreachable(); + case SpillIdRetErrCode: { + LLVMValueRef operand = ir_llvm_value(g, instruction->operand); + LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill); + LLVMBuildStore(g->builder, operand, ptr); + return nullptr; + } + + } + zig_unreachable(); +} + +static LLVMValueRef ir_render_spill_end(CodeGen *g, IrExecutable *executable, IrInstructionSpillEnd *instruction) { + if (!fn_is_async(g->cur_fn)) + return ir_llvm_value(g, instruction->begin->operand); + + switch (instruction->begin->spill_id) { + case SpillIdInvalid: + zig_unreachable(); + case SpillIdRetErrCode: { + LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill); + return LLVMBuildLoad(g->builder, ptr, ""); + } + + } + zig_unreachable(); +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -5445,7 +5654,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdSetRuntimeSafety: case IrInstructionIdSetFloatMode: case IrInstructionIdArrayType: - case IrInstructionIdPromiseType: + case IrInstructionIdAnyFrameType: case IrInstructionIdSliceType: case IrInstructionIdSizeOf: case IrInstructionIdSwitchTarget: @@ -5485,8 +5694,6 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdTagType: case IrInstructionIdExport: case IrInstructionIdErrorUnion: - case IrInstructionIdPromiseResultType: - case IrInstructionIdAwaitBookkeeping: case IrInstructionIdAddImplicitReturnType: case IrInstructionIdIntCast: case IrInstructionIdFloatCast: @@ -5508,17 +5715,19 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdCallSrc: case IrInstructionIdAllocaSrc: case IrInstructionIdEndExpr: - case IrInstructionIdAllocaGen: case IrInstructionIdImplicitCast: case IrInstructionIdResolveResult: case IrInstructionIdResetResult: - case IrInstructionIdResultPtr: case IrInstructionIdContainerInitList: case IrInstructionIdSliceSrc: case IrInstructionIdRef: case IrInstructionIdBitCastSrc: case IrInstructionIdTestErrSrc: case IrInstructionIdUnionInitNamedField: + case IrInstructionIdFrameType: + case IrInstructionIdFrameSizeSrc: + case IrInstructionIdAllocaGen: + case IrInstructionIdAwaitSrc: zig_unreachable(); case IrInstructionIdDeclVarGen: @@ -5597,8 +5806,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_return_address(g, executable, (IrInstructionReturnAddress *)instruction); case IrInstructionIdFrameAddress: return ir_render_frame_address(g, executable, (IrInstructionFrameAddress *)instruction); - case IrInstructionIdHandle: - return ir_render_handle(g, executable, (IrInstructionHandle *)instruction); + case IrInstructionIdFrameHandle: + return ir_render_handle(g, executable, (IrInstructionFrameHandle *)instruction); case IrInstructionIdOverflowOp: return ir_render_overflow_op(g, executable, (IrInstructionOverflowOp *)instruction); case IrInstructionIdTestErrGen: @@ -5641,44 +5850,12 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_align_cast(g, executable, (IrInstructionAlignCast *)instruction); case IrInstructionIdErrorReturnTrace: return ir_render_error_return_trace(g, executable, (IrInstructionErrorReturnTrace *)instruction); - case IrInstructionIdCancel: - return ir_render_cancel(g, executable, (IrInstructionCancel *)instruction); - case IrInstructionIdGetImplicitAllocator: - return ir_render_get_implicit_allocator(g, executable, (IrInstructionGetImplicitAllocator *)instruction); - case IrInstructionIdCoroId: - return ir_render_coro_id(g, executable, (IrInstructionCoroId *)instruction); - case IrInstructionIdCoroAlloc: - return ir_render_coro_alloc(g, executable, (IrInstructionCoroAlloc *)instruction); - case IrInstructionIdCoroSize: - return ir_render_coro_size(g, executable, (IrInstructionCoroSize *)instruction); - case IrInstructionIdCoroBegin: - return ir_render_coro_begin(g, executable, (IrInstructionCoroBegin *)instruction); - case IrInstructionIdCoroAllocFail: - return ir_render_coro_alloc_fail(g, executable, (IrInstructionCoroAllocFail *)instruction); - case IrInstructionIdCoroSuspend: - return ir_render_coro_suspend(g, executable, (IrInstructionCoroSuspend *)instruction); - case IrInstructionIdCoroEnd: - return ir_render_coro_end(g, executable, (IrInstructionCoroEnd *)instruction); - case IrInstructionIdCoroFree: - return ir_render_coro_free(g, executable, (IrInstructionCoroFree *)instruction); - case IrInstructionIdCoroResume: - return ir_render_coro_resume(g, executable, (IrInstructionCoroResume *)instruction); - case IrInstructionIdCoroSave: - return ir_render_coro_save(g, executable, (IrInstructionCoroSave *)instruction); - case IrInstructionIdCoroPromise: - return ir_render_coro_promise(g, executable, (IrInstructionCoroPromise *)instruction); - case IrInstructionIdCoroAllocHelper: - return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction); case IrInstructionIdAtomicRmw: return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); case IrInstructionIdAtomicLoad: return ir_render_atomic_load(g, executable, (IrInstructionAtomicLoad *)instruction); case IrInstructionIdSaveErrRetAddr: return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); - case IrInstructionIdMergeErrRetTraces: - return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction); - case IrInstructionIdMarkErrRetTracePtr: - return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction); case IrInstructionIdFloatOp: return ir_render_float_op(g, executable, (IrInstructionFloatOp *)instruction); case IrInstructionIdMulAdd: @@ -5695,6 +5872,20 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_resize_slice(g, executable, (IrInstructionResizeSlice *)instruction); case IrInstructionIdPtrOfArrayToSlice: return ir_render_ptr_of_array_to_slice(g, executable, (IrInstructionPtrOfArrayToSlice *)instruction); + case IrInstructionIdSuspendBegin: + return ir_render_suspend_begin(g, executable, (IrInstructionSuspendBegin *)instruction); + case IrInstructionIdSuspendFinish: + return ir_render_suspend_finish(g, executable, (IrInstructionSuspendFinish *)instruction); + case IrInstructionIdResume: + return ir_render_resume(g, executable, (IrInstructionResume *)instruction); + case IrInstructionIdFrameSizeGen: + return ir_render_frame_size(g, executable, (IrInstructionFrameSizeGen *)instruction); + case IrInstructionIdAwaitGen: + return ir_render_await(g, executable, (IrInstructionAwaitGen *)instruction); + case IrInstructionIdSpillBegin: + return ir_render_spill_begin(g, executable, (IrInstructionSpillBegin *)instruction); + case IrInstructionIdSpillEnd: + return ir_render_spill_end(g, executable, (IrInstructionSpillEnd *)instruction); } zig_unreachable(); } @@ -5704,6 +5895,7 @@ static void ir_render(CodeGen *g, ZigFn *fn_entry) { IrExecutable *executable = &fn_entry->analyzed_executable; assert(executable->basic_block_list.length > 0); + for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) { IrBasicBlock *current_block = executable->basic_block_list.at(block_i); assert(current_block->llvm_block); @@ -5894,7 +6086,6 @@ static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, Con case ZigTypeIdPointer: case ZigTypeIdFn: case ZigTypeIdOptional: - case ZigTypeIdPromise: { LLVMValueRef ptr_val = gen_const_val(g, const_val, ""); LLVMValueRef ptr_size_int_val = LLVMConstPtrToInt(ptr_val, g->builtin_types.entry_usize->llvm_type); @@ -5957,7 +6148,10 @@ static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, Con } return val; } - + case ZigTypeIdFnFrame: + zig_panic("TODO bit pack an async function frame"); + case ZigTypeIdAnyFrame: + zig_panic("TODO bit pack an anyframe"); } zig_unreachable(); } @@ -6110,6 +6304,9 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c break; } + if ((err = type_resolve(g, type_entry, ResolveStatusLLVMFull))) + zig_unreachable(); + switch (type_entry->id) { case ZigTypeIdInt: return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_bigint); @@ -6181,6 +6378,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c LLVMValueRef *fields = allocate<LLVMValueRef>(type_entry->data.structure.gen_field_count); size_t src_field_count = type_entry->data.structure.src_field_count; bool make_unnamed_struct = false; + assert(type_entry->data.structure.resolve_status == ResolveStatusLLVMFull); if (type_entry->data.structure.layout == ContainerLayoutPacked) { size_t src_field_index = 0; while (src_field_index < src_field_count) { @@ -6250,6 +6448,22 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c LLVMValueRef val = gen_const_val(g, field_val, ""); fields[type_struct_field->gen_index] = val; make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, field_val->type, val); + + size_t end_pad_gen_index = (i + 1 < src_field_count) ? + type_entry->data.structure.fields[i + 1].gen_index : + type_entry->data.structure.gen_field_count; + size_t next_offset = (i + 1 < src_field_count) ? + type_entry->data.structure.fields[i + 1].offset : type_entry->abi_size; + if (end_pad_gen_index != SIZE_MAX) { + for (size_t gen_i = type_struct_field->gen_index + 1; gen_i < end_pad_gen_index; + gen_i += 1) + { + size_t pad_bytes = next_offset - + (type_struct_field->offset + type_struct_field->type_entry->abi_size); + LLVMTypeRef llvm_array_type = LLVMArrayType(LLVMInt8Type(), pad_bytes); + fields[gen_i] = LLVMGetUndef(llvm_array_type); + } + } } } if (make_unnamed_struct) { @@ -6437,13 +6651,18 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c err_payload_value = gen_const_val(g, payload_val, ""); make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_val->type, err_payload_value); } - LLVMValueRef fields[2]; + LLVMValueRef fields[3]; fields[err_union_err_index] = err_tag_value; fields[err_union_payload_index] = err_payload_value; + size_t field_count = 2; + if (type_entry->data.error_union.pad_llvm_type != nullptr) { + fields[2] = LLVMGetUndef(type_entry->data.error_union.pad_llvm_type); + field_count = 3; + } if (make_unnamed_struct) { - return LLVMConstStruct(fields, 2, false); + return LLVMConstStruct(fields, field_count, false); } else { - return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2); + return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, field_count); } } } @@ -6460,9 +6679,11 @@ static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const c case ZigTypeIdBoundFn: case ZigTypeIdArgTuple: case ZigTypeIdOpaque: - case ZigTypeIdPromise: zig_unreachable(); - + case ZigTypeIdFnFrame: + zig_panic("TODO"); + case ZigTypeIdAnyFrame: + zig_panic("TODO"); } zig_unreachable(); } @@ -6546,12 +6767,20 @@ static void generate_error_name_table(CodeGen *g) { static void build_all_basic_blocks(CodeGen *g, ZigFn *fn) { IrExecutable *executable = &fn->analyzed_executable; assert(executable->basic_block_list.length > 0); + LLVMValueRef fn_val = fn_llvm_value(g, fn); + LLVMBasicBlockRef first_bb = nullptr; + if (fn_is_async(fn)) { + first_bb = LLVMAppendBasicBlock(fn_val, "AsyncSwitch"); + g->cur_preamble_llvm_block = first_bb; + } for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) { IrBasicBlock *bb = executable->basic_block_list.at(block_i); - bb->llvm_block = LLVMAppendBasicBlock(fn_llvm_value(g, fn), bb->name_hint); + bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint); } - IrBasicBlock *entry_bb = executable->basic_block_list.at(0); - LLVMPositionBuilderAtEnd(g->builder, entry_bb->llvm_block); + if (first_bb == nullptr) { + first_bb = executable->basic_block_list.at(0)->llvm_block; + } + LLVMPositionBuilderAtEnd(g->builder, first_bb); } static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val, @@ -6728,13 +6957,19 @@ static void do_code_gen(CodeGen *g) { build_all_basic_blocks(g, fn_table_entry); clear_debug_source_node(g); - if (want_sret) { - g->cur_ret_ptr = LLVMGetParam(fn, 0); - } else if (handle_is_ptr(fn_type_id->return_type)) { - g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0); - // TODO add debug info variable for this + bool is_async = fn_is_async(fn_table_entry); + + if (is_async) { + g->cur_frame_ptr = LLVMGetParam(fn, 0); } else { - g->cur_ret_ptr = nullptr; + if (want_sret) { + g->cur_ret_ptr = LLVMGetParam(fn, 0); + } else if (handle_is_ptr(fn_type_id->return_type)) { + g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0); + // TODO add debug info variable for this + } else { + g->cur_ret_ptr = nullptr; + } } uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry); @@ -6746,39 +6981,41 @@ static void do_code_gen(CodeGen *g) { } // error return tracing setup - bool is_async = cc == CallingConventionAsync; - bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && !is_async && !have_err_ret_trace_arg; + bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && + !is_async && !have_err_ret_trace_arg; LLVMValueRef err_ret_array_val = nullptr; if (have_err_ret_trace_stack) { ZigType *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count); err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); - // populate g->stack_trace_type - (void)get_ptr_to_stack_trace_type(g); - g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); + (void)get_llvm_type(g, get_stack_trace_type(g)); + g->cur_err_ret_trace_val_stack = build_alloca(g, get_stack_trace_type(g), "error_return_trace", + get_abi_alignment(g, g->stack_trace_type)); } else { g->cur_err_ret_trace_val_stack = nullptr; } - // allocate temporary stack data - for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) { - IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i); - ZigType *ptr_type = instruction->base.value.type; - assert(ptr_type->id == ZigTypeIdPointer); - ZigType *child_type = ptr_type->data.pointer.child_type; - if (!type_has_bits(child_type)) - continue; - if (instruction->base.ref_count == 0) - continue; - if (instruction->base.value.special != ConstValSpecialRuntime) { - if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special != - ConstValSpecialRuntime) - { + if (!is_async) { + // allocate temporary stack data + for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) { + IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i); + ZigType *ptr_type = instruction->base.value.type; + assert(ptr_type->id == ZigTypeIdPointer); + ZigType *child_type = ptr_type->data.pointer.child_type; + if (!type_has_bits(child_type)) continue; + if (instruction->base.ref_count == 0) + continue; + if (instruction->base.value.special != ConstValSpecialRuntime) { + if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special != + ConstValSpecialRuntime) + { + continue; + } } + instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint, + get_ptr_align(g, ptr_type)); } - instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint, - get_ptr_align(g, ptr_type)); } ZigType *import = get_scope_import(&fn_table_entry->fndef_scope->base); @@ -6816,7 +7053,7 @@ static void do_code_gen(CodeGen *g) { } else if (is_c_abi) { fn_walk_var.data.vars.var = var; iter_function_params_c_abi(g, fn_table_entry->type_entry, &fn_walk_var, var->src_arg_index); - } else { + } else if (!is_async) { ZigType *gen_type; FnGenParamInfo *gen_info = &fn_table_entry->type_entry->data.fn.gen_param_info[var->src_arg_index]; assert(gen_info->gen_index != SIZE_MAX); @@ -6867,14 +7104,76 @@ static void do_code_gen(CodeGen *g) { gen_store(g, LLVMConstInt(usize->llvm_type, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } - // create debug variable declarations for parameters - // rely on the first variables in the variable_list being parameters. - FnWalk fn_walk_init = {}; - fn_walk_init.id = FnWalkIdInits; - fn_walk_init.data.inits.fn = fn_table_entry; - fn_walk_init.data.inits.llvm_fn = fn; - fn_walk_init.data.inits.gen_i = gen_i_init; - walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init); + if (is_async) { + (void)get_llvm_type(g, fn_table_entry->frame_type); + g->cur_resume_block_count = 0; + + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type; + LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false); + ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val); + + if (!g->strip_debug_symbols) { + AstNode *source_node = fn_table_entry->proto_node; + ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1, + (int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope)); + } + IrExecutable *executable = &fn_table_entry->analyzed_executable; + LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume"); + LLVMPositionBuilderAtEnd(g->builder, bad_resume_block); + gen_assertion_scope(g, PanicMsgIdBadResume, fn_table_entry->child_scope); + + LLVMPositionBuilderAtEnd(g->builder, g->cur_preamble_llvm_block); + render_async_spills(g); + g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_awaiter_index, ""); + LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_resume_index, ""); + g->cur_async_resume_index_ptr = resume_index_ptr; + + if (type_has_bits(fn_type_id->return_type)) { + LLVMValueRef cur_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start, ""); + g->cur_ret_ptr = LLVMBuildLoad(g->builder, cur_ret_ptr_ptr, ""); + } + uint32_t trace_field_index_stack = UINT32_MAX; + if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry, true)) { + trace_field_index_stack = frame_index_trace_stack(g, fn_type_id); + g->cur_err_ret_trace_val_stack = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + trace_field_index_stack, ""); + } + + LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, ""); + LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, resume_index, bad_resume_block, 4); + g->cur_async_switch_instr = switch_instr; + + LLVMValueRef zero = LLVMConstNull(usize_type_ref); + IrBasicBlock *entry_block = executable->basic_block_list.at(0); + LLVMAddCase(switch_instr, zero, entry_block->llvm_block); + g->cur_resume_block_count += 1; + LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block); + if (trace_field_index_stack != UINT32_MAX) { + if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) { + LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + frame_index_trace_arg(g, fn_type_id->return_type), ""); + LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(trace_ptr_ptr))); + LLVMBuildStore(g->builder, zero_ptr, trace_ptr_ptr); + } + + LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + trace_field_index_stack, ""); + LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, + trace_field_index_stack + 1, ""); + + gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr); + } + render_async_var_decls(g, entry_block->instruction_list.at(0)->scope); + } else { + // create debug variable declarations for parameters + // rely on the first variables in the variable_list being parameters. + FnWalk fn_walk_init = {}; + fn_walk_init.id = FnWalkIdInits; + fn_walk_init.data.inits.fn = fn_table_entry; + fn_walk_init.data.inits.llvm_fn = fn; + fn_walk_init.data.inits.gen_i = gen_i_init; + walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init); + } ir_render(g, fn_table_entry); @@ -6893,8 +7192,6 @@ static void do_code_gen(CodeGen *g) { LLVMDumpModule(g->module); } - // in release mode, we're sooooo confident that we've generated correct ir, - // that we skip the verify module step in order to get better performance. #ifndef NDEBUG char *error = nullptr; LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error); @@ -7163,16 +7460,8 @@ static void define_builtin_types(CodeGen *g) { g->primitive_type_table.put(&entry->name, entry); } - { - ZigType *entry = get_promise_type(g, nullptr); - g->primitive_type_table.put(&entry->name, entry); - entry->size_in_bits = g->builtin_types.entry_usize->size_in_bits; - entry->abi_align = g->builtin_types.entry_usize->abi_align; - entry->abi_size = g->builtin_types.entry_usize->abi_size; - } } - static BuiltinFnEntry *create_builtin_fn(CodeGen *g, BuiltinFnId id, const char *name, size_t count) { BuiltinFnEntry *builtin_fn = allocate<BuiltinFnEntry>(1); buf_init_from_str(&builtin_fn->name, name); @@ -7185,8 +7474,6 @@ static BuiltinFnEntry *create_builtin_fn(CodeGen *g, BuiltinFnId id, const char static void define_builtin_fns(CodeGen *g) { create_builtin_fn(g, BuiltinFnIdBreakpoint, "breakpoint", 0); create_builtin_fn(g, BuiltinFnIdReturnAddress, "returnAddress", 0); - create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0); - create_builtin_fn(g, BuiltinFnIdHandle, "handle", 0); create_builtin_fn(g, BuiltinFnIdMemcpy, "memcpy", 3); create_builtin_fn(g, BuiltinFnIdMemset, "memset", 3); create_builtin_fn(g, BuiltinFnIdSizeof, "sizeOf", 1); @@ -7262,13 +7549,13 @@ static void define_builtin_fns(CodeGen *g) { create_builtin_fn(g, BuiltinFnIdFloor, "floor", 2); create_builtin_fn(g, BuiltinFnIdCeil, "ceil", 2); create_builtin_fn(g, BuiltinFnIdTrunc, "trunc", 2); - //Needs library support on Windows - //create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 2); + create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 2); create_builtin_fn(g, BuiltinFnIdRound, "round", 2); create_builtin_fn(g, BuiltinFnIdMulAdd, "mulAdd", 4); create_builtin_fn(g, BuiltinFnIdInlineCall, "inlineCall", SIZE_MAX); create_builtin_fn(g, BuiltinFnIdNoInlineCall, "noInlineCall", SIZE_MAX); create_builtin_fn(g, BuiltinFnIdNewStackCall, "newStackCall", SIZE_MAX); + create_builtin_fn(g, BuiltinFnIdAsyncCall, "asyncCall", SIZE_MAX); create_builtin_fn(g, BuiltinFnIdTypeId, "typeId", 1); create_builtin_fn(g, BuiltinFnIdShlExact, "shlExact", 2); create_builtin_fn(g, BuiltinFnIdShrExact, "shrExact", 2); @@ -7287,6 +7574,10 @@ static void define_builtin_fns(CodeGen *g) { create_builtin_fn(g, BuiltinFnIdThis, "This", 0); create_builtin_fn(g, BuiltinFnIdHasDecl, "hasDecl", 2); create_builtin_fn(g, BuiltinFnIdUnionInit, "unionInit", 3); + create_builtin_fn(g, BuiltinFnIdFrameHandle, "frame", 0); + create_builtin_fn(g, BuiltinFnIdFrameType, "Frame", 1); + create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0); + create_builtin_fn(g, BuiltinFnIdFrameSize, "frameSize", 1); } static const char *bool_to_str(bool b) { @@ -7598,7 +7889,8 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { " BoundFn: Fn,\n" " ArgTuple: void,\n" " Opaque: void,\n" - " Promise: Promise,\n" + " Frame: void,\n" + " AnyFrame: AnyFrame,\n" " Vector: Vector,\n" " EnumLiteral: void,\n" "\n\n" @@ -7711,11 +8003,10 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { " is_generic: bool,\n" " is_var_args: bool,\n" " return_type: ?type,\n" - " async_allocator_type: ?type,\n" " args: []FnArg,\n" " };\n" "\n" - " pub const Promise = struct {\n" + " pub const AnyFrame = struct {\n" " child: ?type,\n" " };\n" "\n" @@ -8308,6 +8599,12 @@ void add_cc_args(CodeGen *g, ZigList<const char *> &args, const char *out_dep_pa args.append("-g"); } + if (codegen_have_frame_pointer(g)) { + args.append("-fno-omit-frame-pointer"); + } else { + args.append("-fomit-frame-pointer"); + } + switch (g->build_mode) { case BuildModeDebug: // windows c runtime requires -D_DEBUG if using debug libraries @@ -8320,7 +8617,6 @@ void add_cc_args(CodeGen *g, ZigList<const char *> &args, const char *out_dep_pa } else { args.append("-fno-stack-protector"); } - args.append("-fno-omit-frame-pointer"); break; case BuildModeSafeRelease: // See the comment in the BuildModeFastRelease case for why we pass -O2 rather @@ -8334,7 +8630,6 @@ void add_cc_args(CodeGen *g, ZigList<const char *> &args, const char *out_dep_pa } else { args.append("-fno-stack-protector"); } - args.append("-fomit-frame-pointer"); break; case BuildModeFastRelease: args.append("-DNDEBUG"); @@ -8345,13 +8640,11 @@ void add_cc_args(CodeGen *g, ZigList<const char *> &args, const char *out_dep_pa // running in -O2 and thus the -O3 path has been tested less. args.append("-O2"); args.append("-fno-stack-protector"); - args.append("-fomit-frame-pointer"); break; case BuildModeSmallRelease: args.append("-DNDEBUG"); args.append("-Os"); args.append("-fno-stack-protector"); - args.append("-fomit-frame-pointer"); break; } @@ -8878,7 +9171,8 @@ static void prepend_c_type_to_decl_list(CodeGen *g, GenH *gen_h, ZigType *type_e case ZigTypeIdArgTuple: case ZigTypeIdErrorUnion: case ZigTypeIdErrorSet: - case ZigTypeIdPromise: + case ZigTypeIdFnFrame: + case ZigTypeIdAnyFrame: zig_unreachable(); case ZigTypeIdVoid: case ZigTypeIdUnreachable: @@ -9062,7 +9356,8 @@ static void get_c_type(CodeGen *g, GenH *gen_h, ZigType *type_entry, Buf *out_bu case ZigTypeIdUndefined: case ZigTypeIdNull: case ZigTypeIdArgTuple: - case ZigTypeIdPromise: + case ZigTypeIdFnFrame: + case ZigTypeIdAnyFrame: zig_unreachable(); } } @@ -9229,9 +9524,11 @@ static void gen_h_file(CodeGen *g) { case ZigTypeIdArgTuple: case ZigTypeIdOptional: case ZigTypeIdFn: - case ZigTypeIdPromise: case ZigTypeIdVector: + case ZigTypeIdFnFrame: + case ZigTypeIdAnyFrame: zig_unreachable(); + case ZigTypeIdEnum: if (type_entry->data.enumeration.layout == ContainerLayoutExtern) { fprintf(out_h, "enum %s {\n", buf_ptr(type_h_name(type_entry))); @@ -9770,3 +10067,18 @@ CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget return g; } +bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type) { + return g->have_err_ret_tracing && + (return_type->id == ZigTypeIdErrorUnion || + return_type->id == ZigTypeIdErrorSet); +} + +bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async) { + if (is_async) { + return g->have_err_ret_tracing && (fn->calls_or_awaits_errorable_fn || + codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type)); + } else { + return g->have_err_ret_tracing && fn->calls_or_awaits_errorable_fn && + !codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type); + } +} |