Merge branch 'master' into zen_stdlib

1 files changed, 328 insertions, 96 deletions

diff --git a/src/codegen.cpp b/src/codegen.cpp
index a44091bdbf..2aca143524 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -112,10 +112,10 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out
         // that's for native compilation
         g->zig_target = *target;
         resolve_target_object_format(&g->zig_target);
-        g->dynamic_linker = buf_create_from_str("");
-        g->libc_lib_dir = buf_create_from_str("");
-        g->libc_static_lib_dir = buf_create_from_str("");
-        g->libc_include_dir = buf_create_from_str("");
+        g->dynamic_linker = nullptr;
+        g->libc_lib_dir = nullptr;
+        g->libc_static_lib_dir = nullptr;
+        g->libc_include_dir = nullptr;
         g->msvc_lib_dir = nullptr;
         g->kernel32_lib_dir = nullptr;
         g->each_lib_rpath = false;
@@ -123,16 +123,13 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out
         // native compilation, we can rely on the configuration stuff
         g->is_native_target = true;
         get_native_target(&g->zig_target);
-        g->dynamic_linker = buf_create_from_str(ZIG_DYNAMIC_LINKER);
-        g->libc_lib_dir = buf_create_from_str(ZIG_LIBC_LIB_DIR);
-        g->libc_static_lib_dir = buf_create_from_str(ZIG_LIBC_STATIC_LIB_DIR);
-        g->libc_include_dir = buf_create_from_str(ZIG_LIBC_INCLUDE_DIR);
+        g->dynamic_linker = nullptr; // find it at runtime
+        g->libc_lib_dir = nullptr; // find it at runtime
+        g->libc_static_lib_dir = nullptr; // find it at runtime
+        g->libc_include_dir = nullptr; // find it at runtime
         g->msvc_lib_dir = nullptr; // find it at runtime
         g->kernel32_lib_dir = nullptr; // find it at runtime
-
-#ifdef ZIG_EACH_LIB_RPATH
         g->each_lib_rpath = true;
-#endif
 
         if (g->zig_target.os == OsMacOSX ||
             g->zig_target.os == OsIOS)
@@ -411,6 +408,9 @@ static uint32_t get_err_ret_trace_arg_index(CodeGen *g, FnTableEntry *fn_table_e
     if (!g->have_err_ret_tracing) {
         return UINT32_MAX;
     }
+    if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) {
+        return 0;
+    }
     TypeTableEntry *fn_type = fn_table_entry->type_entry;
     if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) {
         return UINT32_MAX;
@@ -653,6 +653,7 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
         case ScopeIdDeferExpr:
         case ScopeIdLoop:
         case ScopeIdCompTime:
+        case ScopeIdCoroPrelude:
             return get_di_scope(g, scope->parent);
     }
     zig_unreachable();
@@ -1116,22 +1117,19 @@ static LLVMValueRef get_return_address_fn_val(CodeGen *g) {
     return g->return_address_fn_val;
 }
 
-static LLVMValueRef get_return_err_fn(CodeGen *g) {
-    if (g->return_err_fn != nullptr)
-        return g->return_err_fn;
-
-    assert(g->err_tag_type != nullptr);
+static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
+    if (g->add_error_return_trace_addr_fn_val != nullptr)
+        return g->add_error_return_trace_addr_fn_val;
 
     LLVMTypeRef arg_types[] = {
-        // error return trace pointer
         get_ptr_to_stack_trace_type(g)->type_ref,
+        g->builtin_types.entry_usize->type_ref,
     };
-    LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
+    LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
 
-    Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_return_error"), false);
+    Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_add_err_ret_trace_addr"), false);
     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
-    addLLVMFnAttr(fn_val, "noinline"); // so that we can look at return address
-    addLLVMFnAttr(fn_val, "cold");
+    addLLVMFnAttr(fn_val, "alwaysinline");
     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
     addLLVMFnAttr(fn_val, "nounwind");
@@ -1153,6 +1151,8 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
     // stack_trace.instruction_addresses[stack_trace.index % stack_trace.instruction_addresses.len] = return_address;
 
     LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
+    LLVMValueRef address_value = LLVMGetParam(fn_val, 1);
+
     size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
     LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, "");
     size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
@@ -1174,15 +1174,10 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
     LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
     LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, "");
 
-    LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->type_ref);
-    LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
-    LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, "");
-
-    LLVMValueRef address_value = LLVMBuildPtrToInt(g->builder, return_address, usize_type_ref, "");
     gen_store_untyped(g, address_value, address_slot, 0, false);
 
     // stack_trace.index += 1;
-    LLVMValueRef index_plus_one_val = LLVMBuildAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), "");
+    LLVMValueRef index_plus_one_val = LLVMBuildNUWAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), "");
     gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false);
 
     // return;
@@ -1191,6 +1186,187 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) {
     LLVMPositionBuilderAtEnd(g->builder, prev_block);
     LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
 
+    g->add_error_return_trace_addr_fn_val = fn_val;
+    return fn_val;
+}
+
+static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) {
+    if (g->merge_err_ret_traces_fn_val)
+        return g->merge_err_ret_traces_fn_val;
+
+    assert(g->stack_trace_type != nullptr);
+
+    LLVMTypeRef param_types[] = {
+        get_ptr_to_stack_trace_type(g)->type_ref,
+        get_ptr_to_stack_trace_type(g)->type_ref,
+    };
+    LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
+
+    Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false);
+    LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
+    LLVMSetLinkage(fn_val, LLVMInternalLinkage);
+    LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
+    addLLVMFnAttr(fn_val, "nounwind");
+    add_uwtable_attr(g, fn_val);
+    addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
+    addLLVMArgAttr(fn_val, (unsigned)0, "noalias");
+    addLLVMArgAttr(fn_val, (unsigned)0, "writeonly");
+    addLLVMArgAttr(fn_val, (unsigned)1, "nonnull");
+    addLLVMArgAttr(fn_val, (unsigned)1, "noalias");
+    addLLVMArgAttr(fn_val, (unsigned)1, "readonly");
+    if (g->build_mode == BuildModeDebug) {
+        ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
+        ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
+    }
+
+    // this is above the ZigLLVMClearCurrentDebugLocation
+    LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
+
+    LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
+    LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
+    LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
+    LLVMPositionBuilderAtEnd(g->builder, entry_block);
+    ZigLLVMClearCurrentDebugLocation(g->builder);
+
+    // var frame_index: usize = undefined;
+    // var frames_left: usize = undefined;
+    // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) {
+    //     frame_index = 0;
+    //     frames_left = src_stack_trace.index;
+    //     if (frames_left == 0) return;
+    // } else {
+    //     frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len;
+    //     frames_left = src_stack_trace.instruction_addresses.len;
+    // }
+    // while (true) {
+    //     __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]);
+    //     frames_left -= 1;
+    //     if (frames_left == 0) return;
+    //     frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len;
+    // }
+    LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
+
+    LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frame_index");
+    LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frames_left");
+
+    LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0);
+    LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1);
+
+    size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
+    size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
+    LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
+            (unsigned)src_index_field_index, "");
+    LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
+            (unsigned)src_addresses_field_index, "");
+    TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
+    size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
+    LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, "");
+    size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
+    LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, "");
+    LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, "");
+    LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, "");
+    LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, "");
+    LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, "");
+    LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap");
+    LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap");
+    LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop");
+    LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block);
+
+    LLVMPositionBuilderAtEnd(g->builder, no_wrap_block);
+    LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->type_ref);
+    LLVMBuildStore(g->builder, usize_zero, frame_index_ptr);
+    LLVMBuildStore(g->builder, src_index_val, frames_left_ptr);
+    LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, "");
+    LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block);
+
+    LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block);
+    LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->type_ref, 1, false);
+    LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, "");
+    LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, "");
+    LLVMBuildStore(g->builder, mod_len, frame_index_ptr);
+    LLVMBuildStore(g->builder, src_len_val, frames_left_ptr);
+    LLVMBuildBr(g->builder, loop_block);
+
+    LLVMPositionBuilderAtEnd(g->builder, loop_block);
+    LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
+    LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, "");
+    LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, "");
+    LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val};
+    ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
+    LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, "");
+    LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, "");
+    LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, "");
+    LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue");
+    LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block);
+
+    LLVMPositionBuilderAtEnd(g->builder, return_block);
+    LLVMBuildRetVoid(g->builder);
+
+    LLVMPositionBuilderAtEnd(g->builder, continue_block);
+    LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr);
+    LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
+    LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, "");
+    LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, "");
+    LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr);
+    LLVMBuildBr(g->builder, loop_block);
+
+    LLVMPositionBuilderAtEnd(g->builder, prev_block);
+    LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
+
+    g->merge_err_ret_traces_fn_val = fn_val;
+    return fn_val;
+
+}
+
+static LLVMValueRef get_return_err_fn(CodeGen *g) {
+    if (g->return_err_fn != nullptr)
+        return g->return_err_fn;
+
+    assert(g->err_tag_type != nullptr);
+
+    LLVMTypeRef arg_types[] = {
+        // error return trace pointer
+        get_ptr_to_stack_trace_type(g)->type_ref,
+    };
+    LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
+
+    Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_return_error"), false);
+    LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
+    addLLVMFnAttr(fn_val, "noinline"); // so that we can look at return address
+    addLLVMFnAttr(fn_val, "cold");
+    LLVMSetLinkage(fn_val, LLVMInternalLinkage);
+    LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
+    addLLVMFnAttr(fn_val, "nounwind");
+    add_uwtable_attr(g, fn_val);
+    addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
+    if (g->build_mode == BuildModeDebug) {
+        ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
+        ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
+    }
+
+    // this is above the ZigLLVMClearCurrentDebugLocation
+    LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
+
+    LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
+    LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
+    LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
+    LLVMPositionBuilderAtEnd(g->builder, entry_block);
+    ZigLLVMClearCurrentDebugLocation(g->builder);
+
+    LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
+
+    LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref;
+    LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->type_ref);
+    LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
+    LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, "");
+
+    LLVMValueRef args[] = { err_ret_trace_ptr, return_address };
+    ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
+    LLVMBuildRetVoid(g->builder);
+
+    LLVMPositionBuilderAtEnd(g->builder, prev_block);
+    LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
+
     g->return_err_fn = fn_val;
     return fn_val;
 }
@@ -1318,9 +1494,34 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
     return fn_val;
 }
 
-static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val) {
+static bool is_coro_prelude_scope(Scope *scope) {
+    while (scope != nullptr) {
+        if (scope->id == ScopeIdCoroPrelude) {
+            return true;
+        } else if (scope->id == ScopeIdFnDef) {
+            break;
+        }
+        scope = scope->parent;
+    }
+    return false;
+}
+
+static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) {
+    if (!g->have_err_ret_tracing) {
+        return nullptr;
+    }
+    if (g->cur_fn->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) {
+        return is_coro_prelude_scope(scope) ? g->cur_err_ret_trace_val_arg : g->cur_err_ret_trace_val_stack;
+    }
+    if (g->cur_err_ret_trace_val_stack != nullptr) {
+        return g->cur_err_ret_trace_val_stack;
+    }
+    return g->cur_err_ret_trace_val_arg;
+}
+
+static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) {
     LLVMValueRef safety_crash_err_fn = get_safety_crash_err_fn(g);
-    LLVMValueRef err_ret_trace_val = g->cur_err_ret_trace_val;
+    LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope);
     if (err_ret_trace_val == nullptr) {
         TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g);
         err_ret_trace_val = LLVMConstNull(ptr_to_stack_trace_type->type_ref);
@@ -1607,32 +1808,24 @@ static LLVMValueRef ir_llvm_value(CodeGen *g, IrInstruction *instruction) {
     return instruction->llvm_value;
 }
 
+static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *executable,
+        IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction)
+{
+    assert(g->have_err_ret_tracing);
+
+    LLVMValueRef return_err_fn = get_return_err_fn(g);
+    LLVMValueRef args[] = {
+        get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope),
+    };
+    LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1,
+            get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
+    return call_instruction;
+}
+
 static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *return_instruction) {
     LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
     TypeTableEntry *return_type = return_instruction->value->value.type;
 
-    if (g->have_err_ret_tracing) {
-        bool is_err_return = false;
-        if (return_type->id == TypeTableEntryIdErrorUnion) {
-            if (return_instruction->value->value.special == ConstValSpecialStatic) {
-                is_err_return = return_instruction->value->value.data.x_err_union.err != nullptr;
-            } else if (return_instruction->value->value.special == ConstValSpecialRuntime) {
-                is_err_return = return_instruction->value->value.data.rh_error_union == RuntimeHintErrorUnionError;
-                // TODO: emit a branch to check if the return value is an error
-            }
-        } else if (return_type->id == TypeTableEntryIdErrorSet) {
-            is_err_return = true;
-        }
-        if (is_err_return) {
-            LLVMValueRef return_err_fn = get_return_err_fn(g);
-            LLVMValueRef args[] = {
-                g->cur_err_ret_trace_val,
-            };
-            LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1,
-                    get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
-            LLVMSetTailCall(call_instruction, true);
-        }
-    }
     if (handle_is_ptr(return_type)) {
         if (calling_convention_does_first_arg_return(g->cur_fn->type_entry->data.fn.fn_type_id.cc)) {
             assert(g->cur_ret_ptr);
@@ -2732,7 +2925,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
         gen_param_index += 1;
     }
     if (prefix_arg_err_ret_stack) {
-        gen_param_values[gen_param_index] = g->cur_err_ret_trace_val;
+        gen_param_values[gen_param_index] = get_cur_err_ret_trace_val(g, instruction->base.scope);
         gen_param_index += 1;
     }
     if (instruction->is_async) {
@@ -3299,11 +3492,12 @@ static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, I
 static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *executable,
         IrInstructionErrorReturnTrace *instruction)
 {
-    if (g->cur_err_ret_trace_val == nullptr) {
+    LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
+    if (cur_err_ret_trace_val == nullptr) {
         TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g);
         return LLVMConstNull(ptr_to_stack_trace_type->type_ref);
     }
-    return g->cur_err_ret_trace_val;
+    return cur_err_ret_trace_val;
 }
 
 static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) {
@@ -3733,7 +3927,7 @@ static LLVMValueRef ir_render_unwrap_err_payload(CodeGen *g, IrExecutable *execu
         LLVMBuildCondBr(g->builder, cond_val, ok_block, err_block);
 
         LLVMPositionBuilderAtEnd(g->builder, err_block);
-        gen_safety_crash_for_err(g, err_val);
+        gen_safety_crash_for_err(g, err_val, instruction->base.scope);
 
         LLVMPositionBuilderAtEnd(g->builder, ok_block);
     }
@@ -3925,7 +4119,7 @@ static LLVMValueRef ir_render_container_init_list(CodeGen *g, IrExecutable *exec
 }
 
 static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInstructionPanic *instruction) {
-    gen_panic(g, ir_llvm_value(g, instruction->msg), g->cur_err_ret_trace_val);
+    gen_panic(g, ir_llvm_value(g, instruction->msg), get_cur_err_ret_trace_val(g, instruction->base.scope));
     return nullptr;
 }
 
@@ -4187,6 +4381,27 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable,
     return LLVMBuildIntToPtr(g->builder, uncasted_result, operand_type->type_ref, "");
 }
 
+static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *executable,
+        IrInstructionMergeErrRetTraces *instruction)
+{
+    assert(g->have_err_ret_tracing);
+
+    LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->src_err_ret_trace_ptr);
+    LLVMValueRef dest_trace_ptr = ir_llvm_value(g, instruction->dest_err_ret_trace_ptr);
+
+    LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr };
+    ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
+    return nullptr;
+}
+
+static LLVMValueRef ir_render_mark_err_ret_trace_ptr(CodeGen *g, IrExecutable *executable,
+        IrInstructionMarkErrRetTracePtr *instruction)
+{
+    assert(g->have_err_ret_tracing);
+    g->cur_err_ret_trace_val_stack = ir_llvm_value(g, instruction->err_ret_trace_ptr);
+    return nullptr;
+}
+
 static void set_debug_location(CodeGen *g, IrInstruction *instruction) {
     AstNode *source_node = instruction->source_node;
     Scope *scope = instruction->scope;
@@ -4212,6 +4427,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
         case IrInstructionIdSetRuntimeSafety:
         case IrInstructionIdSetFloatMode:
         case IrInstructionIdArrayType:
+        case IrInstructionIdPromiseType:
         case IrInstructionIdSliceType:
         case IrInstructionIdSizeOf:
         case IrInstructionIdSwitchTarget:
@@ -4252,6 +4468,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
         case IrInstructionIdErrorUnion:
         case IrInstructionIdPromiseResultType:
         case IrInstructionIdAwaitBookkeeping:
+        case IrInstructionIdAddImplicitReturnType:
             zig_unreachable();
 
         case IrInstructionIdReturn:
@@ -4400,6 +4617,12 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
             return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction);
         case IrInstructionIdAtomicRmw:
             return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction);
+        case IrInstructionIdSaveErrRetAddr:
+            return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction);
+        case IrInstructionIdMergeErrRetTraces:
+            return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction);
+        case IrInstructionIdMarkErrRetTracePtr:
+            return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction);
     }
     zig_unreachable();
 }
@@ -5282,39 +5505,23 @@ static void do_code_gen(CodeGen *g) {
         clear_debug_source_node(g);
 
         uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry);
-        if (err_ret_trace_arg_index != UINT32_MAX) {
-            g->cur_err_ret_trace_val = LLVMGetParam(fn, err_ret_trace_arg_index);
-        } else if (g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn) {
-            // TODO call graph analysis to find out what this number needs to be for every function
-            static const size_t stack_trace_ptr_count = 30;
-
-            TypeTableEntry *usize = g->builtin_types.entry_usize;
-            TypeTableEntry *array_type = get_array_type(g, usize, stack_trace_ptr_count);
-            LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses",
-                    get_abi_alignment(g, array_type));
-            g->cur_err_ret_trace_val = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type));
-            size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
-            LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)index_field_index, "");
-            gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false);
-
-            size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
-            LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)addresses_field_index, "");
-
-            TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
-            size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
-            LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
-            LLVMValueRef zero = LLVMConstNull(usize->type_ref);
-            LLVMValueRef indices[] = {zero, zero};
-            LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val,
-                    indices, 2, "");
-            gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr,
-                    get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false));
+        bool have_err_ret_trace_arg = err_ret_trace_arg_index != UINT32_MAX;
+        if (have_err_ret_trace_arg) {
+            g->cur_err_ret_trace_val_arg = LLVMGetParam(fn, err_ret_trace_arg_index);
+        } else {
+            g->cur_err_ret_trace_val_arg = nullptr;
+        }
 
-            size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
-            LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
-            gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false));
+        // error return tracing setup
+        bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync;
+        bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && !is_async && !have_err_ret_trace_arg;
+        LLVMValueRef err_ret_array_val = nullptr;
+        if (have_err_ret_trace_stack) {
+            TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count);
+            err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type));
+            g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type));
         } else {
-            g->cur_err_ret_trace_val = nullptr;
+            g->cur_err_ret_trace_val_stack = nullptr;
         }
 
         // allocate temporary stack data
@@ -5407,6 +5614,31 @@ static void do_code_gen(CodeGen *g) {
             }
         }
 
+        // finishing error return trace setup. we have to do this after all the allocas.
+        if (have_err_ret_trace_stack) {
+            TypeTableEntry *usize = g->builtin_types.entry_usize;
+            size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
+            LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, "");
+            gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false);
+
+            size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
+            LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, "");
+
+            TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
+            size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
+            LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
+            LLVMValueRef zero = LLVMConstNull(usize->type_ref);
+            LLVMValueRef indices[] = {zero, zero};
+            LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val,
+                    indices, 2, "");
+            TypeTableEntry *ptr_ptr_usize_type = get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false);
+            gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, ptr_ptr_usize_type);
+
+            size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
+            LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
+            gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false));
+        }
+
         FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id;
 
         // create debug variable declarations for parameters
@@ -5914,6 +6146,8 @@ static void define_builtin_compile_vars(CodeGen *g) {
     os_path_join(g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path);
     Buf *contents = buf_alloc();
 
+    // Modifications to this struct must be coordinated with code that does anything with
+    // g->stack_trace_type. There are hard-coded references to the field indexes.
     buf_append_str(contents,
         "pub const StackTrace = struct {\n"
         "    index: usize,\n"
@@ -6178,7 +6412,9 @@ static void init(CodeGen *g) {
     g->builder = LLVMCreateBuilder();
     g->dbuilder = ZigLLVMCreateDIBuilder(g->module, true);
 
-    Buf *producer = buf_sprintf("zig %s", ZIG_VERSION_STRING);
+    // Don't use ZIG_VERSION_STRING here, llvm misparses it when it includes
+    // the git revision.
+    Buf *producer = buf_sprintf("zig %d.%d.%d", ZIG_VERSION_MAJOR, ZIG_VERSION_MINOR, ZIG_VERSION_PATCH);
     const char *flags = "";
     unsigned runtime_version = 0;
     ZigLLVMDIFile *compile_unit_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(g->root_out_name),
@@ -6257,7 +6493,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package
         zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err));
     }
     Buf *import_code = buf_alloc();
-    if ((err = os_fetch_file_path(abs_full_path, import_code))) {
+    if ((err = os_fetch_file_path(abs_full_path, import_code, false))) {
         zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err));
     }
 
@@ -6345,7 +6581,7 @@ static void gen_root_source(CodeGen *g) {
     }
 
     Buf *source_code = buf_alloc();
-    if ((err = os_fetch_file_path(rel_full_path, source_code))) {
+    if ((err = os_fetch_file_path(rel_full_path, source_code, true))) {
         zig_panic("unable to open '%s': %s", buf_ptr(rel_full_path), err_str(err));
     }
 
@@ -6410,7 +6646,7 @@ static void gen_global_asm(CodeGen *g) {
     int err;
     for (size_t i = 0; i < g->assembly_files.length; i += 1) {
         Buf *asm_file = g->assembly_files.at(i);
-        if ((err = os_fetch_file_path(asm_file, &contents))) {
+        if ((err = os_fetch_file_path(asm_file, &contents,  false))) {
             zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err));
         }
         buf_append_buf(&g->global_asm, &contents);
@@ -6592,6 +6828,7 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf
                 }
             }
         case TypeTableEntryIdStruct:
+        case TypeTableEntryIdOpaque:
             {
                 buf_init_from_str(out_buf, "struct ");
                 buf_append_buf(out_buf, &type_entry->name);
@@ -6609,11 +6846,6 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf
                 buf_append_buf(out_buf, &type_entry->name);
                 return;
             }
-        case TypeTableEntryIdOpaque:
-            {
-                buf_init_from_buf(out_buf, &type_entry->name);
-                return;
-            }
         case TypeTableEntryIdArray:
             {
                 TypeTableEntryArray *array_data = &type_entry->data.array;