Merge pull request #10295 from ifreund/prefetch

Implement @prefetch()

5 files changed, 223 insertions, 0 deletions

diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index 0a3e52f4f4..1c50134194 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -898,6 +898,18 @@ struct AstNodeFnCallExpr {
     bool seen; // used by @compileLog
 };
 
+// Must be kept in sync with std.builtin.PrefetchOptions.Rw
+enum PrefetchRw {
+    PrefetchRwRead,
+    PrefetchRwWrite,
+};
+
+// Must be kept in sync with std.builtin.PrefetchOptions.Cache
+enum PrefetchCache {
+    PrefetchCacheInstruction,
+    PrefetchCacheData,
+};
+
 struct AstNodeArrayAccessExpr {
     AstNode *array_ref_expr;
     AstNode *subscript;
@@ -1818,6 +1830,7 @@ enum BuiltinFnId {
     BuiltinFnIdReduce,
     BuiltinFnIdMaximum,
     BuiltinFnIdMinimum,
+    BuiltinFnIdPrefetch,
 };
 
 struct BuiltinFnEntry {
@@ -2021,6 +2034,7 @@ struct CodeGen {
     LLVMValueRef return_err_fn;
     LLVMValueRef wasm_memory_size;
     LLVMValueRef wasm_memory_grow;
+    LLVMValueRef prefetch;
     LLVMTypeRef anyframe_fn_type;
 
     // reminder: hash tables must be initialized before use
@@ -2647,6 +2661,7 @@ enum Stage1ZirInstId : uint8_t {
     Stage1ZirInstIdWasmMemorySize,
     Stage1ZirInstIdWasmMemoryGrow,
     Stage1ZirInstIdSrc,
+    Stage1ZirInstIdPrefetch,
 };
 
 // ir_render_* functions in codegen.cpp consume Gen instructions and produce LLVM IR.
@@ -2743,6 +2758,7 @@ enum Stage1AirInstId : uint8_t {
     Stage1AirInstIdWasmMemorySize,
     Stage1AirInstIdWasmMemoryGrow,
     Stage1AirInstIdExtern,
+    Stage1AirInstIdPrefetch,
 };
 
 struct Stage1ZirInst {
@@ -3683,6 +3699,24 @@ struct Stage1ZirInstSrc {
     Stage1ZirInst base;
 };
 
+struct Stage1ZirInstPrefetch {
+    Stage1ZirInst base;
+
+    Stage1ZirInst *ptr;
+    Stage1ZirInst *options;
+};
+
+struct Stage1AirInstPrefetch {
+    Stage1AirInst base;
+
+    Stage1AirInst *ptr;
+    PrefetchRw rw;
+    // Must be in the range 0-3 inclusive
+    uint8_t locality;
+    PrefetchCache cache;
+};
+
+
 struct Stage1ZirInstSlice {
     Stage1ZirInst base;
 
diff --git a/src/stage1/astgen.cpp b/src/stage1/astgen.cpp
index 8fbd02c688..ee59aef04a 100644
--- a/src/stage1/astgen.cpp
+++ b/src/stage1/astgen.cpp
@@ -349,6 +349,8 @@ void destroy_instruction_src(Stage1ZirInst *inst) {
             return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstWasmMemoryGrow *>(inst));
         case Stage1ZirInstIdSrc:
             return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstSrc *>(inst));
+        case Stage1ZirInstIdPrefetch:
+            return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstPrefetch *>(inst));
     }
     zig_unreachable();
 }
@@ -941,6 +943,10 @@ static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstSrc *) {
     return Stage1ZirInstIdSrc;
 }
 
+static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstPrefetch *) {
+    return Stage1ZirInstIdPrefetch;
+}
+
 template<typename T>
 static T *ir_create_instruction(Stage1AstGen *ag, Scope *scope, AstNode *source_node) {
     T *special_instruction = heap::c_allocator.create<T>();
@@ -2870,6 +2876,21 @@ static Stage1ZirInst *ir_build_src(Stage1AstGen *ag, Scope *scope, AstNode *sour
     return &instruction->base;
 }
 
+static Stage1ZirInst *ir_build_prefetch(Stage1AstGen *ag, Scope *scope, AstNode *source_node,
+        Stage1ZirInst *ptr, Stage1ZirInst *options)
+{
+    Stage1ZirInstPrefetch *prefetch_instruction = ir_build_instruction<Stage1ZirInstPrefetch>(
+            ag, scope, source_node);
+    prefetch_instruction->ptr = ptr;
+    prefetch_instruction->options = options;
+
+    ir_ref_instruction(ptr, ag->current_basic_block);
+    ir_ref_instruction(options, ag->current_basic_block);
+
+    return &prefetch_instruction->base;
+}
+
+
 static void ir_count_defers(Stage1AstGen *ag, Scope *inner_scope, Scope *outer_scope, size_t *results) {
     results[ReturnKindUnconditional] = 0;
     results[ReturnKindError] = 0;
@@ -5416,6 +5437,29 @@ static Stage1ZirInst *astgen_builtin_fn_call(Stage1AstGen *ag, Scope *scope, Ast
                 Stage1ZirInst *src_inst = ir_build_src(ag, scope, node);
                 return ir_lval_wrap(ag, scope, src_inst, lval, result_loc);
             }
+        case BuiltinFnIdPrefetch:
+            {
+                ZigType *options_type = get_builtin_type(ag->codegen, "PrefetchOptions");
+                Stage1ZirInst *options_type_inst = ir_build_const_type(ag, scope, node, options_type);
+                ResultLocCast *result_loc_cast = ir_build_cast_result_loc(ag, options_type_inst, no_result_loc());
+
+                AstNode *ptr_node = node->data.fn_call_expr.params.at(0);
+                Stage1ZirInst *ptr_value = astgen_node(ag, ptr_node, scope);
+                if (ptr_value == ag->codegen->invalid_inst_src)
+                    return ptr_value;
+
+                AstNode *options_node = node->data.fn_call_expr.params.at(1);
+                Stage1ZirInst *options_value = astgen_node_extra(ag, options_node,
+                    scope, LValNone, &result_loc_cast->base);
+                if (options_value == ag->codegen->invalid_inst_src)
+                    return options_value;
+
+                Stage1ZirInst *casted_options_value = ir_build_implicit_cast(
+                    ag, scope, options_node, options_value, result_loc_cast);
+
+                Stage1ZirInst *ir_extern = ir_build_prefetch(ag, scope, node, ptr_value, casted_options_value);
+                return ir_lval_wrap(ag, scope, ir_extern, lval, result_loc);
+            }
     }
     zig_unreachable();
 }
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index cba820037e..44600ffa45 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -1139,6 +1139,24 @@ static LLVMValueRef gen_wasm_memory_grow(CodeGen *g) {
     return g->wasm_memory_grow;
 }
 
+static LLVMValueRef gen_prefetch(CodeGen *g) {
+    if (g->prefetch)
+        return g->prefetch;
+
+    // declare void @llvm.prefetch(i8*, i32, i32, i32)
+    LLVMTypeRef param_types[] = {
+        LLVMPointerType(LLVMInt8Type(), 0),
+        LLVMInt32Type(),
+        LLVMInt32Type(),
+        LLVMInt32Type(),
+    };
+    LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
+    g->prefetch = LLVMAddFunction(g->module, "llvm.prefetch.p0i8", fn_type);
+    assert(LLVMGetIntrinsicID(g->prefetch));
+
+    return g->prefetch;
+}
+
 static LLVMValueRef get_stacksave_fn_val(CodeGen *g) {
     if (g->stacksave_fn_val)
         return g->stacksave_fn_val;
@@ -5899,6 +5917,52 @@ static LLVMValueRef ir_render_wasm_memory_grow(CodeGen *g, Stage1Air *executable
     return val;
 }
 
+static LLVMValueRef ir_render_prefetch(CodeGen *g, Stage1Air *executable, Stage1AirInstPrefetch *instruction) {
+    static_assert(PrefetchRwRead == 0, "");
+    static_assert(PrefetchRwWrite == 1, "");
+    assert(instruction->rw == PrefetchRwRead || instruction->rw == PrefetchRwWrite);
+
+    assert(instruction->locality >= 0 && instruction->locality <= 3);
+
+    static_assert(PrefetchCacheInstruction == 0, "");
+    static_assert(PrefetchCacheData == 1, "");
+    assert(instruction->cache == PrefetchCacheData || instruction->cache == PrefetchCacheInstruction);
+    
+    // LLVM fails during codegen of instruction cache prefetchs for these architectures.
+    // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported by the target.
+    // To work around this, simply don't emit llvm.prefetch in this case.
+    // See https://bugs.llvm.org/show_bug.cgi?id=21037
+    if (instruction->cache == PrefetchCacheInstruction) {
+        switch (g->zig_target->arch) {
+            case ZigLLVM_x86:
+            case ZigLLVM_x86_64:
+                return nullptr;
+            default:
+                break;
+        }
+    }
+
+    // Another case of the same LLVM bug described above
+    if (instruction->rw == PrefetchRwWrite && instruction->cache == PrefetchCacheInstruction) {
+        switch (g->zig_target->arch) {
+            case ZigLLVM_arm:
+                return nullptr;
+            default:
+                break;
+        }
+
+    }
+
+    LLVMValueRef params[] = {
+        LLVMBuildBitCast(g->builder, ir_llvm_value(g, instruction->ptr), LLVMPointerType(LLVMInt8Type(), 0), ""),
+        LLVMConstInt(LLVMInt32Type(), instruction->rw, false),
+        LLVMConstInt(LLVMInt32Type(), instruction->locality, false),
+        LLVMConstInt(LLVMInt32Type(), instruction->cache, false),
+    };
+    LLVMValueRef val = LLVMBuildCall(g->builder, gen_prefetch(g), params, 4, "");
+    return val;
+}
+
 static LLVMValueRef ir_render_slice(CodeGen *g, Stage1Air *executable, Stage1AirInstSlice *instruction) {
     Error err;
 
@@ -7150,6 +7214,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, Stage1Air *executable, Sta
             return ir_render_wasm_memory_grow(g, executable, (Stage1AirInstWasmMemoryGrow *) instruction);
         case Stage1AirInstIdExtern:
             return ir_render_extern(g, executable, (Stage1AirInstExtern *) instruction);
+        case Stage1AirInstIdPrefetch:
+            return ir_render_prefetch(g, executable, (Stage1AirInstPrefetch *) instruction);
     }
     zig_unreachable();
 }
@@ -9120,6 +9186,7 @@ static void define_builtin_fns(CodeGen *g) {
     create_builtin_fn(g, BuiltinFnIdReduce, "reduce", 2);
     create_builtin_fn(g, BuiltinFnIdMaximum, "maximum", 2);
     create_builtin_fn(g, BuiltinFnIdMinimum, "minimum", 2);
+    create_builtin_fn(g, BuiltinFnIdPrefetch, "prefetch", 2);
 }
 
 static const char *bool_to_str(bool b) {
diff --git a/src/stage1/ir.cpp b/src/stage1/ir.cpp
index 34ff4a4f04..a78b8a3018 100644
--- a/src/stage1/ir.cpp
+++ b/src/stage1/ir.cpp
@@ -467,6 +467,8 @@ void destroy_instruction_gen(Stage1AirInst *inst) {
             return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstWasmMemoryGrow *>(inst));
         case Stage1AirInstIdExtern:
             return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstExtern *>(inst));
+        case Stage1AirInstIdPrefetch:
+            return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstPrefetch *>(inst));
     }
     zig_unreachable();
 }
@@ -1115,6 +1117,10 @@ static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstExtern *) {
     return Stage1AirInstIdExtern;
 }
 
+static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstPrefetch *) {
+    return Stage1AirInstIdPrefetch;
+}
+
 template<typename T>
 static T *ir_create_inst_gen(IrBuilderGen *irb, Scope *scope, AstNode *source_node) {
     T *special_instruction = heap::c_allocator.create<T>();
@@ -24853,6 +24859,52 @@ static Stage1AirInst *ir_analyze_instruction_src(IrAnalyze *ira, Stage1ZirInstSr
     return ir_const_move(ira, instruction->base.scope, instruction->base.source_node, result);
 }
 
+static Stage1AirInst *ir_analyze_instruction_prefetch(IrAnalyze *ira, Stage1ZirInstPrefetch *instruction) {
+    Stage1AirInst *ptr = instruction->ptr->child;
+    if (type_is_invalid(ptr->value->type))
+        return ira->codegen->invalid_inst_gen;
+
+    Stage1AirInst *raw_options_inst = instruction->options->child;
+    if (type_is_invalid(raw_options_inst->value->type))
+        return ira->codegen->invalid_inst_gen;
+
+    ZigType *options_type = get_builtin_type(ira->codegen, "PrefetchOptions");
+    Stage1AirInst *options_inst = ir_implicit_cast(ira, raw_options_inst, options_type);
+    if (type_is_invalid(options_inst->value->type))
+        return ira->codegen->invalid_inst_gen;
+
+    ZigValue *options_val = ir_resolve_const(ira, options_inst, UndefBad);
+    if (options_val == nullptr)
+        return ira->codegen->invalid_inst_gen;
+
+    ZigValue *rw_val = get_const_field(ira, options_inst->source_node, options_val, "rw", 0);
+    if (rw_val == nullptr)
+        return ira->codegen->invalid_inst_gen;
+    PrefetchRw rw = (PrefetchRw)bigint_as_u8(&rw_val->data.x_enum_tag);
+
+    ZigValue *locality_val = get_const_field(ira, options_inst->source_node, options_val, "locality", 1);
+    if (locality_val == nullptr)
+        return ira->codegen->invalid_inst_gen;
+    uint8_t locality = bigint_as_u8(&locality_val->data.x_bigint);
+    assert(locality <= 3);
+
+    ZigValue *cache_val = get_const_field(ira, options_inst->source_node, options_val, "cache", 2);
+    if (cache_val == nullptr)
+        return ira->codegen->invalid_inst_gen;
+    PrefetchCache cache = (PrefetchCache)bigint_as_u8(&cache_val->data.x_enum_tag);
+
+    Stage1AirInstPrefetch *air_instruction = ir_build_inst_void<Stage1AirInstPrefetch>(&ira->new_irb,
+            instruction->base.scope, instruction->base.source_node);
+    air_instruction->ptr = ptr;
+    air_instruction->rw = rw;
+    air_instruction->locality = locality;
+    air_instruction->cache = cache;
+
+    ir_ref_inst_gen(ptr);
+
+    return &air_instruction->base;
+}
+
 static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst *instruction) {
     switch (instruction->id) {
         case Stage1ZirInstIdInvalid:
@@ -25138,6 +25190,8 @@ static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst
             return ir_analyze_instruction_wasm_memory_grow(ira, (Stage1ZirInstWasmMemoryGrow *)instruction);
         case Stage1ZirInstIdSrc:
             return ir_analyze_instruction_src(ira, (Stage1ZirInstSrc *)instruction);
+        case Stage1ZirInstIdPrefetch:
+            return ir_analyze_instruction_prefetch(ira, (Stage1ZirInstPrefetch *)instruction);
     }
     zig_unreachable();
 }
@@ -25305,6 +25359,7 @@ bool ir_inst_gen_has_side_effects(Stage1AirInst *instruction) {
         case Stage1AirInstIdSpillBegin:
         case Stage1AirInstIdWasmMemoryGrow:
         case Stage1AirInstIdExtern:
+        case Stage1AirInstIdPrefetch:
             return true;
 
         case Stage1AirInstIdPhi:
@@ -25444,6 +25499,7 @@ bool ir_inst_src_has_side_effects(Stage1ZirInst *instruction) {
         case Stage1ZirInstIdAwait:
         case Stage1ZirInstIdSpillBegin:
         case Stage1ZirInstIdWasmMemoryGrow:
+        case Stage1ZirInstIdPrefetch:
             return true;
 
         case Stage1ZirInstIdPhi:
diff --git a/src/stage1/ir_print.cpp b/src/stage1/ir_print.cpp
index f92f146d84..5c7727da0c 100644
--- a/src/stage1/ir_print.cpp
+++ b/src/stage1/ir_print.cpp
@@ -371,6 +371,8 @@ const char* ir_inst_src_type_str(Stage1ZirInstId id) {
             return "SrcWasmMemoryGrow";
         case Stage1ZirInstIdSrc:
             return "SrcSrc";
+        case Stage1ZirInstIdPrefetch:
+            return "SrcPrefetch";
     }
     zig_unreachable();
 }
@@ -559,6 +561,8 @@ const char* ir_inst_gen_type_str(Stage1AirInstId id) {
             return "GenWasmMemoryGrow";
         case Stage1AirInstIdExtern:
             return "GenExtern";
+        case Stage1AirInstIdPrefetch:
+            return "GenPrefetch";
     }
     zig_unreachable();
 }
@@ -2436,6 +2440,18 @@ static void ir_print_extern(IrPrintSrc *irp, Stage1ZirInstExtern *instruction) {
     fprintf(irp->f, ")");
 }
 
+static void ir_print_prefetch(IrPrintSrc *irp, Stage1ZirInstPrefetch *instruction) {
+    fprintf(irp->f, "@prefetch(");
+    ir_print_other_inst_src(irp, instruction->ptr);
+    fprintf(irp->f, ",");
+    ir_print_other_inst_src(irp, instruction->options);
+    fprintf(irp->f, ")");
+}
+
+static void ir_print_prefetch(IrPrintGen *irp, Stage1AirInstPrefetch *instruction) {
+    fprintf(irp->f, "@prefetch(...)");
+}
+
 static void ir_print_error_return_trace(IrPrintSrc *irp, Stage1ZirInstErrorReturnTrace *instruction) {
     fprintf(irp->f, "@errorReturnTrace(");
     switch (instruction->optional) {
@@ -3108,6 +3124,9 @@ static void ir_print_inst_src(IrPrintSrc *irp, Stage1ZirInst *instruction, bool
         case Stage1ZirInstIdSrc:
             ir_print_builtin_src(irp, (Stage1ZirInstSrc *)instruction);
             break;
+        case Stage1ZirInstIdPrefetch:
+            ir_print_prefetch(irp, (Stage1ZirInstPrefetch *)instruction);
+            break;
     }
     fprintf(irp->f, "\n");
 }
@@ -3387,6 +3406,9 @@ static void ir_print_inst_gen(IrPrintGen *irp, Stage1AirInst *instruction, bool
         case Stage1AirInstIdExtern:
             ir_print_extern(irp, (Stage1AirInstExtern *)instruction);
             break;
+        case Stage1AirInstIdPrefetch:
+            ir_print_prefetch(irp, (Stage1AirInstPrefetch *)instruction);
+            break;
 
     }
     fprintf(irp->f, "\n");