Merge pull request #19102 from ziglang/decouple-zir

JIT `zig fmt` and `zig reduce`

1 files changed, 595 insertions, 6 deletions

diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index e0787e7312..1437fee2e6 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -1,17 +1,14 @@
-pub const fmt = @import("zig/fmt.zig");
-
 pub const ErrorBundle = @import("zig/ErrorBundle.zig");
 pub const Server = @import("zig/Server.zig");
 pub const Client = @import("zig/Client.zig");
 pub const Token = tokenizer.Token;
 pub const Tokenizer = tokenizer.Tokenizer;
-pub const fmtId = fmt.fmtId;
-pub const fmtEscapes = fmt.fmtEscapes;
-pub const isValidId = fmt.isValidId;
 pub const string_literal = @import("zig/string_literal.zig");
 pub const number_literal = @import("zig/number_literal.zig");
 pub const primitives = @import("zig/primitives.zig");
 pub const Ast = @import("zig/Ast.zig");
+pub const AstGen = @import("zig/AstGen.zig");
+pub const Zir = @import("zig/Zir.zig");
 pub const system = @import("zig/system.zig");
 /// Deprecated: use `std.Target.Query`.
 pub const CrossTarget = std.Target.Query;
@@ -30,6 +27,36 @@ pub const c_translation = @import("zig/c_translation.zig");
 pub const SrcHasher = std.crypto.hash.Blake3;
 pub const SrcHash = [16]u8;
 
+pub const Color = enum {
+    /// Determine whether stderr is a terminal or not automatically.
+    auto,
+    /// Assume stderr is not a terminal.
+    off,
+    /// Assume stderr is a terminal.
+    on,
+
+    pub fn get_tty_conf(color: Color) std.io.tty.Config {
+        return switch (color) {
+            .auto => std.io.tty.detectConfig(std.io.getStdErr()),
+            .on => .escape_codes,
+            .off => .no_color,
+        };
+    }
+
+    pub fn renderOptions(color: Color) std.zig.ErrorBundle.RenderOptions {
+        const ttyconf = get_tty_conf(color);
+        return .{
+            .ttyconf = ttyconf,
+            .include_source_line = ttyconf != .no_color,
+            .include_reference_trace = ttyconf != .no_color,
+        };
+    }
+};
+
+/// There are many assumptions in the entire codebase that Zig source files can
+/// be byte-indexed with a u32 integer.
+pub const max_src_size = std.math.maxInt(u32);
+
 pub fn hashSrc(src: []const u8) SrcHash {
     var out: SrcHash = undefined;
     SrcHasher.hash(src, &out, .{});
@@ -315,11 +342,573 @@ pub fn serializeCpuAlloc(ally: Allocator, cpu: std.Target.Cpu) Allocator.Error![
     return buffer.toOwnedSlice();
 }
 
+pub const DeclIndex = enum(u32) {
+    _,
+
+    pub fn toOptional(i: DeclIndex) OptionalDeclIndex {
+        return @enumFromInt(@intFromEnum(i));
+    }
+};
+
+pub const OptionalDeclIndex = enum(u32) {
+    none = std.math.maxInt(u32),
+    _,
+
+    pub fn init(oi: ?DeclIndex) OptionalDeclIndex {
+        return @enumFromInt(@intFromEnum(oi orelse return .none));
+    }
+
+    pub fn unwrap(oi: OptionalDeclIndex) ?DeclIndex {
+        if (oi == .none) return null;
+        return @enumFromInt(@intFromEnum(oi));
+    }
+};
+
+/// Resolving a source location into a byte offset may require doing work
+/// that we would rather not do unless the error actually occurs.
+/// Therefore we need a data structure that contains the information necessary
+/// to lazily produce a `SrcLoc` as required.
+/// Most of the offsets in this data structure are relative to the containing Decl.
+/// This makes the source location resolve properly even when a Decl gets
+/// shifted up or down in the file, as long as the Decl's contents itself
+/// do not change.
+pub const LazySrcLoc = union(enum) {
+    /// When this tag is set, the code that constructed this `LazySrcLoc` is asserting
+    /// that all code paths which would need to resolve the source location are
+    /// unreachable. If you are debugging this tag incorrectly being this value,
+    /// look into using reverse-continue with a memory watchpoint to see where the
+    /// value is being set to this tag.
+    unneeded,
+    /// Means the source location points to an entire file; not any particular
+    /// location within the file. `file_scope` union field will be active.
+    entire_file,
+    /// The source location points to a byte offset within a source file,
+    /// offset from 0. The source file is determined contextually.
+    /// Inside a `SrcLoc`, the `file_scope` union field will be active.
+    byte_abs: u32,
+    /// The source location points to a token within a source file,
+    /// offset from 0. The source file is determined contextually.
+    /// Inside a `SrcLoc`, the `file_scope` union field will be active.
+    token_abs: u32,
+    /// The source location points to an AST node within a source file,
+    /// offset from 0. The source file is determined contextually.
+    /// Inside a `SrcLoc`, the `file_scope` union field will be active.
+    node_abs: u32,
+    /// The source location points to a byte offset within a source file,
+    /// offset from the byte offset of the Decl within the file.
+    /// The Decl is determined contextually.
+    byte_offset: u32,
+    /// This data is the offset into the token list from the Decl token.
+    /// The Decl is determined contextually.
+    token_offset: u32,
+    /// The source location points to an AST node, which is this value offset
+    /// from its containing Decl node AST index.
+    /// The Decl is determined contextually.
+    node_offset: TracedOffset,
+    /// The source location points to the main token of an AST node, found
+    /// by taking this AST node index offset from the containing Decl AST node.
+    /// The Decl is determined contextually.
+    node_offset_main_token: i32,
+    /// The source location points to the beginning of a struct initializer.
+    /// The Decl is determined contextually.
+    node_offset_initializer: i32,
+    /// The source location points to a variable declaration type expression,
+    /// found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a variable declaration AST node. Next, navigate
+    /// to the type expression.
+    /// The Decl is determined contextually.
+    node_offset_var_decl_ty: i32,
+    /// The source location points to the alignment expression of a var decl.
+    /// The Decl is determined contextually.
+    node_offset_var_decl_align: i32,
+    /// The source location points to the linksection expression of a var decl.
+    /// The Decl is determined contextually.
+    node_offset_var_decl_section: i32,
+    /// The source location points to the addrspace expression of a var decl.
+    /// The Decl is determined contextually.
+    node_offset_var_decl_addrspace: i32,
+    /// The source location points to the initializer of a var decl.
+    /// The Decl is determined contextually.
+    node_offset_var_decl_init: i32,
+    /// The source location points to the first parameter of a builtin
+    /// function call, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a builtin call AST node. Next, navigate
+    /// to the first parameter.
+    /// The Decl is determined contextually.
+    node_offset_builtin_call_arg0: i32,
+    /// Same as `node_offset_builtin_call_arg0` except arg index 1.
+    node_offset_builtin_call_arg1: i32,
+    node_offset_builtin_call_arg2: i32,
+    node_offset_builtin_call_arg3: i32,
+    node_offset_builtin_call_arg4: i32,
+    node_offset_builtin_call_arg5: i32,
+    /// Like `node_offset_builtin_call_arg0` but recurses through arbitrarily many calls
+    /// to pointer cast builtins.
+    node_offset_ptrcast_operand: i32,
+    /// The source location points to the index expression of an array access
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to an array access AST node. Next, navigate
+    /// to the index expression.
+    /// The Decl is determined contextually.
+    node_offset_array_access_index: i32,
+    /// The source location points to the LHS of a slice expression
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a slice AST node. Next, navigate
+    /// to the sentinel expression.
+    /// The Decl is determined contextually.
+    node_offset_slice_ptr: i32,
+    /// The source location points to start expression of a slice expression
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a slice AST node. Next, navigate
+    /// to the sentinel expression.
+    /// The Decl is determined contextually.
+    node_offset_slice_start: i32,
+    /// The source location points to the end expression of a slice
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a slice AST node. Next, navigate
+    /// to the sentinel expression.
+    /// The Decl is determined contextually.
+    node_offset_slice_end: i32,
+    /// The source location points to the sentinel expression of a slice
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a slice AST node. Next, navigate
+    /// to the sentinel expression.
+    /// The Decl is determined contextually.
+    node_offset_slice_sentinel: i32,
+    /// The source location points to the callee expression of a function
+    /// call expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function call AST node. Next, navigate
+    /// to the callee expression.
+    /// The Decl is determined contextually.
+    node_offset_call_func: i32,
+    /// The payload is offset from the containing Decl AST node.
+    /// The source location points to the field name of:
+    ///  * a field access expression (`a.b`), or
+    ///  * the callee of a method call (`a.b()`)
+    /// The Decl is determined contextually.
+    node_offset_field_name: i32,
+    /// The payload is offset from the containing Decl AST node.
+    /// The source location points to the field name of the operand ("b" node)
+    /// of a field initialization expression (`.a = b`)
+    /// The Decl is determined contextually.
+    node_offset_field_name_init: i32,
+    /// The source location points to the pointer of a pointer deref expression,
+    /// found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a pointer deref AST node. Next, navigate
+    /// to the pointer expression.
+    /// The Decl is determined contextually.
+    node_offset_deref_ptr: i32,
+    /// The source location points to the assembly source code of an inline assembly
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to inline assembly AST node. Next, navigate
+    /// to the asm template source code.
+    /// The Decl is determined contextually.
+    node_offset_asm_source: i32,
+    /// The source location points to the return type of an inline assembly
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to inline assembly AST node. Next, navigate
+    /// to the return type expression.
+    /// The Decl is determined contextually.
+    node_offset_asm_ret_ty: i32,
+    /// The source location points to the condition expression of an if
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to an if expression AST node. Next, navigate
+    /// to the condition expression.
+    /// The Decl is determined contextually.
+    node_offset_if_cond: i32,
+    /// The source location points to a binary expression, such as `a + b`, found
+    /// by taking this AST node index offset from the containing Decl AST node.
+    /// The Decl is determined contextually.
+    node_offset_bin_op: i32,
+    /// The source location points to the LHS of a binary expression, found
+    /// by taking this AST node index offset from the containing Decl AST node,
+    /// which points to a binary expression AST node. Next, navigate to the LHS.
+    /// The Decl is determined contextually.
+    node_offset_bin_lhs: i32,
+    /// The source location points to the RHS of a binary expression, found
+    /// by taking this AST node index offset from the containing Decl AST node,
+    /// which points to a binary expression AST node. Next, navigate to the RHS.
+    /// The Decl is determined contextually.
+    node_offset_bin_rhs: i32,
+    /// The source location points to the operand of a switch expression, found
+    /// by taking this AST node index offset from the containing Decl AST node,
+    /// which points to a switch expression AST node. Next, navigate to the operand.
+    /// The Decl is determined contextually.
+    node_offset_switch_operand: i32,
+    /// The source location points to the else/`_` prong of a switch expression, found
+    /// by taking this AST node index offset from the containing Decl AST node,
+    /// which points to a switch expression AST node. Next, navigate to the else/`_` prong.
+    /// The Decl is determined contextually.
+    node_offset_switch_special_prong: i32,
+    /// The source location points to all the ranges of a switch expression, found
+    /// by taking this AST node index offset from the containing Decl AST node,
+    /// which points to a switch expression AST node. Next, navigate to any of the
+    /// range nodes. The error applies to all of them.
+    /// The Decl is determined contextually.
+    node_offset_switch_range: i32,
+    /// The source location points to the capture of a switch_prong.
+    /// The Decl is determined contextually.
+    node_offset_switch_prong_capture: i32,
+    /// The source location points to the tag capture of a switch_prong.
+    /// The Decl is determined contextually.
+    node_offset_switch_prong_tag_capture: i32,
+    /// The source location points to the align expr of a function type
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function type AST node. Next, navigate to
+    /// the calling convention node.
+    /// The Decl is determined contextually.
+    node_offset_fn_type_align: i32,
+    /// The source location points to the addrspace expr of a function type
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function type AST node. Next, navigate to
+    /// the calling convention node.
+    /// The Decl is determined contextually.
+    node_offset_fn_type_addrspace: i32,
+    /// The source location points to the linksection expr of a function type
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function type AST node. Next, navigate to
+    /// the calling convention node.
+    /// The Decl is determined contextually.
+    node_offset_fn_type_section: i32,
+    /// The source location points to the calling convention of a function type
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function type AST node. Next, navigate to
+    /// the calling convention node.
+    /// The Decl is determined contextually.
+    node_offset_fn_type_cc: i32,
+    /// The source location points to the return type of a function type
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function type AST node. Next, navigate to
+    /// the return type node.
+    /// The Decl is determined contextually.
+    node_offset_fn_type_ret_ty: i32,
+    node_offset_param: i32,
+    token_offset_param: i32,
+    /// The source location points to the type expression of an `anyframe->T`
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a `anyframe->T` expression AST node. Next, navigate
+    /// to the type expression.
+    /// The Decl is determined contextually.
+    node_offset_anyframe_type: i32,
+    /// The source location points to the string literal of `extern "foo"`, found
+    /// by taking this AST node index offset from the containing
+    /// Decl AST node, which points to a function prototype or variable declaration
+    /// expression AST node. Next, navigate to the string literal of the `extern "foo"`.
+    /// The Decl is determined contextually.
+    node_offset_lib_name: i32,
+    /// The source location points to the len expression of an `[N:S]T`
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to an `[N:S]T` expression AST node. Next, navigate
+    /// to the len expression.
+    /// The Decl is determined contextually.
+    node_offset_array_type_len: i32,
+    /// The source location points to the sentinel expression of an `[N:S]T`
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to an `[N:S]T` expression AST node. Next, navigate
+    /// to the sentinel expression.
+    /// The Decl is determined contextually.
+    node_offset_array_type_sentinel: i32,
+    /// The source location points to the elem expression of an `[N:S]T`
+    /// expression, found by taking this AST node index offset from the containing
+    /// Decl AST node, which points to an `[N:S]T` expression AST node. Next, navigate
+    /// to the elem expression.
+    /// The Decl is determined contextually.
+    node_offset_array_type_elem: i32,
+    /// The source location points to the operand of an unary expression.
+    /// The Decl is determined contextually.
+    node_offset_un_op: i32,
+    /// The source location points to the elem type of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_elem: i32,
+    /// The source location points to the sentinel of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_sentinel: i32,
+    /// The source location points to the align expr of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_align: i32,
+    /// The source location points to the addrspace expr of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_addrspace: i32,
+    /// The source location points to the bit-offset of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_bitoffset: i32,
+    /// The source location points to the host size of a pointer.
+    /// The Decl is determined contextually.
+    node_offset_ptr_hostsize: i32,
+    /// The source location points to the tag type of an union or an enum.
+    /// The Decl is determined contextually.
+    node_offset_container_tag: i32,
+    /// The source location points to the default value of a field.
+    /// The Decl is determined contextually.
+    node_offset_field_default: i32,
+    /// The source location points to the type of an array or struct initializer.
+    /// The Decl is determined contextually.
+    node_offset_init_ty: i32,
+    /// The source location points to the LHS of an assignment.
+    /// The Decl is determined contextually.
+    node_offset_store_ptr: i32,
+    /// The source location points to the RHS of an assignment.
+    /// The Decl is determined contextually.
+    node_offset_store_operand: i32,
+    /// The source location points to the operand of a `return` statement, or
+    /// the `return` itself if there is no explicit operand.
+    /// The Decl is determined contextually.
+    node_offset_return_operand: i32,
+    /// The source location points to a for loop input.
+    /// The Decl is determined contextually.
+    for_input: struct {
+        /// Points to the for loop AST node.
+        for_node_offset: i32,
+        /// Picks one of the inputs from the condition.
+        input_index: u32,
+    },
+    /// The source location points to one of the captures of a for loop, found
+    /// by taking this AST node index offset from the containing
+    /// Decl AST node, which points to one of the input nodes of a for loop.
+    /// Next, navigate to the corresponding capture.
+    /// The Decl is determined contextually.
+    for_capture_from_input: i32,
+    /// The source location points to the argument node of a function call.
+    call_arg: struct {
+        decl: DeclIndex,
+        /// Points to the function call AST node.
+        call_node_offset: i32,
+        /// The index of the argument the source location points to.
+        arg_index: u32,
+    },
+    fn_proto_param: struct {
+        decl: DeclIndex,
+        /// Points to the function prototype AST node.
+        fn_proto_node_offset: i32,
+        /// The index of the parameter the source location points to.
+        param_index: u32,
+    },
+    array_cat_lhs: ArrayCat,
+    array_cat_rhs: ArrayCat,
+
+    const ArrayCat = struct {
+        /// Points to the array concat AST node.
+        array_cat_offset: i32,
+        /// The index of the element the source location points to.
+        elem_index: u32,
+    };
+
+    pub const nodeOffset = if (TracedOffset.want_tracing) nodeOffsetDebug else nodeOffsetRelease;
+
+    noinline fn nodeOffsetDebug(node_offset: i32) LazySrcLoc {
+        var result: LazySrcLoc = .{ .node_offset = .{ .x = node_offset } };
+        result.node_offset.trace.addAddr(@returnAddress(), "init");
+        return result;
+    }
+
+    fn nodeOffsetRelease(node_offset: i32) LazySrcLoc {
+        return .{ .node_offset = .{ .x = node_offset } };
+    }
+
+    /// This wraps a simple integer in debug builds so that later on we can find out
+    /// where in semantic analysis the value got set.
+    pub const TracedOffset = struct {
+        x: i32,
+        trace: std.debug.Trace = .{},
+
+        const want_tracing = false;
+    };
+};
+
 const std = @import("std.zig");
 const tokenizer = @import("zig/tokenizer.zig");
 const assert = std.debug.assert;
 const Allocator = std.mem.Allocator;
 
+/// Return a Formatter for a Zig identifier
+pub fn fmtId(bytes: []const u8) std.fmt.Formatter(formatId) {
+    return .{ .data = bytes };
+}
+
+/// Print the string as a Zig identifier escaping it with @"" syntax if needed.
+fn formatId(
+    bytes: []const u8,
+    comptime unused_format_string: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    _ = unused_format_string;
+    if (isValidId(bytes)) {
+        return writer.writeAll(bytes);
+    }
+    try writer.writeAll("@\"");
+    try stringEscape(bytes, "", options, writer);
+    try writer.writeByte('"');
+}
+
+/// Return a Formatter for Zig Escapes of a double quoted string.
+/// The format specifier must be one of:
+///  * `{}` treats contents as a double-quoted string.
+///  * `{'}` treats contents as a single-quoted string.
+pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(stringEscape) {
+    return .{ .data = bytes };
+}
+
+test "escape invalid identifiers" {
+    const expectFmt = std.testing.expectFmt;
+    try expectFmt("@\"while\"", "{}", .{fmtId("while")});
+    try expectFmt("hello", "{}", .{fmtId("hello")});
+    try expectFmt("@\"11\\\"23\"", "{}", .{fmtId("11\"23")});
+    try expectFmt("@\"11\\x0f23\"", "{}", .{fmtId("11\x0F23")});
+    try expectFmt("\\x0f", "{}", .{fmtEscapes("\x0f")});
+    try expectFmt(
+        \\" \\ hi \x07 \x11 " derp \'"
+    , "\"{'}\"", .{fmtEscapes(" \\ hi \x07 \x11 \" derp '")});
+    try expectFmt(
+        \\" \\ hi \x07 \x11 \" derp '"
+    , "\"{}\"", .{fmtEscapes(" \\ hi \x07 \x11 \" derp '")});
+}
+
+/// Print the string as escaped contents of a double quoted or single-quoted string.
+/// Format `{}` treats contents as a double-quoted string.
+/// Format `{'}` treats contents as a single-quoted string.
+pub fn stringEscape(
+    bytes: []const u8,
+    comptime f: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    _ = options;
+    for (bytes) |byte| switch (byte) {
+        '\n' => try writer.writeAll("\\n"),
+        '\r' => try writer.writeAll("\\r"),
+        '\t' => try writer.writeAll("\\t"),
+        '\\' => try writer.writeAll("\\\\"),
+        '"' => {
+            if (f.len == 1 and f[0] == '\'') {
+                try writer.writeByte('"');
+            } else if (f.len == 0) {
+                try writer.writeAll("\\\"");
+            } else {
+                @compileError("expected {} or {'}, found {" ++ f ++ "}");
+            }
+        },
+        '\'' => {
+            if (f.len == 1 and f[0] == '\'') {
+                try writer.writeAll("\\'");
+            } else if (f.len == 0) {
+                try writer.writeByte('\'');
+            } else {
+                @compileError("expected {} or {'}, found {" ++ f ++ "}");
+            }
+        },
+        ' ', '!', '#'...'&', '('...'[', ']'...'~' => try writer.writeByte(byte),
+        // Use hex escapes for rest any unprintable characters.
+        else => {
+            try writer.writeAll("\\x");
+            try std.fmt.formatInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }, writer);
+        },
+    };
+}
+
+pub fn isValidId(bytes: []const u8) bool {
+    if (bytes.len == 0) return false;
+    if (std.mem.eql(u8, bytes, "_")) return false;
+    for (bytes, 0..) |c, i| {
+        switch (c) {
+            '_', 'a'...'z', 'A'...'Z' => {},
+            '0'...'9' => if (i == 0) return false,
+            else => return false,
+        }
+    }
+    return std.zig.Token.getKeyword(bytes) == null;
+}
+
+test isValidId {
+    try std.testing.expect(!isValidId(""));
+    try std.testing.expect(isValidId("foobar"));
+    try std.testing.expect(!isValidId("a b c"));
+    try std.testing.expect(!isValidId("3d"));
+    try std.testing.expect(!isValidId("enum"));
+    try std.testing.expect(isValidId("i386"));
+}
+
+pub fn readSourceFileToEndAlloc(
+    allocator: Allocator,
+    input: std.fs.File,
+    size_hint: ?usize,
+) ![:0]u8 {
+    const source_code = input.readToEndAllocOptions(
+        allocator,
+        max_src_size,
+        size_hint,
+        @alignOf(u16),
+        0,
+    ) catch |err| switch (err) {
+        error.ConnectionResetByPeer => unreachable,
+        error.ConnectionTimedOut => unreachable,
+        error.NotOpenForReading => unreachable,
+        else => |e| return e,
+    };
+    errdefer allocator.free(source_code);
+
+    // Detect unsupported file types with their Byte Order Mark
+    const unsupported_boms = [_][]const u8{
+        "\xff\xfe\x00\x00", // UTF-32 little endian
+        "\xfe\xff\x00\x00", // UTF-32 big endian
+        "\xfe\xff", // UTF-16 big endian
+    };
+    for (unsupported_boms) |bom| {
+        if (std.mem.startsWith(u8, source_code, bom)) {
+            return error.UnsupportedEncoding;
+        }
+    }
+
+    // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
+    if (std.mem.startsWith(u8, source_code, "\xff\xfe")) {
+        const source_code_utf16_le = std.mem.bytesAsSlice(u16, source_code);
+        const source_code_utf8 = std.unicode.utf16LeToUtf8AllocZ(allocator, source_code_utf16_le) catch |err| switch (err) {
+            error.DanglingSurrogateHalf => error.UnsupportedEncoding,
+            error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
+            error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,
+            else => |e| return e,
+        };
+
+        allocator.free(source_code);
+        return source_code_utf8;
+    }
+
+    return source_code;
+}
+
+pub fn printAstErrorsToStderr(gpa: Allocator, tree: Ast, path: []const u8, color: Color) !void {
+    var wip_errors: std.zig.ErrorBundle.Wip = undefined;
+    try wip_errors.init(gpa);
+    defer wip_errors.deinit();
+
+    try putAstErrorsIntoBundle(gpa, tree, path, &wip_errors);
+
+    var error_bundle = try wip_errors.toOwnedBundle("");
+    defer error_bundle.deinit(gpa);
+    error_bundle.renderToStdErr(color.renderOptions());
+}
+
+pub fn putAstErrorsIntoBundle(
+    gpa: Allocator,
+    tree: Ast,
+    path: []const u8,
+    wip_errors: *std.zig.ErrorBundle.Wip,
+) Allocator.Error!void {
+    var zir = try AstGen.generate(gpa, tree);
+    defer zir.deinit(gpa);
+
+    try wip_errors.addZirErrorMessages(zir, tree, tree.source, path);
+}
+
 test {
-    @import("std").testing.refAllDecls(@This());
+    _ = Ast;
+    _ = AstRlAnnotate;
+    _ = BuiltinFn;
+    _ = Client;
+    _ = ErrorBundle;
+    _ = Server;
+    _ = number_literal;
+    _ = primitives;
+    _ = string_literal;
+    _ = system;
 }