From 52de2802c457140f3d9923cf014b51bb8c16689f Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Sat, 2 Mar 2024 00:50:53 -0800 Subject: Lazily compile the `zig rc` subcommand and use it during `zig build-exe` This moves .rc/.manifest compilation out of the main Zig binary, contributing towards #19063 Also: - Make resinator use Aro as its preprocessor instead of clang - Sync resinator with upstream --- src/Compilation.zig | 566 +++++++++++----------------------------------------- 1 file changed, 121 insertions(+), 445 deletions(-) (limited to 'src/Compilation.zig') diff --git a/src/Compilation.zig b/src/Compilation.zig index 13fb672042..46fd25735d 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -36,7 +36,6 @@ const Cache = std.Build.Cache; const c_codegen = @import("codegen/c.zig"); const libtsan = @import("libtsan.zig"); const Zir = std.zig.Zir; -const resinator = @import("resinator.zig"); const Builtin = @import("Builtin.zig"); const LlvmObject = @import("codegen/llvm.zig").Object; @@ -174,7 +173,7 @@ local_cache_directory: Directory, global_cache_directory: Directory, libc_include_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, -rc_include_dir_list: []const []const u8, +rc_includes: RcIncludes, thread_pool: *ThreadPool, /// Populated when we build the libc++ static library. A Job to build this is placed in the queue @@ -1243,68 +1242,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil options.libc_installation, ); - // The include directories used when preprocessing .rc files are separate from the - // target. Which include directories are used is determined by `options.rc_includes`. - // - // Note: It should be okay that the include directories used when compiling .rc - // files differ from the include directories used when compiling the main - // binary, since the .res format is not dependent on anything ABI-related. The - // only relevant differences would be things like `#define` constants being - // different in the MinGW headers vs the MSVC headers, but any such - // differences would likely be a MinGW bug. - const rc_dirs: std.zig.LibCDirs = b: { - // Set the includes to .none here when there are no rc files to compile - var includes = if (options.rc_source_files.len > 0) options.rc_includes else .none; - const target = options.root_mod.resolved_target.result; - if (!options.root_mod.resolved_target.is_native_os or target.os.tag != .windows) { - switch (includes) { - // MSVC can't be found when the host isn't Windows, so short-circuit. - .msvc => return error.WindowsSdkNotFound, - // Skip straight to gnu since we won't be able to detect - // MSVC on non-Windows hosts. - .any => includes = .gnu, - .none, .gnu => {}, - } - } - while (true) switch (includes) { - .any, .msvc => break :b std.zig.LibCDirs.detect( - arena, - options.zig_lib_directory.path.?, - .{ - .cpu = target.cpu, - .os = target.os, - .abi = .msvc, - .ofmt = target.ofmt, - }, - options.root_mod.resolved_target.is_native_abi, - // The .rc preprocessor will need to know the libc include dirs even if we - // are not linking libc, so force 'link_libc' to true - true, - options.libc_installation, - ) catch |err| { - if (includes == .any) { - // fall back to mingw - includes = .gnu; - continue; - } - return err; - }, - .gnu => break :b try std.zig.LibCDirs.detectFromBuilding(arena, options.zig_lib_directory.path.?, .{ - .cpu = target.cpu, - .os = target.os, - .abi = .gnu, - .ofmt = target.ofmt, - }), - .none => break :b .{ - .libc_include_dir_list = &[0][]u8{}, - .libc_installation = null, - .libc_framework_dir_list = &.{}, - .sysroot = null, - .darwin_sdk_layout = null, - }, - }; - }; - const sysroot = options.sysroot orelse libc_dirs.sysroot; const include_compiler_rt = options.want_compiler_rt orelse @@ -1492,7 +1429,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .self_exe_path = options.self_exe_path, .libc_include_dir_list = libc_dirs.libc_include_dir_list, .libc_framework_dir_list = libc_dirs.libc_framework_dir_list, - .rc_include_dir_list = rc_dirs.libc_include_dir_list, + .rc_includes = options.rc_includes, .thread_pool = options.thread_pool, .clang_passthrough_mode = options.clang_passthrough_mode, .clang_preprocessor_mode = options.clang_preprocessor_mode, @@ -2506,7 +2443,7 @@ fn addNonIncrementalStuffToCacheManifest( man.hash.add(comp.link_eh_frame_hdr); man.hash.add(comp.skip_linker_dependencies); man.hash.add(comp.include_compiler_rt); - man.hash.addListOfBytes(comp.rc_include_dir_list); + man.hash.add(comp.rc_includes); man.hash.addListOfBytes(comp.force_undefined_symbols.keys()); man.hash.addListOfBytes(comp.framework_dirs); try link.hashAddSystemLibs(man, comp.system_libs); @@ -4172,7 +4109,7 @@ pub fn obtainCObjectCacheManifest( pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest { var man = comp.cache_parent.obtain(); - man.hash.addListOfBytes(comp.rc_include_dir_list); + man.hash.add(comp.rc_includes); return man; } @@ -4812,11 +4749,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P } fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32_resource_prog_node: *std.Progress.Node) !void { - if (!build_options.have_llvm) { - return comp.failWin32Resource(win32_resource, "clang not available: compiler built without LLVM extensions", .{}); + if (!std.process.can_spawn) { + return comp.failWin32Resource(win32_resource, "{s} does not support spawning a child process", .{@tagName(builtin.os.tag)}); } + const self_exe_path = comp.self_exe_path orelse - return comp.failWin32Resource(win32_resource, "clang compilation disabled", .{}); + return comp.failWin32Resource(win32_resource, "unable to find self exe path", .{}); const tracy_trace = trace(@src()); defer tracy_trace.end(); @@ -4856,6 +4794,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 if (win32_resource.src == .manifest) { _ = try man.addFile(src_path, null); + const rc_basename = try std.fmt.allocPrint(arena, "{s}.rc", .{src_basename}); const res_basename = try std.fmt.allocPrint(arena, "{s}.res", .{src_basename}); const digest = if (try man.hit()) man.final() else blk: { @@ -4867,17 +4806,12 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); defer o_dir.close(); - var output_file = o_dir.createFile(res_basename, .{}) catch |err| { - const output_file_path = try comp.local_cache_directory.join(arena, &.{ o_sub_path, res_basename }); - return comp.failWin32Resource(win32_resource, "failed to create output file '{s}': {s}", .{ output_file_path, @errorName(err) }); - }; - var output_file_closed = false; - defer if (!output_file_closed) output_file.close(); - - var diagnostics = resinator.errors.Diagnostics.init(arena); - defer diagnostics.deinit(); - - var output_buffered_stream = std.io.bufferedWriter(output_file.writer()); + const in_rc_path = try comp.local_cache_directory.join(comp.gpa, &.{ + o_sub_path, rc_basename, + }); + const out_res_path = try comp.local_cache_directory.join(comp.gpa, &.{ + o_sub_path, res_basename, + }); // In .rc files, a " within a quoted string is escaped as "" const fmtRcEscape = struct { @@ -4899,28 +4833,47 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // 1 is CREATEPROCESS_MANIFEST_RESOURCE_ID which is the default ID used for RT_MANIFEST resources // 24 is RT_MANIFEST const input = try std.fmt.allocPrint(arena, "1 24 \"{s}\"", .{fmtRcEscape(src_path)}); + try o_dir.writeFile(rc_basename, input); + + var argv = std.ArrayList([]const u8).init(comp.gpa); + defer argv.deinit(); + + try argv.appendSlice(&.{ + self_exe_path, + "rc", + "/:no-preprocess", + "/x", // ignore INCLUDE environment variable + "/c65001", // UTF-8 codepage + "/:auto-includes", + "none", + }); + try argv.appendSlice(&.{ "--", in_rc_path, out_res_path }); - resinator.compile.compile(arena, input, output_buffered_stream.writer(), .{ - .cwd = std.fs.cwd(), - .diagnostics = &diagnostics, - .ignore_include_env_var = true, - .default_code_page = .utf8, - }) catch |err| switch (err) { - error.ParseError, error.CompileError => { - // Delete the output file on error - output_file.close(); - output_file_closed = true; - // Failing to delete is not really a big deal, so swallow any errors - o_dir.deleteFile(res_basename) catch { - const output_file_path = try comp.local_cache_directory.join(arena, &.{ o_sub_path, res_basename }); - log.warn("failed to delete '{s}': {s}", .{ output_file_path, @errorName(err) }); - }; - return comp.failWin32ResourceCompile(win32_resource, input, &diagnostics, null); - }, - else => |e| return e, + var child = std.ChildProcess.init(argv.items, arena); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stderr_reader = child.stderr.?.reader(); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + const term = child.wait() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); }; - try output_buffered_stream.flush(); + switch (term) { + .Exited => |code| { + if (code != 0) { + log.err("zig rc failed with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); + } + }, + else => { + log.err("zig rc terminated with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); + }, + } break :blk digest; }; @@ -4951,9 +4904,6 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 const rc_basename_noext = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const digest = if (try man.hit()) man.final() else blk: { - const rcpp_filename = try std.fmt.allocPrint(arena, "{s}.rcpp", .{rc_basename_noext}); - - const out_rcpp_path = try comp.tmpFilePath(arena, rcpp_filename); var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath("tmp", .{}); defer zig_cache_tmp_dir.close(); @@ -4963,193 +4913,89 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 // so we need a temporary filename. const out_res_path = try comp.tmpFilePath(arena, res_filename); - var options = options: { - var resinator_args = try std.ArrayListUnmanaged([]const u8).initCapacity(comp.gpa, rc_src.extra_flags.len + 4); - defer resinator_args.deinit(comp.gpa); - - resinator_args.appendAssumeCapacity(""); // dummy 'process name' arg - resinator_args.appendSliceAssumeCapacity(rc_src.extra_flags); - resinator_args.appendSliceAssumeCapacity(&.{ "--", out_rcpp_path, out_res_path }); - - var cli_diagnostics = resinator.cli.Diagnostics.init(comp.gpa); - defer cli_diagnostics.deinit(); - const options = resinator.cli.parse(comp.gpa, resinator_args.items, &cli_diagnostics) catch |err| switch (err) { - error.ParseError => { - return comp.failWin32ResourceCli(win32_resource, &cli_diagnostics); - }, - else => |e| return e, - }; - break :options options; - }; - defer options.deinit(); - - // We never want to read the INCLUDE environment variable, so - // unconditionally set `ignore_include_env_var` to true - options.ignore_include_env_var = true; - - if (options.preprocess != .yes) { - return comp.failWin32Resource(win32_resource, "the '{s}' option is not supported in this context", .{switch (options.preprocess) { - .no => "/:no-preprocess", - .only => "/p", - .yes => unreachable, - }}); - } - var argv = std.ArrayList([]const u8).init(comp.gpa); defer argv.deinit(); - try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang" }); - - try resinator.preprocess.appendClangArgs(arena, &argv, options, .{ - .clang_target = null, // handled by addCCArgs - .system_include_paths = &.{}, // handled by addCCArgs - .needs_gnu_workaround = comp.getTarget().isGnu(), - .nostdinc = false, // handled by addCCArgs + const depfile_filename = try std.fmt.allocPrint(arena, "{s}.d.json", .{rc_basename_noext}); + const out_dep_path = try comp.tmpFilePath(arena, depfile_filename); + try argv.appendSlice(&.{ + self_exe_path, + "rc", + "/:depfile", + out_dep_path, + "/:depfile-fmt", + "json", + "/x", // ignore INCLUDE environment variable + "/:auto-includes", + @tagName(comp.rc_includes), }); - - try argv.append(rc_src.src_path); - try argv.appendSlice(&[_][]const u8{ - "-o", - out_rcpp_path, - }); - - const out_dep_path = try std.fmt.allocPrint(arena, "{s}.d", .{out_rcpp_path}); - // Note: addCCArgs will implicitly add _DEBUG/NDEBUG depending on the optimization - // mode. While these defines are not normally present when calling rc.exe directly, + // While these defines are not normally present when calling rc.exe directly, // them being defined matches the behavior of how MSVC calls rc.exe which is the more // relevant behavior in this case. - try comp.addCCArgs(arena, &argv, .rc, out_dep_path, rc_src.owner); - - if (comp.verbose_cc) { - dump_argv(argv.items); + switch (rc_src.owner.optimize_mode) { + .Debug => try argv.append("-D_DEBUG"), + .ReleaseSafe => {}, + .ReleaseFast, .ReleaseSmall => try argv.append("-DNDEBUG"), } + try argv.appendSlice(rc_src.extra_flags); + try argv.appendSlice(&.{ "--", rc_src.src_path, out_res_path }); - if (std.process.can_spawn) { - var child = std.ChildProcess.init(argv.items, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; - - try child.spawn(); + var child = std.ChildProcess.init(argv.items, arena); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; - const stderr_reader = child.stderr.?.reader(); - - const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); - - const term = child.wait() catch |err| { - return comp.failWin32Resource(win32_resource, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - }; - - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse clang stderr and turn it into an error message - // and then call failCObjWithOwnedErrorMsg - log.err("clang preprocessor failed with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "clang preprocessor exited with code {d}", .{code}); - } - }, - else => { - log.err("clang preprocessor terminated with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "clang preprocessor terminated unexpectedly", .{}); - }, - } - } else { - const exit_code = try clangMain(arena, argv.items); - if (exit_code != 0) { - return comp.failWin32Resource(win32_resource, "clang preprocessor exited with code {d}", .{exit_code}); - } - } + try child.spawn(); - const dep_basename = std.fs.path.basename(out_dep_path); - // Add the files depended on to the cache system. - try man.addDepFilePost(zig_cache_tmp_dir, dep_basename); - switch (comp.cache_use) { - .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { - whole.cache_manifest_mutex.lock(); - defer whole.cache_manifest_mutex.unlock(); - try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename); - }, - .incremental => {}, - } - // Just to save disk space, we delete the file because it is never needed again. - zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { - log.warn("failed to delete '{s}': {s}", .{ out_dep_path, @errorName(err) }); + const stderr_reader = child.stderr.?.reader(); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + const term = child.wait() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); }; - const full_input = std.fs.cwd().readFileAlloc(arena, out_rcpp_path, std.math.maxInt(usize)) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => |e| { - return comp.failWin32Resource(win32_resource, "failed to read preprocessed file '{s}': {s}", .{ out_rcpp_path, @errorName(e) }); + switch (term) { + .Exited => |code| { + if (code != 0) { + log.err("zig rc failed with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); + } + }, + else => { + log.err("zig rc terminated with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); }, - }; - - var mapping_results = try resinator.source_mapping.parseAndRemoveLineCommands(arena, full_input, full_input, .{ .initial_filename = rc_src.src_path }); - defer mapping_results.mappings.deinit(arena); - - const final_input = resinator.comments.removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings); - - var output_file = zig_cache_tmp_dir.createFile(out_res_path, .{}) catch |err| { - return comp.failWin32Resource(win32_resource, "failed to create output file '{s}': {s}", .{ out_res_path, @errorName(err) }); - }; - var output_file_closed = false; - defer if (!output_file_closed) output_file.close(); - - var diagnostics = resinator.errors.Diagnostics.init(arena); - defer diagnostics.deinit(); - - var dependencies_list = std.ArrayList([]const u8).init(comp.gpa); - defer { - for (dependencies_list.items) |item| { - comp.gpa.free(item); - } - dependencies_list.deinit(); } - var output_buffered_stream = std.io.bufferedWriter(output_file.writer()); - - resinator.compile.compile(arena, final_input, output_buffered_stream.writer(), .{ - .cwd = std.fs.cwd(), - .diagnostics = &diagnostics, - .source_mappings = &mapping_results.mappings, - .dependencies_list = &dependencies_list, - .system_include_paths = comp.rc_include_dir_list, - .ignore_include_env_var = true, - // options - .extra_include_paths = options.extra_include_paths.items, - .default_language_id = options.default_language_id, - .default_code_page = options.default_code_page orelse .windows1252, - .verbose = options.verbose, - .null_terminate_string_table_strings = options.null_terminate_string_table_strings, - .max_string_literal_codepoints = options.max_string_literal_codepoints, - .silent_duplicate_control_ids = options.silent_duplicate_control_ids, - .warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page, - }) catch |err| switch (err) { - error.ParseError, error.CompileError => { - // Delete the output file on error - output_file.close(); - output_file_closed = true; - // Failing to delete is not really a big deal, so swallow any errors - zig_cache_tmp_dir.deleteFile(out_res_path) catch { - log.warn("failed to delete '{s}': {s}", .{ out_res_path, @errorName(err) }); - }; - return comp.failWin32ResourceCompile(win32_resource, final_input, &diagnostics, mapping_results.mappings); - }, - else => |e| return e, - }; + // Read depfile and update cache manifest + { + const dep_basename = std.fs.path.basename(out_dep_path); + const dep_file_contents = try zig_cache_tmp_dir.readFileAlloc(arena, dep_basename, 50 * 1024 * 1024); + defer arena.free(dep_file_contents); - try output_buffered_stream.flush(); + const value = try std.json.parseFromSliceLeaky(std.json.Value, arena, dep_file_contents, .{}); + if (value != .array) { + return comp.failWin32Resource(win32_resource, "depfile from zig rc has unexpected format", .{}); + } - for (dependencies_list.items) |dep_file_path| { - try man.addFilePost(dep_file_path); - switch (comp.cache_use) { - .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { - whole.cache_manifest_mutex.lock(); - defer whole.cache_manifest_mutex.unlock(); - try whole_cache_manifest.addFilePost(dep_file_path); - }, - .incremental => {}, + for (value.array.items) |element| { + if (element != .string) { + return comp.failWin32Resource(win32_resource, "depfile from zig rc has unexpected format", .{}); + } + const dep_file_path = element.string; + try man.addFilePost(dep_file_path); + switch (comp.cache_use) { + .whole => |whole| if (whole.cache_manifest) |whole_cache_manifest| { + whole.cache_manifest_mutex.lock(); + defer whole.cache_manifest_mutex.unlock(); + try whole_cache_manifest.addFilePost(dep_file_path); + }, + .incremental => {}, + } } + // Just to save disk space, we delete the file because it is never needed again. + zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| { + log.warn("failed to delete '{s}': {s}", .{ out_dep_path, @errorName(err) }); + }; } // Rename into place. @@ -5159,8 +5005,6 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 defer o_dir.close(); const tmp_basename = std.fs.path.basename(out_res_path); try std.fs.rename(zig_cache_tmp_dir, tmp_basename, o_dir, res_filename); - const tmp_rcpp_basename = std.fs.path.basename(out_rcpp_path); - try std.fs.rename(zig_cache_tmp_dir, tmp_rcpp_basename, o_dir, rcpp_filename); break :blk digest; }; @@ -5352,16 +5196,9 @@ pub fn addCCArgs( try argv.append("-isystem"); try argv.append(c_headers_dir); - if (ext == .rc) { - for (comp.rc_include_dir_list) |include_dir| { - try argv.append("-isystem"); - try argv.append(include_dir); - } - } else { - for (comp.libc_include_dir_list) |include_dir| { - try argv.append("-isystem"); - try argv.append(include_dir); - } + for (comp.libc_include_dir_list) |include_dir| { + try argv.append("-isystem"); + try argv.append(include_dir); } if (target.cpu.model.llvm_name) |llvm_name| { @@ -5726,167 +5563,6 @@ fn failWin32ResourceWithOwnedBundle( return error.AnalysisFail; } -fn failWin32ResourceCli( - comp: *Compilation, - win32_resource: *Win32Resource, - diagnostics: *resinator.cli.Diagnostics, -) SemaError { - @setCold(true); - - var bundle: ErrorBundle.Wip = undefined; - try bundle.init(comp.gpa); - errdefer bundle.deinit(); - - try bundle.addRootErrorMessage(.{ - .msg = try bundle.addString("invalid command line option(s)"), - .src_loc = try bundle.addSourceLocation(.{ - .src_path = try bundle.addString(switch (win32_resource.src) { - .rc => |rc_src| rc_src.src_path, - .manifest => |manifest_src| manifest_src, - }), - .line = 0, - .column = 0, - .span_start = 0, - .span_main = 0, - .span_end = 0, - }), - }); - - var cur_err: ?ErrorBundle.ErrorMessage = null; - var cur_notes: std.ArrayListUnmanaged(ErrorBundle.ErrorMessage) = .{}; - defer cur_notes.deinit(comp.gpa); - for (diagnostics.errors.items) |err_details| { - switch (err_details.type) { - .err => { - if (cur_err) |err| { - try win32ResourceFlushErrorMessage(&bundle, err, cur_notes.items); - } - cur_err = .{ - .msg = try bundle.addString(err_details.msg.items), - }; - cur_notes.clearRetainingCapacity(); - }, - .warning => cur_err = null, - .note => { - if (cur_err == null) continue; - cur_err.?.notes_len += 1; - try cur_notes.append(comp.gpa, .{ - .msg = try bundle.addString(err_details.msg.items), - }); - }, - } - } - if (cur_err) |err| { - try win32ResourceFlushErrorMessage(&bundle, err, cur_notes.items); - } - - const finished_bundle = try bundle.toOwnedBundle(""); - return comp.failWin32ResourceWithOwnedBundle(win32_resource, finished_bundle); -} - -fn failWin32ResourceCompile( - comp: *Compilation, - win32_resource: *Win32Resource, - source: []const u8, - diagnostics: *resinator.errors.Diagnostics, - opt_mappings: ?resinator.source_mapping.SourceMappings, -) SemaError { - @setCold(true); - - var bundle: ErrorBundle.Wip = undefined; - try bundle.init(comp.gpa); - errdefer bundle.deinit(); - - var msg_buf: std.ArrayListUnmanaged(u8) = .{}; - defer msg_buf.deinit(comp.gpa); - var cur_err: ?ErrorBundle.ErrorMessage = null; - var cur_notes: std.ArrayListUnmanaged(ErrorBundle.ErrorMessage) = .{}; - defer cur_notes.deinit(comp.gpa); - for (diagnostics.errors.items) |err_details| { - switch (err_details.type) { - .hint => continue, - // Clear the current error so that notes don't bleed into unassociated errors - .warning => { - cur_err = null; - continue; - }, - .note => if (cur_err == null) continue, - .err => {}, - } - const err_line, const err_filename = blk: { - if (opt_mappings) |mappings| { - const corresponding_span = mappings.get(err_details.token.line_number); - const corresponding_file = mappings.files.get(corresponding_span.filename_offset); - const err_line = corresponding_span.start_line; - break :blk .{ err_line, corresponding_file }; - } else { - break :blk .{ err_details.token.line_number, "" }; - } - }; - - const source_line_start = err_details.token.getLineStart(source); - const column = err_details.token.calculateColumn(source, 1, source_line_start); - - msg_buf.clearRetainingCapacity(); - try err_details.render(msg_buf.writer(comp.gpa), source, diagnostics.strings.items); - - const src_loc = src_loc: { - var src_loc: ErrorBundle.SourceLocation = .{ - .src_path = try bundle.addString(err_filename), - .line = @intCast(err_line - 1), // 1-based -> 0-based - .column = @intCast(column), - .span_start = 0, - .span_main = 0, - .span_end = 0, - }; - if (err_details.print_source_line) { - const source_line = err_details.token.getLine(source, source_line_start); - const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len); - src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len); - src_loc.span_main = @intCast(visual_info.point_offset); - src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len); - src_loc.source_line = try bundle.addString(source_line); - } - break :src_loc try bundle.addSourceLocation(src_loc); - }; - - switch (err_details.type) { - .err => { - if (cur_err) |err| { - try win32ResourceFlushErrorMessage(&bundle, err, cur_notes.items); - } - cur_err = .{ - .msg = try bundle.addString(msg_buf.items), - .src_loc = src_loc, - }; - cur_notes.clearRetainingCapacity(); - }, - .note => { - cur_err.?.notes_len += 1; - try cur_notes.append(comp.gpa, .{ - .msg = try bundle.addString(msg_buf.items), - .src_loc = src_loc, - }); - }, - .warning, .hint => unreachable, - } - } - if (cur_err) |err| { - try win32ResourceFlushErrorMessage(&bundle, err, cur_notes.items); - } - - const finished_bundle = try bundle.toOwnedBundle(""); - return comp.failWin32ResourceWithOwnedBundle(win32_resource, finished_bundle); -} - -fn win32ResourceFlushErrorMessage(wip: *ErrorBundle.Wip, msg: ErrorBundle.ErrorMessage, notes: []const ErrorBundle.ErrorMessage) !void { - try wip.addRootErrorMessage(msg); - const notes_start = try wip.reserveNotes(@intCast(notes.len)); - for (notes_start.., notes) |i, note| { - wip.extra.items[i] = @intFromEnum(wip.addErrorMessageAssumeCapacity(note)); - } -} - pub const FileExt = enum { c, cpp, -- cgit v1.2.3 From 7c05330287b453a0095bc179c7541ab6ea53d146 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Wed, 6 Mar 2024 21:28:49 -0800 Subject: Integrate resinator errors with Zig using std.zig.Server and ErrorBundle This takes the code that was previously in src/Compilation.zig to turn resinator diagnostics into Zig error bundles and puts it in resinator/main.zig, and then makes resinator emit the resulting error bundles via std.zig.Server (which is used by the build runner, etc). Also adds support for turning Aro diagnostics into ErrorBundles. --- lib/compiler/resinator/main.zig | 460 ++++++++++++++++++++++++++++++++++++--- lib/compiler/resinator/utils.zig | 4 +- src/Compilation.zig | 89 ++++++-- 3 files changed, 507 insertions(+), 46 deletions(-) (limited to 'src/Compilation.zig') diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index e598a45122..08a3ea3fc3 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -25,26 +25,48 @@ pub fn main() !void { std.os.exit(1); } const zig_lib_dir = args[1]; + var cli_args = args[2..]; + + var zig_integration = false; + if (cli_args.len > 0 and std.mem.eql(u8, cli_args[0], "--zig-integration")) { + zig_integration = true; + cli_args = args[3..]; + } + + var error_handler: ErrorHandler = switch (zig_integration) { + true => .{ + .server = .{ + .out = std.io.getStdOut(), + .in = undefined, // won't be receiving messages + .receive_fifo = undefined, // won't be receiving messages + }, + }, + false => .{ + .tty = stderr_config, + }, + }; var options = options: { var cli_diagnostics = cli.Diagnostics.init(allocator); defer cli_diagnostics.deinit(); - var options = cli.parse(allocator, args[2..], &cli_diagnostics) catch |err| switch (err) { + var options = cli.parse(allocator, cli_args, &cli_diagnostics) catch |err| switch (err) { error.ParseError => { - cli_diagnostics.renderToStdErr(args, stderr_config); + try error_handler.emitCliDiagnostics(allocator, cli_args, &cli_diagnostics); std.os.exit(1); }, else => |e| return e, }; try options.maybeAppendRC(std.fs.cwd()); - // print any warnings/notes - cli_diagnostics.renderToStdErr(args, stderr_config); - // If there was something printed, then add an extra newline separator - // so that there is a clear separation between the cli diagnostics and whatever - // gets printed after - if (cli_diagnostics.errors.items.len > 0) { - try stderr.writeAll("\n"); + if (!zig_integration) { + // print any warnings/notes + cli_diagnostics.renderToStdErr(args, stderr_config); + // If there was something printed, then add an extra newline separator + // so that there is a clear separation between the cli diagnostics and whatever + // gets printed after + if (cli_diagnostics.errors.items.len > 0) { + try stderr.writeAll("\n"); + } } break :options options; }; @@ -55,6 +77,9 @@ pub fn main() !void { return; } + // Don't allow verbose when integrating with Zig via stdout + options.verbose = false; + const stdout_writer = std.io.getStdOut().writer(); if (options.verbose) { try options.dumpVerbose(stdout_writer); @@ -86,13 +111,13 @@ pub fn main() !void { else => |e| { switch (e) { error.MsvcIncludesNotFound => { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "MSVC include paths could not be automatically detected", .{}); + try error_handler.emitMessage(allocator, .err, "MSVC include paths could not be automatically detected", .{}); }, error.MingwIncludesNotFound => { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "MinGW include paths could not be automatically detected", .{}); + try error_handler.emitMessage(allocator, .err, "MinGW include paths could not be automatically detected", .{}); }, } - try renderErrorMessage(stderr.writer(), stderr_config, .note, "to disable auto includes, use the option /:auto-includes none", .{}); + try error_handler.emitMessage(allocator, .note, "to disable auto includes, use the option /:auto-includes none", .{}); std.os.exit(1); }, }; @@ -117,20 +142,16 @@ pub fn main() !void { preprocess.preprocess(&comp, preprocessed_buf.writer(), argv.items, maybe_dependencies_list) catch |err| switch (err) { error.GeneratedSourceError => { - // extra newline to separate this line from the aro errors - try renderErrorMessage(stderr.writer(), stderr_config, .err, "failed during preprocessor setup (this is always a bug):\n", .{}); - aro.Diagnostics.render(&comp, stderr_config); + try error_handler.emitAroDiagnostics(allocator, "failed during preprocessor setup (this is always a bug):", &comp); std.os.exit(1); }, // ArgError can occur if e.g. the .rc file is not found error.ArgError, error.PreprocessError => { - // extra newline to separate this line from the aro errors - try renderErrorMessage(stderr.writer(), stderr_config, .err, "failed during preprocessing:\n", .{}); - aro.Diagnostics.render(&comp, stderr_config); + try error_handler.emitAroDiagnostics(allocator, "failed during preprocessing:", &comp); std.os.exit(1); }, error.StreamTooLong => { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "failed during preprocessing: maximum file size exceeded", .{}); + try error_handler.emitMessage(allocator, .err, "failed during preprocessing: maximum file size exceeded", .{}); std.os.exit(1); }, error.OutOfMemory => |e| return e, @@ -139,7 +160,7 @@ pub fn main() !void { break :full_input try preprocessed_buf.toOwnedSlice(); } else { break :full_input std.fs.cwd().readFileAlloc(allocator, options.input_filename, std.math.maxInt(usize)) catch |err| { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "unable to read input file path '{s}': {s}", .{ options.input_filename, @errorName(err) }); + try error_handler.emitMessage(allocator, .err, "unable to read input file path '{s}': {s}", .{ options.input_filename, @errorName(err) }); std.os.exit(1); }; } @@ -159,14 +180,14 @@ pub fn main() !void { const final_input = removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings) catch |err| switch (err) { error.InvalidSourceMappingCollapse => { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "failed during comment removal; this is a known bug", .{}); + try error_handler.emitMessage(allocator, .err, "failed during comment removal; this is a known bug", .{}); std.os.exit(1); }, else => |e| return e, }; var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch |err| { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) }); + try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) }); std.os.exit(1); }; var output_file_closed = false; @@ -193,7 +214,7 @@ pub fn main() !void { .warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page, }) catch |err| switch (err) { error.ParseError, error.CompileError => { - diagnostics.renderToStdErr(std.fs.cwd(), final_input, stderr_config, mapping_results.mappings); + try error_handler.emitDiagnostics(allocator, std.fs.cwd(), final_input, &diagnostics, mapping_results.mappings); // Delete the output file on error output_file.close(); output_file_closed = true; @@ -207,12 +228,14 @@ pub fn main() !void { try output_buffered_stream.flush(); // print any warnings/notes - diagnostics.renderToStdErr(std.fs.cwd(), final_input, stderr_config, mapping_results.mappings); + if (!zig_integration) { + diagnostics.renderToStdErr(std.fs.cwd(), final_input, stderr_config, mapping_results.mappings); + } // write the depfile if (options.depfile_path) |depfile_path| { var depfile = std.fs.cwd().createFile(depfile_path, .{}) catch |err| { - try renderErrorMessage(stderr.writer(), stderr_config, .err, "unable to create depfile '{s}': {s}", .{ depfile_path, @errorName(err) }); + try error_handler.emitMessage(allocator, .err, "unable to create depfile '{s}': {s}", .{ depfile_path, @errorName(err) }); std.os.exit(1); }; defer depfile.close(); @@ -296,3 +319,390 @@ fn getIncludePaths(arena: std.mem.Allocator, auto_includes_option: cli.Options.A } } } + +const ErrorBundle = std.zig.ErrorBundle; +const SourceMappings = @import("source_mapping.zig").SourceMappings; + +const ErrorHandler = union(enum) { + server: std.zig.Server, + tty: std.io.tty.Config, + + pub fn emitCliDiagnostics( + self: *ErrorHandler, + allocator: std.mem.Allocator, + args: []const []const u8, + diagnostics: *cli.Diagnostics, + ) !void { + switch (self.*) { + .server => |*server| { + var error_bundle = try cliDiagnosticsToErrorBundle(allocator, diagnostics); + defer error_bundle.deinit(allocator); + + try server.serveErrorBundle(error_bundle); + }, + .tty => { + diagnostics.renderToStdErr(args, self.tty); + }, + } + } + + pub fn emitAroDiagnostics( + self: *ErrorHandler, + allocator: std.mem.Allocator, + fail_msg: []const u8, + comp: *aro.Compilation, + ) !void { + switch (self.*) { + .server => |*server| { + var error_bundle = try aroDiagnosticsToErrorBundle(allocator, fail_msg, comp); + defer error_bundle.deinit(allocator); + + try server.serveErrorBundle(error_bundle); + }, + .tty => { + // extra newline to separate this line from the aro errors + try renderErrorMessage(std.io.getStdErr().writer(), self.tty, .err, "{s}\n", .{fail_msg}); + aro.Diagnostics.render(comp, self.tty); + }, + } + } + + pub fn emitDiagnostics( + self: *ErrorHandler, + allocator: std.mem.Allocator, + cwd: std.fs.Dir, + source: []const u8, + diagnostics: *Diagnostics, + mappings: SourceMappings, + ) !void { + switch (self.*) { + .server => |*server| { + var error_bundle = try diagnosticsToErrorBundle(allocator, source, diagnostics, mappings); + defer error_bundle.deinit(allocator); + + try server.serveErrorBundle(error_bundle); + }, + .tty => { + diagnostics.renderToStdErr(cwd, source, self.tty, mappings); + }, + } + } + + pub fn emitMessage( + self: *ErrorHandler, + allocator: std.mem.Allocator, + msg_type: @import("utils.zig").ErrorMessageType, + comptime format: []const u8, + args: anytype, + ) !void { + switch (self.*) { + .server => |*server| { + // only emit errors + if (msg_type != .err) return; + + var error_bundle = try errorStringToErrorBundle(allocator, format, args); + defer error_bundle.deinit(allocator); + + try server.serveErrorBundle(error_bundle); + }, + .tty => { + try renderErrorMessage(std.io.getStdErr().writer(), self.tty, msg_type, format, args); + }, + } + } +}; + +fn cliDiagnosticsToErrorBundle( + gpa: std.mem.Allocator, + diagnostics: *cli.Diagnostics, +) !ErrorBundle { + @setCold(true); + + var bundle: ErrorBundle.Wip = undefined; + try bundle.init(gpa); + errdefer bundle.deinit(); + + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString("invalid command line option(s)"), + }); + + var cur_err: ?ErrorBundle.ErrorMessage = null; + var cur_notes: std.ArrayListUnmanaged(ErrorBundle.ErrorMessage) = .{}; + defer cur_notes.deinit(gpa); + for (diagnostics.errors.items) |err_details| { + switch (err_details.type) { + .err => { + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + cur_err = .{ + .msg = try bundle.addString(err_details.msg.items), + }; + cur_notes.clearRetainingCapacity(); + }, + .warning => cur_err = null, + .note => { + if (cur_err == null) continue; + cur_err.?.notes_len += 1; + try cur_notes.append(gpa, .{ + .msg = try bundle.addString(err_details.msg.items), + }); + }, + } + } + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + + return try bundle.toOwnedBundle(""); +} + +fn diagnosticsToErrorBundle( + gpa: std.mem.Allocator, + source: []const u8, + diagnostics: *Diagnostics, + mappings: SourceMappings, +) !ErrorBundle { + @setCold(true); + + var bundle: ErrorBundle.Wip = undefined; + try bundle.init(gpa); + errdefer bundle.deinit(); + + var msg_buf: std.ArrayListUnmanaged(u8) = .{}; + defer msg_buf.deinit(gpa); + var cur_err: ?ErrorBundle.ErrorMessage = null; + var cur_notes: std.ArrayListUnmanaged(ErrorBundle.ErrorMessage) = .{}; + defer cur_notes.deinit(gpa); + for (diagnostics.errors.items) |err_details| { + switch (err_details.type) { + .hint => continue, + // Clear the current error so that notes don't bleed into unassociated errors + .warning => { + cur_err = null; + continue; + }, + .note => if (cur_err == null) continue, + .err => {}, + } + const corresponding_span = mappings.getCorrespondingSpan(err_details.token.line_number).?; + const err_line = corresponding_span.start_line; + const err_filename = mappings.files.get(corresponding_span.filename_offset); + + const source_line_start = err_details.token.getLineStartForErrorDisplay(source); + // Treat tab stops as 1 column wide for error display purposes, + // and add one to get a 1-based column + const column = err_details.token.calculateColumn(source, 1, source_line_start) + 1; + + msg_buf.clearRetainingCapacity(); + try err_details.render(msg_buf.writer(gpa), source, diagnostics.strings.items); + + const src_loc = src_loc: { + var src_loc: ErrorBundle.SourceLocation = .{ + .src_path = try bundle.addString(err_filename), + .line = @intCast(err_line - 1), // 1-based -> 0-based + .column = @intCast(column - 1), // 1-based -> 0-based + .span_start = 0, + .span_main = 0, + .span_end = 0, + }; + if (err_details.print_source_line) { + const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start); + const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len); + src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len); + src_loc.span_main = @intCast(visual_info.point_offset); + src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len); + src_loc.source_line = try bundle.addString(source_line); + } + break :src_loc try bundle.addSourceLocation(src_loc); + }; + + switch (err_details.type) { + .err => { + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + cur_err = .{ + .msg = try bundle.addString(msg_buf.items), + .src_loc = src_loc, + }; + cur_notes.clearRetainingCapacity(); + }, + .note => { + cur_err.?.notes_len += 1; + try cur_notes.append(gpa, .{ + .msg = try bundle.addString(msg_buf.items), + .src_loc = src_loc, + }); + }, + .warning, .hint => unreachable, + } + } + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + + return try bundle.toOwnedBundle(""); +} + +fn flushErrorMessageIntoBundle(wip: *ErrorBundle.Wip, msg: ErrorBundle.ErrorMessage, notes: []const ErrorBundle.ErrorMessage) !void { + try wip.addRootErrorMessage(msg); + const notes_start = try wip.reserveNotes(@intCast(notes.len)); + for (notes_start.., notes) |i, note| { + wip.extra.items[i] = @intFromEnum(wip.addErrorMessageAssumeCapacity(note)); + } +} + +fn errorStringToErrorBundle(allocator: std.mem.Allocator, comptime format: []const u8, args: anytype) !ErrorBundle { + @setCold(true); + var bundle: ErrorBundle.Wip = undefined; + try bundle.init(allocator); + errdefer bundle.deinit(); + try bundle.addRootErrorMessage(.{ + .msg = try bundle.printString(format, args), + }); + return try bundle.toOwnedBundle(""); +} + +fn aroDiagnosticsToErrorBundle( + gpa: std.mem.Allocator, + fail_msg: []const u8, + comp: *aro.Compilation, +) !ErrorBundle { + @setCold(true); + + var bundle: ErrorBundle.Wip = undefined; + try bundle.init(gpa); + errdefer bundle.deinit(); + + try bundle.addRootErrorMessage(.{ + .msg = try bundle.addString(fail_msg), + }); + + var msg_writer = MsgWriter.init(gpa); + defer msg_writer.deinit(); + var cur_err: ?ErrorBundle.ErrorMessage = null; + var cur_notes: std.ArrayListUnmanaged(ErrorBundle.ErrorMessage) = .{}; + defer cur_notes.deinit(gpa); + for (comp.diagnostics.list.items) |msg| { + switch (msg.kind) { + // Clear the current error so that notes don't bleed into unassociated errors + .off, .warning => { + cur_err = null; + continue; + }, + .note => if (cur_err == null) continue, + .@"fatal error", .@"error" => {}, + .default => unreachable, + } + msg_writer.resetRetainingCapacity(); + aro.Diagnostics.renderMessage(comp, &msg_writer, msg); + + const src_loc = src_loc: { + if (msg_writer.path) |src_path| { + var src_loc: ErrorBundle.SourceLocation = .{ + .src_path = try bundle.addString(src_path), + .line = msg_writer.line - 1, // 1-based -> 0-based + .column = msg_writer.col - 1, // 1-based -> 0-based + .span_start = 0, + .span_main = 0, + .span_end = 0, + }; + if (msg_writer.source_line) |source_line| { + src_loc.span_start = msg_writer.span_main; + src_loc.span_main = msg_writer.span_main; + src_loc.span_end = msg_writer.span_main; + src_loc.source_line = try bundle.addString(source_line); + } + break :src_loc try bundle.addSourceLocation(src_loc); + } + break :src_loc ErrorBundle.SourceLocationIndex.none; + }; + + switch (msg.kind) { + .@"fatal error", .@"error" => { + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + cur_err = .{ + .msg = try bundle.addString(msg_writer.buf.items), + .src_loc = src_loc, + }; + cur_notes.clearRetainingCapacity(); + }, + .note => { + cur_err.?.notes_len += 1; + try cur_notes.append(gpa, .{ + .msg = try bundle.addString(msg_writer.buf.items), + .src_loc = src_loc, + }); + }, + .off, .warning, .default => unreachable, + } + } + if (cur_err) |err| { + try flushErrorMessageIntoBundle(&bundle, err, cur_notes.items); + } + + return try bundle.toOwnedBundle(""); +} + +// Similar to aro.Diagnostics.MsgWriter but: +// - Writers to an ArrayList +// - Only prints the message itself (no location, source line, error: prefix, etc) +// - Keeps track of source path/line/col instead +const MsgWriter = struct { + buf: std.ArrayList(u8), + path: ?[]const u8 = null, + // 1-indexed + line: u32 = undefined, + col: u32 = undefined, + source_line: ?[]const u8 = null, + span_main: u32 = undefined, + + fn init(allocator: std.mem.Allocator) MsgWriter { + return .{ + .buf = std.ArrayList(u8).init(allocator), + }; + } + + fn deinit(m: *MsgWriter) void { + m.buf.deinit(); + } + + fn resetRetainingCapacity(m: *MsgWriter) void { + m.buf.clearRetainingCapacity(); + m.path = null; + m.source_line = null; + } + + pub fn print(m: *MsgWriter, comptime fmt: []const u8, args: anytype) void { + m.buf.writer().print(fmt, args) catch {}; + } + + pub fn write(m: *MsgWriter, msg: []const u8) void { + m.buf.writer().writeAll(msg) catch {}; + } + + pub fn setColor(m: *MsgWriter, color: std.io.tty.Color) void { + _ = m; + _ = color; + } + + pub fn location(m: *MsgWriter, path: []const u8, line: u32, col: u32) void { + m.path = path; + m.line = line; + m.col = col; + } + + pub fn start(m: *MsgWriter, kind: aro.Diagnostics.Kind) void { + _ = m; + _ = kind; + } + + pub fn end(m: *MsgWriter, maybe_line: ?[]const u8, col: u32, end_with_splice: bool) void { + _ = end_with_splice; + m.source_line = maybe_line; + m.span_main = col; + } +}; diff --git a/lib/compiler/resinator/utils.zig b/lib/compiler/resinator/utils.zig index d216f07838..b69222fe03 100644 --- a/lib/compiler/resinator/utils.zig +++ b/lib/compiler/resinator/utils.zig @@ -82,9 +82,11 @@ pub fn isNonAsciiDigit(c: u21) bool { }; } +pub const ErrorMessageType = enum { err, warning, note }; + /// Used for generic colored errors/warnings/notes, more context-specific error messages /// are handled elsewhere. -pub fn renderErrorMessage(writer: anytype, config: std.io.tty.Config, msg_type: enum { err, warning, note }, comptime format: []const u8, args: anytype) !void { +pub fn renderErrorMessage(writer: anytype, config: std.io.tty.Config, msg_type: ErrorMessageType, comptime format: []const u8, args: anytype) !void { switch (msg_type) { .err => { try config.setColor(writer, .bold); diff --git a/src/Compilation.zig b/src/Compilation.zig index 46fd25735d..d978dc5280 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -4921,6 +4921,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try argv.appendSlice(&.{ self_exe_path, "rc", + "--zig-integration", "/:depfile", out_dep_path, "/:depfile-fmt", @@ -4940,30 +4941,78 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try argv.appendSlice(rc_src.extra_flags); try argv.appendSlice(&.{ "--", rc_src.src_path, out_res_path }); - var child = std.ChildProcess.init(argv.items, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; + { + var child = std.ChildProcess.init(argv.items, arena); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; - try child.spawn(); + child.spawn() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to spawn {s} rc: {s}", .{ argv.items[0], @errorName(err) }); + }; - const stderr_reader = child.stderr.?.reader(); - const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); - const term = child.wait() catch |err| { - return comp.failWin32Resource(win32_resource, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - }; + var poller = std.io.poll(comp.gpa, enum { stdout }, .{ + .stdout = child.stdout.?, + }); + defer poller.deinit(); - switch (term) { - .Exited => |code| { - if (code != 0) { - log.err("zig rc failed with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); + const stdout = poller.fifo(.stdout); + + poll: while (true) { + while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) { + if (!(try poller.poll())) break :poll; } - }, - else => { - log.err("zig rc terminated with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); - }, + const header = stdout.reader().readStruct(std.zig.Server.Message.Header) catch unreachable; + while (stdout.readableLength() < header.bytes_len) { + if (!(try poller.poll())) break :poll; + } + const body = stdout.readableSliceOfLen(header.bytes_len); + + switch (header.tag) { + // We expect exactly one ErrorBundle, and if any error_bundle header is + // sent then it's a fatal error. + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try comp.gpa.alloc(u32, unaligned_extra.len); + @memcpy(extra_array, unaligned_extra); + const error_bundle = .{ + .string_bytes = try comp.gpa.dupe(u8, string_bytes), + .extra = extra_array, + }; + return comp.failWin32ResourceWithOwnedBundle(win32_resource, error_bundle); + }, + else => {}, // ignore other messages + } + + stdout.discard(body.len); + } + + // Just in case there's a failure that didn't send an ErrorBundle (e.g. an error return trace) + const stderr_reader = child.stderr.?.reader(); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to wait for {s} rc: {s}", .{ argv.items[0], @errorName(err) }); + }; + + switch (term) { + .Exited => |code| { + if (code != 0) { + log.err("zig rc failed with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); + } + }, + else => { + log.err("zig rc terminated with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); + }, + } } // Read depfile and update cache manifest -- cgit v1.2.3 From 8799f7466d68405d715bf567c3de9935b619fcf6 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Fri, 8 Mar 2024 11:54:56 -0800 Subject: Report the progress of lazily building zig rc jitCmd now takes a `server` option that will emit progress/errors via std.zig.Server when enabled. --- lib/compiler/resinator/main.zig | 6 ++ src/Compilation.zig | 195 ++++++++++++++++++++-------------------- src/main.zig | 54 ++++++++++- 3 files changed, 153 insertions(+), 102 deletions(-) (limited to 'src/Compilation.zig') diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 08a3ea3fc3..8e6859978e 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -46,6 +46,12 @@ pub fn main() !void { }, }; + if (zig_integration) { + // Send progress with an empty string to indicate that the building of the + // resinator binary is finished and we've moved on to actually compiling the .rc file + try error_handler.server.serveStringMessage(.progress, ""); + } + var options = options: { var cli_diagnostics = cli.Diagnostics.init(allocator); defer cli_diagnostics.deinit(); diff --git a/src/Compilation.zig b/src/Compilation.zig index d978dc5280..3f312e9956 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -4841,6 +4841,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try argv.appendSlice(&.{ self_exe_path, "rc", + "--zig-integration", "/:no-preprocess", "/x", // ignore INCLUDE environment variable "/c65001", // UTF-8 codepage @@ -4849,31 +4850,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 }); try argv.appendSlice(&.{ "--", in_rc_path, out_res_path }); - var child = std.ChildProcess.init(argv.items, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; - - try child.spawn(); - - const stderr_reader = child.stderr.?.reader(); - const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); - const term = child.wait() catch |err| { - return comp.failWin32Resource(win32_resource, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - }; - - switch (term) { - .Exited => |code| { - if (code != 0) { - log.err("zig rc failed with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); - } - }, - else => { - log.err("zig rc terminated with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); - }, - } + try spawnZigRc(comp, win32_resource, src_basename, arena, argv.items, &child_progress_node); break :blk digest; }; @@ -4941,79 +4918,7 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 try argv.appendSlice(rc_src.extra_flags); try argv.appendSlice(&.{ "--", rc_src.src_path, out_res_path }); - { - var child = std.ChildProcess.init(argv.items, arena); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Pipe; - child.stderr_behavior = .Pipe; - - child.spawn() catch |err| { - return comp.failWin32Resource(win32_resource, "unable to spawn {s} rc: {s}", .{ argv.items[0], @errorName(err) }); - }; - - var poller = std.io.poll(comp.gpa, enum { stdout }, .{ - .stdout = child.stdout.?, - }); - defer poller.deinit(); - - const stdout = poller.fifo(.stdout); - - poll: while (true) { - while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) { - if (!(try poller.poll())) break :poll; - } - const header = stdout.reader().readStruct(std.zig.Server.Message.Header) catch unreachable; - while (stdout.readableLength() < header.bytes_len) { - if (!(try poller.poll())) break :poll; - } - const body = stdout.readableSliceOfLen(header.bytes_len); - - switch (header.tag) { - // We expect exactly one ErrorBundle, and if any error_bundle header is - // sent then it's a fatal error. - .error_bundle => { - const EbHdr = std.zig.Server.Message.ErrorBundle; - const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); - const extra_bytes = - body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; - const string_bytes = - body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; - const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); - const extra_array = try comp.gpa.alloc(u32, unaligned_extra.len); - @memcpy(extra_array, unaligned_extra); - const error_bundle = .{ - .string_bytes = try comp.gpa.dupe(u8, string_bytes), - .extra = extra_array, - }; - return comp.failWin32ResourceWithOwnedBundle(win32_resource, error_bundle); - }, - else => {}, // ignore other messages - } - - stdout.discard(body.len); - } - - // Just in case there's a failure that didn't send an ErrorBundle (e.g. an error return trace) - const stderr_reader = child.stderr.?.reader(); - const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); - - const term = child.wait() catch |err| { - return comp.failWin32Resource(win32_resource, "unable to wait for {s} rc: {s}", .{ argv.items[0], @errorName(err) }); - }; - - switch (term) { - .Exited => |code| { - if (code != 0) { - log.err("zig rc failed with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); - } - }, - else => { - log.err("zig rc terminated with stderr:\n{s}", .{stderr}); - return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); - }, - } - } + try spawnZigRc(comp, win32_resource, src_basename, arena, argv.items, &child_progress_node); // Read depfile and update cache manifest { @@ -5079,6 +4984,100 @@ fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32 }; } +fn spawnZigRc( + comp: *Compilation, + win32_resource: *Win32Resource, + src_basename: []const u8, + arena: Allocator, + argv: []const []const u8, + child_progress_node: *std.Progress.Node, +) !void { + var node_name: std.ArrayListUnmanaged(u8) = .{}; + defer node_name.deinit(arena); + + var child = std.ChildProcess.init(argv, arena); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + child.spawn() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to spawn {s} rc: {s}", .{ argv[0], @errorName(err) }); + }; + + var poller = std.io.poll(comp.gpa, enum { stdout }, .{ + .stdout = child.stdout.?, + }); + defer poller.deinit(); + + const stdout = poller.fifo(.stdout); + + poll: while (true) { + while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) { + if (!(try poller.poll())) break :poll; + } + const header = stdout.reader().readStruct(std.zig.Server.Message.Header) catch unreachable; + while (stdout.readableLength() < header.bytes_len) { + if (!(try poller.poll())) break :poll; + } + const body = stdout.readableSliceOfLen(header.bytes_len); + + switch (header.tag) { + // We expect exactly one ErrorBundle, and if any error_bundle header is + // sent then it's a fatal error. + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try comp.gpa.alloc(u32, unaligned_extra.len); + @memcpy(extra_array, unaligned_extra); + const error_bundle = std.zig.ErrorBundle{ + .string_bytes = try comp.gpa.dupe(u8, string_bytes), + .extra = extra_array, + }; + return comp.failWin32ResourceWithOwnedBundle(win32_resource, error_bundle); + }, + .progress => { + node_name.clearRetainingCapacity(); + if (body.len > 0) { + try node_name.appendSlice(arena, "build 'zig rc'... "); + try node_name.appendSlice(arena, body); + child_progress_node.setName(node_name.items); + } else { + child_progress_node.setName(src_basename); + } + }, + else => {}, // ignore other messages + } + + stdout.discard(body.len); + } + + // Just in case there's a failure that didn't send an ErrorBundle (e.g. an error return trace) + const stderr_reader = child.stderr.?.reader(); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + return comp.failWin32Resource(win32_resource, "unable to wait for {s} rc: {s}", .{ argv[0], @errorName(err) }); + }; + + switch (term) { + .Exited => |code| { + if (code != 0) { + log.err("zig rc failed with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc exited with code {d}", .{code}); + } + }, + else => { + log.err("zig rc terminated with stderr:\n{s}", .{stderr}); + return comp.failWin32Resource(win32_resource, "zig rc terminated unexpectedly", .{}); + }, + } +} + pub fn tmpFilePath(comp: *Compilation, ally: Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; const rand_int = std.crypto.random.int(u64); diff --git a/src/main.zig b/src/main.zig index a11bf4b808..7fe08698f1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -291,11 +291,13 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "translate-c")) { return buildOutputType(gpa, arena, args, .translate_c); } else if (mem.eql(u8, cmd, "rc")) { + const use_server = cmd_args.len > 0 and std.mem.eql(u8, cmd_args[0], "--zig-integration"); return jitCmd(gpa, arena, cmd_args, .{ .cmd_name = "resinator", .root_src_path = "resinator/main.zig", .depend_on_aro = true, .prepend_zig_lib_dir_path = true, + .server = use_server, }); } else if (mem.eql(u8, cmd, "fmt")) { return jitCmd(gpa, arena, cmd_args, .{ @@ -5304,6 +5306,8 @@ const JitCmdOptions = struct { prepend_zig_exe_path: bool = false, depend_on_aro: bool = false, capture: ?*[]u8 = null, + /// Send progress and error bundles via std.zig.Server over stdout + server: bool = false, }; fn jitCmd( @@ -5449,10 +5453,52 @@ fn jitCmd( }; defer comp.destroy(); - updateModule(comp, color) catch |err| switch (err) { - error.SemanticAnalyzeFail => process.exit(2), - else => |e| return e, - }; + if (options.server and !builtin.single_threaded) { + var reset: std.Thread.ResetEvent = .{}; + var progress: std.Progress = .{ + .terminal = null, + .root = .{ + .context = undefined, + .parent = null, + .name = "", + .unprotected_estimated_total_items = 0, + .unprotected_completed_items = 0, + }, + .columns_written = 0, + .prev_refresh_timestamp = 0, + .timer = null, + .done = false, + }; + const main_progress_node = &progress.root; + main_progress_node.context = &progress; + var server = std.zig.Server{ + .out = std.io.getStdOut(), + .in = undefined, // won't be receiving messages + .receive_fifo = undefined, // won't be receiving messages + }; + + var progress_thread = try std.Thread.spawn(.{}, progressThread, .{ + &progress, &server, &reset, + }); + defer { + reset.set(); + progress_thread.join(); + } + + try comp.update(main_progress_node); + + var error_bundle = try comp.getAllErrorsAlloc(); + defer error_bundle.deinit(comp.gpa); + if (error_bundle.errorMessageCount() > 0) { + try server.serveErrorBundle(error_bundle); + process.exit(2); + } + } else { + updateModule(comp, color) catch |err| switch (err) { + error.SemanticAnalyzeFail => process.exit(2), + else => |e| return e, + }; + } const exe_path = try global_cache_directory.join(arena, &.{comp.cache_use.whole.bin_sub_path.?}); child_argv.appendAssumeCapacity(exe_path); -- cgit v1.2.3 From c32e0d3000f4ee539c00577b75e18c683e84f2c4 Mon Sep 17 00:00:00 2001 From: Ryan Liptak Date: Sat, 9 Mar 2024 17:47:56 -0800 Subject: Fix progress when multiple zig rc child processes are building resinator --- lib/compiler/resinator/main.zig | 4 ++-- src/Compilation.zig | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'src/Compilation.zig') diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 8e6859978e..c0b99855dc 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -47,9 +47,9 @@ pub fn main() !void { }; if (zig_integration) { - // Send progress with an empty string to indicate that the building of the + // Send progress with a special string to indicate that the building of the // resinator binary is finished and we've moved on to actually compiling the .rc file - try error_handler.server.serveStringMessage(.progress, ""); + try error_handler.server.serveStringMessage(.progress, ""); } var options = options: { diff --git a/src/Compilation.zig b/src/Compilation.zig index 3f312e9956..2fce786856 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -5042,12 +5042,18 @@ fn spawnZigRc( }, .progress => { node_name.clearRetainingCapacity(); - if (body.len > 0) { + // is a special string that indicates that the child + // process has reached resinator's main function + if (std.mem.eql(u8, body, "")) { + child_progress_node.setName(src_basename); + } + // Ignore 0-length strings since if multiple zig rc commands + // are executed at the same time, only one will send progress strings + // while the other(s) will send empty strings. + else if (body.len > 0) { try node_name.appendSlice(arena, "build 'zig rc'... "); try node_name.appendSlice(arena, body); child_progress_node.setName(node_name.items); - } else { - child_progress_node.setName(src_basename); } }, else => {}, // ignore other messages -- cgit v1.2.3