tokenizer: clean up invalid token error

It now displays the byte with proper printability handling. This makes
the relevant compile error test case no longer a regression in quality
from stage1 to stage2.

3 files changed, 21 insertions, 14 deletions

diff --git a/src/Module.zig b/src/Module.zig
index 2e421ea65b..4b82a6df07 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -2480,11 +2480,12 @@ pub fn astGenFile(mod: *Module, file: *Scope.File) !void {
         };
         if (token_tags[parse_err.token] == .invalid) {
             const bad_off = @intCast(u32, file.tree.tokenSlice(parse_err.token).len);
+            const byte_abs = token_starts[parse_err.token] + bad_off;
             try mod.errNoteNonLazy(.{
                 .file_scope = file,
                 .parent_decl_node = 0,
-                .lazy = .{ .byte_abs = token_starts[parse_err.token] + bad_off },
-            }, err_msg, "invalid byte here", .{});
+                .lazy = .{ .byte_abs = byte_abs },
+            }, err_msg, "invalid byte: '{'}'", .{ std.zig.fmtEscapes(source[byte_abs..][0..1]) });
         }
 
         {
diff --git a/src/main.zig b/src/main.zig
index f4ca11a96a..37044d0b99 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -233,7 +233,7 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
     } else if (mem.eql(u8, cmd, "build")) {
         return cmdBuild(gpa, arena, cmd_args);
     } else if (mem.eql(u8, cmd, "fmt")) {
-        return cmdFmt(gpa, cmd_args);
+        return cmdFmt(gpa, arena, cmd_args);
     } else if (mem.eql(u8, cmd, "libc")) {
         return cmdLibC(gpa, cmd_args);
     } else if (mem.eql(u8, cmd, "init-exe")) {
@@ -3039,12 +3039,13 @@ const Fmt = struct {
     check_ast: bool,
     color: Color,
     gpa: *Allocator,
+    arena: *Allocator,
     out_buffer: std.ArrayList(u8),
 
     const SeenMap = std.AutoHashMap(fs.File.INode, void);
 };
 
-pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
+pub fn cmdFmt(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !void {
     var color: Color = .auto;
     var stdin_flag: bool = false;
     var check_flag: bool = false;
@@ -3102,7 +3103,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
         defer tree.deinit(gpa);
 
         for (tree.errors) |parse_error| {
-            try printErrMsgToStdErr(gpa, parse_error, tree, "<stdin>", color);
+            try printErrMsgToStdErr(gpa, arena, parse_error, tree, "<stdin>", color);
         }
         var has_ast_error = false;
         if (check_ast_flag) {
@@ -3170,6 +3171,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
 
     var fmt = Fmt{
         .gpa = gpa,
+        .arena = arena,
         .seen = Fmt.SeenMap.init(gpa),
         .any_error = false,
         .check_ast = check_ast_flag,
@@ -3293,7 +3295,7 @@ fn fmtPathFile(
     defer tree.deinit(fmt.gpa);
 
     for (tree.errors) |parse_error| {
-        try printErrMsgToStdErr(fmt.gpa, parse_error, tree, file_path, fmt.color);
+        try printErrMsgToStdErr(fmt.gpa, fmt.arena, parse_error, tree, file_path, fmt.color);
     }
     if (tree.errors.len != 0) {
         fmt.any_error = true;
@@ -3374,6 +3376,7 @@ fn fmtPathFile(
 
 fn printErrMsgToStdErr(
     gpa: *mem.Allocator,
+    arena: *mem.Allocator,
     parse_error: ast.Error,
     tree: ast.Tree,
     path: []const u8,
@@ -3395,11 +3398,14 @@ fn printErrMsgToStdErr(
 
     if (token_tags[parse_error.token] == .invalid) {
         const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token).len);
+        const byte_offset = @intCast(u32, start_loc.line_start) + bad_off;
         notes_buffer[notes_len] = .{
             .src = .{
                 .src_path = path,
-                .msg = "invalid byte here",
-                .byte_offset = @intCast(u32, start_loc.line_start) + bad_off,
+                .msg = try std.fmt.allocPrint(arena, "invalid byte: '{'}'", .{
+                    std.zig.fmtEscapes(tree.source[byte_offset..][0..1]),
+                }),
+                .byte_offset = byte_offset,
                 .line = @intCast(u32, start_loc.line),
                 .column = @intCast(u32, start_loc.column) + bad_off,
                 .source_line = source_line,
@@ -3943,7 +3949,7 @@ pub fn cmdAstCheck(
     defer file.tree.deinit(gpa);
 
     for (file.tree.errors) |parse_error| {
-        try printErrMsgToStdErr(gpa, parse_error, file.tree, file.sub_file_path, color);
+        try printErrMsgToStdErr(gpa, arena, parse_error, file.tree, file.sub_file_path, color);
     }
     if (file.tree.errors.len != 0) {
         process.exit(1);
@@ -4069,7 +4075,7 @@ pub fn cmdChangelist(
     defer file.tree.deinit(gpa);
 
     for (file.tree.errors) |parse_error| {
-        try printErrMsgToStdErr(gpa, parse_error, file.tree, old_source_file, .auto);
+        try printErrMsgToStdErr(gpa, arena, parse_error, file.tree, old_source_file, .auto);
     }
     if (file.tree.errors.len != 0) {
         process.exit(1);
@@ -4108,7 +4114,7 @@ pub fn cmdChangelist(
     defer new_tree.deinit(gpa);
 
     for (new_tree.errors) |parse_error| {
-        try printErrMsgToStdErr(gpa, parse_error, new_tree, new_source_file, .auto);
+        try printErrMsgToStdErr(gpa, arena, parse_error, new_tree, new_source_file, .auto);
     }
     if (new_tree.errors.len != 0) {
         process.exit(1);
diff --git a/src/stage1/analyze.cpp b/src/stage1/analyze.cpp
index 3fb0cb55b7..5daad0213d 100644
--- a/src/stage1/analyze.cpp
+++ b/src/stage1/analyze.cpp
@@ -3915,7 +3915,7 @@ static void add_top_level_decl(CodeGen *g, ScopeDecls *decls_scope, Tld *tld) {
                 }
             }
             ErrorMsg *msg = add_node_error(g, tld->source_node, buf_sprintf("redefinition of '%s'", buf_ptr(tld->name)));
-            add_error_note(g, msg, other_tld->source_node, buf_sprintf("previous definition is here"));
+            add_error_note(g, msg, other_tld->source_node, buf_sprintf("previous definition here"));
             return;
         }
 
@@ -4176,7 +4176,7 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf
             if (existing_var->var_type == nullptr || !type_is_invalid(existing_var->var_type)) {
                 ErrorMsg *msg = add_node_error(g, source_node,
                         buf_sprintf("redeclaration of variable '%s'", buf_ptr(name)));
-                add_error_note(g, msg, existing_var->decl_node, buf_sprintf("previous declaration is here"));
+                add_error_note(g, msg, existing_var->decl_node, buf_sprintf("previous declaration here"));
             }
             variable_entry->var_type = g->builtin_types.entry_invalid;
         } else {
@@ -4205,7 +4205,7 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf
                         if (want_err_msg) {
                             ErrorMsg *msg = add_node_error(g, source_node,
                                     buf_sprintf("redefinition of '%s'", buf_ptr(name)));
-                            add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition is here"));
+                            add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition here"));
                         }
                         variable_entry->var_type = g->builtin_types.entry_invalid;
                     }