stage2: improve compile errors from tokenizer

In order to not regress the quality of compile errors, some improvements
had to be made.

 * std.zig.parseCharLiteral is improved to return more detailed parse
   failure information.
 * tokenizer is improved to handle null bytes in the middle of strings,
   character literals, and line comments.
 * validating how many unicode escape digits in string literals is moved
   to std.zig.parseStringLiteral rather than handled in the tokenizer.
 * when a tokenizer error occurs, if the reported token is the 'invalid'
   tag, an error note is added to point to the invalid byte location.
   Further improvements would be:
   - Mention the expected set of allowed bytes at this location.
   - Display the invalid byte (if printable, print it, otherwise
     escape-print it).

1 files changed, 20 insertions, 0 deletions

diff --git a/src/main.zig b/src/main.zig
index 760ad79a3a..f4ca11a96a 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -3380,6 +3380,7 @@ fn printErrMsgToStdErr(
     color: Color,
 ) !void {
     const lok_token = parse_error.token;
+    const token_tags = tree.tokens.items(.tag);
     const start_loc = tree.tokenLocation(0, lok_token);
     const source_line = tree.source[start_loc.line_start..start_loc.line_end];
 
@@ -3389,6 +3390,24 @@ fn printErrMsgToStdErr(
     try tree.renderError(parse_error, writer);
     const text = text_buf.items;
 
+    var notes_buffer: [1]Compilation.AllErrors.Message = undefined;
+    var notes_len: usize = 0;
+
+    if (token_tags[parse_error.token] == .invalid) {
+        const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token).len);
+        notes_buffer[notes_len] = .{
+            .src = .{
+                .src_path = path,
+                .msg = "invalid byte here",
+                .byte_offset = @intCast(u32, start_loc.line_start) + bad_off,
+                .line = @intCast(u32, start_loc.line),
+                .column = @intCast(u32, start_loc.column) + bad_off,
+                .source_line = source_line,
+            },
+        };
+        notes_len += 1;
+    }
+
     const message: Compilation.AllErrors.Message = .{
         .src = .{
             .src_path = path,
@@ -3397,6 +3416,7 @@ fn printErrMsgToStdErr(
             .line = @intCast(u32, start_loc.line),
             .column = @intCast(u32, start_loc.column),
             .source_line = source_line,
+            .notes = notes_buffer[0..notes_len],
         },
     };