std.zig.tokenizer Tokens now don't contain a line and column field.

* Instead, this information is optained by asking the tokenizer.
* getTokenLocation takes a start_index, so relative loc can be optained

2 files changed, 25 insertions, 42 deletions

diff --git a/std/zig/parser.zig b/std/zig/parser.zig
index 8a4999a4ee..f7db8fd93d 100644
--- a/std/zig/parser.zig
+++ b/std/zig/parser.zig
@@ -1571,12 +1571,12 @@ pub const Parser = struct {
     }
 
     fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
-        const loc = self.tokenizer.getTokenLocation(token);
-        warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);
+        const loc = self.tokenizer.getTokenLocation(0, token);
+        warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
         warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
         {
             var i: usize = 0;
-            while (i < token.column) : (i += 1) {
+            while (i < loc.column) : (i += 1) {
                 warn(" ");
             }
         }
@@ -1679,9 +1679,8 @@ pub const Parser = struct {
                     try stack.append(RenderState {
                         .Text = blk: {
                             const prev_node = root_node.decls.at(i - 1);
-                            const prev_line_index = prev_node.lastToken().line;
-                            const this_line_index = decl.firstToken().line;
-                            if (this_line_index - prev_line_index >= 2) {
+                            const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken());
+                            if (loc.line >= 2) {
                                 break :blk "\n\n";
                             }
                             break :blk "\n";
@@ -1858,10 +1857,9 @@ pub const Parser = struct {
                                 try stack.append(RenderState {
                                     .Text = blk: {
                                         if (i != 0) {
-                                            const prev_statement_node = block.statements.items[i - 1];
-                                            const prev_line_index = prev_statement_node.lastToken().line;
-                                            const this_line_index = statement_node.firstToken().line;
-                                            if (this_line_index - prev_line_index >= 2) {
+                                            const prev_node = block.statements.items[i - 1];
+                                            const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken());
+                                            if (loc.line >= 2) {
                                                 break :blk "\n\n";
                                             }
                                         }
@@ -2083,9 +2081,8 @@ pub const Parser = struct {
                                 .Text = blk: {
                                     if (i != 0) {
                                         const prev_node = fields_and_decls[i - 1];
-                                        const prev_line_index = prev_node.lastToken().line;
-                                        const this_line_index = node.firstToken().line;
-                                        if (this_line_index - prev_line_index >= 2) {
+                                        const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken());
+                                        if (loc.line >= 2) {
                                             break :blk "\n\n";
                                         }
                                     }
diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig
index 2e40999920..3427678341 100644
--- a/std/zig/tokenizer.zig
+++ b/std/zig/tokenizer.zig
@@ -5,8 +5,6 @@ pub const Token = struct {
     id: Id,
     start: usize,
     end: usize,
-    line: usize,
-    column: usize,
 
     const KeywordId = struct {
         bytes: []const u8,
@@ -180,28 +178,34 @@ pub const Token = struct {
 pub const Tokenizer = struct {
     buffer: []const u8,
     index: usize,
-    line: usize,
-    column: usize,
     pending_invalid_token: ?Token,
 
-    pub const LineLocation = struct {
+    pub const Location = struct {
+        line: usize,
+        column: usize,
         line_start: usize,
         line_end: usize,
     };
 
-    pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {
-        var loc = LineLocation {
-            .line_start = 0,
+    pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location {
+        var loc = Location {
+            .line = 0,
+            .column = 0,
+            .line_start = start_index,
             .line_end = self.buffer.len,
         };
-        for (self.buffer) |c, i| {
-            if (i == token.start) {
-                loc.line_end = i;
+        for (self.buffer[start_index..]) |c, i| {
+            if (i + start_index == token.start) {
+                loc.line_end = i + start_index;
                 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
                 return loc;
             }
             if (c == '\n') {
+                loc.line += 1;
+                loc.column = 0;
                 loc.line_start = i + 1;
+            } else {
+                loc.column += 1;
             }
         }
         return loc;
@@ -216,8 +220,6 @@ pub const Tokenizer = struct {
         return Tokenizer {
             .buffer = buffer,
             .index = 0,
-            .line = 0,
-            .column = 0,
             .pending_invalid_token = null,
         };
     }
@@ -277,8 +279,6 @@ pub const Tokenizer = struct {
             .id = Token.Id.Eof,
             .start = self.index,
             .end = undefined,
-            .line = self.line,
-            .column = self.column,
         };
         while (self.index < self.buffer.len) : (self.index += 1) {
             const c = self.buffer[self.index];
@@ -286,12 +286,9 @@ pub const Tokenizer = struct {
                 State.Start => switch (c) {
                     ' ' => {
                         result.start = self.index + 1;
-                        result.column += 1;
                     },
                     '\n' => {
                         result.start = self.index + 1;
-                        result.line += 1;
-                        result.column = 0;
                     },
                     'c' => {
                         state = State.C;
@@ -977,15 +974,6 @@ pub const Tokenizer = struct {
             }
         }
 
-        for (self.buffer[start_index..self.index]) |c| {
-            if (c == '\n') {
-                self.line += 1;
-                self.column = 0;
-            } else {
-                self.column += 1;
-            }
-        }
-
         if (result.id == Token.Id.Eof) {
             if (self.pending_invalid_token) |token| {
                 self.pending_invalid_token = null;
@@ -1009,8 +997,6 @@ pub const Tokenizer = struct {
             .id = Token.Id.Invalid,
             .start = self.index,
             .end = self.index + invalid_length,
-            .line = self.line,
-            .column = self.column,
         };
     }