introduce [*] for unknown length pointers

See #770

Currently it does not have any different behavior than `*`
but it is now recommended to use `[*]` for unknown length
pointers to be future-proof.

Instead of [ * ] being separate tokens as the proposal
suggested, this commit implements `[*]` as a single token.

4 files changed, 51 insertions, 8 deletions

diff --git a/std/cstr.zig b/std/cstr.zig
index dfbfb8047f..d60adf8faa 100644
--- a/std/cstr.zig
+++ b/std/cstr.zig
@@ -9,13 +9,13 @@ pub const line_sep = switch (builtin.os) {
     else => "\n",
 };
 
-pub fn len(ptr: *const u8) usize {
+pub fn len(ptr: [*]const u8) usize {
     var count: usize = 0;
     while (ptr[count] != 0) : (count += 1) {}
     return count;
 }
 
-pub fn cmp(a: *const u8, b: *const u8) i8 {
+pub fn cmp(a: [*]const u8, b: [*]const u8) i8 {
     var index: usize = 0;
     while (a[index] == b[index] and a[index] != 0) : (index += 1) {}
     if (a[index] > b[index]) {
@@ -27,11 +27,11 @@ pub fn cmp(a: *const u8, b: *const u8) i8 {
     }
 }
 
-pub fn toSliceConst(str: *const u8) []const u8 {
+pub fn toSliceConst(str: [*]const u8) []const u8 {
     return str[0..len(str)];
 }
 
-pub fn toSlice(str: *u8) []u8 {
+pub fn toSlice(str: [*]u8) []u8 {
     return str[0..len(str)];
 }
 
diff --git a/std/zig/parse.zig b/std/zig/parse.zig
index 6adcf34c95..7faca8e11b 100644
--- a/std/zig/parse.zig
+++ b/std/zig/parse.zig
@@ -3292,7 +3292,7 @@ fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {
         Token.Id.Minus => ast.Node.PrefixOp.Op{ .Negation = void{} },
         Token.Id.MinusPercent => ast.Node.PrefixOp.Op{ .NegationWrap = void{} },
         Token.Id.Ampersand => ast.Node.PrefixOp.Op{ .AddressOf = void{} },
-        Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.Node.PrefixOp.Op{
+        Token.Id.Asterisk, Token.Id.AsteriskAsterisk, Token.Id.BracketStarBracket => ast.Node.PrefixOp.Op{
             .PtrType = ast.Node.PrefixOp.PtrInfo{
                 .align_info = null,
                 .const_token = null,
diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig
index bad677580c..c28a70b770 100644
--- a/std/zig/parser_test.zig
+++ b/std/zig/parser_test.zig
@@ -1,3 +1,10 @@
+test "zig fmt: pointer of unknown length" {
+    try testCanonical(
+        \\fn foo(ptr: [*]u8) void {}
+        \\
+    );
+}
+
 test "zig fmt: spaces around slice operator" {
     try testCanonical(
         \\var a = b[c..d];
diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig
index 8378a9011d..b288a3adb7 100644
--- a/std/zig/tokenizer.zig
+++ b/std/zig/tokenizer.zig
@@ -143,6 +143,7 @@ pub const Token = struct {
         FloatLiteral,
         LineComment,
         DocComment,
+        BracketStarBracket,
         Keyword_align,
         Keyword_and,
         Keyword_asm,
@@ -263,6 +264,8 @@ pub const Tokenizer = struct {
         Period,
         Period2,
         SawAtSign,
+        LBracket,
+        LBracketStar,
     };
 
     pub fn next(self: *Tokenizer) Token {
@@ -325,9 +328,7 @@ pub const Tokenizer = struct {
                         break;
                     },
                     '[' => {
-                        result.id = Token.Id.LBracket;
-                        self.index += 1;
-                        break;
+                        state = State.LBracket;
                     },
                     ']' => {
                         result.id = Token.Id.RBracket;
@@ -429,6 +430,28 @@ pub const Tokenizer = struct {
                     },
                 },
 
+                State.LBracket => switch (c) {
+                    '*' => {
+                        state = State.LBracketStar;
+                    },
+                    else => {
+                        result.id = Token.Id.LBracket;
+                        break;
+                    },
+                },
+
+                State.LBracketStar => switch (c) {
+                    ']' => {
+                        result.id = Token.Id.BracketStarBracket;
+                        self.index += 1;
+                        break;
+                    },
+                    else => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
+                },
+
                 State.Ampersand => switch (c) {
                     '=' => {
                         result.id = Token.Id.AmpersandEqual;
@@ -1008,6 +1031,7 @@ pub const Tokenizer = struct {
                 State.CharLiteralEscape2,
                 State.CharLiteralEnd,
                 State.StringLiteralBackslash,
+                State.LBracketStar,
                 => {
                     result.id = Token.Id.Invalid;
                 },
@@ -1024,6 +1048,9 @@ pub const Tokenizer = struct {
                 State.Slash => {
                     result.id = Token.Id.Slash;
                 },
+                State.LBracket => {
+                    result.id = Token.Id.LBracket;
+                },
                 State.Zero => {
                     result.id = Token.Id.IntegerLiteral;
                 },
@@ -1142,6 +1169,15 @@ test "tokenizer" {
     testTokenize("test", []Token.Id{Token.Id.Keyword_test});
 }
 
+test "tokenizer - unknown length pointer" {
+    testTokenize(
+        \\[*]u8
+    , []Token.Id{
+        Token.Id.BracketStarBracket,
+        Token.Id.Identifier,
+    });
+}
+
 test "tokenizer - char literal with hex escape" {
     testTokenize(
         \\'\x1b'