Merge remote-tracking branch 'origin/master' into pointer-reform

5 files changed, 214 insertions, 52 deletions

diff --git a/std/fmt/index.zig b/std/fmt/index.zig
index d494ec1fd0..0af772b7dc 100644
--- a/std/fmt/index.zig
+++ b/std/fmt/index.zig
@@ -26,6 +26,7 @@ pub fn format(context: var, comptime Errors: type, output: fn(@typeOf(context),
         Buf,
         BufWidth,
         Bytes,
+        BytesBase,
         BytesWidth,
     };
 
@@ -97,6 +98,7 @@ pub fn format(context: var, comptime Errors: type, output: fn(@typeOf(context),
                 },
                 'B' => {
                     width = 0;
+                    radix = 1000;
                     state = State.Bytes;
                 },
                 else => @compileError("Unknown format character: " ++ []u8{c}),
@@ -212,7 +214,24 @@ pub fn format(context: var, comptime Errors: type, output: fn(@typeOf(context),
             },
             State.Bytes => switch (c) {
                 '}' => {
-                    try formatBytes(args[next_arg], 0, context, Errors, output);
+                    try formatBytes(args[next_arg], 0, radix, context, Errors, output);
+                    next_arg += 1;
+                    state = State.Start;
+                    start_index = i + 1;
+                },
+                'i' => {
+                    radix = 1024;
+                    state = State.BytesBase;
+                },
+                '0' ... '9' => {
+                    width_start = i;
+                    state = State.BytesWidth;
+                },
+                else => @compileError("Unexpected character in format string: " ++ []u8{c}),
+            },
+            State.BytesBase => switch (c) {
+                '}' => {
+                    try formatBytes(args[next_arg], 0, radix, context, Errors, output);
                     next_arg += 1;
                     state = State.Start;
                     start_index = i + 1;
@@ -226,7 +245,7 @@ pub fn format(context: var, comptime Errors: type, output: fn(@typeOf(context),
             State.BytesWidth => switch (c) {
                 '}' => {
                     width = comptime (parseUnsigned(usize, fmt[width_start..i], 10) catch unreachable);
-                    try formatBytes(args[next_arg], width, context, Errors, output);
+                    try formatBytes(args[next_arg], width, radix, context, Errors, output);
                     next_arg += 1;
                     state = State.Start;
                     start_index = i + 1;
@@ -543,7 +562,7 @@ pub fn formatFloatDecimal(value: var, maybe_precision: ?usize, context: var, com
     }
 }
 
-pub fn formatBytes(value: var, width: ?usize,
+pub fn formatBytes(value: var, width: ?usize, comptime radix: usize,
     context: var, comptime Errors: type, output: fn(@typeOf(context), []const u8)Errors!void) Errors!void
 {
     if (value == 0) {
@@ -551,16 +570,26 @@ pub fn formatBytes(value: var, width: ?usize,
     }
 
     const mags = " KMGTPEZY";
-    const magnitude = math.min(math.log2(value) / 10, mags.len - 1);
-    const new_value = f64(value) / math.pow(f64, 1024, f64(magnitude));
+    const magnitude = switch (radix) {
+        1000 => math.min(math.log2(value) / comptime math.log2(1000), mags.len - 1),
+        1024 => math.min(math.log2(value) / 10, mags.len - 1),
+        else => unreachable,
+    };
+    const new_value = f64(value) / math.pow(f64, f64(radix), f64(magnitude));
     const suffix = mags[magnitude];
 
     try formatFloatDecimal(new_value, width, context, Errors, output);
 
-    if (suffix != ' ') {
-        try output(context, (&suffix)[0..1]);
+    if (suffix == ' ') {
+        return output(context, "B");
     }
-    return output(context, "B");
+
+    const buf = switch (radix) {
+        1000 => []u8 { suffix, 'B' },
+        1024 => []u8 { suffix, 'i', 'B' },
+        else => unreachable,
+    };
+    return output(context, buf);
 }
 
 pub fn formatInt(value: var, base: u8, uppercase: bool, width: usize,
@@ -773,41 +802,27 @@ test "parse unsigned comptime" {
 
 test "fmt.format" {
     {
-        var buf1: [32]u8 = undefined;
         const value: ?i32 = 1234;
-        const result = try bufPrint(buf1[0..], "nullable: {}\n", value);
-        assert(mem.eql(u8, result, "nullable: 1234\n"));
+        try testFmt("nullable: 1234\n", "nullable: {}\n", value);
     }
     {
-        var buf1: [32]u8 = undefined;
         const value: ?i32 = null;
-        const result = try bufPrint(buf1[0..], "nullable: {}\n", value);
-        assert(mem.eql(u8, result, "nullable: null\n"));
+        try testFmt("nullable: null\n", "nullable: {}\n", value);
     }
     {
-        var buf1: [32]u8 = undefined;
         const value: error!i32 = 1234;
-        const result = try bufPrint(buf1[0..], "error union: {}\n", value);
-        assert(mem.eql(u8, result, "error union: 1234\n"));
+        try testFmt("error union: 1234\n", "error union: {}\n", value);
     }
     {
-        var buf1: [32]u8 = undefined;
         const value: error!i32 = error.InvalidChar;
-        const result = try bufPrint(buf1[0..], "error union: {}\n", value);
-        assert(mem.eql(u8, result, "error union: error.InvalidChar\n"));
+        try testFmt("error union: error.InvalidChar\n", "error union: {}\n", value);
     }
     {
-        var buf1: [32]u8 = undefined;
         const value: u3 = 0b101;
-        const result = try bufPrint(buf1[0..], "u3: {}\n", value);
-        assert(mem.eql(u8, result, "u3: 5\n"));
-    }
-    {
-        var buf1: [32]u8 = undefined;
-        const value: usize = 63 * 1024 * 1024;
-        const result = try bufPrint(buf1[0..], "file size: {B}\n", value);
-        assert(mem.eql(u8, result, "file size: 63MB\n"));
+        try testFmt("u3: 5\n", "u3: {}\n", value);
     }
+    try testFmt("file size: 63MiB\n", "file size: {Bi}\n", usize(63 * 1024 * 1024));
+    try testFmt("file size: 66.06MB\n", "file size: {B2}\n", usize(63 * 1024 * 1024));
     {
         // Dummy field because of https://github.com/zig-lang/zig/issues/557.
         const Struct = struct {
@@ -1025,6 +1040,20 @@ test "fmt.format" {
     }
 }
 
+fn testFmt(expected: []const u8, comptime template: []const u8, args: ...) !void {
+    var buf: [100]u8 = undefined;
+    const result = try bufPrint(buf[0..], template, args);
+    if (mem.eql(u8, result, expected))
+        return;
+
+    std.debug.warn("\n====== expected this output: =========\n");
+    std.debug.warn("{}", expected);
+    std.debug.warn("\n======== instead found this: =========\n");
+    std.debug.warn("{}", result);
+    std.debug.warn("\n======================================\n");
+    return error.TestFailed;
+}
+
 pub fn trim(buf: []const u8) []const u8 {
     var start: usize = 0;
     while (start < buf.len and isWhiteSpace(buf[start])) : (start += 1) {}
diff --git a/std/segmented_list.zig b/std/segmented_list.zig
index 280c6a7f9b..098378de4e 100644
--- a/std/segmented_list.zig
+++ b/std/segmented_list.zig
@@ -294,21 +294,25 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type
 
                 return &it.list.dynamic_segments[it.shelf_index][it.box_index];
             }
+
+            pub fn set(it: &Iterator, index: usize) void {
+                it.index = index;
+                if (index < prealloc_item_count) return;
+                it.shelf_index = shelfIndex(index);
+                it.box_index = boxIndex(index, it.shelf_index);
+                it.shelf_size = shelfSize(it.shelf_index);
+            }
         };
 
         pub fn iterator(self: &Self, start_index: usize) Iterator {
             var it = Iterator{
                 .list = self,
-                .index = start_index,
+                .index = undefined,
                 .shelf_index = undefined,
                 .box_index = undefined,
                 .shelf_size = undefined,
             };
-            if (start_index >= prealloc_item_count) {
-                it.shelf_index = shelfIndex(start_index);
-                it.box_index = boxIndex(start_index, it.shelf_index);
-                it.shelf_size = shelfSize(it.shelf_index);
-            }
+            it.set(start_index);
             return it;
         }
     };
diff --git a/std/zig/parse.zig b/std/zig/parse.zig
index 04aeeccaeb..154cfc0e46 100644
--- a/std/zig/parse.zig
+++ b/std/zig/parse.zig
@@ -927,6 +927,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
                 continue;
             },
             State.Else => |dest| {
+                const old_index = tok_it.index;
+                var need_index_restore = false;
+                while (try eatLineComment(arena, &tok_it, &tree)) |_| {
+                    need_index_restore = true;
+                }
                 if (eatToken(&tok_it, &tree, Token.Id.Keyword_else)) |else_token| {
                     const node = try arena.construct(ast.Node.Else{
                         .base = ast.Node{ .id = ast.Node.Id.Else },
@@ -940,6 +945,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
                     try stack.append(State{ .Payload = OptionalCtx{ .Optional = &node.payload } });
                     continue;
                 } else {
+                    if (need_index_restore) {
+                        tok_it.set(old_index);
+                    }
                     continue;
                 }
             },
@@ -1297,7 +1305,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
             },
 
             State.SwitchCaseCommaOrEnd => |list_state| {
-                switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RParen)) {
+                switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
                     ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
                         (list_state.ptr).* = end;
                         continue;
diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig
index f95ecfaee3..094c4d51f7 100644
--- a/std/zig/parser_test.zig
+++ b/std/zig/parser_test.zig
@@ -1,3 +1,95 @@
+test "zig fmt: comment after if before another if" {
+    try testCanonical(
+        \\test "aoeu" {
+        \\    if (x) {
+        \\        foo();
+        \\    }
+        \\    // comment
+        \\    if (x) {
+        \\        bar();
+        \\    }
+        \\}
+        \\
+    );
+}
+
+test "zig fmt: line comment between if block and else keyword" {
+    try testTransform(
+        \\test "aoeu" {
+        \\    // cexp(finite|nan +- i inf|nan) = nan + i nan
+        \\    if ((hx & 0x7fffffff) != 0x7f800000) {
+        \\        return Complex(f32).new(y - y, y - y);
+        \\    }
+        \\    // cexp(-inf +- i inf|nan) = 0 + i0
+        \\    else if (hx & 0x80000000 != 0) {
+        \\        return Complex(f32).new(0, 0);
+        \\    }
+        \\    // cexp(+inf +- i inf|nan) = inf + i nan
+        \\    // another comment
+        \\    else {
+        \\        return Complex(f32).new(x, y - y);
+        \\    }
+        \\}
+    ,
+        \\test "aoeu" {
+        \\    // cexp(finite|nan +- i inf|nan) = nan + i nan
+        \\    if ((hx & 0x7fffffff) != 0x7f800000) {
+        \\        return Complex(f32).new(y - y, y - y);
+        \\    } // cexp(-inf +- i inf|nan) = 0 + i0
+        \\    else if (hx & 0x80000000 != 0) {
+        \\        return Complex(f32).new(0, 0);
+        \\    } // cexp(+inf +- i inf|nan) = inf + i nan
+        \\    // another comment
+        \\    else {
+        \\        return Complex(f32).new(x, y - y);
+        \\    }
+        \\}
+        \\
+    );
+}
+
+test "zig fmt: same line comments in expression" {
+    try testCanonical(
+        \\test "aoeu" {
+        \\    const x = ( // a
+        \\        0 // b
+        \\    ); // c
+        \\}
+        \\
+    );
+}
+
+test "zig fmt: add comma on last switch prong" {
+    try testTransform(
+        \\test "aoeu" {
+        \\switch (self.init_arg_expr) {
+        \\    InitArg.Type => |t| { },
+        \\    InitArg.None,
+        \\    InitArg.Enum => { }
+        \\}
+        \\ switch (self.init_arg_expr) {
+        \\     InitArg.Type => |t| { },
+        \\     InitArg.None,
+        \\     InitArg.Enum => { }//line comment
+        \\ }
+        \\}
+    ,
+        \\test "aoeu" {
+        \\    switch (self.init_arg_expr) {
+        \\        InitArg.Type => |t| {},
+        \\        InitArg.None,
+        \\        InitArg.Enum => {},
+        \\    }
+        \\    switch (self.init_arg_expr) {
+        \\        InitArg.Type => |t| {},
+        \\        InitArg.None,
+        \\        InitArg.Enum => {}, //line comment
+        \\    }
+        \\}
+        \\
+    );
+}
+
 test "zig fmt: same-line doc comment on variable declaration" {
     try testTransform(
         \\pub const MAP_ANONYMOUS = 0x1000; /// allocated from memory, swap space
diff --git a/std/zig/render.zig b/std/zig/render.zig
index 52bc074d5d..f240be8716 100644
--- a/std/zig/render.zig
+++ b/std/zig/render.zig
@@ -81,7 +81,7 @@ fn renderTopLevelDecl(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, i
             }
             try stream.print("{}: ", tree.tokenSlice(field.name_token));
             try renderExpression(allocator, stream, tree, indent, field.type_expr);
-            try renderToken(tree, stream, field.lastToken() + 1, indent, true);
+            try renderToken(tree, stream, field.lastToken() + 1, indent, true, true);
         },
         ast.Node.Id.UnionTag => {
             const tag = @fieldParentPtr(ast.Node.UnionTag, "base", decl);
@@ -514,9 +514,9 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
         ast.Node.Id.GroupedExpression => {
             const grouped_expr = @fieldParentPtr(ast.Node.GroupedExpression, "base", base);
 
-            try stream.write("(");
+            try renderToken(tree, stream, grouped_expr.lparen, indent, false, false);
             try renderExpression(allocator, stream, tree, indent, grouped_expr.expr);
-            try stream.write(")");
+            try renderToken(tree, stream, grouped_expr.rparen, indent, false, false);
         },
 
         ast.Node.Id.FieldInitializer => {
@@ -528,7 +528,7 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
 
         ast.Node.Id.IntegerLiteral => {
             const integer_literal = @fieldParentPtr(ast.Node.IntegerLiteral, "base", base);
-            try stream.print("{}", tree.tokenSlice(integer_literal.token));
+            try renderToken(tree, stream, integer_literal.token, indent, false, false);
         },
         ast.Node.Id.FloatLiteral => {
             const float_literal = @fieldParentPtr(ast.Node.FloatLiteral, "base", base);
@@ -536,7 +536,7 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
         },
         ast.Node.Id.StringLiteral => {
             const string_literal = @fieldParentPtr(ast.Node.StringLiteral, "base", base);
-            try stream.print("{}", tree.tokenSlice(string_literal.token));
+            try renderToken(tree, stream, string_literal.token, indent, false, false);
         },
         ast.Node.Id.CharLiteral => {
             const char_literal = @fieldParentPtr(ast.Node.CharLiteral, "base", base);
@@ -830,7 +830,20 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
             }
 
             try renderExpression(allocator, stream, tree, indent, switch_case.expr);
-            try renderToken(tree, stream, switch_case.lastToken() + 1, indent, true);
+            {
+                // Handle missing comma after last switch case
+                var index = switch_case.lastToken() + 1;
+                switch (tree.tokens.at(index).id) {
+                    Token.Id.RBrace => {
+                        try stream.write(",");
+                    },
+                    Token.Id.LineComment => {
+                        try stream.write(", ");
+                        try renderToken(tree, stream, index, indent, true, true);
+                    },
+                    else => try renderToken(tree, stream, index, indent, true, true),
+                }
+            }
         },
         ast.Node.Id.SwitchElse => {
             const switch_else = @fieldParentPtr(ast.Node.SwitchElse, "base", base);
@@ -838,6 +851,15 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
         },
         ast.Node.Id.Else => {
             const else_node = @fieldParentPtr(ast.Node.Else, "base", base);
+
+            var prev_tok_index = else_node.else_token - 1;
+            while (tree.tokens.at(prev_tok_index).id == Token.Id.LineComment) : (prev_tok_index -= 1) { }
+            prev_tok_index += 1;
+            while (prev_tok_index < else_node.else_token) : (prev_tok_index += 1) {
+                try stream.print("{}\n", tree.tokenSlice(prev_tok_index));
+                try stream.writeByteNTimes(' ', indent);
+            }
+
             try stream.print("{}", tree.tokenSlice(else_node.else_token));
 
             const block_body = switch (else_node.body.id) {
@@ -959,7 +981,7 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
             try stream.print("{} (", tree.tokenSlice(if_node.if_token));
 
             try renderExpression(allocator, stream, tree, indent, if_node.condition);
-            try renderToken(tree, stream, if_node.condition.lastToken() + 1, indent, false);
+            try renderToken(tree, stream, if_node.condition.lastToken() + 1, indent, false, true);
 
             if (if_node.payload) |payload| {
                 try renderExpression(allocator, stream, tree, indent, payload);
@@ -1118,11 +1140,11 @@ fn renderExpression(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, ind
 
 fn renderVarDecl(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, indent: usize, var_decl: &ast.Node.VarDecl) (@typeOf(stream).Child.Error || Error)!void {
     if (var_decl.visib_token) |visib_token| {
-        try stream.print("{} ", tree.tokenSlice(visib_token));
+        try renderToken(tree, stream, visib_token, indent, false, true);
     }
 
     if (var_decl.extern_export_token) |extern_export_token| {
-        try stream.print("{} ", tree.tokenSlice(extern_export_token));
+        try renderToken(tree, stream, extern_export_token, indent, false, true);
 
         if (var_decl.lib_name) |lib_name| {
             try renderExpression(allocator, stream, tree, indent, lib_name);
@@ -1131,10 +1153,11 @@ fn renderVarDecl(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, indent
     }
 
     if (var_decl.comptime_token) |comptime_token| {
-        try stream.print("{} ", tree.tokenSlice(comptime_token));
+        try renderToken(tree, stream, comptime_token, indent, false, true);
     }
 
-    try stream.print("{} {}", tree.tokenSlice(var_decl.mut_token), tree.tokenSlice(var_decl.name_token));
+    try renderToken(tree, stream, var_decl.mut_token, indent, false, true);
+    try renderToken(tree, stream, var_decl.name_token, indent, false, false);
 
     if (var_decl.type_node) |type_node| {
         try stream.write(": ");
@@ -1153,14 +1176,14 @@ fn renderVarDecl(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, indent
         try renderExpression(allocator, stream, tree, indent, init_node);
     }
 
-    try renderToken(tree, stream, var_decl.semicolon_token, indent, true);
+    try renderToken(tree, stream, var_decl.semicolon_token, indent, true, false);
 }
 
 fn maybeRenderSemicolon(stream: var, tree: &ast.Tree, indent: usize, base: &ast.Node) (@typeOf(stream).Child.Error || Error)!void {
     if (base.requireSemiColon()) {
         const semicolon_index = base.lastToken() + 1;
         assert(tree.tokens.at(semicolon_index).id == Token.Id.Semicolon);
-        try renderToken(tree, stream, semicolon_index, indent, true);
+        try renderToken(tree, stream, semicolon_index, indent, true, true);
     }
 }
 
@@ -1195,7 +1218,7 @@ fn renderStatement(allocator: &mem.Allocator, stream: var, tree: &ast.Tree, inde
     }
 }
 
-fn renderToken(tree: &ast.Tree, stream: var, token_index: ast.TokenIndex, indent: usize, line_break: bool) (@typeOf(stream).Child.Error || Error)!void {
+fn renderToken(tree: &ast.Tree, stream: var, token_index: ast.TokenIndex, indent: usize, line_break: bool, space: bool) (@typeOf(stream).Child.Error || Error)!void {
     const token = tree.tokens.at(token_index);
     try stream.write(tree.tokenSlicePtr(token));
 
@@ -1206,13 +1229,19 @@ fn renderToken(tree: &ast.Tree, stream: var, token_index: ast.TokenIndex, indent
             try stream.print(" {}", tree.tokenSlicePtr(next_token));
             if (!line_break) {
                 try stream.write("\n");
-                try stream.writeByteNTimes(' ', indent + indent_delta);
+
+                const after_comment_token = tree.tokens.at(token_index + 2);
+                const next_line_indent = switch (after_comment_token.id) {
+                    Token.Id.RParen, Token.Id.RBrace, Token.Id.RBracket => indent,
+                    else => indent + indent_delta,
+                };
+                try stream.writeByteNTimes(' ', next_line_indent);
                 return;
             }
         }
     }
 
-    if (!line_break) {
+    if (!line_break and space) {
         try stream.writeByte(' ');
     }
 }