Merge pull request #4053 from ziglang/test-run-translated-c

add test harness for "run translated C" tests

4 files changed, 521 insertions, 0 deletions

diff --git a/lib/std/build/check_file.zig b/lib/std/build/check_file.zig
new file mode 100644
index 0000000000..782cdb5600
--- /dev/null
+++ b/lib/std/build/check_file.zig
@@ -0,0 +1,52 @@
+const std = @import("../std.zig");
+const build = std.build;
+const Step = build.Step;
+const Builder = build.Builder;
+const fs = std.fs;
+const mem = std.mem;
+const warn = std.debug.warn;
+
+pub const CheckFileStep = struct {
+    step: Step,
+    builder: *Builder,
+    expected_matches: []const []const u8,
+    source: build.FileSource,
+    max_bytes: usize = 20 * 1024 * 1024,
+
+    pub fn create(
+        builder: *Builder,
+        source: build.FileSource,
+        expected_matches: []const []const u8,
+    ) *CheckFileStep {
+        const self = builder.allocator.create(CheckFileStep) catch unreachable;
+        self.* = CheckFileStep{
+            .builder = builder,
+            .step = Step.init("CheckFile", builder.allocator, make),
+            .source = source,
+            .expected_matches = expected_matches,
+        };
+        self.source.addStepDependencies(&self.step);
+        return self;
+    }
+
+    fn make(step: *Step) !void {
+        const self = @fieldParentPtr(CheckFileStep, "step", step);
+
+        const src_path = self.source.getPath(self.builder);
+        const contents = try fs.cwd().readFileAlloc(self.builder.allocator, src_path, self.max_bytes);
+
+        for (self.expected_matches) |expected_match| {
+            if (mem.indexOf(u8, contents, expected_match) == null) {
+                warn(
+                    \\
+                    \\========= Expected to find: ===================
+                    \\{}
+                    \\========= But file does not contain it: =======
+                    \\{}
+                    \\
+                , .{ expected_match, contents });
+                return error.TestFailed;
+            }
+        }
+    }
+};
diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig
new file mode 100644
index 0000000000..75809bde03
--- /dev/null
+++ b/lib/std/build/run.zig
@@ -0,0 +1,302 @@
+const std = @import("../std.zig");
+const builtin = std.builtin;
+const build = std.build;
+const Step = build.Step;
+const Builder = build.Builder;
+const LibExeObjStep = build.LibExeObjStep;
+const fs = std.fs;
+const mem = std.mem;
+const process = std.process;
+const ArrayList = std.ArrayList;
+const BufMap = std.BufMap;
+const Buffer = std.Buffer;
+const warn = std.debug.warn;
+
+const max_stdout_size = 1 * 1024 * 1024; // 1 MiB
+
+pub const RunStep = struct {
+    step: Step,
+    builder: *Builder,
+
+    /// See also addArg and addArgs to modifying this directly
+    argv: ArrayList(Arg),
+
+    /// Set this to modify the current working directory
+    cwd: ?[]const u8,
+
+    /// Override this field to modify the environment, or use setEnvironmentVariable
+    env_map: ?*BufMap,
+
+    stdout_action: StdIoAction = .inherit,
+    stderr_action: StdIoAction = .inherit,
+
+    expected_exit_code: u8 = 0,
+
+    pub const StdIoAction = union(enum) {
+        inherit,
+        ignore,
+        expect_exact: []const u8,
+        expect_matches: []const []const u8,
+    };
+
+    pub const Arg = union(enum) {
+        Artifact: *LibExeObjStep,
+        Bytes: []u8,
+    };
+
+    pub fn create(builder: *Builder, name: []const u8) *RunStep {
+        const self = builder.allocator.create(RunStep) catch unreachable;
+        self.* = RunStep{
+            .builder = builder,
+            .step = Step.init(name, builder.allocator, make),
+            .argv = ArrayList(Arg).init(builder.allocator),
+            .cwd = null,
+            .env_map = null,
+        };
+        return self;
+    }
+
+    pub fn addArtifactArg(self: *RunStep, artifact: *LibExeObjStep) void {
+        self.argv.append(Arg{ .Artifact = artifact }) catch unreachable;
+        self.step.dependOn(&artifact.step);
+    }
+
+    pub fn addArg(self: *RunStep, arg: []const u8) void {
+        self.argv.append(Arg{ .Bytes = self.builder.dupe(arg) }) catch unreachable;
+    }
+
+    pub fn addArgs(self: *RunStep, args: []const []const u8) void {
+        for (args) |arg| {
+            self.addArg(arg);
+        }
+    }
+
+    pub fn clearEnvironment(self: *RunStep) void {
+        const new_env_map = self.builder.allocator.create(BufMap) catch unreachable;
+        new_env_map.* = BufMap.init(self.builder.allocator);
+        self.env_map = new_env_map;
+    }
+
+    pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
+        const env_map = self.getEnvMap();
+
+        var key: []const u8 = undefined;
+        var prev_path: ?[]const u8 = undefined;
+        if (builtin.os == .windows) {
+            key = "Path";
+            prev_path = env_map.get(key);
+            if (prev_path == null) {
+                key = "PATH";
+                prev_path = env_map.get(key);
+            }
+        } else {
+            key = "PATH";
+            prev_path = env_map.get(key);
+        }
+
+        if (prev_path) |pp| {
+            const new_path = self.builder.fmt("{}" ++ [1]u8{fs.path.delimiter} ++ "{}", .{ pp, search_path });
+            env_map.set(key, new_path) catch unreachable;
+        } else {
+            env_map.set(key, search_path) catch unreachable;
+        }
+    }
+
+    pub fn getEnvMap(self: *RunStep) *BufMap {
+        return self.env_map orelse {
+            const env_map = self.builder.allocator.create(BufMap) catch unreachable;
+            env_map.* = process.getEnvMap(self.builder.allocator) catch unreachable;
+            self.env_map = env_map;
+            return env_map;
+        };
+    }
+
+    pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void {
+        const env_map = self.getEnvMap();
+        env_map.set(key, value) catch unreachable;
+    }
+
+    pub fn expectStdErrEqual(self: *RunStep, bytes: []const u8) void {
+        self.stderr_action = .{ .expect_exact = bytes };
+    }
+
+    pub fn expectStdOutEqual(self: *RunStep, bytes: []const u8) void {
+        self.stdout_action = .{ .expect_exact = bytes };
+    }
+
+    fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
+        return switch (action) {
+            .ignore => .Ignore,
+            .inherit => .Inherit,
+            .expect_exact, .expect_matches => .Pipe,
+        };
+    }
+
+    fn make(step: *Step) !void {
+        const self = @fieldParentPtr(RunStep, "step", step);
+
+        const cwd = if (self.cwd) |cwd| self.builder.pathFromRoot(cwd) else self.builder.build_root;
+
+        var argv_list = ArrayList([]const u8).init(self.builder.allocator);
+        for (self.argv.toSlice()) |arg| {
+            switch (arg) {
+                Arg.Bytes => |bytes| try argv_list.append(bytes),
+                Arg.Artifact => |artifact| {
+                    if (artifact.target.isWindows()) {
+                        // On Windows we don't have rpaths so we have to add .dll search paths to PATH
+                        self.addPathForDynLibs(artifact);
+                    }
+                    const executable_path = artifact.installed_path orelse artifact.getOutputPath();
+                    try argv_list.append(executable_path);
+                },
+            }
+        }
+
+        const argv = argv_list.toSliceConst();
+
+        const child = std.ChildProcess.init(argv, self.builder.allocator) catch unreachable;
+        defer child.deinit();
+
+        child.cwd = cwd;
+        child.env_map = self.env_map orelse self.builder.env_map;
+
+        child.stdin_behavior = .Ignore;
+        child.stdout_behavior = stdIoActionToBehavior(self.stdout_action);
+        child.stderr_behavior = stdIoActionToBehavior(self.stderr_action);
+
+        child.spawn() catch |err| {
+            warn("Unable to spawn {}: {}\n", .{ argv[0], @errorName(err) });
+            return err;
+        };
+
+        var stdout = Buffer.initNull(self.builder.allocator);
+        var stderr = Buffer.initNull(self.builder.allocator);
+
+        // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O).
+
+        switch (self.stdout_action) {
+            .expect_exact, .expect_matches => {
+                var stdout_file_in_stream = child.stdout.?.inStream();
+                stdout_file_in_stream.stream.readAllBuffer(&stdout, max_stdout_size) catch unreachable;
+            },
+            .inherit, .ignore => {},
+        }
+
+        switch (self.stdout_action) {
+            .expect_exact, .expect_matches => {
+                var stderr_file_in_stream = child.stderr.?.inStream();
+                stderr_file_in_stream.stream.readAllBuffer(&stderr, max_stdout_size) catch unreachable;
+            },
+            .inherit, .ignore => {},
+        }
+
+        const term = child.wait() catch |err| {
+            warn("Unable to spawn {}: {}\n", .{ argv[0], @errorName(err) });
+            return err;
+        };
+
+        switch (term) {
+            .Exited => |code| {
+                if (code != self.expected_exit_code) {
+                    warn("The following command exited with error code {} (expected {}):\n", .{
+                        code,
+                        self.expected_exit_code,
+                    });
+                    printCmd(cwd, argv);
+                    return error.UncleanExit;
+                }
+            },
+            else => {
+                warn("The following command terminated unexpectedly:\n", .{});
+                printCmd(cwd, argv);
+                return error.UncleanExit;
+            },
+        }
+
+        switch (self.stderr_action) {
+            .inherit, .ignore => {},
+            .expect_exact => |expected_bytes| {
+                if (!mem.eql(u8, expected_bytes, stderr.toSliceConst())) {
+                    warn(
+                        \\
+                        \\========= Expected this stderr: =========
+                        \\{}
+                        \\========= But found: ====================
+                        \\{}
+                        \\
+                    , .{ expected_bytes, stderr.toSliceConst() });
+                    printCmd(cwd, argv);
+                    return error.TestFailed;
+                }
+            },
+            .expect_matches => |matches| for (matches) |match| {
+                if (mem.indexOf(u8, stderr.toSliceConst(), match) == null) {
+                    warn(
+                        \\
+                        \\========= Expected to find in stderr: =========
+                        \\{}
+                        \\========= But stderr does not contain it: =====
+                        \\{}
+                        \\
+                    , .{ match, stderr.toSliceConst() });
+                    printCmd(cwd, argv);
+                    return error.TestFailed;
+                }
+            },
+        }
+
+        switch (self.stdout_action) {
+            .inherit, .ignore => {},
+            .expect_exact => |expected_bytes| {
+                if (!mem.eql(u8, expected_bytes, stdout.toSliceConst())) {
+                    warn(
+                        \\
+                        \\========= Expected this stdout: =========
+                        \\{}
+                        \\========= But found: ====================
+                        \\{}
+                        \\
+                    , .{ expected_bytes, stdout.toSliceConst() });
+                    printCmd(cwd, argv);
+                    return error.TestFailed;
+                }
+            },
+            .expect_matches => |matches| for (matches) |match| {
+                if (mem.indexOf(u8, stdout.toSliceConst(), match) == null) {
+                    warn(
+                        \\
+                        \\========= Expected to find in stdout: =========
+                        \\{}
+                        \\========= But stdout does not contain it: =====
+                        \\{}
+                        \\
+                    , .{ match, stdout.toSliceConst() });
+                    printCmd(cwd, argv);
+                    return error.TestFailed;
+                }
+            },
+        }
+    }
+
+    fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
+        if (cwd) |yes_cwd| warn("cd {} && ", .{yes_cwd});
+        for (argv) |arg| {
+            warn("{} ", .{arg});
+        }
+        warn("\n", .{});
+    }
+
+    fn addPathForDynLibs(self: *RunStep, artifact: *LibExeObjStep) void {
+        for (artifact.link_objects.toSliceConst()) |link_object| {
+            switch (link_object) {
+                .OtherStep => |other| {
+                    if (other.target.isWindows() and other.isDynamicLibrary()) {
+                        self.addPathDir(fs.path.dirname(other.getOutputPath()).?);
+                        self.addPathForDynLibs(other);
+                    }
+                },
+                else => {},
+            }
+        }
+    }
+};
diff --git a/lib/std/build/translate_c.zig b/lib/std/build/translate_c.zig
new file mode 100644
index 0000000000..7574dbe331
--- /dev/null
+++ b/lib/std/build/translate_c.zig
@@ -0,0 +1,73 @@
+const std = @import("../std.zig");
+const build = std.build;
+const Step = build.Step;
+const Builder = build.Builder;
+const WriteFileStep = build.WriteFileStep;
+const LibExeObjStep = build.LibExeObjStep;
+const CheckFileStep = build.CheckFileStep;
+const fs = std.fs;
+const mem = std.mem;
+
+pub const TranslateCStep = struct {
+    step: Step,
+    builder: *Builder,
+    source: build.FileSource,
+    output_dir: ?[]const u8,
+    out_basename: []const u8,
+
+    pub fn create(builder: *Builder, source: build.FileSource) *TranslateCStep {
+        const self = builder.allocator.create(TranslateCStep) catch unreachable;
+        self.* = TranslateCStep{
+            .step = Step.init("zig translate-c", builder.allocator, make),
+            .builder = builder,
+            .source = source,
+            .output_dir = null,
+            .out_basename = undefined,
+        };
+        source.addStepDependencies(&self.step);
+        return self;
+    }
+
+    /// Unless setOutputDir was called, this function must be called only in
+    /// the make step, from a step that has declared a dependency on this one.
+    /// To run an executable built with zig build, use `run`, or create an install step and invoke it.
+    pub fn getOutputPath(self: *TranslateCStep) []const u8 {
+        return fs.path.join(
+            self.builder.allocator,
+            &[_][]const u8{ self.output_dir.?, self.out_basename },
+        ) catch unreachable;
+    }
+
+    /// Creates a step to build an executable from the translated source.
+    pub fn addExecutable(self: *TranslateCStep) *LibExeObjStep {
+        return self.builder.addExecutableSource("translated_c", @as(build.FileSource, .{ .translate_c = self }));
+    }
+
+    pub fn addCheckFile(self: *TranslateCStep, expected_matches: []const []const u8) *CheckFileStep {
+        return CheckFileStep.create(self.builder, .{ .translate_c = self }, expected_matches);
+    }
+
+    fn make(step: *Step) !void {
+        const self = @fieldParentPtr(TranslateCStep, "step", step);
+
+        const argv = [_][]const u8{
+            self.builder.zig_exe,
+            "translate-c",
+            "-lc",
+            "--cache",
+            "on",
+            self.source.getPath(self.builder),
+        };
+
+        const output_path_nl = try self.builder.exec(&argv);
+        const output_path = mem.trimRight(u8, output_path_nl, "\r\n");
+
+        self.out_basename = fs.path.basename(output_path);
+        if (self.output_dir) |output_dir| {
+            const full_dest = try fs.path.join(self.builder.allocator, &[_][]const u8{ output_dir, self.out_basename });
+            try self.builder.updateFile(output_path, full_dest);
+        } else {
+            self.output_dir = fs.path.dirname(output_path).?;
+        }
+    }
+};
diff --git a/lib/std/build/write_file.zig b/lib/std/build/write_file.zig
new file mode 100644
index 0000000000..13d131ac61
--- /dev/null
+++ b/lib/std/build/write_file.zig
@@ -0,0 +1,94 @@
+const std = @import("../std.zig");
+const build = @import("../build.zig");
+const Step = build.Step;
+const Builder = build.Builder;
+const fs = std.fs;
+const warn = std.debug.warn;
+const ArrayList = std.ArrayList;
+
+pub const WriteFileStep = struct {
+    step: Step,
+    builder: *Builder,
+    output_dir: []const u8,
+    files: ArrayList(File),
+
+    pub const File = struct {
+        basename: []const u8,
+        bytes: []const u8,
+    };
+
+    pub fn init(builder: *Builder) WriteFileStep {
+        return WriteFileStep{
+            .builder = builder,
+            .step = Step.init("writefile", builder.allocator, make),
+            .files = ArrayList(File).init(builder.allocator),
+            .output_dir = undefined,
+        };
+    }
+
+    pub fn add(self: *WriteFileStep, basename: []const u8, bytes: []const u8) void {
+        self.files.append(.{ .basename = basename, .bytes = bytes }) catch unreachable;
+    }
+
+    /// Unless setOutputDir was called, this function must be called only in
+    /// the make step, from a step that has declared a dependency on this one.
+    /// To run an executable built with zig build, use `run`, or create an install step and invoke it.
+    pub fn getOutputPath(self: *WriteFileStep, basename: []const u8) []const u8 {
+        return fs.path.join(
+            self.builder.allocator,
+            &[_][]const u8{ self.output_dir, basename },
+        ) catch unreachable;
+    }
+
+    fn make(step: *Step) !void {
+        const self = @fieldParentPtr(WriteFileStep, "step", step);
+
+        // The cache is used here not really as a way to speed things up - because writing
+        // the data to a file would probably be very fast - but as a way to find a canonical
+        // location to put build artifacts.
+
+        // If, for example, a hard-coded path was used as the location to put WriteFileStep
+        // files, then two WriteFileSteps executing in parallel might clobber each other.
+
+        // TODO port the cache system from stage1 to zig std lib. Until then we use blake2b
+        // directly and construct the path, and no "cache hit" detection happens; the files
+        // are always written.
+        var hash = std.crypto.Blake2b384.init();
+
+        // Random bytes to make WriteFileStep unique. Refresh this with
+        // new random bytes when WriteFileStep implementation is modified
+        // in a non-backwards-compatible way.
+        hash.update("eagVR1dYXoE7ARDP");
+        for (self.files.toSliceConst()) |file| {
+            hash.update(file.basename);
+            hash.update(file.bytes);
+            hash.update("|");
+        }
+        var digest: [48]u8 = undefined;
+        hash.final(&digest);
+        var hash_basename: [64]u8 = undefined;
+        fs.base64_encoder.encode(&hash_basename, &digest);
+        self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
+            self.builder.cache_root,
+            "o",
+            &hash_basename,
+        });
+        // TODO replace with something like fs.makePathAndOpenDir
+        fs.makePath(self.builder.allocator, self.output_dir) catch |err| {
+            warn("unable to make path {}: {}\n", .{ self.output_dir, @errorName(err) });
+            return err;
+        };
+        var dir = try fs.cwd().openDirTraverse(self.output_dir);
+        defer dir.close();
+        for (self.files.toSliceConst()) |file| {
+            dir.writeFile(file.basename, file.bytes) catch |err| {
+                warn("unable to write {} into {}: {}\n", .{
+                    file.basename,
+                    self.output_dir,
+                    @errorName(err),
+                });
+                return err;
+            };
+        }
+    }
+};