aboutsummaryrefslogtreecommitdiff
path: root/lib/std/Build
diff options
context:
space:
mode:
Diffstat (limited to 'lib/std/Build')
-rw-r--r--lib/std/Build/Cache.zig1253
-rw-r--r--lib/std/Build/Cache/DepTokenizer.zig1069
-rw-r--r--lib/std/Build/CompileStep.zig89
-rw-r--r--lib/std/Build/ConfigHeaderStep.zig249
-rw-r--r--lib/std/Build/OptionsStep.zig34
-rw-r--r--lib/std/Build/RunStep.zig139
-rw-r--r--lib/std/Build/TranslateCStep.zig11
-rw-r--r--lib/std/Build/WriteFileStep.zig14
8 files changed, 2715 insertions, 143 deletions
diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig
new file mode 100644
index 0000000000..c459fca633
--- /dev/null
+++ b/lib/std/Build/Cache.zig
@@ -0,0 +1,1253 @@
+//! Manages `zig-cache` directories.
+//! This is not a general-purpose cache. It is designed to be fast and simple,
+//! not to withstand attacks using specially-crafted input.
+
+pub const Directory = struct {
+ /// This field is redundant for operations that can act on the open directory handle
+ /// directly, but it is needed when passing the directory to a child process.
+ /// `null` means cwd.
+ path: ?[]const u8,
+ handle: std.fs.Dir,
+
+ pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
+ if (self.path) |p| {
+ // TODO clean way to do this with only 1 allocation
+ const part2 = try std.fs.path.join(allocator, paths);
+ defer allocator.free(part2);
+ return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
+ } else {
+ return std.fs.path.join(allocator, paths);
+ }
+ }
+
+ pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
+ if (self.path) |p| {
+ // TODO clean way to do this with only 1 allocation
+ const part2 = try std.fs.path.join(allocator, paths);
+ defer allocator.free(part2);
+ return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
+ } else {
+ return std.fs.path.joinZ(allocator, paths);
+ }
+ }
+
+ /// Whether or not the handle should be closed, or the path should be freed
+ /// is determined by usage, however this function is provided for convenience
+ /// if it happens to be what the caller needs.
+ pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
+ self.handle.close();
+ if (self.path) |p| gpa.free(p);
+ self.* = undefined;
+ }
+};
+
+gpa: Allocator,
+manifest_dir: fs.Dir,
+hash: HashHelper = .{},
+/// This value is accessed from multiple threads, protected by mutex.
+recent_problematic_timestamp: i128 = 0,
+mutex: std.Thread.Mutex = .{},
+
+/// A set of strings such as the zig library directory or project source root, which
+/// are stripped from the file paths before putting into the cache. They
+/// are replaced with single-character indicators. This is not to save
+/// space but to eliminate absolute file paths. This improves portability
+/// and usefulness of the cache for advanced use cases.
+prefixes_buffer: [4]Directory = undefined,
+prefixes_len: usize = 0,
+
+pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
+
+const Cache = @This();
+const std = @import("std");
+const builtin = @import("builtin");
+const crypto = std.crypto;
+const fs = std.fs;
+const assert = std.debug.assert;
+const testing = std.testing;
+const mem = std.mem;
+const fmt = std.fmt;
+const Allocator = std.mem.Allocator;
+const log = std.log.scoped(.cache);
+
+pub fn addPrefix(cache: *Cache, directory: Directory) void {
+ cache.prefixes_buffer[cache.prefixes_len] = directory;
+ cache.prefixes_len += 1;
+}
+
+/// Be sure to call `Manifest.deinit` after successful initialization.
+pub fn obtain(cache: *Cache) Manifest {
+ return Manifest{
+ .cache = cache,
+ .hash = cache.hash,
+ .manifest_file = null,
+ .manifest_dirty = false,
+ .hex_digest = undefined,
+ };
+}
+
+pub fn prefixes(cache: *const Cache) []const Directory {
+ return cache.prefixes_buffer[0..cache.prefixes_len];
+}
+
+const PrefixedPath = struct {
+ prefix: u8,
+ sub_path: []u8,
+};
+
+fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
+ const gpa = cache.gpa;
+ const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
+ errdefer gpa.free(resolved_path);
+ return findPrefixResolved(cache, resolved_path);
+}
+
+/// Takes ownership of `resolved_path` on success.
+fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
+ const gpa = cache.gpa;
+ const prefixes_slice = cache.prefixes();
+ var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
+ while (i < prefixes_slice.len) : (i += 1) {
+ const p = prefixes_slice[i].path.?;
+ if (mem.startsWith(u8, resolved_path, p)) {
+ // +1 to skip over the path separator here
+ const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
+ gpa.free(resolved_path);
+ return PrefixedPath{
+ .prefix = @intCast(u8, i),
+ .sub_path = sub_path,
+ };
+ }
+ }
+
+ return PrefixedPath{
+ .prefix = 0,
+ .sub_path = resolved_path,
+ };
+}
+
+/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
+pub const bin_digest_len = 16;
+pub const hex_digest_len = bin_digest_len * 2;
+pub const BinDigest = [bin_digest_len]u8;
+
+const manifest_file_size_max = 50 * 1024 * 1024;
+
+/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
+/// provides enough collision resistance for the Manifest use cases, while being one of our
+/// fastest options right now.
+pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
+
+/// Initial state, that can be copied.
+pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
+
+pub const File = struct {
+ prefixed_path: ?PrefixedPath,
+ max_file_size: ?usize,
+ stat: Stat,
+ bin_digest: BinDigest,
+ contents: ?[]const u8,
+
+ pub const Stat = struct {
+ inode: fs.File.INode,
+ size: u64,
+ mtime: i128,
+ };
+
+ pub fn deinit(self: *File, gpa: Allocator) void {
+ if (self.prefixed_path) |pp| {
+ gpa.free(pp.sub_path);
+ self.prefixed_path = null;
+ }
+ if (self.contents) |contents| {
+ gpa.free(contents);
+ self.contents = null;
+ }
+ self.* = undefined;
+ }
+};
+
+pub const HashHelper = struct {
+ hasher: Hasher = hasher_init,
+
+ /// Record a slice of bytes as an dependency of the process being cached
+ pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
+ hh.hasher.update(mem.asBytes(&bytes.len));
+ hh.hasher.update(bytes);
+ }
+
+ pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
+ hh.add(optional_bytes != null);
+ hh.addBytes(optional_bytes orelse return);
+ }
+
+ pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
+ hh.add(list_of_bytes.len);
+ for (list_of_bytes) |bytes| hh.addBytes(bytes);
+ }
+
+ /// Convert the input value into bytes and record it as a dependency of the process being cached.
+ pub fn add(hh: *HashHelper, x: anytype) void {
+ switch (@TypeOf(x)) {
+ std.builtin.Version => {
+ hh.add(x.major);
+ hh.add(x.minor);
+ hh.add(x.patch);
+ },
+ std.Target.Os.TaggedVersionRange => {
+ switch (x) {
+ .linux => |linux| {
+ hh.add(linux.range.min);
+ hh.add(linux.range.max);
+ hh.add(linux.glibc);
+ },
+ .windows => |windows| {
+ hh.add(windows.min);
+ hh.add(windows.max);
+ },
+ .semver => |semver| {
+ hh.add(semver.min);
+ hh.add(semver.max);
+ },
+ .none => {},
+ }
+ },
+ else => switch (@typeInfo(@TypeOf(x))) {
+ .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
+ else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
+ },
+ }
+ }
+
+ pub fn addOptional(hh: *HashHelper, optional: anytype) void {
+ hh.add(optional != null);
+ hh.add(optional orelse return);
+ }
+
+ /// Returns a hex encoded hash of the inputs, without modifying state.
+ pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
+ var copy = hh;
+ return copy.final();
+ }
+
+ pub fn peekBin(hh: HashHelper) BinDigest {
+ var copy = hh;
+ var bin_digest: BinDigest = undefined;
+ copy.hasher.final(&bin_digest);
+ return bin_digest;
+ }
+
+ /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
+ pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
+ var bin_digest: BinDigest = undefined;
+ hh.hasher.final(&bin_digest);
+
+ var out_digest: [hex_digest_len]u8 = undefined;
+ _ = std.fmt.bufPrint(
+ &out_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+ return out_digest;
+ }
+};
+
+pub const Lock = struct {
+ manifest_file: fs.File,
+
+ pub fn release(lock: *Lock) void {
+ if (builtin.os.tag == .windows) {
+ // Windows does not guarantee that locks are immediately unlocked when
+ // the file handle is closed. See LockFileEx documentation.
+ lock.manifest_file.unlock();
+ }
+
+ lock.manifest_file.close();
+ lock.* = undefined;
+ }
+};
+
+pub const Manifest = struct {
+ cache: *Cache,
+ /// Current state for incremental hashing.
+ hash: HashHelper,
+ manifest_file: ?fs.File,
+ manifest_dirty: bool,
+ /// Set this flag to true before calling hit() in order to indicate that
+ /// upon a cache hit, the code using the cache will not modify the files
+ /// within the cache directory. This allows multiple processes to utilize
+ /// the same cache directory at the same time.
+ want_shared_lock: bool = true,
+ have_exclusive_lock: bool = false,
+ // Indicate that we want isProblematicTimestamp to perform a filesystem write in
+ // order to obtain a problematic timestamp for the next call. Calls after that
+ // will then use the same timestamp, to avoid unnecessary filesystem writes.
+ want_refresh_timestamp: bool = true,
+ files: std.ArrayListUnmanaged(File) = .{},
+ hex_digest: [hex_digest_len]u8,
+ /// Populated when hit() returns an error because of one
+ /// of the files listed in the manifest.
+ failed_file_index: ?usize = null,
+ /// Keeps track of the last time we performed a file system write to observe
+ /// what time the file system thinks it is, according to its own granularity.
+ recent_problematic_timestamp: i128 = 0,
+
+ /// Add a file as a dependency of process being cached. When `hit` is
+ /// called, the file's contents will be checked to ensure that it matches
+ /// the contents from previous times.
+ ///
+ /// Max file size will be used to determine the amount of space the file contents
+ /// are allowed to take up in memory. If max_file_size is null, then the contents
+ /// will not be loaded into memory.
+ ///
+ /// Returns the index of the entry in the `files` array list. You can use it
+ /// to access the contents of the file after calling `hit()` like so:
+ ///
+ /// ```
+ /// var file_contents = cache_hash.files.items[file_index].contents.?;
+ /// ```
+ pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
+ assert(self.manifest_file == null);
+
+ const gpa = self.cache.gpa;
+ try self.files.ensureUnusedCapacity(gpa, 1);
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ self.files.addOneAssumeCapacity().* = .{
+ .prefixed_path = prefixed_path,
+ .contents = null,
+ .max_file_size = max_file_size,
+ .stat = undefined,
+ .bin_digest = undefined,
+ };
+
+ self.hash.add(prefixed_path.prefix);
+ self.hash.addBytes(prefixed_path.sub_path);
+
+ return self.files.items.len - 1;
+ }
+
+ pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
+ self.hash.add(optional_file_path != null);
+ const file_path = optional_file_path orelse return;
+ _ = try self.addFile(file_path, null);
+ }
+
+ pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
+ self.hash.add(list_of_files.len);
+ for (list_of_files) |file_path| {
+ _ = try self.addFile(file_path, null);
+ }
+ }
+
+ /// Check the cache to see if the input exists in it. If it exists, returns `true`.
+ /// A hex encoding of its hash is available by calling `final`.
+ ///
+ /// This function will also acquire an exclusive lock to the manifest file. This means
+ /// that a process holding a Manifest will block any other process attempting to
+ /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
+ /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
+ /// file to be locked in exclusive mode.
+ ///
+ /// The lock on the manifest file is released when `deinit` is called. As another
+ /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
+ /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
+ pub fn hit(self: *Manifest) !bool {
+ const gpa = self.cache.gpa;
+ assert(self.manifest_file == null);
+
+ self.failed_file_index = null;
+
+ const ext = ".txt";
+ var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
+
+ var bin_digest: BinDigest = undefined;
+ self.hash.hasher.final(&bin_digest);
+
+ _ = std.fmt.bufPrint(
+ &self.hex_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+
+ self.hash.hasher = hasher_init;
+ self.hash.hasher.update(&bin_digest);
+
+ mem.copy(u8, &manifest_file_path, &self.hex_digest);
+ manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
+
+ if (self.files.items.len == 0) {
+ // If there are no file inputs, we check if the manifest file exists instead of
+ // comparing the hashes on the files used for the cached item
+ while (true) {
+ if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .mode = .read_write,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ break;
+ } else |open_err| switch (open_err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
+ });
+ break;
+ },
+ error.FileNotFound => {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
+ .read = true,
+ .truncate = false,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.manifest_dirty = true;
+ self.have_exclusive_lock = true;
+ return false; // cache miss; exclusive lock already held
+ } else |err| switch (err) {
+ error.WouldBlock => continue,
+ else => |e| return e,
+ }
+ },
+ else => |e| return e,
+ }
+ }
+ } else {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
+ .read = true,
+ .truncate = false,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ } else |err| switch (err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
+ });
+ },
+ else => |e| return e,
+ }
+ }
+
+ self.want_refresh_timestamp = true;
+
+ const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
+ defer gpa.free(file_contents);
+
+ const input_file_count = self.files.items.len;
+ var any_file_changed = false;
+ var line_iter = mem.tokenize(u8, file_contents, "\n");
+ var idx: usize = 0;
+ while (line_iter.next()) |line| {
+ defer idx += 1;
+
+ const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
+ const new = try self.files.addOne(gpa);
+ new.* = .{
+ .prefixed_path = null,
+ .contents = null,
+ .max_file_size = null,
+ .stat = undefined,
+ .bin_digest = undefined,
+ };
+ break :blk new;
+ };
+
+ var iter = mem.tokenize(u8, line, " ");
+ const size = iter.next() orelse return error.InvalidFormat;
+ const inode = iter.next() orelse return error.InvalidFormat;
+ const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
+ const digest_str = iter.next() orelse return error.InvalidFormat;
+ const prefix_str = iter.next() orelse return error.InvalidFormat;
+ const file_path = iter.rest();
+
+ cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
+ cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
+ cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
+ _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
+ const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
+ if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
+
+ if (file_path.len == 0) {
+ return error.InvalidFormat;
+ }
+ if (cache_hash_file.prefixed_path) |pp| {
+ if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
+ return error.InvalidFormat;
+ }
+ }
+
+ if (cache_hash_file.prefixed_path == null) {
+ cache_hash_file.prefixed_path = .{
+ .prefix = prefix,
+ .sub_path = try gpa.dupe(u8, file_path),
+ };
+ }
+
+ const pp = cache_hash_file.prefixed_path.?;
+ const dir = self.cache.prefixes()[pp.prefix].handle;
+ const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
+ error.FileNotFound => {
+ try self.upgradeToExclusiveLock();
+ return false;
+ },
+ else => return error.CacheUnavailable,
+ };
+ defer this_file.close();
+
+ const actual_stat = this_file.stat() catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+ const size_match = actual_stat.size == cache_hash_file.stat.size;
+ const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
+ const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
+
+ if (!size_match or !mtime_match or !inode_match) {
+ self.manifest_dirty = true;
+
+ cache_hash_file.stat = .{
+ .size = actual_stat.size,
+ .mtime = actual_stat.mtime,
+ .inode = actual_stat.inode,
+ };
+
+ if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ cache_hash_file.stat.mtime = 0;
+ cache_hash_file.stat.inode = 0;
+ }
+
+ var actual_digest: BinDigest = undefined;
+ hashFile(this_file, &actual_digest) catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+
+ if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
+ cache_hash_file.bin_digest = actual_digest;
+ // keep going until we have the input file digests
+ any_file_changed = true;
+ }
+ }
+
+ if (!any_file_changed) {
+ self.hash.hasher.update(&cache_hash_file.bin_digest);
+ }
+ }
+
+ if (any_file_changed) {
+ // cache miss
+ // keep the manifest file open
+ self.unhit(bin_digest, input_file_count);
+ try self.upgradeToExclusiveLock();
+ return false;
+ }
+
+ if (idx < input_file_count) {
+ self.manifest_dirty = true;
+ while (idx < input_file_count) : (idx += 1) {
+ const ch_file = &self.files.items[idx];
+ self.populateFileHash(ch_file) catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+ }
+ try self.upgradeToExclusiveLock();
+ return false;
+ }
+
+ if (self.want_shared_lock) {
+ try self.downgradeToSharedLock();
+ }
+
+ return true;
+ }
+
+ pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
+ // Reset the hash.
+ self.hash.hasher = hasher_init;
+ self.hash.hasher.update(&bin_digest);
+
+ // Remove files not in the initial hash.
+ for (self.files.items[input_file_count..]) |*file| {
+ file.deinit(self.cache.gpa);
+ }
+ self.files.shrinkRetainingCapacity(input_file_count);
+
+ for (self.files.items) |file| {
+ self.hash.hasher.update(&file.bin_digest);
+ }
+ }
+
+ fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
+ // If the file_time is prior to the most recent problematic timestamp
+ // then we don't need to access the filesystem.
+ if (file_time < man.recent_problematic_timestamp)
+ return false;
+
+ // Next we will check the globally shared Cache timestamp, which is accessed
+ // from multiple threads.
+ man.cache.mutex.lock();
+ defer man.cache.mutex.unlock();
+
+ // Save the global one to our local one to avoid locking next time.
+ man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
+ if (file_time < man.recent_problematic_timestamp)
+ return false;
+
+ // This flag prevents multiple filesystem writes for the same hit() call.
+ if (man.want_refresh_timestamp) {
+ man.want_refresh_timestamp = false;
+
+ var file = man.cache.manifest_dir.createFile("timestamp", .{
+ .read = true,
+ .truncate = true,
+ }) catch return true;
+ defer file.close();
+
+ // Save locally and also save globally (we still hold the global lock).
+ man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
+ man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
+ }
+
+ return file_time >= man.recent_problematic_timestamp;
+ }
+
+ fn populateFileHash(self: *Manifest, ch_file: *File) !void {
+ const pp = ch_file.prefixed_path.?;
+ const dir = self.cache.prefixes()[pp.prefix].handle;
+ const file = try dir.openFile(pp.sub_path, .{});
+ defer file.close();
+
+ const actual_stat = try file.stat();
+ ch_file.stat = .{
+ .size = actual_stat.size,
+ .mtime = actual_stat.mtime,
+ .inode = actual_stat.inode,
+ };
+
+ if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ ch_file.stat.mtime = 0;
+ ch_file.stat.inode = 0;
+ }
+
+ if (ch_file.max_file_size) |max_file_size| {
+ if (ch_file.stat.size > max_file_size) {
+ return error.FileTooBig;
+ }
+
+ const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
+ errdefer self.cache.gpa.free(contents);
+
+ // Hash while reading from disk, to keep the contents in the cpu cache while
+ // doing hashing.
+ var hasher = hasher_init;
+ var off: usize = 0;
+ while (true) {
+ // give me everything you've got, captain
+ const bytes_read = try file.read(contents[off..]);
+ if (bytes_read == 0) break;
+ hasher.update(contents[off..][0..bytes_read]);
+ off += bytes_read;
+ }
+ hasher.final(&ch_file.bin_digest);
+
+ ch_file.contents = contents;
+ } else {
+ try hashFile(file, &ch_file.bin_digest);
+ }
+
+ self.hash.hasher.update(&ch_file.bin_digest);
+ }
+
+ /// Add a file as a dependency of process being cached, after the initial hash has been
+ /// calculated. This is useful for processes that don't know all the files that
+ /// are depended on ahead of time. For example, a source file that can import other files
+ /// will need to be recompiled if the imported file is changed.
+ pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
+ assert(self.manifest_file != null);
+
+ const gpa = self.cache.gpa;
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ const new_ch_file = try self.files.addOne(gpa);
+ new_ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = max_file_size,
+ .stat = undefined,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ try self.populateFileHash(new_ch_file);
+
+ return new_ch_file.contents.?;
+ }
+
+ /// Add a file as a dependency of process being cached, after the initial hash has been
+ /// calculated. This is useful for processes that don't know the all the files that
+ /// are depended on ahead of time. For example, a source file that can import other files
+ /// will need to be recompiled if the imported file is changed.
+ pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
+ assert(self.manifest_file != null);
+
+ const gpa = self.cache.gpa;
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ const new_ch_file = try self.files.addOne(gpa);
+ new_ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = null,
+ .stat = undefined,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ try self.populateFileHash(new_ch_file);
+ }
+
+ /// Like `addFilePost` but when the file contents have already been loaded from disk.
+ /// On success, cache takes ownership of `resolved_path`.
+ pub fn addFilePostContents(
+ self: *Manifest,
+ resolved_path: []u8,
+ bytes: []const u8,
+ stat: File.Stat,
+ ) error{OutOfMemory}!void {
+ assert(self.manifest_file != null);
+ const gpa = self.cache.gpa;
+
+ const ch_file = try self.files.addOne(gpa);
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = null,
+ .stat = stat,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+
+ if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ ch_file.stat.mtime = 0;
+ ch_file.stat.inode = 0;
+ }
+
+ {
+ var hasher = hasher_init;
+ hasher.update(bytes);
+ hasher.final(&ch_file.bin_digest);
+ }
+
+ self.hash.hasher.update(&ch_file.bin_digest);
+ }
+
+ pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
+ assert(self.manifest_file != null);
+
+ const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
+ defer self.cache.gpa.free(dep_file_contents);
+
+ var error_buf = std.ArrayList(u8).init(self.cache.gpa);
+ defer error_buf.deinit();
+
+ var it: DepTokenizer = .{ .bytes = dep_file_contents };
+
+ // Skip first token: target.
+ switch (it.next() orelse return) { // Empty dep file OK.
+ .target, .target_must_resolve, .prereq => {},
+ else => |err| {
+ try err.printError(error_buf.writer());
+ log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
+ return error.InvalidDepFile;
+ },
+ }
+ // Process 0+ preqreqs.
+ // Clang is invoked in single-source mode so we never get more targets.
+ while (true) {
+ switch (it.next() orelse return) {
+ .target, .target_must_resolve => return,
+ .prereq => |file_path| try self.addFilePost(file_path),
+ else => |err| {
+ try err.printError(error_buf.writer());
+ log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
+ return error.InvalidDepFile;
+ },
+ }
+ }
+ }
+
+ /// Returns a hex encoded hash of the inputs.
+ pub fn final(self: *Manifest) [hex_digest_len]u8 {
+ assert(self.manifest_file != null);
+
+ // We don't close the manifest file yet, because we want to
+ // keep it locked until the API user is done using it.
+ // We also don't write out the manifest yet, because until
+ // cache_release is called we still might be working on creating
+ // the artifacts to cache.
+
+ var bin_digest: BinDigest = undefined;
+ self.hash.hasher.final(&bin_digest);
+
+ var out_digest: [hex_digest_len]u8 = undefined;
+ _ = std.fmt.bufPrint(
+ &out_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+
+ return out_digest;
+ }
+
+ /// If `want_shared_lock` is true, this function automatically downgrades the
+ /// lock from exclusive to shared.
+ pub fn writeManifest(self: *Manifest) !void {
+ assert(self.have_exclusive_lock);
+
+ const manifest_file = self.manifest_file.?;
+ if (self.manifest_dirty) {
+ self.manifest_dirty = false;
+
+ var contents = std.ArrayList(u8).init(self.cache.gpa);
+ defer contents.deinit();
+
+ const writer = contents.writer();
+ var encoded_digest: [hex_digest_len]u8 = undefined;
+
+ for (self.files.items) |file| {
+ _ = std.fmt.bufPrint(
+ &encoded_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
+ ) catch unreachable;
+ try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
+ file.stat.size,
+ file.stat.inode,
+ file.stat.mtime,
+ &encoded_digest,
+ file.prefixed_path.?.prefix,
+ file.prefixed_path.?.sub_path,
+ });
+ }
+
+ try manifest_file.setEndPos(contents.items.len);
+ try manifest_file.pwriteAll(contents.items, 0);
+ }
+
+ if (self.want_shared_lock) {
+ try self.downgradeToSharedLock();
+ }
+ }
+
+ fn downgradeToSharedLock(self: *Manifest) !void {
+ if (!self.have_exclusive_lock) return;
+
+ // WASI does not currently support flock, so we bypass it here.
+ // TODO: If/when flock is supported on WASI, this check should be removed.
+ // See https://github.com/WebAssembly/wasi-filesystem/issues/2
+ if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
+ const manifest_file = self.manifest_file.?;
+ try manifest_file.downgradeLock();
+ }
+
+ self.have_exclusive_lock = false;
+ }
+
+ fn upgradeToExclusiveLock(self: *Manifest) !void {
+ if (self.have_exclusive_lock) return;
+ assert(self.manifest_file != null);
+
+ // WASI does not currently support flock, so we bypass it here.
+ // TODO: If/when flock is supported on WASI, this check should be removed.
+ // See https://github.com/WebAssembly/wasi-filesystem/issues/2
+ if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
+ const manifest_file = self.manifest_file.?;
+ // Here we intentionally have a period where the lock is released, in case there are
+ // other processes holding a shared lock.
+ manifest_file.unlock();
+ try manifest_file.lock(.Exclusive);
+ }
+ self.have_exclusive_lock = true;
+ }
+
+ /// Obtain only the data needed to maintain a lock on the manifest file.
+ /// The `Manifest` remains safe to deinit.
+ /// Don't forget to call `writeManifest` before this!
+ pub fn toOwnedLock(self: *Manifest) Lock {
+ const lock: Lock = .{
+ .manifest_file = self.manifest_file.?,
+ };
+
+ self.manifest_file = null;
+ return lock;
+ }
+
+ /// Releases the manifest file and frees any memory the Manifest was using.
+ /// `Manifest.hit` must be called first.
+ /// Don't forget to call `writeManifest` before this!
+ pub fn deinit(self: *Manifest) void {
+ if (self.manifest_file) |file| {
+ if (builtin.os.tag == .windows) {
+ // See Lock.release for why this is required on Windows
+ file.unlock();
+ }
+
+ file.close();
+ }
+ for (self.files.items) |*file| {
+ file.deinit(self.cache.gpa);
+ }
+ self.files.deinit(self.cache.gpa);
+ }
+};
+
+/// On operating systems that support symlinks, does a readlink. On other operating systems,
+/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
+/// it is treated as not supporting symlinks.
+pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
+ if (builtin.os.tag == .windows) {
+ return dir.readFile(sub_path, buffer);
+ } else {
+ return dir.readLink(sub_path, buffer);
+ }
+}
+
+/// On operating systems that support symlinks, does a symlink. On other operating systems,
+/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
+/// it is treated as not supporting symlinks.
+/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
+pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
+ assert(data.len <= 255);
+ if (builtin.os.tag == .windows) {
+ return dir.writeFile(sub_path, data);
+ } else {
+ return dir.symLink(data, sub_path, .{});
+ }
+}
+
+fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
+ var buf: [1024]u8 = undefined;
+
+ var hasher = hasher_init;
+ while (true) {
+ const bytes_read = try file.read(&buf);
+ if (bytes_read == 0) break;
+ hasher.update(buf[0..bytes_read]);
+ }
+
+ hasher.final(bin_digest);
+}
+
+// Create/Write a file, close it, then grab its stat.mtime timestamp.
+fn testGetCurrentFileTimestamp() !i128 {
+ var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
+ .read = true,
+ .truncate = true,
+ });
+ defer file.close();
+
+ return (try file.stat()).mtime;
+}
+
+test "cache file and then recall it" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+
+ const cwd = fs.cwd();
+
+ const temp_file = "test.txt";
+ const temp_manifest_dir = "temp_manifest_dir";
+
+ try cwd.writeFile(temp_file, "Hello, world!\n");
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.add(true);
+ ch.hash.add(@as(u16, 1234));
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file, null);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ digest1 = ch.final();
+ try ch.writeManifest();
+ }
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.add(true);
+ ch.hash.add(@as(u16, 1234));
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file, null);
+
+ // Cache hit! We just "built" the same file
+ try testing.expect(try ch.hit());
+ digest2 = ch.final();
+
+ try testing.expectEqual(false, ch.have_exclusive_lock);
+ }
+
+ try testing.expectEqual(digest1, digest2);
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteFile(temp_file);
+}
+
+test "check that changing a file makes cache fail" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+
+ const temp_file = "cache_hash_change_file_test.txt";
+ const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
+ const original_temp_file_contents = "Hello, world!\n";
+ const updated_temp_file_contents = "Hello, world; but updated!\n";
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteTree(temp_file);
+
+ try cwd.writeFile(temp_file, original_temp_file_contents);
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ const temp_file_idx = try ch.addFile(temp_file, 100);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
+
+ digest1 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try cwd.writeFile(temp_file, updated_temp_file_contents);
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ const temp_file_idx = try ch.addFile(temp_file, 100);
+
+ // A file that we depend on has been updated, so the cache should not contain an entry for it
+ try testing.expectEqual(false, try ch.hit());
+
+ // The cache system does not keep the contents of re-hashed input files.
+ try testing.expect(ch.files.items[temp_file_idx].contents == null);
+
+ digest2 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteTree(temp_file);
+}
+
+test "no file inputs" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+ const temp_manifest_dir = "no_file_inputs_manifest_dir";
+ defer cwd.deleteTree(temp_manifest_dir) catch {};
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var man = cache.obtain();
+ defer man.deinit();
+
+ man.hash.addBytes("1234");
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try man.hit());
+
+ digest1 = man.final();
+
+ try man.writeManifest();
+ }
+ {
+ var man = cache.obtain();
+ defer man.deinit();
+
+ man.hash.addBytes("1234");
+
+ try testing.expect(try man.hit());
+ digest2 = man.final();
+ try testing.expectEqual(false, man.have_exclusive_lock);
+ }
+
+ try testing.expectEqual(digest1, digest2);
+}
+
+test "Manifest with files added after initial hash work" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+
+ const temp_file1 = "cache_hash_post_file_test1.txt";
+ const temp_file2 = "cache_hash_post_file_test2.txt";
+ const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
+
+ try cwd.writeFile(temp_file1, "Hello, world!\n");
+ try cwd.writeFile(temp_file2, "Hello world the second!\n");
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+ var digest3: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ _ = try ch.addFilePost(temp_file2);
+
+ digest1 = ch.final();
+ try ch.writeManifest();
+ }
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ try testing.expect(try ch.hit());
+ digest2 = ch.final();
+
+ try testing.expectEqual(false, ch.have_exclusive_lock);
+ }
+ try testing.expect(mem.eql(u8, &digest1, &digest2));
+
+ // Modify the file added after initial hash
+ try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
+
+ // Wait for file timestamps to tick
+ const initial_time2 = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time2) {
+ std.time.sleep(1);
+ }
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ // A file that we depend on has been updated, so the cache should not contain an entry for it
+ try testing.expectEqual(false, try ch.hit());
+
+ _ = try ch.addFilePost(temp_file2);
+
+ digest3 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try testing.expect(!mem.eql(u8, &digest1, &digest3));
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteFile(temp_file1);
+ try cwd.deleteFile(temp_file2);
+}
diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig
new file mode 100644
index 0000000000..8f9f2f81cd
--- /dev/null
+++ b/lib/std/Build/Cache/DepTokenizer.zig
@@ -0,0 +1,1069 @@
+const Tokenizer = @This();
+
+index: usize = 0,
+bytes: []const u8,
+state: State = .lhs,
+
+const std = @import("std");
+const testing = std.testing;
+const assert = std.debug.assert;
+
+pub fn next(self: *Tokenizer) ?Token {
+ var start = self.index;
+ var must_resolve = false;
+ while (self.index < self.bytes.len) {
+ const char = self.bytes[self.index];
+ switch (self.state) {
+ .lhs => switch (char) {
+ '\t', '\n', '\r', ' ' => {
+ // silently ignore whitespace
+ self.index += 1;
+ },
+ else => {
+ start = self.index;
+ self.state = .target;
+ },
+ },
+ .target => switch (char) {
+ '\t', '\n', '\r', ' ' => {
+ return errorIllegalChar(.invalid_target, self.index, char);
+ },
+ '$' => {
+ self.state = .target_dollar_sign;
+ self.index += 1;
+ },
+ '\\' => {
+ self.state = .target_reverse_solidus;
+ self.index += 1;
+ },
+ ':' => {
+ self.state = .target_colon;
+ self.index += 1;
+ },
+ else => {
+ self.index += 1;
+ },
+ },
+ .target_reverse_solidus => switch (char) {
+ '\t', '\n', '\r' => {
+ return errorIllegalChar(.bad_target_escape, self.index, char);
+ },
+ ' ', '#', '\\' => {
+ must_resolve = true;
+ self.state = .target;
+ self.index += 1;
+ },
+ '$' => {
+ self.state = .target_dollar_sign;
+ self.index += 1;
+ },
+ else => {
+ self.state = .target;
+ self.index += 1;
+ },
+ },
+ .target_dollar_sign => switch (char) {
+ '$' => {
+ must_resolve = true;
+ self.state = .target;
+ self.index += 1;
+ },
+ else => {
+ return errorIllegalChar(.expected_dollar_sign, self.index, char);
+ },
+ },
+ .target_colon => switch (char) {
+ '\n', '\r' => {
+ const bytes = self.bytes[start .. self.index - 1];
+ if (bytes.len != 0) {
+ self.state = .lhs;
+ return finishTarget(must_resolve, bytes);
+ }
+ // silently ignore null target
+ self.state = .lhs;
+ },
+ '/', '\\' => {
+ self.state = .target_colon_reverse_solidus;
+ self.index += 1;
+ },
+ else => {
+ const bytes = self.bytes[start .. self.index - 1];
+ if (bytes.len != 0) {
+ self.state = .rhs;
+ return finishTarget(must_resolve, bytes);
+ }
+ // silently ignore null target
+ self.state = .lhs;
+ },
+ },
+ .target_colon_reverse_solidus => switch (char) {
+ '\n', '\r' => {
+ const bytes = self.bytes[start .. self.index - 2];
+ if (bytes.len != 0) {
+ self.state = .lhs;
+ return finishTarget(must_resolve, bytes);
+ }
+ // silently ignore null target
+ self.state = .lhs;
+ },
+ else => {
+ self.state = .target;
+ },
+ },
+ .rhs => switch (char) {
+ '\t', ' ' => {
+ // silently ignore horizontal whitespace
+ self.index += 1;
+ },
+ '\n', '\r' => {
+ self.state = .lhs;
+ },
+ '\\' => {
+ self.state = .rhs_continuation;
+ self.index += 1;
+ },
+ '"' => {
+ self.state = .prereq_quote;
+ self.index += 1;
+ start = self.index;
+ },
+ else => {
+ start = self.index;
+ self.state = .prereq;
+ },
+ },
+ .rhs_continuation => switch (char) {
+ '\n' => {
+ self.state = .rhs;
+ self.index += 1;
+ },
+ '\r' => {
+ self.state = .rhs_continuation_linefeed;
+ self.index += 1;
+ },
+ else => {
+ return errorIllegalChar(.continuation_eol, self.index, char);
+ },
+ },
+ .rhs_continuation_linefeed => switch (char) {
+ '\n' => {
+ self.state = .rhs;
+ self.index += 1;
+ },
+ else => {
+ return errorIllegalChar(.continuation_eol, self.index, char);
+ },
+ },
+ .prereq_quote => switch (char) {
+ '"' => {
+ self.index += 1;
+ self.state = .rhs;
+ return Token{ .prereq = self.bytes[start .. self.index - 1] };
+ },
+ else => {
+ self.index += 1;
+ },
+ },
+ .prereq => switch (char) {
+ '\t', ' ' => {
+ self.state = .rhs;
+ return Token{ .prereq = self.bytes[start..self.index] };
+ },
+ '\n', '\r' => {
+ self.state = .lhs;
+ return Token{ .prereq = self.bytes[start..self.index] };
+ },
+ '\\' => {
+ self.state = .prereq_continuation;
+ self.index += 1;
+ },
+ else => {
+ self.index += 1;
+ },
+ },
+ .prereq_continuation => switch (char) {
+ '\n' => {
+ self.index += 1;
+ self.state = .rhs;
+ return Token{ .prereq = self.bytes[start .. self.index - 2] };
+ },
+ '\r' => {
+ self.state = .prereq_continuation_linefeed;
+ self.index += 1;
+ },
+ else => {
+ // not continuation
+ self.state = .prereq;
+ self.index += 1;
+ },
+ },
+ .prereq_continuation_linefeed => switch (char) {
+ '\n' => {
+ self.index += 1;
+ self.state = .rhs;
+ return Token{ .prereq = self.bytes[start .. self.index - 1] };
+ },
+ else => {
+ return errorIllegalChar(.continuation_eol, self.index, char);
+ },
+ },
+ }
+ } else {
+ switch (self.state) {
+ .lhs,
+ .rhs,
+ .rhs_continuation,
+ .rhs_continuation_linefeed,
+ => return null,
+ .target => {
+ return errorPosition(.incomplete_target, start, self.bytes[start..]);
+ },
+ .target_reverse_solidus,
+ .target_dollar_sign,
+ => {
+ const idx = self.index - 1;
+ return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
+ },
+ .target_colon => {
+ const bytes = self.bytes[start .. self.index - 1];
+ if (bytes.len != 0) {
+ self.index += 1;
+ self.state = .rhs;
+ return finishTarget(must_resolve, bytes);
+ }
+ // silently ignore null target
+ self.state = .lhs;
+ return null;
+ },
+ .target_colon_reverse_solidus => {
+ const bytes = self.bytes[start .. self.index - 2];
+ if (bytes.len != 0) {
+ self.index += 1;
+ self.state = .rhs;
+ return finishTarget(must_resolve, bytes);
+ }
+ // silently ignore null target
+ self.state = .lhs;
+ return null;
+ },
+ .prereq_quote => {
+ return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
+ },
+ .prereq => {
+ self.state = .lhs;
+ return Token{ .prereq = self.bytes[start..] };
+ },
+ .prereq_continuation => {
+ self.state = .lhs;
+ return Token{ .prereq = self.bytes[start .. self.index - 1] };
+ },
+ .prereq_continuation_linefeed => {
+ self.state = .lhs;
+ return Token{ .prereq = self.bytes[start .. self.index - 2] };
+ },
+ }
+ }
+ unreachable;
+}
+
+fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token {
+ return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
+}
+
+fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token {
+ return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
+}
+
+fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
+ return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
+}
+
+const State = enum {
+ lhs,
+ target,
+ target_reverse_solidus,
+ target_dollar_sign,
+ target_colon,
+ target_colon_reverse_solidus,
+ rhs,
+ rhs_continuation,
+ rhs_continuation_linefeed,
+ prereq_quote,
+ prereq,
+ prereq_continuation,
+ prereq_continuation_linefeed,
+};
+
+pub const Token = union(enum) {
+ target: []const u8,
+ target_must_resolve: []const u8,
+ prereq: []const u8,
+
+ incomplete_quoted_prerequisite: IndexAndBytes,
+ incomplete_target: IndexAndBytes,
+
+ invalid_target: IndexAndChar,
+ bad_target_escape: IndexAndChar,
+ expected_dollar_sign: IndexAndChar,
+ continuation_eol: IndexAndChar,
+ incomplete_escape: IndexAndChar,
+
+ pub const IndexAndChar = struct {
+ index: usize,
+ char: u8,
+ };
+
+ pub const IndexAndBytes = struct {
+ index: usize,
+ bytes: []const u8,
+ };
+
+ /// Resolve escapes in target. Only valid with .target_must_resolve.
+ pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
+ const bytes = self.target_must_resolve; // resolve called on incorrect token
+
+ var state: enum { start, escape, dollar } = .start;
+ for (bytes) |c| {
+ switch (state) {
+ .start => {
+ switch (c) {
+ '\\' => state = .escape,
+ '$' => state = .dollar,
+ else => try writer.writeByte(c),
+ }
+ },
+ .escape => {
+ switch (c) {
+ ' ', '#', '\\' => {},
+ '$' => {
+ try writer.writeByte('\\');
+ state = .dollar;
+ continue;
+ },
+ else => try writer.writeByte('\\'),
+ }
+ try writer.writeByte(c);
+ state = .start;
+ },
+ .dollar => {
+ try writer.writeByte('$');
+ switch (c) {
+ '$' => {},
+ else => try writer.writeByte(c),
+ }
+ state = .start;
+ },
+ }
+ }
+ }
+
+ pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
+ switch (self) {
+ .target, .target_must_resolve, .prereq => unreachable, // not an error
+ .incomplete_quoted_prerequisite,
+ .incomplete_target,
+ => |index_and_bytes| {
+ try writer.print("{s} '", .{self.errStr()});
+ if (self == .incomplete_target) {
+ const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
+ try tmp.resolve(writer);
+ } else {
+ try printCharValues(writer, index_and_bytes.bytes);
+ }
+ try writer.print("' at position {d}", .{index_and_bytes.index});
+ },
+ .invalid_target,
+ .bad_target_escape,
+ .expected_dollar_sign,
+ .continuation_eol,
+ .incomplete_escape,
+ => |index_and_char| {
+ try writer.writeAll("illegal char ");
+ try printUnderstandableChar(writer, index_and_char.char);
+ try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
+ },
+ }
+ }
+
+ fn errStr(self: Token) []const u8 {
+ return switch (self) {
+ .target, .target_must_resolve, .prereq => unreachable, // not an error
+ .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
+ .incomplete_target => "incomplete target",
+ .invalid_target => "invalid target",
+ .bad_target_escape => "bad target escape",
+ .expected_dollar_sign => "expecting '$'",
+ .continuation_eol => "continuation expecting end-of-line",
+ .incomplete_escape => "incomplete escape",
+ };
+ }
+};
+
+test "empty file" {
+ try depTokenizer("", "");
+}
+
+test "empty whitespace" {
+ try depTokenizer("\n", "");
+ try depTokenizer("\r", "");
+ try depTokenizer("\r\n", "");
+ try depTokenizer(" ", "");
+}
+
+test "empty colon" {
+ try depTokenizer(":", "");
+ try depTokenizer("\n:", "");
+ try depTokenizer("\r:", "");
+ try depTokenizer("\r\n:", "");
+ try depTokenizer(" :", "");
+}
+
+test "empty target" {
+ try depTokenizer("foo.o:", "target = {foo.o}");
+ try depTokenizer(
+ \\foo.o:
+ \\bar.o:
+ \\abcd.o:
+ ,
+ \\target = {foo.o}
+ \\target = {bar.o}
+ \\target = {abcd.o}
+ );
+}
+
+test "whitespace empty target" {
+ try depTokenizer("\nfoo.o:", "target = {foo.o}");
+ try depTokenizer("\rfoo.o:", "target = {foo.o}");
+ try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
+ try depTokenizer(" foo.o:", "target = {foo.o}");
+}
+
+test "escape empty target" {
+ try depTokenizer("\\ foo.o:", "target = { foo.o}");
+ try depTokenizer("\\#foo.o:", "target = {#foo.o}");
+ try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
+ try depTokenizer("$$foo.o:", "target = {$foo.o}");
+}
+
+test "empty target linefeeds" {
+ try depTokenizer("\n", "");
+ try depTokenizer("\r\n", "");
+
+ const expect = "target = {foo.o}";
+ try depTokenizer(
+ \\foo.o:
+ , expect);
+ try depTokenizer(
+ \\foo.o:
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o:
+ , expect);
+ try depTokenizer(
+ \\foo.o:
+ \\
+ , expect);
+}
+
+test "empty target linefeeds + continuations" {
+ const expect = "target = {foo.o}";
+ try depTokenizer(
+ \\foo.o:\
+ , expect);
+ try depTokenizer(
+ \\foo.o:\
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o:\
+ , expect);
+ try depTokenizer(
+ \\foo.o:\
+ \\
+ , expect);
+}
+
+test "empty target linefeeds + hspace + continuations" {
+ const expect = "target = {foo.o}";
+ try depTokenizer(
+ \\foo.o: \
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\
+ , expect);
+}
+
+test "prereq" {
+ const expect =
+ \\target = {foo.o}
+ \\prereq = {foo.c}
+ ;
+ try depTokenizer("foo.o: foo.c", expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\foo.c
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\ foo.c
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\ foo.c
+ , expect);
+}
+
+test "prereq continuation" {
+ const expect =
+ \\target = {foo.o}
+ \\prereq = {foo.h}
+ \\prereq = {bar.h}
+ ;
+ try depTokenizer(
+ \\foo.o: foo.h\
+ \\bar.h
+ , expect);
+ try depTokenizer(
+ \\foo.o: foo.h\
+ \\bar.h
+ , expect);
+}
+
+test "multiple prereqs" {
+ const expect =
+ \\target = {foo.o}
+ \\prereq = {foo.c}
+ \\prereq = {foo.h}
+ \\prereq = {bar.h}
+ ;
+ try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\foo.c foo.h bar.h
+ , expect);
+ try depTokenizer(
+ \\foo.o: foo.c foo.h bar.h\
+ , expect);
+ try depTokenizer(
+ \\foo.o: foo.c foo.h bar.h\
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\foo.c \
+ \\ foo.h\
+ \\bar.h
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\foo.c \
+ \\ foo.h\
+ \\bar.h\
+ \\
+ , expect);
+ try depTokenizer(
+ \\foo.o: \
+ \\foo.c \
+ \\ foo.h\
+ \\bar.h\
+ , expect);
+}
+
+test "multiple targets and prereqs" {
+ try depTokenizer(
+ \\foo.o: foo.c
+ \\bar.o: bar.c a.h b.h c.h
+ \\abc.o: abc.c \
+ \\ one.h two.h \
+ \\ three.h four.h
+ ,
+ \\target = {foo.o}
+ \\prereq = {foo.c}
+ \\target = {bar.o}
+ \\prereq = {bar.c}
+ \\prereq = {a.h}
+ \\prereq = {b.h}
+ \\prereq = {c.h}
+ \\target = {abc.o}
+ \\prereq = {abc.c}
+ \\prereq = {one.h}
+ \\prereq = {two.h}
+ \\prereq = {three.h}
+ \\prereq = {four.h}
+ );
+ try depTokenizer(
+ \\ascii.o: ascii.c
+ \\base64.o: base64.c stdio.h
+ \\elf.o: elf.c a.h b.h c.h
+ \\macho.o: \
+ \\ macho.c\
+ \\ a.h b.h c.h
+ ,
+ \\target = {ascii.o}
+ \\prereq = {ascii.c}
+ \\target = {base64.o}
+ \\prereq = {base64.c}
+ \\prereq = {stdio.h}
+ \\target = {elf.o}
+ \\prereq = {elf.c}
+ \\prereq = {a.h}
+ \\prereq = {b.h}
+ \\prereq = {c.h}
+ \\target = {macho.o}
+ \\prereq = {macho.c}
+ \\prereq = {a.h}
+ \\prereq = {b.h}
+ \\prereq = {c.h}
+ );
+ try depTokenizer(
+ \\a$$scii.o: ascii.c
+ \\\\base64.o: "\base64.c" "s t#dio.h"
+ \\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
+ \\macho.o: \
+ \\ "macho!.c" \
+ \\ a.h b.h c.h
+ ,
+ \\target = {a$scii.o}
+ \\prereq = {ascii.c}
+ \\target = {\base64.o}
+ \\prereq = {\base64.c}
+ \\prereq = {s t#dio.h}
+ \\target = {e\lf.o}
+ \\prereq = {e\lf.c}
+ \\prereq = {a.h$$}
+ \\prereq = {$$b.h c.h$$}
+ \\target = {macho.o}
+ \\prereq = {macho!.c}
+ \\prereq = {a.h}
+ \\prereq = {b.h}
+ \\prereq = {c.h}
+ );
+}
+
+test "windows quoted prereqs" {
+ try depTokenizer(
+ \\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
+ \\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
+ ,
+ \\target = {c:\foo.o}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
+ \\target = {c:\foo2.o}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
+ );
+}
+
+test "windows mixed prereqs" {
+ try depTokenizer(
+ \\cimport.o: \
+ \\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
+ \\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
+ \\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
+ \\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
+ ,
+ \\target = {cimport.o}
+ \\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
+ \\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
+ \\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
+ \\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
+ );
+}
+
+test "windows funky targets" {
+ try depTokenizer(
+ \\C:\Users\anon\foo.o:
+ \\C:\Users\anon\foo\ .o:
+ \\C:\Users\anon\foo\#.o:
+ \\C:\Users\anon\foo$$.o:
+ \\C:\Users\anon\\\ foo.o:
+ \\C:\Users\anon\\#foo.o:
+ \\C:\Users\anon\$$foo.o:
+ \\C:\Users\anon\\\ \ \ \ \ foo.o:
+ ,
+ \\target = {C:\Users\anon\foo.o}
+ \\target = {C:\Users\anon\foo .o}
+ \\target = {C:\Users\anon\foo#.o}
+ \\target = {C:\Users\anon\foo$.o}
+ \\target = {C:\Users\anon\ foo.o}
+ \\target = {C:\Users\anon\#foo.o}
+ \\target = {C:\Users\anon\$foo.o}
+ \\target = {C:\Users\anon\ foo.o}
+ );
+}
+
+test "windows drive and forward slashes" {
+ try depTokenizer(
+ \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
+ \\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c
+ ,
+ \\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj}
+ \\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c}
+ );
+}
+
+test "error incomplete escape - reverse_solidus" {
+ try depTokenizer("\\",
+ \\ERROR: illegal char '\' at position 0: incomplete escape
+ );
+ try depTokenizer("\t\\",
+ \\ERROR: illegal char '\' at position 1: incomplete escape
+ );
+ try depTokenizer("\n\\",
+ \\ERROR: illegal char '\' at position 1: incomplete escape
+ );
+ try depTokenizer("\r\\",
+ \\ERROR: illegal char '\' at position 1: incomplete escape
+ );
+ try depTokenizer("\r\n\\",
+ \\ERROR: illegal char '\' at position 2: incomplete escape
+ );
+ try depTokenizer(" \\",
+ \\ERROR: illegal char '\' at position 1: incomplete escape
+ );
+}
+
+test "error incomplete escape - dollar_sign" {
+ try depTokenizer("$",
+ \\ERROR: illegal char '$' at position 0: incomplete escape
+ );
+ try depTokenizer("\t$",
+ \\ERROR: illegal char '$' at position 1: incomplete escape
+ );
+ try depTokenizer("\n$",
+ \\ERROR: illegal char '$' at position 1: incomplete escape
+ );
+ try depTokenizer("\r$",
+ \\ERROR: illegal char '$' at position 1: incomplete escape
+ );
+ try depTokenizer("\r\n$",
+ \\ERROR: illegal char '$' at position 2: incomplete escape
+ );
+ try depTokenizer(" $",
+ \\ERROR: illegal char '$' at position 1: incomplete escape
+ );
+}
+
+test "error incomplete target" {
+ try depTokenizer("foo.o",
+ \\ERROR: incomplete target 'foo.o' at position 0
+ );
+ try depTokenizer("\tfoo.o",
+ \\ERROR: incomplete target 'foo.o' at position 1
+ );
+ try depTokenizer("\nfoo.o",
+ \\ERROR: incomplete target 'foo.o' at position 1
+ );
+ try depTokenizer("\rfoo.o",
+ \\ERROR: incomplete target 'foo.o' at position 1
+ );
+ try depTokenizer("\r\nfoo.o",
+ \\ERROR: incomplete target 'foo.o' at position 2
+ );
+ try depTokenizer(" foo.o",
+ \\ERROR: incomplete target 'foo.o' at position 1
+ );
+
+ try depTokenizer("\\ foo.o",
+ \\ERROR: incomplete target ' foo.o' at position 0
+ );
+ try depTokenizer("\\#foo.o",
+ \\ERROR: incomplete target '#foo.o' at position 0
+ );
+ try depTokenizer("\\\\foo.o",
+ \\ERROR: incomplete target '\foo.o' at position 0
+ );
+ try depTokenizer("$$foo.o",
+ \\ERROR: incomplete target '$foo.o' at position 0
+ );
+}
+
+test "error illegal char at position - bad target escape" {
+ try depTokenizer("\\\t",
+ \\ERROR: illegal char \x09 at position 1: bad target escape
+ );
+ try depTokenizer("\\\n",
+ \\ERROR: illegal char \x0A at position 1: bad target escape
+ );
+ try depTokenizer("\\\r",
+ \\ERROR: illegal char \x0D at position 1: bad target escape
+ );
+ try depTokenizer("\\\r\n",
+ \\ERROR: illegal char \x0D at position 1: bad target escape
+ );
+}
+
+test "error illegal char at position - execting dollar_sign" {
+ try depTokenizer("$\t",
+ \\ERROR: illegal char \x09 at position 1: expecting '$'
+ );
+ try depTokenizer("$\n",
+ \\ERROR: illegal char \x0A at position 1: expecting '$'
+ );
+ try depTokenizer("$\r",
+ \\ERROR: illegal char \x0D at position 1: expecting '$'
+ );
+ try depTokenizer("$\r\n",
+ \\ERROR: illegal char \x0D at position 1: expecting '$'
+ );
+}
+
+test "error illegal char at position - invalid target" {
+ try depTokenizer("foo\t.o",
+ \\ERROR: illegal char \x09 at position 3: invalid target
+ );
+ try depTokenizer("foo\n.o",
+ \\ERROR: illegal char \x0A at position 3: invalid target
+ );
+ try depTokenizer("foo\r.o",
+ \\ERROR: illegal char \x0D at position 3: invalid target
+ );
+ try depTokenizer("foo\r\n.o",
+ \\ERROR: illegal char \x0D at position 3: invalid target
+ );
+}
+
+test "error target - continuation expecting end-of-line" {
+ try depTokenizer("foo.o: \\\t",
+ \\target = {foo.o}
+ \\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
+ );
+ try depTokenizer("foo.o: \\ ",
+ \\target = {foo.o}
+ \\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line
+ );
+ try depTokenizer("foo.o: \\x",
+ \\target = {foo.o}
+ \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
+ );
+ try depTokenizer("foo.o: \\\x0dx",
+ \\target = {foo.o}
+ \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
+ );
+}
+
+test "error prereq - continuation expecting end-of-line" {
+ try depTokenizer("foo.o: foo.h\\\x0dx",
+ \\target = {foo.o}
+ \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
+ );
+}
+
+// - tokenize input, emit textual representation, and compare to expect
+fn depTokenizer(input: []const u8, expect: []const u8) !void {
+ var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
+ const arena = arena_allocator.allocator();
+ defer arena_allocator.deinit();
+
+ var it: Tokenizer = .{ .bytes = input };
+ var buffer = std.ArrayList(u8).init(arena);
+ var resolve_buf = std.ArrayList(u8).init(arena);
+ var i: usize = 0;
+ while (it.next()) |token| {
+ if (i != 0) try buffer.appendSlice("\n");
+ switch (token) {
+ .target, .prereq => |bytes| {
+ try buffer.appendSlice(@tagName(token));
+ try buffer.appendSlice(" = {");
+ for (bytes) |b| {
+ try buffer.append(printable_char_tab[b]);
+ }
+ try buffer.appendSlice("}");
+ },
+ .target_must_resolve => {
+ try buffer.appendSlice("target = {");
+ try token.resolve(resolve_buf.writer());
+ for (resolve_buf.items) |b| {
+ try buffer.append(printable_char_tab[b]);
+ }
+ resolve_buf.items.len = 0;
+ try buffer.appendSlice("}");
+ },
+ else => {
+ try buffer.appendSlice("ERROR: ");
+ try token.printError(buffer.writer());
+ break;
+ },
+ }
+ i += 1;
+ }
+
+ if (std.mem.eql(u8, expect, buffer.items)) {
+ try testing.expect(true);
+ return;
+ }
+
+ const out = std.io.getStdErr().writer();
+
+ try out.writeAll("\n");
+ try printSection(out, "<<<< input", input);
+ try printSection(out, "==== expect", expect);
+ try printSection(out, ">>>> got", buffer.items);
+ try printRuler(out);
+
+ try testing.expect(false);
+}
+
+fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
+ try printLabel(out, label, bytes);
+ try hexDump(out, bytes);
+ try printRuler(out);
+ try out.writeAll(bytes);
+ try out.writeAll("\n");
+}
+
+fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
+ var buf: [80]u8 = undefined;
+ var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
+ try out.writeAll(text);
+ var i: usize = text.len;
+ const end = 79;
+ while (i < end) : (i += 1) {
+ try out.writeAll(&[_]u8{label[0]});
+ }
+ try out.writeAll("\n");
+}
+
+fn printRuler(out: anytype) !void {
+ var i: usize = 0;
+ const end = 79;
+ while (i < end) : (i += 1) {
+ try out.writeAll("-");
+ }
+ try out.writeAll("\n");
+}
+
+fn hexDump(out: anytype, bytes: []const u8) !void {
+ const n16 = bytes.len >> 4;
+ var line: usize = 0;
+ var offset: usize = 0;
+ while (line < n16) : (line += 1) {
+ try hexDump16(out, offset, bytes[offset .. offset + 16]);
+ offset += 16;
+ }
+
+ const n = bytes.len & 0x0f;
+ if (n > 0) {
+ try printDecValue(out, offset, 8);
+ try out.writeAll(":");
+ try out.writeAll(" ");
+ var end1 = std.math.min(offset + n, offset + 8);
+ for (bytes[offset..end1]) |b| {
+ try out.writeAll(" ");
+ try printHexValue(out, b, 2);
+ }
+ var end2 = offset + n;
+ if (end2 > end1) {
+ try out.writeAll(" ");
+ for (bytes[end1..end2]) |b| {
+ try out.writeAll(" ");
+ try printHexValue(out, b, 2);
+ }
+ }
+ const short = 16 - n;
+ var i: usize = 0;
+ while (i < short) : (i += 1) {
+ try out.writeAll(" ");
+ }
+ if (end2 > end1) {
+ try out.writeAll(" |");
+ } else {
+ try out.writeAll(" |");
+ }
+ try printCharValues(out, bytes[offset..end2]);
+ try out.writeAll("|\n");
+ offset += n;
+ }
+
+ try printDecValue(out, offset, 8);
+ try out.writeAll(":");
+ try out.writeAll("\n");
+}
+
+fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
+ try printDecValue(out, offset, 8);
+ try out.writeAll(":");
+ try out.writeAll(" ");
+ for (bytes[0..8]) |b| {
+ try out.writeAll(" ");
+ try printHexValue(out, b, 2);
+ }
+ try out.writeAll(" ");
+ for (bytes[8..16]) |b| {
+ try out.writeAll(" ");
+ try printHexValue(out, b, 2);
+ }
+ try out.writeAll(" |");
+ try printCharValues(out, bytes);
+ try out.writeAll("|\n");
+}
+
+fn printDecValue(out: anytype, value: u64, width: u8) !void {
+ var buffer: [20]u8 = undefined;
+ const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' });
+ try out.writeAll(buffer[0..len]);
+}
+
+fn printHexValue(out: anytype, value: u64, width: u8) !void {
+ var buffer: [16]u8 = undefined;
+ const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' });
+ try out.writeAll(buffer[0..len]);
+}
+
+fn printCharValues(out: anytype, bytes: []const u8) !void {
+ for (bytes) |b| {
+ try out.writeAll(&[_]u8{printable_char_tab[b]});
+ }
+}
+
+fn printUnderstandableChar(out: anytype, char: u8) !void {
+ if (std.ascii.isPrint(char)) {
+ try out.print("'{c}'", .{char});
+ } else {
+ try out.print("\\x{X:0>2}", .{char});
+ }
+}
+
+// zig fmt: off
+const printable_char_tab: [256]u8 = (
+ "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
+ "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++
+ "................................................................" ++
+ "................................................................"
+).*;
diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig
index 879793f781..1f145f8171 100644
--- a/lib/std/Build/CompileStep.zig
+++ b/lib/std/Build/CompileStep.zig
@@ -83,7 +83,7 @@ max_memory: ?u64 = null,
shared_memory: bool = false,
global_base: ?u64 = null,
c_std: std.Build.CStd,
-override_lib_dir: ?[]const u8,
+zig_lib_dir: ?[]const u8,
main_pkg_path: ?[]const u8,
exec_cmd_args: ?[]const ?[]const u8,
name_prefix: []const u8,
@@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
.installed_headers = ArrayList(*Step).init(builder.allocator),
.object_src = undefined,
.c_std = std.Build.CStd.C99,
- .override_lib_dir = null,
+ .zig_lib_dir = null,
.main_pkg_path = null,
.exec_cmd_args = null,
.name_prefix = "",
@@ -442,6 +442,26 @@ pub fn installHeader(a: *CompileStep, src_path: []const u8, dest_rel_path: []con
a.installed_headers.append(&install_file.step) catch @panic("OOM");
}
+pub const InstallConfigHeaderOptions = struct {
+ install_dir: InstallDir = .header,
+ dest_rel_path: ?[]const u8 = null,
+};
+
+pub fn installConfigHeader(
+ cs: *CompileStep,
+ config_header: *ConfigHeaderStep,
+ options: InstallConfigHeaderOptions,
+) void {
+ const dest_rel_path = options.dest_rel_path orelse config_header.include_path;
+ const install_file = cs.builder.addInstallFileWithDir(
+ .{ .generated = &config_header.output_file },
+ options.install_dir,
+ dest_rel_path,
+ );
+ cs.builder.getInstallStep().dependOn(&install_file.step);
+ cs.installed_headers.append(&install_file.step) catch @panic("OOM");
+}
+
pub fn installHeadersDirectory(
a: *CompileStep,
src_dir_path: []const u8,
@@ -486,26 +506,11 @@ pub fn installLibraryHeaders(a: *CompileStep, l: *CompileStep) void {
a.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM");
}
-/// Creates a `RunStep` with an executable built with `addExecutable`.
-/// Add command line arguments with `addArg`.
+/// Deprecated: use `std.Build.addRunArtifact`
+/// This function will run in the context of the package that created the executable,
+/// which is undesirable when running an executable provided by a dependency package.
pub fn run(exe: *CompileStep) *RunStep {
- assert(exe.kind == .exe or exe.kind == .test_exe);
-
- // It doesn't have to be native. We catch that if you actually try to run it.
- // Consider that this is declarative; the run step may not be run unless a user
- // option is supplied.
- const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}));
- run_step.addArtifactArg(exe);
-
- if (exe.kind == .test_exe) {
- run_step.addArg(exe.builder.zig_exe);
- }
-
- if (exe.vcpkg_bin_path) |path| {
- run_step.addPathDir(path);
- }
-
- return run_step;
+ return exe.builder.addRunArtifact(exe);
}
/// Creates an `EmulatableRunStep` with an executable built with `addExecutable`.
@@ -852,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
}
pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
- self.override_lib_dir = self.builder.dupePath(dir_path);
+ self.zig_lib_dir = self.builder.dupePath(dir_path);
}
pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
@@ -1345,10 +1350,10 @@ fn make(step: *Step) !void {
}
try zig_args.append("--cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.cache_root));
+ try zig_args.append(builder.cache_root.path orelse ".");
try zig_args.append("--global-cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.global_cache_root));
+ try zig_args.append(builder.global_cache_root.path orelse ".");
try zig_args.append("--name");
try zig_args.append(self.name);
@@ -1622,8 +1627,9 @@ fn make(step: *Step) !void {
}
},
.config_header_step => |config_header| {
- try zig_args.append("-I");
- try zig_args.append(config_header.output_dir);
+ const full_file_path = config_header.output_file.path.?;
+ const header_dir_path = full_file_path[0 .. full_file_path.len - config_header.include_path.len];
+ try zig_args.appendSlice(&.{ "-I", header_dir_path });
},
}
}
@@ -1697,12 +1703,12 @@ fn make(step: *Step) !void {
try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
try addFlag(&zig_args, "build-id", self.build_id);
- if (self.override_lib_dir) |dir| {
+ if (self.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
try zig_args.append(builder.pathFromRoot(dir));
- } else if (builder.override_lib_dir) |dir| {
+ } else if (builder.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
- try zig_args.append(builder.pathFromRoot(dir));
+ try zig_args.append(dir);
}
if (self.main_pkg_path) |dir| {
@@ -1739,23 +1745,15 @@ fn make(step: *Step) !void {
args_length += arg.len + 1; // +1 to account for null terminator
}
if (args_length >= 30 * 1024) {
- const args_dir = try fs.path.join(
- builder.allocator,
- &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
- );
- try std.fs.cwd().makePath(args_dir);
-
- var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
- defer args_arena.deinit();
+ try builder.cache_root.handle.makePath("args");
const args_to_escape = zig_args.items[2..];
- var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);
-
+ var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
arg_blk: for (args_to_escape) |arg| {
for (arg) |c, arg_idx| {
if (c == '\\' or c == '"') {
// Slow path for arguments that need to be escaped. We'll need to allocate and copy
- var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);
+ var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
const writer = escaped.writer();
try writer.writeAll(arg[0..arg_idx]);
for (arg[arg_idx..]) |to_escape| {
@@ -1783,11 +1781,16 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&args_hash)},
);
- const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
- try std.fs.cwd().writeFile(args_file, args);
+ const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
+ try builder.cache_root.handle.writeFile(args_file, args);
+
+ const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
+ "@",
+ try builder.cache_root.join(builder.allocator, &.{args_file}),
+ });
zig_args.shrinkRetainingCapacity(2);
- try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
+ try zig_args.append(resolved_args_file);
}
const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);
diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig
index 58a78b939d..f8d6f7bd57 100644
--- a/lib/std/Build/ConfigHeaderStep.zig
+++ b/lib/std/Build/ConfigHeaderStep.zig
@@ -4,13 +4,24 @@ const Step = std.Build.Step;
pub const base_id: Step.Id = .config_header;
-pub const Style = enum {
+pub const Style = union(enum) {
/// The configure format supported by autotools. It uses `#undef foo` to
/// mark lines that can be substituted with different values.
- autoconf,
+ autoconf: std.Build.FileSource,
/// The configure format supported by CMake. It uses `@@FOO@@` and
/// `#cmakedefine` for template substitution.
- cmake,
+ cmake: std.Build.FileSource,
+ /// Instead of starting with an input file, start with nothing.
+ blank,
+ /// Start with nothing, like blank, and output a nasm .asm file.
+ nasm,
+
+ pub fn getFileSource(style: Style) ?std.Build.FileSource {
+ switch (style) {
+ .autoconf, .cmake => |s| return s,
+ .blank, .nasm => return null,
+ }
+ }
};
pub const Value = union(enum) {
@@ -24,34 +35,50 @@ pub const Value = union(enum) {
step: Step,
builder: *std.Build,
-source: std.Build.FileSource,
+values: std.StringArrayHashMap(Value),
+output_file: std.Build.GeneratedFile,
+
style: Style,
-values: std.StringHashMap(Value),
-max_bytes: usize = 2 * 1024 * 1024,
-output_dir: []const u8,
-output_basename: []const u8,
+max_bytes: usize,
+include_path: []const u8,
+
+pub const Options = struct {
+ style: Style = .blank,
+ max_bytes: usize = 2 * 1024 * 1024,
+ include_path: ?[]const u8 = null,
+};
-pub fn create(builder: *std.Build, source: std.Build.FileSource, style: Style) *ConfigHeaderStep {
+pub fn create(builder: *std.Build, options: Options) *ConfigHeaderStep {
const self = builder.allocator.create(ConfigHeaderStep) catch @panic("OOM");
- const name = builder.fmt("configure header {s}", .{source.getDisplayName()});
+ const name = if (options.style.getFileSource()) |s|
+ builder.fmt("configure {s} header {s}", .{ @tagName(options.style), s.getDisplayName() })
+ else
+ builder.fmt("configure {s} header", .{@tagName(options.style)});
self.* = .{
.builder = builder,
.step = Step.init(base_id, name, builder.allocator, make),
- .source = source,
- .style = style,
- .values = std.StringHashMap(Value).init(builder.allocator),
- .output_dir = undefined,
- .output_basename = "config.h",
+ .style = options.style,
+ .values = std.StringArrayHashMap(Value).init(builder.allocator),
+
+ .max_bytes = options.max_bytes,
+ .include_path = "config.h",
+ .output_file = .{ .step = &self.step },
};
- switch (source) {
+
+ if (options.style.getFileSource()) |s| switch (s) {
.path => |p| {
const basename = std.fs.path.basename(p);
if (std.mem.endsWith(u8, basename, ".h.in")) {
- self.output_basename = basename[0 .. basename.len - 3];
+ self.include_path = basename[0 .. basename.len - 3];
}
},
else => {},
+ };
+
+ if (options.include_path) |include_path| {
+ self.include_path = include_path;
}
+
return self;
}
@@ -59,6 +86,10 @@ pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
return addValuesInner(self, values) catch @panic("OOM");
}
+pub fn getFileSource(self: *ConfigHeaderStep) std.Build.FileSource {
+ return .{ .generated = &self.output_file };
+}
+
fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
try putValue(self, field.name, field.type, @field(values, field.name));
@@ -100,6 +131,12 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
return;
}
},
+ .Int => {
+ if (ptr.size == .Slice and ptr.child == u8) {
+ try self.values.put(field_name, .{ .string = v });
+ return;
+ }
+ },
else => {},
}
@@ -112,8 +149,6 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
fn make(step: *Step) !void {
const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
const gpa = self.builder.allocator;
- const src_path = self.source.getPath(self.builder);
- const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
// The cache is used here not really as a way to speed things up - because writing
// the data to a file would probably be very fast - but as a way to find a canonical
@@ -130,9 +165,39 @@ fn make(step: *Step) !void {
// Random bytes to make ConfigHeaderStep unique. Refresh this with new
// random bytes when ConfigHeaderStep implementation is modified in a
// non-backwards-compatible way.
- var hash = Hasher.init("X1pQzdDt91Zlh7Eh");
- hash.update(self.source.getDisplayName());
- hash.update(contents);
+ var hash = Hasher.init("PGuDTpidxyMqnkGM");
+
+ var output = std.ArrayList(u8).init(gpa);
+ defer output.deinit();
+
+ const header_text = "This file was generated by ConfigHeaderStep using the Zig Build System.";
+ const c_generated_line = "/* " ++ header_text ++ " */\n";
+ const asm_generated_line = "; " ++ header_text ++ "\n";
+
+ switch (self.style) {
+ .autoconf => |file_source| {
+ try output.appendSlice(c_generated_line);
+ const src_path = file_source.getPath(self.builder);
+ const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
+ try render_autoconf(contents, &output, self.values, src_path);
+ },
+ .cmake => |file_source| {
+ try output.appendSlice(c_generated_line);
+ const src_path = file_source.getPath(self.builder);
+ const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
+ try render_cmake(contents, &output, self.values, src_path);
+ },
+ .blank => {
+ try output.appendSlice(c_generated_line);
+ try render_blank(&output, self.values, self.include_path);
+ },
+ .nasm => {
+ try output.appendSlice(asm_generated_line);
+ try render_nasm(&output, self.values);
+ },
+ }
+
+ hash.update(output.items);
var digest: [16]u8 = undefined;
hash.final(&digest);
@@ -143,38 +208,40 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- self.output_dir = try std.fs.path.join(gpa, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
- });
- var dir = std.fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| {
- std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });
+ const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
+
+ // If output_path has directory parts, deal with them. Example:
+ // output_dir is zig-cache/o/HASH
+ // output_path is libavutil/avconfig.h
+ // We want to open directory zig-cache/o/HASH/libavutil/
+ // but keep output_dir as zig-cache/o/HASH for -I include
+ const sub_dir_path = if (std.fs.path.dirname(self.include_path)) |d|
+ try std.fs.path.join(gpa, &.{ output_dir, d })
+ else
+ output_dir;
+
+ var dir = std.fs.cwd().makeOpenPath(sub_dir_path, .{}) catch |err| {
+ std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
return err;
};
defer dir.close();
- var values_copy = try self.values.clone();
- defer values_copy.deinit();
-
- var output = std.ArrayList(u8).init(gpa);
- defer output.deinit();
- try output.ensureTotalCapacity(contents.len);
-
- try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
+ try dir.writeFile(std.fs.path.basename(self.include_path), output.items);
- switch (self.style) {
- .autoconf => try render_autoconf(contents, &output, &values_copy, src_path),
- .cmake => try render_cmake(contents, &output, &values_copy, src_path),
- }
-
- try dir.writeFile(self.output_basename, output.items);
+ self.output_file.path = try std.fs.path.join(self.builder.allocator, &.{
+ output_dir, self.include_path,
+ });
}
fn render_autoconf(
contents: []const u8,
output: *std.ArrayList(u8),
- values_copy: *std.StringHashMap(Value),
+ values: std.StringArrayHashMap(Value),
src_path: []const u8,
) !void {
+ var values_copy = try values.clone();
+ defer values_copy.deinit();
+
var any_errors = false;
var line_index: u32 = 0;
var line_it = std.mem.split(u8, contents, "\n");
@@ -192,22 +259,18 @@ fn render_autoconf(
continue;
}
const name = it.rest();
- const kv = values_copy.fetchRemove(name) orelse {
+ const kv = values_copy.fetchSwapRemove(name) orelse {
std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{
src_path, line_index + 1, name,
});
any_errors = true;
continue;
};
- try renderValue(output, name, kv.value);
+ try renderValueC(output, name, kv.value);
}
- {
- var it = values_copy.iterator();
- while (it.next()) |entry| {
- const name = entry.key_ptr.*;
- std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
- }
+ for (values_copy.keys()) |name| {
+ std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
}
if (any_errors) {
@@ -218,9 +281,12 @@ fn render_autoconf(
fn render_cmake(
contents: []const u8,
output: *std.ArrayList(u8),
- values_copy: *std.StringHashMap(Value),
+ values: std.StringArrayHashMap(Value),
src_path: []const u8,
) !void {
+ var values_copy = try values.clone();
+ defer values_copy.deinit();
+
var any_errors = false;
var line_index: u32 = 0;
var line_it = std.mem.split(u8, contents, "\n");
@@ -244,22 +310,18 @@ fn render_cmake(
any_errors = true;
continue;
};
- const kv = values_copy.fetchRemove(name) orelse {
+ const kv = values_copy.fetchSwapRemove(name) orelse {
std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{
src_path, line_index + 1, name,
});
any_errors = true;
continue;
};
- try renderValue(output, name, kv.value);
+ try renderValueC(output, name, kv.value);
}
- {
- var it = values_copy.iterator();
- while (it.next()) |entry| {
- const name = entry.key_ptr.*;
- std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
- }
+ for (values_copy.keys()) |name| {
+ std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name });
}
if (any_errors) {
@@ -267,7 +329,44 @@ fn render_cmake(
}
}
-fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+fn render_blank(
+ output: *std.ArrayList(u8),
+ defines: std.StringArrayHashMap(Value),
+ include_path: []const u8,
+) !void {
+ const include_guard_name = try output.allocator.dupe(u8, include_path);
+ for (include_guard_name) |*byte| {
+ switch (byte.*) {
+ 'a'...'z' => byte.* = byte.* - 'a' + 'A',
+ 'A'...'Z', '0'...'9' => continue,
+ else => byte.* = '_',
+ }
+ }
+
+ try output.appendSlice("#ifndef ");
+ try output.appendSlice(include_guard_name);
+ try output.appendSlice("\n#define ");
+ try output.appendSlice(include_guard_name);
+ try output.appendSlice("\n");
+
+ const values = defines.values();
+ for (defines.keys()) |name, i| {
+ try renderValueC(output, name, values[i]);
+ }
+
+ try output.appendSlice("#endif /* ");
+ try output.appendSlice(include_guard_name);
+ try output.appendSlice(" */\n");
+}
+
+fn render_nasm(output: *std.ArrayList(u8), defines: std.StringArrayHashMap(Value)) !void {
+ const values = defines.values();
+ for (defines.keys()) |name, i| {
+ try renderValueNasm(output, name, values[i]);
+ }
+}
+
+fn renderValueC(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
switch (value) {
.undef => {
try output.appendSlice("/* #undef ");
@@ -297,3 +396,33 @@ fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void
},
}
}
+
+fn renderValueNasm(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+ switch (value) {
+ .undef => {
+ try output.appendSlice("; %undef ");
+ try output.appendSlice(name);
+ try output.appendSlice("\n");
+ },
+ .defined => {
+ try output.appendSlice("%define ");
+ try output.appendSlice(name);
+ try output.appendSlice("\n");
+ },
+ .boolean => |b| {
+ try output.appendSlice("%define ");
+ try output.appendSlice(name);
+ try output.appendSlice(if (b) " 1\n" else " 0\n");
+ },
+ .int => |i| {
+ try output.writer().print("%define {s} {d}\n", .{ name, i });
+ },
+ .ident => |ident| {
+ try output.writer().print("%define {s} {s}\n", .{ name, ident });
+ },
+ .string => |string| {
+ // TODO: use nasm-specific escaping instead of zig string literals
+ try output.writer().print("%define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) });
+ },
+ }
+}
diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig
index 8a50456539..e5c3e23821 100644
--- a/lib/std/Build/OptionsStep.zig
+++ b/lib/std/Build/OptionsStep.zig
@@ -234,26 +234,20 @@ fn make(step: *Step) !void {
);
}
- const options_directory = self.builder.pathFromRoot(
- try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ self.builder.cache_root, "options" },
- ),
- );
-
- try fs.cwd().makePath(options_directory);
+ var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
+ defer options_dir.close();
- const options_file = try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ options_directory, &self.hashContentsToFileName() },
- );
+ const basename = self.hashContentsToFileName();
- try fs.cwd().writeFile(options_file, self.contents.items);
+ try options_dir.writeFile(&basename, self.contents.items);
- self.generated_file.path = options_file;
+ self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "options", &basename,
+ });
}
fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
+ // TODO update to use the cache system instead of this
// This implementation is copied from `WriteFileStep.make`
var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
@@ -289,13 +283,19 @@ test "OptionsStep" {
const host = try std.zig.system.NativeTargetInfo.detect(.{});
+ var cache: std.Build.Cache = .{
+ .gpa = arena.allocator(),
+ .manifest_dir = std.fs.cwd(),
+ };
+
var builder = try std.Build.create(
arena.allocator(),
"test",
- "test",
- "test",
- "test",
+ .{ .path = "test", .handle = std.fs.cwd() },
+ .{ .path = "test", .handle = std.fs.cwd() },
+ .{ .path = "test", .handle = std.fs.cwd() },
host,
+ &cache,
);
defer builder.destroy();
diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig
index 07f2363623..5bc271409a 100644
--- a/lib/std/Build/RunStep.zig
+++ b/lib/std/Build/RunStep.zig
@@ -39,6 +39,14 @@ expected_exit_code: ?u8 = 0,
/// Print the command before running it
print: bool,
+/// Controls whether execution is skipped if the output file is up-to-date.
+/// The default is to always run if there is no output file, and to skip
+/// running if all output files are up-to-date.
+condition: enum { output_outdated, always } = .output_outdated,
+
+/// Additional file paths relative to build.zig that, when modified, indicate
+/// that the RunStep should be re-executed.
+extra_file_dependencies: []const []const u8 = &.{},
pub const StdIoAction = union(enum) {
inherit,
@@ -51,6 +59,12 @@ pub const Arg = union(enum) {
artifact: *CompileStep,
file_source: std.Build.FileSource,
bytes: []u8,
+ output: Output,
+
+ pub const Output = struct {
+ generated_file: *std.Build.GeneratedFile,
+ basename: []const u8,
+ };
};
pub fn create(builder: *std.Build, name: []const u8) *RunStep {
@@ -71,6 +85,20 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void {
self.step.dependOn(&artifact.step);
}
+/// This provides file path as a command line argument to the command being
+/// run, and returns a FileSource which can be used as inputs to other APIs
+/// throughout the build system.
+pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource {
+ const generated_file = rs.builder.allocator.create(std.Build.GeneratedFile) catch @panic("OOM");
+ generated_file.* = .{ .step = &rs.step };
+ rs.argv.append(.{ .output = .{
+ .generated_file = generated_file,
+ .basename = rs.builder.dupe(basename),
+ } }) catch @panic("OOM");
+
+ return .{ .generated = generated_file };
+}
+
pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {
self.argv.append(Arg{
.file_source = file_source.dupe(self.builder),
@@ -159,25 +187,105 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
};
}
+fn needOutputCheck(self: RunStep) bool {
+ if (self.extra_file_dependencies.len > 0) return true;
+
+ for (self.argv.items) |arg| switch (arg) {
+ .output => return true,
+ else => continue,
+ };
+
+ return switch (self.condition) {
+ .always => false,
+ .output_outdated => true,
+ };
+}
+
fn make(step: *Step) !void {
const self = @fieldParentPtr(RunStep, "step", step);
+ const need_output_check = self.needOutputCheck();
var argv_list = ArrayList([]const u8).init(self.builder.allocator);
+ var output_placeholders = ArrayList(struct {
+ index: usize,
+ output: Arg.Output,
+ }).init(self.builder.allocator);
+
+ var man = self.builder.cache.obtain();
+ defer man.deinit();
+
for (self.argv.items) |arg| {
switch (arg) {
- .bytes => |bytes| try argv_list.append(bytes),
- .file_source => |file| try argv_list.append(file.getPath(self.builder)),
+ .bytes => |bytes| {
+ try argv_list.append(bytes);
+ man.hash.addBytes(bytes);
+ },
+ .file_source => |file| {
+ const file_path = file.getPath(self.builder);
+ try argv_list.append(file_path);
+ _ = try man.addFile(file_path, null);
+ },
.artifact => |artifact| {
if (artifact.target.isWindows()) {
// On Windows we don't have rpaths so we have to add .dll search paths to PATH
self.addPathForDynLibs(artifact);
}
- const executable_path = artifact.installed_path orelse artifact.getOutputSource().getPath(self.builder);
- try argv_list.append(executable_path);
+ const file_path = artifact.installed_path orelse
+ artifact.getOutputSource().getPath(self.builder);
+
+ try argv_list.append(file_path);
+
+ _ = try man.addFile(file_path, null);
+ },
+ .output => |output| {
+ man.hash.addBytes(output.basename);
+ // Add a placeholder into the argument list because we need the
+ // manifest hash to be updated with all arguments before the
+ // object directory is computed.
+ try argv_list.append("");
+ try output_placeholders.append(.{
+ .index = argv_list.items.len - 1,
+ .output = output,
+ });
},
}
}
+ if (need_output_check) {
+ for (self.extra_file_dependencies) |file_path| {
+ _ = try man.addFile(self.builder.pathFromRoot(file_path), null);
+ }
+
+ if (man.hit() catch |err| failWithCacheError(man, err)) {
+ // cache hit, skip running command
+ const digest = man.final();
+ for (output_placeholders.items) |placeholder| {
+ placeholder.output.generated_file.path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ }
+ return;
+ }
+
+ const digest = man.final();
+
+ for (output_placeholders.items) |placeholder| {
+ const output_path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ const output_dir = fs.path.dirname(output_path).?;
+ fs.cwd().makePath(output_dir) catch |err| {
+ std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
+ return err;
+ };
+
+ placeholder.output.generated_file.path = output_path;
+ argv_list.items[placeholder.index] = output_path;
+ }
+ }
+
try runCommand(
argv_list.items,
self.builder,
@@ -189,6 +297,10 @@ fn make(step: *Step) !void {
self.cwd,
self.print,
);
+
+ if (need_output_check) {
+ try man.writeManifest();
+ }
}
pub fn runCommand(
@@ -202,11 +314,13 @@ pub fn runCommand(
maybe_cwd: ?[]const u8,
print: bool,
) !void {
- const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root;
+ const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path;
if (!std.process.can_spawn) {
const cmd = try std.mem.join(builder.allocator, " ", argv);
- std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
+ std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
+ @tagName(builtin.os.tag), cmd,
+ });
builder.allocator.free(cmd);
return ExecError.ExecNotSupported;
}
@@ -347,6 +461,19 @@ pub fn runCommand(
}
}
+fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
+ const i = man.failed_file_index orelse failWithSimpleError(err);
+ const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
+ const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
+ std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
+ std.process.exit(1);
+}
+
+fn failWithSimpleError(err: anyerror) noreturn {
+ std.debug.print("{s}\n", .{@errorName(err)});
+ std.process.exit(1);
+}
+
fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});
for (argv) |arg| {
diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/TranslateCStep.zig
index d9874142d8..fb0adfd0ae 100644
--- a/lib/std/Build/TranslateCStep.zig
+++ b/lib/std/Build/TranslateCStep.zig
@@ -15,7 +15,6 @@ builder: *std.Build,
source: std.Build.FileSource,
include_dirs: std.ArrayList([]const u8),
c_macros: std.ArrayList([]const u8),
-output_dir: ?[]const u8,
out_basename: []const u8,
target: CrossTarget,
optimize: std.builtin.OptimizeMode,
@@ -36,7 +35,6 @@ pub fn create(builder: *std.Build, options: Options) *TranslateCStep {
.source = source,
.include_dirs = std.ArrayList([]const u8).init(builder.allocator),
.c_macros = std.ArrayList([]const u8).init(builder.allocator),
- .output_dir = null,
.out_basename = undefined,
.target = options.target,
.optimize = options.optimize,
@@ -122,15 +120,10 @@ fn make(step: *Step) !void {
const output_path = mem.trimRight(u8, output_path_nl, "\r\n");
self.out_basename = fs.path.basename(output_path);
- if (self.output_dir) |output_dir| {
- const full_dest = try fs.path.join(self.builder.allocator, &[_][]const u8{ output_dir, self.out_basename });
- try self.builder.updateFile(output_path, full_dest);
- } else {
- self.output_dir = fs.path.dirname(output_path).?;
- }
+ const output_dir = fs.path.dirname(output_path).?;
self.output_file.path = try fs.path.join(
self.builder.allocator,
- &[_][]const u8{ self.output_dir.?, self.out_basename },
+ &[_][]const u8{ output_dir, self.out_basename },
);
}
diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig
index 9e8fcdc203..1621295ad8 100644
--- a/lib/std/Build/WriteFileStep.zig
+++ b/lib/std/Build/WriteFileStep.zig
@@ -9,7 +9,6 @@ pub const base_id = .write_file;
step: Step,
builder: *std.Build,
-output_dir: []const u8,
files: std.TailQueue(File),
pub const File = struct {
@@ -23,7 +22,6 @@ pub fn init(builder: *std.Build) WriteFileStep {
.builder = builder,
.step = Step.init(.write_file, "writefile", builder.allocator, make),
.files = .{},
- .output_dir = undefined,
};
}
@@ -87,11 +85,11 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
+ const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "o", &hash_basename,
});
- var dir = fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| {
- std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });
+ var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
+ std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
return err;
};
defer dir.close();
@@ -101,14 +99,14 @@ fn make(step: *Step) !void {
dir.writeFile(node.data.basename, node.data.bytes) catch |err| {
std.debug.print("unable to write {s} into {s}: {s}\n", .{
node.data.basename,
- self.output_dir,
+ output_dir,
@errorName(err),
});
return err;
};
node.data.source.path = try fs.path.join(
self.builder.allocator,
- &[_][]const u8{ self.output_dir, node.data.basename },
+ &[_][]const u8{ output_dir, node.data.basename },
);
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-28 05:04:11 +0000