From d0bcc390e8f61ada470b524e3fd203c1af521a99 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 17:05:58 -0700 Subject: get `zig fetch` working with the new system * start renaming "package" to "module" (see #14307) - build system gains `main_mod_path` and `main_pkg_path` is still there but it is deprecated. * eliminate the object-oriented memory management style of what was previously `*Package`. Now it is `*Package.Module` and all pointers point to externally managed memory. * fixes to get the new Fetch.zig code working. The previous commit was work-in-progress. There are still two commented out code paths, the one that leads to `Compilation.create` and the one for `zig build` that fetches the entire dependency tree and creates the required modules for the build runner. --- lib/std/Build/Cache.zig | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib/std/Build/Cache.zig') diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f76985a01a..bb36bb978f 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,13 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn cwd() Directory { + return .{ + .path = null, + .handle = fs.cwd(), + }; + } + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { if (self.path) |p| { // TODO clean way to do this with only 1 allocation @@ -53,6 +60,10 @@ pub const Directory = struct { try writer.writeAll(fs.path.sep_str); } } + + pub fn eql(self: Directory, other: Directory) bool { + return self.handle.fd == other.handle.fd; + } }; gpa: Allocator, -- cgit v1.2.3 From e5c2a7dbca103b0c61bebaff95c5d5a5b777bd59 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Oct 2023 22:34:45 -0700 Subject: finish hooking up new dependency tree logic * add Module instances for each package's build.zig and attach it to the dependencies.zig module with the hash digest hex string as the name. * fix incorrectly skipping the wrong packages for creating dependencies.zig * a couple more renaming of "package" to "module" --- lib/std/Build/Cache.zig | 7 +++++ src/Module.zig | 6 ++-- src/Package.zig | 7 +++++ src/Package/Fetch.zig | 7 +++-- src/Sema.zig | 10 +++---- src/main.zig | 79 +++++++++++++++++++++++++++++++------------------ 6 files changed, 77 insertions(+), 39 deletions(-) (limited to 'lib/std/Build/Cache.zig') diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index bb36bb978f..1ad111f2e3 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,13 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory { + return .{ + .path = if (d.path) |p| try arena.dupe(u8, p) else null, + .handle = d.handle, + }; + } + pub fn cwd() Directory { return .{ .path = null, diff --git a/src/Module.zig b/src/Module.zig index 153edeafef..41f4ec2b41 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4074,7 +4074,7 @@ pub fn importFile( return mod.importPkg(pkg); } if (!mem.endsWith(u8, import_string, ".zig")) { - return error.PackageNotFound; + return error.ModuleNotFound; } const gpa = mod.gpa; @@ -4120,7 +4120,7 @@ pub fn importFile( { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); @@ -4206,7 +4206,7 @@ pub fn embedFile(mod: *Module, cur_file: *File, import_string: []const u8) !*Emb { break :p try gpa.dupe(u8, resolved_path); } - return error.ImportOutsidePkgPath; + return error.ImportOutsideModulePath; }; errdefer gpa.free(sub_file_path); diff --git a/src/Package.zig b/src/Package.zig index 5de914e114..84ba10bfa5 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -9,6 +9,13 @@ pub const Path = struct { /// Empty string means the root_dir is the path. sub_path: []const u8 = "", + pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path { + return .{ + .root_dir = try p.root_dir.clone(arena), + .sub_path = try arena.dupe(u8, p.sub_path), + }; + } + pub fn cwd() Path { return .{ .root_dir = Cache.Directory.cwd() }; } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 7ea1fee733..ace1871d42 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -109,7 +109,6 @@ pub const JobQueue = struct { /// build runner to obtain via `@import("@dependencies")`. pub fn createDependenciesModule(jq: *JobQueue, buf: *std.ArrayList(u8)) Allocator.Error!void { const keys = jq.table.keys(); - const values = jq.table.values(); if (keys.len == 0) return createEmptyDependenciesModule(buf); @@ -124,7 +123,11 @@ pub const JobQueue = struct { } }, .{ .keys = keys })); - for (keys[1..], values[1..]) |hash, fetch| { + for (keys, jq.table.values()) |hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } try buf.writer().print( \\ pub const {} = struct {{ \\ pub const build_root = "{q}"; diff --git a/src/Sema.zig b/src/Sema.zig index 5e428c9f77..2df8b8570e 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -13075,13 +13075,13 @@ fn zirImport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air. const operand = inst_data.get(sema.code); const result = mod.importFile(block.getFileScope(mod), operand) catch |err| switch (err) { - error.ImportOutsidePkgPath => { - return sema.fail(block, operand_src, "import of file outside package path: '{s}'", .{operand}); + error.ImportOutsideModulePath => { + return sema.fail(block, operand_src, "import of file outside module path: '{s}'", .{operand}); }, - error.PackageNotFound => { + error.ModuleNotFound => { //const name = try block.getFileScope(mod).mod.getName(sema.gpa, mod.*); //defer sema.gpa.free(name); - return sema.fail(block, operand_src, "no package named '{s}' available within package '{}'", .{ + return sema.fail(block, operand_src, "no module named '{s}' available within module '{}'", .{ operand, block.getFileScope(mod).mod.root, }); }, @@ -13112,7 +13112,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A } const embed_file = mod.embedFile(block.getFileScope(mod), name) catch |err| switch (err) { - error.ImportOutsidePkgPath => { + error.ImportOutsideModulePath => { return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name}); }, else => { diff --git a/src/main.zig b/src/main.zig index bf4c37781f..df32783807 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4885,39 +4885,60 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi } try job_queue.createDependenciesModule(&dependencies_zig_src); - } - { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Package.Manifest.hex64(rand_int); - { - var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, dependencies_zig_src.items); - } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(dependencies_zig_src.items); - const hex_digest = hh.final(); - const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); - try Package.Fetch.renameTmpIntoCache( - local_cache_directory.handle, - tmp_dir_sub_path, - o_dir_sub_path, - ); + const deps_mod = m: { + // Atomically create the file in a directory named after the hash of its contents. + const basename = "dependencies.zig"; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ + Package.Manifest.hex64(rand_int); + { + var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); + defer tmp_dir.close(); + try tmp_dir.writeFile(basename, dependencies_zig_src.items); + } - const deps_mod = try Package.Module.create(arena, .{ - .root = .{ - .root_dir = local_cache_directory, - .sub_path = o_dir_sub_path, - }, - .root_src_path = basename, - }); + var hh: Cache.HashHelper = .{}; + hh.addBytes(build_options.version); + hh.addBytes(dependencies_zig_src.items); + const hex_digest = hh.final(); + + const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); + try Package.Fetch.renameTmpIntoCache( + local_cache_directory.handle, + tmp_dir_sub_path, + o_dir_sub_path, + ); + + break :m try Package.Module.create(arena, .{ + .root = .{ + .root_dir = local_cache_directory, + .sub_path = o_dir_sub_path, + }, + .root_src_path = basename, + }); + }; + { + // We need a Module for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + const m = try Package.Module.create(arena, .{ + .root = try f.package_root.clone(arena), + .root_src_path = if (f.has_build_zig) Package.build_zig_basename else "", + }); + const hash_cloned = try arena.dupe(u8, &hash); + deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); + } + } try main_mod.deps.put(arena, "@dependencies", deps_mod); } + try main_mod.deps.put(arena, "@build", &build_mod); const comp = Compilation.create(gpa, .{ -- cgit v1.2.3 From 47a413361dd6702dd0412ed58003e9ca9d8ba928 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Oct 2023 14:33:39 -0700 Subject: Package.Fetch: fix handling of relative paths --- lib/std/Build/Cache.zig | 2 +- src/Package/Fetch.zig | 81 ++++++++++++++++++++++++++++--------------------- src/main.zig | 8 ++++- 3 files changed, 54 insertions(+), 37 deletions(-) (limited to 'lib/std/Build/Cache.zig') diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 1ad111f2e3..b1ceaa1b09 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -61,7 +61,7 @@ pub const Directory = struct { writer: anytype, ) !void { _ = options; - if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self); + if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self); if (self.path) |p| { try writer.writeAll(p); try writer.writeAll(fs.path.sep_str); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 3d4b58324c..9b4fc13f6f 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -200,7 +200,7 @@ pub const JobQueue = struct { pub const Location = union(enum) { remote: Remote, /// A directory found inside the parent package. - relative_path: []const u8, + relative_path: Package.Path, /// Recursive Fetch tasks will never use this Location, but it may be /// passed in by the CLI. Indicates the file contents here should be copied /// into the global package cache. It may be a file relative to the cwd or @@ -239,8 +239,8 @@ pub fn run(f: *Fetch) RunError!void { // relative path, treat this the same as a cache hit. Otherwise, proceed. const remote = switch (f.location) { - .relative_path => |sub_path| { - if (fs.path.isAbsolute(sub_path)) return f.fail( + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( f.location_tok, try eb.addString("expected path relative to build root; found absolute path"), ); @@ -248,31 +248,19 @@ pub fn run(f: *Fetch) RunError!void { f.hash_tok, try eb.addString("path-based dependencies are not hashed"), ); - f.package_root = try f.parent_package_root.resolvePosix(arena, sub_path); - if (std.mem.startsWith(u8, f.package_root.sub_path, "../")) { + if (std.mem.startsWith(u8, pkg_root.sub_path, "../")) { return f.fail( f.location_tok, - try eb.addString("dependency path outside package"), + try eb.printString("dependency path outside project: '{}{s}'", .{ + pkg_root.root_dir, pkg_root.sub_path, + }), ); } - try loadManifest(f, f.package_root); + f.package_root = pkg_root; + try loadManifest(f, pkg_root); try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; - // Package hashes are used as unique identifiers for packages, so - // we still need one for relative paths. - const digest = h: { - var hasher = Manifest.Hash.init(.{}); - // This hash is a tuple of: - // * whether it relative to the global cache directory or to the root package - // * the relative file path from there to the build root of the package - hasher.update(if (f.package_root.root_dir.eql(cache_root)) - &package_hash_prefix_cached - else - &package_hash_prefix_project); - hasher.update(f.package_root.sub_path); - break :h hasher.finalResult(); - }; - return queueJobsForDeps(f, Manifest.hexDigest(digest)); + return queueJobsForDeps(f); }, .remote => |remote| remote, .path_or_url => |path_or_url| { @@ -310,7 +298,7 @@ pub fn run(f: *Fetch) RunError!void { try loadManifest(f, f.package_root); try checkBuildFileExistence(f); if (!f.job_queue.recursive) return; - return queueJobsForDeps(f, expected_hash); + return queueJobsForDeps(f); } else |err| switch (err) { error.FileNotFound => {}, else => |e| { @@ -450,7 +438,7 @@ fn runResource( // Spawn a new fetch job for each dependency in the manifest file. Use // a mutex and a hash map so that redundant jobs do not get queued up. if (!f.job_queue.recursive) return; - return queueJobsForDeps(f, actual_hex); + return queueJobsForDeps(f); } /// `computeHash` gets a free check for the existence of `build.zig`, but when @@ -534,27 +522,29 @@ fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { } } -fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void { +fn queueJobsForDeps(f: *Fetch) RunError!void { assert(f.job_queue.recursive); // If the package does not have a build.zig.zon file then there are no dependencies. const manifest = f.manifest orelse return; const new_fetches = nf: { - const deps = manifest.dependencies.values(); + const parent_arena = f.arena.allocator(); const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const deps = manifest.dependencies.values(); // Grab the new tasks into a temporary buffer so we can unlock that mutex // as fast as possible. // This overallocates any fetches that get skipped by the `continue` in the // loop below. - const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len); + const new_fetches = try parent_arena.alloc(Fetch, deps.len); var new_fetch_index: usize = 0; f.job_queue.mutex.lock(); defer f.job_queue.mutex.unlock(); try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); - try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1)); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); // There are four cases here: // * Correct hash is provided by manifest. @@ -564,12 +554,8 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void // * Hash is not provided by manifest. // - Hash missing error emitted; `queueJobsForDeps` is not called. // * path-based location is used without a hash. - // - We need to add `hash` to the table now. - switch (f.location) { - .remote => assert(f.job_queue.table.get(hash) == f), - .relative_path => f.job_queue.table.putAssumeCapacityNoClobber(hash, f), - .path_or_url => unreachable, - } + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. for (deps) |dep| { const new_fetch = &new_fetches[new_fetch_index]; @@ -586,7 +572,16 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void break :h multihash_digest; }, } }, - .path => |path| .{ .relative_path = path }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const multihash_digest = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, }; new_fetch_index += 1; f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); @@ -630,6 +625,22 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void } } +pub fn relativePathDigest( + pkg_root: Package.Path, + cache_root: Cache.Directory, +) Manifest.MultiHashHexDigest { + var hasher = Manifest.Hash.init(.{}); + // This hash is a tuple of: + // * whether it relative to the global cache directory or to the root package + // * the relative file path from there to the build root of the package + hasher.update(if (pkg_root.root_dir.eql(cache_root)) + &package_hash_prefix_cached + else + &package_hash_prefix_project); + hasher.update(pkg_root.sub_path); + return Manifest.hexDigest(hasher.finalResult()); +} + pub fn workerRun(f: *Fetch) void { defer f.job_queue.wait_group.finish(); run(f) catch |err| switch (err) { diff --git a/src/main.zig b/src/main.zig index 6c7469cc93..5dc4b0c9c9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4851,10 +4851,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi defer job_queue.deinit(); try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); var fetch: Package.Fetch = .{ .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = "" }, + .location = .{ .relative_path = build_mod.root }, .location_tok = 0, .hash_tok = 0, .parent_package_root = build_mod.root, @@ -4874,6 +4875,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi }; job_queue.all_fetches.appendAssumeCapacity(&fetch); + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory), + &fetch, + ); + job_queue.wait_group.start(); try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{&fetch}); job_queue.wait_group.wait(); -- cgit v1.2.3