get `zig fetch` working with the new system

* start renaming "package" to "module" (see #14307)
  - build system gains `main_mod_path` and `main_pkg_path` is still
    there but it is deprecated.
* eliminate the object-oriented memory management style of what was
  previously `*Package`. Now it is `*Package.Module` and all pointers
  point to externally managed memory.
* fixes to get the new Fetch.zig code working. The previous commit was
  work-in-progress. There are still two commented out code paths, the
  one that leads to `Compilation.create` and the one for `zig build`
  that fetches the entire dependency tree and creates the required
  modules for the build runner.

1 files changed, 84 insertions, 86 deletions

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 28b67ff734..3827789fed 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -273,8 +273,8 @@ const Job = union(enum) {
     /// The source file containing the Decl has been updated, and so the
     /// Decl may need its line number information updated in the debug info.
     update_line_number: Module.Decl.Index,
-    /// The main source file for the package needs to be analyzed.
-    analyze_pkg: *Package,
+    /// The main source file for the module needs to be analyzed.
+    analyze_mod: *Package.Module,
 
     /// one of the glibc static objects
     glibc_crt_file: glibc.CRTFile,
@@ -414,7 +414,7 @@ pub const MiscTask = enum {
     compiler_rt,
     libssp,
     zig_libc,
-    analyze_pkg,
+    analyze_mod,
 
     @"musl crti.o",
     @"musl crtn.o",
@@ -544,7 +544,7 @@ pub const InitOptions = struct {
     global_cache_directory: Directory,
     target: Target,
     root_name: []const u8,
-    main_pkg: ?*Package,
+    main_mod: ?*Package.Module,
     output_mode: std.builtin.OutputMode,
     thread_pool: *ThreadPool,
     dynamic_linker: ?[]const u8 = null,
@@ -736,53 +736,53 @@ pub const InitOptions = struct {
     pdb_out_path: ?[]const u8 = null,
 };
 
-fn addPackageTableToCacheHash(
+fn addModuleTableToCacheHash(
     hash: *Cache.HashHelper,
     arena: *std.heap.ArenaAllocator,
-    pkg_table: Package.Table,
-    seen_table: *std.AutoHashMap(*Package, void),
+    mod_table: Package.Module.Deps,
+    seen_table: *std.AutoHashMap(*Package.Module, void),
     hash_type: union(enum) { path_bytes, files: *Cache.Manifest },
 ) (error{OutOfMemory} || std.os.GetCwdError)!void {
     const allocator = arena.allocator();
 
-    const packages = try allocator.alloc(Package.Table.KV, pkg_table.count());
+    const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count());
     {
         // Copy over the hashmap entries to our slice
-        var table_it = pkg_table.iterator();
+        var table_it = mod_table.iterator();
         var idx: usize = 0;
         while (table_it.next()) |entry| : (idx += 1) {
-            packages[idx] = .{
+            modules[idx] = .{
                 .key = entry.key_ptr.*,
                 .value = entry.value_ptr.*,
             };
         }
     }
     // Sort the slice by package name
-    mem.sort(Package.Table.KV, packages, {}, struct {
-        fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool {
+    mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct {
+        fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool {
             return std.mem.lessThan(u8, lhs.key, rhs.key);
         }
     }.lessThan);
 
-    for (packages) |pkg| {
-        if ((try seen_table.getOrPut(pkg.value)).found_existing) continue;
+    for (modules) |mod| {
+        if ((try seen_table.getOrPut(mod.value)).found_existing) continue;
 
         // Finally insert the package name and path to the cache hash.
-        hash.addBytes(pkg.key);
+        hash.addBytes(mod.key);
         switch (hash_type) {
             .path_bytes => {
-                hash.addBytes(pkg.value.root_src_path);
-                hash.addOptionalBytes(pkg.value.root_src_directory.path);
+                hash.addBytes(mod.value.root_src_path);
+                hash.addOptionalBytes(mod.value.root_src_directory.path);
             },
             .files => |man| {
-                const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{
-                    pkg.value.root_src_path,
+                const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{
+                    mod.value.root_src_path,
                 });
                 _ = try man.addFile(pkg_zig_file, null);
             },
         }
-        // Recurse to handle the package's dependencies
-        try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type);
+        // Recurse to handle the module's dependencies
+        try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type);
     }
 }
 
@@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 break :blk true;
 
             // If we have no zig code to compile, no need for LLVM.
-            if (options.main_pkg == null)
+            if (options.main_mod == null)
                 break :blk false;
 
             // If LLVM does not support the target, then we can't use it.
@@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         // compiler state, the second clause here can be removed so that incremental
         // cache mode is used for LLVM backend too. We need some fuzz testing before
         // that can be enabled.
-        const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching)
+        const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching)
             CacheMode.whole
         else
             options.cache_mode;
@@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             if (use_llvm) {
                 // If stage1 generates an object file, self-hosted linker is not
                 // yet sophisticated enough to handle that.
-                break :blk options.main_pkg != null;
+                break :blk options.main_mod != null;
             }
 
             break :blk false;
@@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
         // TODO audit this and make sure everything is in it
 
-        const module: ?*Module = if (options.main_pkg) |main_pkg| blk: {
+        const module: ?*Module = if (options.main_mod) |main_mod| blk: {
             // Options that are specific to zig source files, that cannot be
             // modified between incremental updates.
             var hash = cache.hash;
@@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                     // do want to namespace different source file names because they are
                     // likely different compilations and therefore this would be likely to
                     // cause cache hits.
-                    hash.addBytes(main_pkg.root_src_path);
-                    hash.addOptionalBytes(main_pkg.root_src_directory.path);
+                    hash.addBytes(main_mod.root_src_path);
+                    hash.addOptionalBytes(main_mod.root.root_dir.path);
+                    hash.addBytes(main_mod.root.sub_path);
                     {
-                        var seen_table = std.AutoHashMap(*Package, void).init(arena);
-                        try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
+                        var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
+                        try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes);
                     }
                 },
                 .whole => {
@@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
             };
 
-            const builtin_pkg = try Package.createWithDir(
-                gpa,
-                zig_cache_artifact_directory,
-                null,
-                "builtin.zig",
-            );
-            errdefer builtin_pkg.destroy(gpa);
+            const builtin_mod = try Package.Module.create(arena, .{
+                .root = .{ .root_dir = zig_cache_artifact_directory },
+                .root_src_path = "builtin.zig",
+            });
 
-            // When you're testing std, the main module is std. In that case, we'll just set the std
-            // module to the main one, since avoiding the errors caused by duplicating it is more
-            // effort than it's worth.
-            const main_pkg_is_std = m: {
+            // When you're testing std, the main module is std. In that case,
+            // we'll just set the std module to the main one, since avoiding
+            // the errors caused by duplicating it is more effort than it's
+            // worth.
+            const main_mod_is_std = m: {
                 const std_path = try std.fs.path.resolve(arena, &[_][]const u8{
                     options.zig_lib_directory.path orelse ".",
                     "std",
                     "std.zig",
                 });
-                defer arena.free(std_path);
                 const main_path = try std.fs.path.resolve(arena, &[_][]const u8{
-                    main_pkg.root_src_directory.path orelse ".",
-                    main_pkg.root_src_path,
+                    main_mod.root.root_dir.path orelse ".",
+                    main_mod.root.sub_path,
+                    main_mod.root_src_path,
                 });
-                defer arena.free(main_path);
                 break :m mem.eql(u8, main_path, std_path);
             };
 
-            const std_pkg = if (main_pkg_is_std)
-                main_pkg
+            const std_mod = if (main_mod_is_std)
+                main_mod
             else
                 try Package.createWithDir(
                     gpa,
@@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                     "std.zig",
                 );
 
-            errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa);
+            errdefer if (!main_mod_is_std) std_mod.destroy(gpa);
 
-            const root_pkg = if (options.is_test) root_pkg: {
+            const root_mod = if (options.is_test) root_mod: {
                 const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: {
                     const test_dir = std.fs.path.dirname(test_runner);
                     const basename = std.fs.path.basename(test_runner);
                     const pkg = try Package.create(gpa, test_dir, basename);
 
-                    // copy package table from main_pkg to root_pkg
-                    pkg.table = try main_pkg.table.clone(gpa);
+                    // copy module table from main_mod to root_mod
+                    pkg.deps = try main_mod.deps.clone(gpa);
                     break :test_pkg pkg;
                 } else try Package.createWithDir(
                     gpa,
@@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 );
                 errdefer test_pkg.destroy(gpa);
 
-                break :root_pkg test_pkg;
-            } else main_pkg;
-            errdefer if (options.is_test) root_pkg.destroy(gpa);
+                break :root_mod test_pkg;
+            } else main_mod;
+            errdefer if (options.is_test) root_mod.destroy(gpa);
 
-            const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: {
-                break :compiler_rt_pkg try Package.createWithDir(
+            const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: {
+                break :compiler_rt_mod try Package.createWithDir(
                     gpa,
                     options.zig_lib_directory,
                     null,
                     "compiler_rt.zig",
                 );
             } else null;
-            errdefer if (compiler_rt_pkg) |p| p.destroy(gpa);
+            errdefer if (compiler_rt_mod) |p| p.destroy(gpa);
 
-            try main_pkg.add(gpa, "builtin", builtin_pkg);
-            try main_pkg.add(gpa, "root", root_pkg);
-            try main_pkg.add(gpa, "std", std_pkg);
+            try main_mod.add(gpa, "builtin", builtin_mod);
+            try main_mod.add(gpa, "root", root_mod);
+            try main_mod.add(gpa, "std", std_mod);
 
-            if (compiler_rt_pkg) |p| {
-                try main_pkg.add(gpa, "compiler_rt", p);
+            if (compiler_rt_mod) |p| {
+                try main_mod.add(gpa, "compiler_rt", p);
             }
 
             // Pre-open the directory handles for cached ZIR code so that it does not need
@@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             module.* = .{
                 .gpa = gpa,
                 .comp = comp,
-                .main_pkg = main_pkg,
-                .root_pkg = root_pkg,
+                .main_mod = main_mod,
+                .root_mod = root_mod,
                 .zig_cache_artifact_directory = zig_cache_artifact_directory,
                 .global_zir_cache = global_zir_cache,
                 .local_zir_cache = local_zir_cache,
@@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director
     // This is only for cleanup purposes; Module.deinit calls close
     // on the handle of zig_cache_artifact_directory.
     if (comp.bin_file.options.module) |module| {
-        const builtin_pkg = module.main_pkg.table.get("builtin").?;
-        module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
+        const builtin_mod = module.main_mod.deps.get("builtin").?;
+        module.zig_cache_artifact_directory = builtin_mod.root_src_directory;
     }
 }
 
@@ -2148,8 +2146,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
 
         // Make sure std.zig is inside the import_table. We unconditionally need
         // it for start.zig.
-        const std_pkg = module.main_pkg.table.get("std").?;
-        _ = try module.importPkg(std_pkg);
+        const std_mod = module.main_mod.deps.get("std").?;
+        _ = try module.importPkg(std_mod);
 
         // Normally we rely on importing std to in turn import the root source file
         // in the start code, but when using the stage1 backend that won't happen,
@@ -2158,11 +2156,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
         // Likewise, in the case of `zig test`, the test runner is the root source file,
         // and so there is nothing to import the main file.
         if (comp.bin_file.options.is_test) {
-            _ = try module.importPkg(module.main_pkg);
+            _ = try module.importPkg(module.main_mod);
         }
 
-        if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
-            _ = try module.importPkg(compiler_rt_pkg);
+        if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
+            _ = try module.importPkg(compiler_rt_mod);
         }
 
         // Put a work item in for every known source file to detect if
@@ -2185,13 +2183,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
             }
         }
 
-        try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
+        try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
         if (comp.bin_file.options.is_test) {
-            try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
+            try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod });
         }
 
-        if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
-            try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg });
+        if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
+            try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod });
         }
     }
 
@@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
     comptime assert(link_hash_implementation_version == 10);
 
     if (comp.bin_file.options.module) |mod| {
-        const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
-            mod.main_pkg.root_src_path,
+        const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{
+            mod.main_mod.root_src_path,
         });
         _ = try man.addFile(main_zig_file, null);
         {
-            var seen_table = std.AutoHashMap(*Package, void).init(arena);
+            var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
 
             // Skip builtin.zig; it is useless as an input, and we don't want to have to
             // write it before checking for a cache hit.
-            const builtin_pkg = mod.main_pkg.table.get("builtin").?;
-            try seen_table.put(builtin_pkg, {});
+            const builtin_mod = mod.main_mod.deps.get("builtin").?;
+            try seen_table.put(builtin_mod, {});
 
-            try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
+            try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man });
         }
 
         // Synchronize with other matching comments: ZigOnlyHashStuff
@@ -3564,8 +3562,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
                 decl.analysis = .codegen_failure_retryable;
             };
         },
-        .analyze_pkg => |pkg| {
-            const named_frame = tracy.namedFrame("analyze_pkg");
+        .analyze_mod => |pkg| {
+            const named_frame = tracy.namedFrame("analyze_mod");
             defer named_frame.end();
 
             const module = comp.bin_file.options.module.?;
@@ -6379,11 +6377,11 @@ fn buildOutputFromZig(
 
     std.debug.assert(output_mode != .Exe);
 
-    var main_pkg: Package = .{
+    var main_mod: Package = .{
         .root_src_directory = comp.zig_lib_directory,
         .root_src_path = src_basename,
     };
-    defer main_pkg.deinitTable(comp.gpa);
+    defer main_mod.deinitTable(comp.gpa);
     const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
     const target = comp.getTarget();
     const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
@@ -6404,7 +6402,7 @@ fn buildOutputFromZig(
         .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
-        .main_pkg = &main_pkg,
+        .main_mod = &main_mod,
         .output_mode = output_mode,
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
@@ -6481,7 +6479,7 @@ pub fn build_crt_file(
         .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
-        .main_pkg = null,
+        .main_mod = null,
         .output_mode = output_mode,
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,