stage1 is now a hybrid of C++ and Zig

This modifies the build process of Zig to put all of the source files
into libcompiler.a, except main.cpp and userland.cpp.

Next, the build process links main.cpp, userland.cpp, and libcompiler.a
into zig1. userland.cpp is a shim for functions that will later be
replaced with self-hosted implementations.

Next, the build process uses zig1 to build src-self-hosted/stage1.zig
into libuserland.a, which does not depend on any of the things that
are shimmed in userland.cpp, such as translate-c.

Finally, the build process re-links main.cpp and libcompiler.a, except
with libuserland.a instead of userland.cpp. Now the shims are replaced
with .zig code. This provides all of the Zig standard library to the
stage1 C++ compiler, and enables us to move certain things to userland,
such as translate-c.

As a proof of concept I have made the `zig zen` command use text defined
in userland. I added `zig translate-c-2` which is a work-in-progress
reimplementation of translate-c in userland, which currently calls
`std.debug.panic("unimplemented")` and you can see the stack trace makes
it all the way back into the C++ main() function (Thanks LemonBoy for
improving that!).

This could potentially let us move other things into userland, such as
hashing algorithms, the entire cache system, .d file parsing, pretty
much anything that libuserland.a itself doesn't need to depend on.

This can also let us have `zig fmt` in stage1 without the overhead
of child process execution, and without the initial compilation delay
before it gets cached.

See #1964

1 files changed, 61 insertions, 40 deletions

diff --git a/src/main.cpp b/src/main.cpp
index 5123512d1c..03cf3aad68 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -14,6 +14,7 @@
 #include "os.hpp"
 #include "target.hpp"
 #include "libc_installation.hpp"
+#include "userland.h"
 
 #include <stdio.h>
 
@@ -40,6 +41,7 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) {
         "  libc [paths_file]            Display native libc paths file or validate one\n"
         "  run [source] [-- [args]]     create executable and run immediately\n"
         "  translate-c [source]         convert c code to zig code\n"
+        "  translate-c-2 [source]       experimental self-hosted translate-c\n"
         "  targets                      list available compilation targets\n"
         "  test [source]                create and run a test build\n"
         "  version                      print version number and exit\n"
@@ -131,19 +133,6 @@ static int print_libc_usage(const char *arg0, FILE *file, int return_code) {
     return return_code;
 }
 
-static const char *ZIG_ZEN = "\n"
-" * Communicate intent precisely.\n"
-" * Edge cases matter.\n"
-" * Favor reading code over writing code.\n"
-" * Only one obvious way to do things.\n"
-" * Runtime crashes are better than bugs.\n"
-" * Compile errors are better than runtime crashes.\n"
-" * Incremental improvements.\n"
-" * Avoid local maximums.\n"
-" * Reduce the amount one must remember.\n"
-" * Minimize energy spent on coding style.\n"
-" * Together we serve end users.\n";
-
 static bool arch_available_in_llvm(ZigLLVM_ArchType arch) {
     LLVMTargetRef target_ref;
     char *err_msg = nullptr;
@@ -211,6 +200,7 @@ enum Cmd {
     CmdTargets,
     CmdTest,
     CmdTranslateC,
+    CmdTranslateCUserland,
     CmdVersion,
     CmdZen,
     CmdLibC,
@@ -324,7 +314,7 @@ int main(int argc, char **argv) {
                 return print_error_usage(arg0);
             }
             Buf *cmd_template_path = buf_alloc();
-            os_path_join(get_zig_special_dir(), buf_create_from_str(init_cmd), cmd_template_path);
+            os_path_join(get_zig_special_dir(get_zig_lib_dir()), buf_create_from_str(init_cmd), cmd_template_path);
             Buf *build_zig_path = buf_alloc();
             os_path_join(cmd_template_path, buf_create_from_str("build.zig"), build_zig_path);
             Buf *src_dir_path = buf_alloc();
@@ -453,6 +443,7 @@ int main(int argc, char **argv) {
     bool want_single_threaded = false;
     bool disable_gen_h = false;
     Buf *override_std_dir = nullptr;
+    Buf *override_lib_dir = nullptr;
     Buf *main_pkg_path = nullptr;
     ValgrindSupport valgrind_support = ValgrindSupportAuto;
     WantPIC want_pic = WantPICAuto;
@@ -486,13 +477,27 @@ int main(int argc, char **argv) {
             } else if (i + 1 < argc && strcmp(argv[i], "--cache-dir") == 0) {
                 cache_dir = argv[i + 1];
                 i += 1;
+            } else if (i + 1 < argc && strcmp(argv[i], "--override-std-dir") == 0) {
+                override_std_dir = buf_create_from_str(argv[i + 1]);
+                i += 1;
+
+                args.append("--override-std-dir");
+                args.append(buf_ptr(override_std_dir));
+            } else if (i + 1 < argc && strcmp(argv[i], "--override-lib-dir") == 0) {
+                override_lib_dir = buf_create_from_str(argv[i + 1]);
+                i += 1;
+
+                args.append("--override-lib-dir");
+                args.append(buf_ptr(override_lib_dir));
             } else {
                 args.append(argv[i]);
             }
         }
 
+        Buf *zig_lib_dir = (override_lib_dir == nullptr) ? get_zig_lib_dir() : override_lib_dir;
+
         Buf *build_runner_path = buf_alloc();
-        os_path_join(get_zig_special_dir(), buf_create_from_str("build_runner.zig"), build_runner_path);
+        os_path_join(get_zig_special_dir(zig_lib_dir), buf_create_from_str("build_runner.zig"), build_runner_path);
 
         ZigTarget target;
         get_native_target(&target);
@@ -512,7 +517,7 @@ int main(int argc, char **argv) {
         }
 
         CodeGen *g = codegen_create(main_pkg_path, build_runner_path, &target, OutTypeExe,
-                BuildModeDebug, get_zig_lib_dir(), override_std_dir, nullptr, &full_cache_dir);
+                BuildModeDebug, override_lib_dir, override_std_dir, nullptr, &full_cache_dir);
         g->valgrind_support = valgrind_support;
         g->enable_time_report = timing_info;
         codegen_set_out_name(g, buf_create_from_str("build"));
@@ -532,23 +537,25 @@ int main(int argc, char **argv) {
                         "Usage: %s build [options]\n"
                         "\n"
                         "General Options:\n"
-                        "  --help                 Print this help and exit\n"
-                        "  --verbose              Print commands before executing them\n"
-                        "  --prefix [path]        Override default install prefix\n"
-                        "  --search-prefix [path] Add a path to look for binaries, libraries, headers\n"
+                        "  --help                   Print this help and exit\n"
+                        "  --verbose                Print commands before executing them\n"
+                        "  --prefix [path]          Override default install prefix\n"
+                        "  --search-prefix [path]   Add a path to look for binaries, libraries, headers\n"
                         "\n"
                         "Project-specific options become available when the build file is found.\n"
                         "\n"
                         "Advanced Options:\n"
-                        "  --build-file [file]    Override path to build.zig\n"
-                        "  --cache-dir [path]     Override path to cache directory\n"
-                        "  --verbose-tokenize     Enable compiler debug output for tokenization\n"
-                        "  --verbose-ast          Enable compiler debug output for parsing into an AST\n"
-                        "  --verbose-link         Enable compiler debug output for linking\n"
-                        "  --verbose-ir           Enable compiler debug output for Zig IR\n"
-                        "  --verbose-llvm-ir      Enable compiler debug output for LLVM IR\n"
-                        "  --verbose-cimport      Enable compiler debug output for C imports\n"
-                        "  --verbose-cc           Enable compiler debug output for C compilation\n"
+                        "  --build-file [file]      Override path to build.zig\n"
+                        "  --cache-dir [path]       Override path to cache directory\n"
+                        "  --override-std-dir [arg] Override path to Zig standard library\n"
+                        "  --override-lib-dir [arg] Override path to Zig lib library\n"
+                        "  --verbose-tokenize       Enable compiler debug output for tokenization\n"
+                        "  --verbose-ast            Enable compiler debug output for parsing into an AST\n"
+                        "  --verbose-link           Enable compiler debug output for linking\n"
+                        "  --verbose-ir             Enable compiler debug output for Zig IR\n"
+                        "  --verbose-llvm-ir        Enable compiler debug output for LLVM IR\n"
+                        "  --verbose-cimport        Enable compiler debug output for C imports\n"
+                        "  --verbose-cc             Enable compiler debug output for C compilation\n"
                         "\n"
                 , zig_exe_path);
                 return EXIT_SUCCESS;
@@ -584,11 +591,12 @@ int main(int argc, char **argv) {
         init_all_targets();
         ZigTarget target;
         get_native_target(&target);
+        Buf *zig_lib_dir = (override_lib_dir == nullptr) ? get_zig_lib_dir() : override_lib_dir;
         Buf *fmt_runner_path = buf_alloc();
-        os_path_join(get_zig_special_dir(), buf_create_from_str("fmt_runner.zig"), fmt_runner_path);
+        os_path_join(get_zig_special_dir(zig_lib_dir), buf_create_from_str("fmt_runner.zig"), fmt_runner_path);
         Buf *cache_dir_buf = buf_create_from_str(cache_dir ? cache_dir : default_zig_cache_name);
         CodeGen *g = codegen_create(main_pkg_path, fmt_runner_path, &target, OutTypeExe,
-                BuildModeDebug, get_zig_lib_dir(), nullptr, nullptr, cache_dir_buf);
+                BuildModeDebug, zig_lib_dir, nullptr, nullptr, cache_dir_buf);
         g->valgrind_support = valgrind_support;
         g->want_single_threaded = true;
         codegen_set_out_name(g, buf_create_from_str("fmt"));
@@ -757,6 +765,8 @@ int main(int argc, char **argv) {
                     llvm_argv.append(argv[i]);
                 } else if (strcmp(arg, "--override-std-dir") == 0) {
                     override_std_dir = buf_create_from_str(argv[i]);
+                } else if (strcmp(arg, "--override-lib-dir") == 0) {
+                    override_lib_dir = buf_create_from_str(argv[i]);
                 } else if (strcmp(arg, "--main-pkg-path") == 0) {
                     main_pkg_path = buf_create_from_str(argv[i]);
                 } else if (strcmp(arg, "--library-path") == 0 || strcmp(arg, "-L") == 0) {
@@ -867,6 +877,8 @@ int main(int argc, char **argv) {
                 cmd = CmdLibC;
             } else if (strcmp(arg, "translate-c") == 0) {
                 cmd = CmdTranslateC;
+            } else if (strcmp(arg, "translate-c-2") == 0) {
+                cmd = CmdTranslateCUserland;
             } else if (strcmp(arg, "test") == 0) {
                 cmd = CmdTest;
                 out_type = OutTypeExe;
@@ -883,6 +895,7 @@ int main(int argc, char **argv) {
                 case CmdBuild:
                 case CmdRun:
                 case CmdTranslateC:
+                case CmdTranslateCUserland:
                 case CmdTest:
                 case CmdLibC:
                     if (!in_file) {
@@ -959,7 +972,7 @@ int main(int argc, char **argv) {
     }
     case CmdBuiltin: {
         CodeGen *g = codegen_create(main_pkg_path, nullptr, &target,
-                out_type, build_mode, get_zig_lib_dir(), override_std_dir, nullptr, nullptr);
+                out_type, build_mode, override_lib_dir, override_std_dir, nullptr, nullptr);
         g->valgrind_support = valgrind_support;
         g->want_pic = want_pic;
         g->want_single_threaded = want_single_threaded;
@@ -973,6 +986,7 @@ int main(int argc, char **argv) {
     case CmdRun:
     case CmdBuild:
     case CmdTranslateC:
+    case CmdTranslateCUserland:
     case CmdTest:
         {
             if (cmd == CmdBuild && !in_file && objects.length == 0 && asm_files.length == 0 &&
@@ -985,14 +999,16 @@ int main(int argc, char **argv) {
                     " * --assembly argument\n"
                     " * --c-source argument\n");
                 return print_error_usage(arg0);
-            } else if ((cmd == CmdTranslateC || cmd == CmdTest || cmd == CmdRun) && !in_file) {
+            } else if ((cmd == CmdTranslateC || cmd == CmdTranslateCUserland ||
+                        cmd == CmdTest || cmd == CmdRun) && !in_file)
+            {
                 fprintf(stderr, "Expected source file argument.\n");
                 return print_error_usage(arg0);
             }
 
             assert(cmd != CmdBuild || out_type != OutTypeUnknown);
 
-            bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC);
+            bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC || cmd == CmdTranslateCUserland);
 
             if (cmd == CmdRun) {
                 out_name = "run";
@@ -1026,7 +1042,8 @@ int main(int argc, char **argv) {
                 return print_error_usage(arg0);
             }
 
-            Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf;
+            Buf *zig_root_source_file = (cmd == CmdTranslateC || cmd == CmdTranslateCUserland) ?
+                nullptr : in_file_buf;
 
             if (cmd == CmdRun && buf_out_name == nullptr) {
                 buf_out_name = buf_create_from_str("run");
@@ -1050,7 +1067,7 @@ int main(int argc, char **argv) {
                 cache_dir_buf = buf_create_from_str(cache_dir);
             }
             CodeGen *g = codegen_create(main_pkg_path, zig_root_source_file, &target, out_type, build_mode,
-                    get_zig_lib_dir(), override_std_dir, libc, cache_dir_buf);
+                    override_lib_dir, override_std_dir, libc, cache_dir_buf);
             if (llvm_argv.length >= 2) codegen_set_llvm_argv(g, llvm_argv.items + 1, llvm_argv.length - 2);
             g->valgrind_support = valgrind_support;
             g->want_pic = want_pic;
@@ -1170,8 +1187,8 @@ int main(int argc, char **argv) {
                 } else {
                     zig_unreachable();
                 }
-            } else if (cmd == CmdTranslateC) {
-                AstNode *root_node = codegen_translate_c(g, in_file_buf);
+            } else if (cmd == CmdTranslateC || cmd == CmdTranslateCUserland) {
+                AstNode *root_node = codegen_translate_c(g, in_file_buf, cmd == CmdTranslateCUserland);
                 ast_render(g, stdout, root_node, 4);
                 if (timing_info)
                     codegen_print_timing_report(g, stderr);
@@ -1229,9 +1246,13 @@ int main(int argc, char **argv) {
     case CmdVersion:
         printf("%s\n", ZIG_VERSION_STRING);
         return EXIT_SUCCESS;
-    case CmdZen:
-        printf("%s\n", ZIG_ZEN);
+    case CmdZen: {
+        const char *ptr;
+        size_t len;
+        stage2_zen(&ptr, &len);
+        fwrite(ptr, len, 1, stdout);
         return EXIT_SUCCESS;
+    }
     case CmdTargets:
         return print_target_list(stdout);
     case CmdNone: