rewrite how importing works

 * Introduce the concept of packages. Closes #3
 * Add support for error notes.
 * Introduce `@import` and `@c_import` builtin functions and
   remove the `import` and `c_import` top level declarations.
 * Introduce the `use` top level declaration.
 * Add `--check-unused` parameter to perform semantic
   analysis and codegen on all top level declarations, not
   just exported ones and ones referenced by exported ones.
 * Delete the root export node and add `--library` argument.

1 files changed, 95 insertions, 252 deletions

diff --git a/src/codegen.cpp b/src/codegen.cpp
index 8b8023a31f..4ee6074e21 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -47,19 +47,30 @@ static void init_darwin_native(CodeGen *g) {
     }
 }
 
+static PackageTableEntry *new_package(const char *root_src_dir, const char *root_src_path) {
+    PackageTableEntry *entry = allocate<PackageTableEntry>(1);
+    entry->package_table.init(4);
+    buf_init_from_str(&entry->root_src_dir, root_src_dir);
+    buf_init_from_str(&entry->root_src_path, root_src_path);
+    return entry;
+}
+
 CodeGen *codegen_create(Buf *root_source_dir, const ZigTarget *target) {
     CodeGen *g = allocate<CodeGen>(1);
     g->import_table.init(32);
     g->builtin_fn_table.init(32);
     g->primitive_type_table.init(32);
-    g->unresolved_top_level_decls.init(32);
     g->fn_type_table.init(32);
     g->error_table.init(16);
     g->is_release_build = false;
     g->is_test_build = false;
-    g->root_source_dir = root_source_dir;
     g->error_value_count = 1;
 
+    g->root_package = new_package(buf_ptr(root_source_dir), "");
+    g->std_package = new_package(ZIG_STD_DIR, "index.zig");
+    g->root_package->package_table.put(buf_create_from_str("std"), g->std_package);
+
+
     if (target) {
         // cross compiling, so we can't rely on all the configured stuff since
         // that's for native compilation
@@ -117,6 +128,10 @@ void codegen_set_verbose(CodeGen *g, bool verbose) {
     g->verbose = verbose;
 }
 
+void codegen_set_check_unused(CodeGen *g, bool check_unused) {
+    g->check_unused = check_unused;
+}
+
 void codegen_set_errmsg_color(CodeGen *g, ErrColor err_color) {
     g->err_color = err_color;
 }
@@ -157,6 +172,14 @@ void codegen_add_lib_dir(CodeGen *g, const char *dir) {
     g->lib_dirs.append(dir);
 }
 
+void codegen_add_link_lib(CodeGen *g, const char *lib) {
+    if (strcmp(lib, "c") == 0) {
+        g->link_libc = true;
+    } else {
+        g->link_libs.append(buf_create_from_str(lib));
+    }
+}
+
 void codegen_set_windows_subsystem(CodeGen *g, bool mwindows, bool mconsole) {
     g->windows_subsystem_windows = mwindows;
     g->windows_subsystem_console = mconsole;
@@ -316,6 +339,8 @@ static LLVMValueRef gen_builtin_fn_call_expr(CodeGen *g, AstNode *node) {
         case BuiltinFnIdCInclude:
         case BuiltinFnIdCDefine:
         case BuiltinFnIdCUndef:
+        case BuiltinFnIdImport:
+        case BuiltinFnIdCImport:
             zig_unreachable();
         case BuiltinFnIdCtz:
         case BuiltinFnIdClz:
@@ -844,7 +869,7 @@ static LLVMValueRef gen_field_ptr(CodeGen *g, AstNode *node, TypeTableEntry **ou
 
     LLVMValueRef struct_ptr;
     if (struct_expr_node->type == NodeTypeSymbol) {
-        VariableTableEntry *var = struct_expr_node->data.symbol_expr.variable;
+        VariableTableEntry *var = get_resolved_expr(struct_expr_node)->variable;
         assert(var);
 
         if (var->is_ptr && var->type->id == TypeTableEntryIdPointer) {
@@ -983,6 +1008,17 @@ static LLVMValueRef gen_array_access_expr(CodeGen *g, AstNode *node, bool is_lva
     }
 }
 
+static LLVMValueRef gen_variable(CodeGen *g, AstNode *source_node, VariableTableEntry *variable) {
+    if (!type_has_bits(variable->type)) {
+        return nullptr;
+    } else if (variable->is_ptr) {
+        assert(variable->value_ref);
+        return get_handle_value(g, source_node, variable->value_ref, variable->type);
+    } else {
+        return variable->value_ref;
+    }
+}
+
 static LLVMValueRef gen_field_access_expr(CodeGen *g, AstNode *node, bool is_lvalue) {
     assert(node->type == NodeTypeFieldAccessExpr);
 
@@ -1016,6 +1052,10 @@ static LLVMValueRef gen_field_access_expr(CodeGen *g, AstNode *node, bool is_lva
         } else {
             zig_unreachable();
         }
+    } else if (struct_type->id == TypeTableEntryIdNamespace) {
+        VariableTableEntry *variable = get_resolved_expr(node)->variable;
+        assert(variable);
+        return gen_variable(g, node, variable);
     } else {
         zig_unreachable();
     }
@@ -1027,7 +1067,7 @@ static LLVMValueRef gen_lvalue(CodeGen *g, AstNode *expr_node, AstNode *node,
     LLVMValueRef target_ref;
 
     if (node->type == NodeTypeSymbol) {
-        VariableTableEntry *var = node->data.symbol_expr.variable;
+        VariableTableEntry *var = get_resolved_expr(node)->variable;
         assert(var);
 
         *out_type_entry = var->type;
@@ -2468,21 +2508,18 @@ static LLVMValueRef gen_var_decl_expr(CodeGen *g, AstNode *node) {
 
 static LLVMValueRef gen_symbol(CodeGen *g, AstNode *node) {
     assert(node->type == NodeTypeSymbol);
-    VariableTableEntry *variable = node->data.symbol_expr.variable;
+    VariableTableEntry *variable = get_resolved_expr(node)->variable;
     if (variable) {
-        if (!type_has_bits(variable->type)) {
-            return nullptr;
-        } else if (variable->is_ptr) {
-            assert(variable->value_ref);
-            return get_handle_value(g, node, variable->value_ref, variable->type);
-        } else {
-            return variable->value_ref;
-        }
+        return gen_variable(g, node, variable);
     }
 
+    zig_unreachable();
+
+    /* TODO delete
     FnTableEntry *fn_entry = node->data.symbol_expr.fn_entry;
     assert(fn_entry);
     return fn_entry->fn_value;
+    */
 }
 
 static LLVMValueRef gen_switch_expr(CodeGen *g, AstNode *node) {
@@ -2703,14 +2740,12 @@ static LLVMValueRef gen_expr(CodeGen *g, AstNode *node) {
             // caught by constant expression eval codegen
             zig_unreachable();
         case NodeTypeRoot:
-        case NodeTypeRootExportDecl:
         case NodeTypeFnProto:
         case NodeTypeFnDef:
         case NodeTypeFnDecl:
         case NodeTypeParamDecl:
         case NodeTypeDirective:
-        case NodeTypeImport:
-        case NodeTypeCImport:
+        case NodeTypeUse:
         case NodeTypeStructDecl:
         case NodeTypeStructField:
         case NodeTypeStructValueField:
@@ -2891,6 +2926,7 @@ static LLVMValueRef gen_const_val(CodeGen *g, TypeTableEntry *type_entry, ConstE
         case TypeTableEntryIdNumLitInt:
         case TypeTableEntryIdUndefLit:
         case TypeTableEntryIdVoid:
+        case TypeTableEntryIdNamespace:
             zig_unreachable();
 
     }
@@ -2943,14 +2979,14 @@ static bool skip_fn_codegen(CodeGen *g, FnTableEntry *fn_entry) {
         if (fn_entry == g->main_fn) {
             return true;
         }
-        return fn_entry->ref_count == 0;
+        return false;
     }
 
     if (fn_entry->is_test) {
         return true;
     }
 
-    return fn_entry->ref_count == 0;
+    return false;
 }
 
 static LLVMValueRef gen_test_fn_val(CodeGen *g, FnTableEntry *fn_entry) {
@@ -3297,6 +3333,12 @@ static void define_builtin_types(CodeGen *g) {
         g->builtin_types.entry_invalid = entry;
     }
     {
+        TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdNamespace);
+        buf_init_from_str(&entry->name, "(namespace)");
+        entry->zero_bits = true;
+        g->builtin_types.entry_namespace = entry;
+    }
+    {
         TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdNumLitFloat);
         buf_init_from_str(&entry->name, "(float literal)");
         entry->zero_bits = true;
@@ -3499,14 +3541,6 @@ static void define_builtin_types(CodeGen *g) {
         g->builtin_types.entry_type = entry;
         g->primitive_type_table.put(&entry->name, entry);
     }
-    {
-        // partially complete the error type. we complete it later after we know
-        // error_value_count.
-        TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdPureError);
-        buf_init_from_str(&entry->name, "error");
-        g->builtin_types.entry_pure_error = entry;
-        g->primitive_type_table.put(&entry->name, entry);
-    }
 
     g->builtin_types.entry_u8 = get_int_type(g, false, 8);
     g->builtin_types.entry_u16 = get_int_type(g, false, 16);
@@ -3518,6 +3552,21 @@ static void define_builtin_types(CodeGen *g) {
     g->builtin_types.entry_i64 = get_int_type(g, true, 64);
 
     {
+        TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdPureError);
+        buf_init_from_str(&entry->name, "error");
+
+        // TODO allow overriding this type and keep track of max value and emit an
+        // error if there are too many errors declared
+        g->err_tag_type = g->builtin_types.entry_u16;
+
+        g->builtin_types.entry_pure_error = entry;
+        entry->type_ref = g->err_tag_type->type_ref;
+        entry->di_type = g->err_tag_type->di_type;
+
+        g->primitive_type_table.put(&entry->name, entry);
+    }
+
+    {
         TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdEnum);
         entry->zero_bits = true; // only allowed at compile time
         buf_init_from_str(&entry->name, "@OS");
@@ -3685,12 +3734,11 @@ static void define_builtin_fns(CodeGen *g) {
     create_builtin_fn_with_arg_count(g, BuiltinFnIdConstEval, "const_eval", 1);
     create_builtin_fn_with_arg_count(g, BuiltinFnIdCtz, "ctz", 2);
     create_builtin_fn_with_arg_count(g, BuiltinFnIdClz, "clz", 2);
+    create_builtin_fn_with_arg_count(g, BuiltinFnIdImport, "import", 1);
+    create_builtin_fn_with_arg_count(g, BuiltinFnIdCImport, "c_import", 1);
 }
 
 static void init(CodeGen *g, Buf *source_path) {
-    g->lib_search_paths.append(g->root_source_dir);
-    g->lib_search_paths.append(buf_create_from_str(ZIG_STD_DIR));
-
     g->module = LLVMModuleCreateWithName(buf_ptr(source_path));
 
     get_target_triple(&g->triple_str, &g->zig_target);
@@ -3741,7 +3789,7 @@ static void init(CodeGen *g, Buf *source_path) {
     const char *flags = "";
     unsigned runtime_version = 0;
     g->compile_unit = LLVMZigCreateCompileUnit(g->dbuilder, LLVMZigLang_DW_LANG_C99(),
-            buf_ptr(source_path), buf_ptr(g->root_source_dir),
+            buf_ptr(source_path), buf_ptr(&g->root_package->root_src_dir),
             buf_ptr(producer), is_optimized, flags, runtime_version,
             "", 0, !g->strip_debug_symbols);
 
@@ -3761,9 +3809,6 @@ void codegen_parseh(CodeGen *g, Buf *src_dirname, Buf *src_basename, Buf *source
     ImportTableEntry *import = allocate<ImportTableEntry>(1);
     import->source_code = source_code;
     import->path = full_path;
-    import->fn_table.init(32);
-    import->type_table.init(8);
-    import->error_table.init(8);
     g->root_import = import;
 
     init(g, full_path);
@@ -3791,214 +3836,7 @@ void codegen_render_ast(CodeGen *g, FILE *f, int indent_size) {
 }
 
 
-static int parse_version_string(Buf *buf, int *major, int *minor, int *patch) {
-    char *dot1 = strstr(buf_ptr(buf), ".");
-    if (!dot1)
-        return ErrorInvalidFormat;
-    char *dot2 = strstr(dot1 + 1, ".");
-    if (!dot2)
-        return ErrorInvalidFormat;
-
-    *major = (int)strtol(buf_ptr(buf), nullptr, 10);
-    *minor = (int)strtol(dot1 + 1, nullptr, 10);
-    *patch = (int)strtol(dot2 + 1, nullptr, 10);
-
-    return ErrorNone;
-}
-
-static void set_root_export_version(CodeGen *g, Buf *version_buf, AstNode *node) {
-    int err;
-    if ((err = parse_version_string(version_buf, &g->version_major, &g->version_minor, &g->version_patch))) {
-        add_node_error(g, node,
-                buf_sprintf("invalid version string"));
-    }
-}
-
-
-static ImportTableEntry *codegen_add_code(CodeGen *g, Buf *abs_full_path,
-        Buf *src_dirname, Buf *src_basename, Buf *source_code)
-{
-    int err;
-    Buf *full_path = buf_alloc();
-    os_path_join(src_dirname, src_basename, full_path);
-
-    if (g->verbose) {
-        fprintf(stderr, "\nOriginal Source (%s):\n", buf_ptr(full_path));
-        fprintf(stderr, "----------------\n");
-        fprintf(stderr, "%s\n", buf_ptr(source_code));
-
-        fprintf(stderr, "\nTokens:\n");
-        fprintf(stderr, "---------\n");
-    }
-
-    Tokenization tokenization = {0};
-    tokenize(source_code, &tokenization);
-
-    if (tokenization.err) {
-        ErrorMsg *err = err_msg_create_with_line(full_path, tokenization.err_line, tokenization.err_column,
-                source_code, tokenization.line_offsets, tokenization.err);
-
-        print_err_msg(err, g->err_color);
-        exit(1);
-    }
-
-    if (g->verbose) {
-        print_tokens(source_code, tokenization.tokens);
-
-        fprintf(stderr, "\nAST:\n");
-        fprintf(stderr, "------\n");
-    }
-
-    ImportTableEntry *import_entry = allocate<ImportTableEntry>(1);
-    import_entry->source_code = source_code;
-    import_entry->line_offsets = tokenization.line_offsets;
-    import_entry->path = full_path;
-    import_entry->fn_table.init(32);
-    import_entry->type_table.init(8);
-    import_entry->error_table.init(8);
-
-    import_entry->root = ast_parse(source_code, tokenization.tokens, import_entry, g->err_color,
-            &g->next_node_index);
-    assert(import_entry->root);
-    if (g->verbose) {
-        ast_print(stderr, import_entry->root, 0);
-    }
-
-    import_entry->di_file = LLVMZigCreateFile(g->dbuilder, buf_ptr(src_basename), buf_ptr(src_dirname));
-    g->import_table.put(abs_full_path, import_entry);
-
-    import_entry->block_context = new_block_context(import_entry->root, nullptr);
-    import_entry->block_context->di_scope = LLVMZigFileToScope(import_entry->di_file);
-
-
-    assert(import_entry->root->type == NodeTypeRoot);
-    for (int decl_i = 0; decl_i < import_entry->root->data.root.top_level_decls.length; decl_i += 1) {
-        AstNode *top_level_decl = import_entry->root->data.root.top_level_decls.at(decl_i);
-
-        if (top_level_decl->type == NodeTypeRootExportDecl) {
-            if (g->root_import) {
-                add_node_error(g, top_level_decl,
-                        buf_sprintf("root export declaration only valid in root source file"));
-            } else {
-                ZigList<AstNode *> *directives = top_level_decl->data.root_export_decl.directives;
-                if (directives) {
-                    for (int i = 0; i < directives->length; i += 1) {
-                        AstNode *directive_node = directives->at(i);
-                        Buf *name = &directive_node->data.directive.name;
-                        AstNode *param_node = directive_node->data.directive.expr;
-                        assert(param_node->type == NodeTypeStringLiteral);
-                        Buf *param = &param_node->data.string_literal.buf;
-
-                        if (param) {
-                            if (buf_eql_str(name, "version")) {
-                                set_root_export_version(g, param, directive_node);
-                            } else if (buf_eql_str(name, "link")) {
-                                if (buf_eql_str(param, "c")) {
-                                    g->link_libc = true;
-                                } else {
-                                    g->link_libs.append(param);
-                                }
-                            } else {
-                                add_node_error(g, directive_node,
-                                        buf_sprintf("invalid directive: '%s'", buf_ptr(name)));
-                            }
-                        }
-                    }
-                }
-
-                if (g->root_export_decl) {
-                    add_node_error(g, top_level_decl,
-                            buf_sprintf("only one root export declaration allowed"));
-                } else {
-                    g->root_export_decl = top_level_decl;
-
-                    if (!g->root_out_name)
-                        g->root_out_name = &top_level_decl->data.root_export_decl.name;
-
-                    Buf *out_type = &top_level_decl->data.root_export_decl.type;
-                    OutType export_out_type;
-                    if (buf_eql_str(out_type, "executable")) {
-                        export_out_type = OutTypeExe;
-                    } else if (buf_eql_str(out_type, "library")) {
-                        export_out_type = OutTypeLib;
-                    } else if (buf_eql_str(out_type, "object")) {
-                        export_out_type = OutTypeObj;
-                    } else {
-                        add_node_error(g, top_level_decl,
-                                buf_sprintf("invalid export type: '%s'", buf_ptr(out_type)));
-                    }
-                    if (g->out_type == OutTypeUnknown) {
-                        g->out_type = export_out_type;
-                    }
-                }
-            }
-        } else if (top_level_decl->type == NodeTypeImport) {
-            Buf *import_target_path = &top_level_decl->data.import.path;
-            Buf full_path = BUF_INIT;
-            Buf *import_code = buf_alloc();
-            bool found_it = false;
-
-            for (int path_i = 0; path_i < g->lib_search_paths.length; path_i += 1) {
-                Buf *search_path = g->lib_search_paths.at(path_i);
-                os_path_join(search_path, import_target_path, &full_path);
-
-                Buf *abs_full_path = buf_alloc();
-                if ((err = os_path_real(&full_path, abs_full_path))) {
-                    if (err == ErrorFileNotFound) {
-                        continue;
-                    } else {
-                        g->error_during_imports = true;
-                        add_node_error(g, top_level_decl,
-                                buf_sprintf("unable to open '%s': %s", buf_ptr(&full_path), err_str(err)));
-                        goto done_looking_at_imports;
-                    }
-                }
-
-                auto entry = g->import_table.maybe_get(abs_full_path);
-                if (entry) {
-                    found_it = true;
-                    top_level_decl->data.import.import = entry->value;
-                } else {
-                    if ((err = os_fetch_file_path(abs_full_path, import_code))) {
-                        if (err == ErrorFileNotFound) {
-                            continue;
-                        } else {
-                            g->error_during_imports = true;
-                            add_node_error(g, top_level_decl,
-                                    buf_sprintf("unable to open '%s': %s", buf_ptr(&full_path), err_str(err)));
-                            goto done_looking_at_imports;
-                        }
-                    }
-                    top_level_decl->data.import.import = codegen_add_code(g,
-                            abs_full_path, search_path, &top_level_decl->data.import.path, import_code);
-                    found_it = true;
-                }
-                break;
-            }
-            if (!found_it) {
-                g->error_during_imports = true;
-                add_node_error(g, top_level_decl,
-                        buf_sprintf("unable to find '%s'", buf_ptr(import_target_path)));
-            }
-        } else if (top_level_decl->type == NodeTypeFnDef) {
-            AstNode *proto_node = top_level_decl->data.fn_def.fn_proto;
-            assert(proto_node->type == NodeTypeFnProto);
-            Buf *proto_name = &proto_node->data.fn_proto.name;
-
-            bool is_private = (proto_node->data.fn_proto.visib_mod == VisibModPrivate);
-
-            if (buf_eql_str(proto_name, "main") && !is_private) {
-                g->have_exported_main = true;
-            }
-        }
-    }
-
-done_looking_at_imports:
-
-    return import_entry;
-}
-
-static ImportTableEntry *add_special_code(CodeGen *g, const char *basename) {
+static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package, const char *basename) {
     Buf *std_dir = buf_create_from_str(ZIG_STD_DIR);
     Buf *code_basename = buf_create_from_str(basename);
     Buf path_to_code_src = BUF_INIT;
@@ -4013,12 +3851,22 @@ static ImportTableEntry *add_special_code(CodeGen *g, const char *basename) {
         zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err));
     }
 
-    return codegen_add_code(g, abs_full_path, std_dir, code_basename, import_code);
+    return add_source_file(g, package, abs_full_path, std_dir, code_basename, import_code);
+}
+
+static PackageTableEntry *create_bootstrap_pkg(CodeGen *g) {
+    PackageTableEntry *package = new_package(ZIG_STD_DIR, "");
+    package->package_table.put(buf_create_from_str("std"), g->std_package);
+    package->package_table.put(buf_create_from_str("@root"), g->root_package);
+    return package;
 }
 
 void codegen_add_root_code(CodeGen *g, Buf *src_dir, Buf *src_basename, Buf *source_code) {
     Buf source_path = BUF_INIT;
     os_path_join(src_dir, src_basename, &source_path);
+
+    buf_init_from_buf(&g->root_package->root_src_path, src_basename);
+
     init(g, &source_path);
 
     Buf *abs_full_path = buf_alloc();
@@ -4027,19 +3875,14 @@ void codegen_add_root_code(CodeGen *g, Buf *src_dir, Buf *src_basename, Buf *sou
         zig_panic("unable to open '%s': %s", buf_ptr(&source_path), err_str(err));
     }
 
-    g->root_import = codegen_add_code(g, abs_full_path, src_dir, src_basename, source_code);
+    g->root_import = add_source_file(g, g->root_package, abs_full_path, src_dir, src_basename, source_code);
 
-    if (!g->root_out_name) {
-        add_node_error(g, g->root_import->root,
-                buf_sprintf("missing export declaration and output name not provided"));
-    } else if (g->out_type == OutTypeUnknown) {
-        add_node_error(g, g->root_import->root,
-                buf_sprintf("missing export declaration and export type not provided"));
-    }
+    assert(g->root_out_name);
+    assert(g->out_type != OutTypeUnknown);
 
     if (!g->link_libc && !g->is_test_build) {
         if (g->have_exported_main && (g->out_type == OutTypeObj || g->out_type == OutTypeExe)) {
-            g->bootstrap_import = add_special_code(g, "bootstrap.zig");
+            g->bootstrap_import = add_special_code(g, create_bootstrap_pkg(g), "bootstrap.zig");
         }
     }
 
@@ -4120,7 +3963,7 @@ void codegen_generate_h_file(CodeGen *g) {
         assert(proto_node->type == NodeTypeFnProto);
         AstNodeFnProto *fn_proto = &proto_node->data.fn_proto;
 
-        if (fn_proto->visib_mod != VisibModExport)
+        if (fn_proto->top_level_decl.visib_mod != VisibModExport)
             continue;
 
         Buf return_type_c = BUF_INIT;