Merge pull request #1924 from ziglang/tls

Implement Thread Local Variables

12 files changed, 114 insertions, 48 deletions

diff --git a/src/all_types.hpp b/src/all_types.hpp
index c4c9e13cfb..5af4e71157 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -544,12 +544,7 @@ struct AstNodeDefer {
 };
 
 struct AstNodeVariableDeclaration {
-    VisibMod visib_mod;
     Buf *symbol;
-    bool is_const;
-    bool is_comptime;
-    bool is_export;
-    bool is_extern;
     // one or both of type and expr will be non null
     AstNode *type;
     AstNode *expr;
@@ -559,6 +554,13 @@ struct AstNodeVariableDeclaration {
     AstNode *align_expr;
     // populated if the "section(S)" is present
     AstNode *section_expr;
+    Token *threadlocal_tok;
+
+    VisibMod visib_mod;
+    bool is_const;
+    bool is_comptime;
+    bool is_export;
+    bool is_extern;
 };
 
 struct AstNodeTestDecl {
@@ -1873,6 +1875,7 @@ struct ZigVar {
     bool shadowable;
     bool src_is_const;
     bool gen_is_const;
+    bool is_thread_local;
 };
 
 struct ErrorTableEntry {
diff --git a/src/analyze.cpp b/src/analyze.cpp
index d92d337c69..83a576554a 100644
--- a/src/analyze.cpp
+++ b/src/analyze.cpp
@@ -28,28 +28,10 @@ static Error ATTRIBUTE_MUST_USE resolve_enum_zero_bits(CodeGen *g, ZigType *enum
 static Error ATTRIBUTE_MUST_USE resolve_union_zero_bits(CodeGen *g, ZigType *union_type);
 static void analyze_fn_body(CodeGen *g, ZigFn *fn_table_entry);
 
-ErrorMsg *add_node_error(CodeGen *g, AstNode *node, Buf *msg) {
-    if (node->owner->c_import_node != nullptr) {
-        // if this happens, then translate_c generated code that
-        // failed semantic analysis, which isn't supposed to happen
-        ErrorMsg *err = add_node_error(g, node->owner->c_import_node,
-            buf_sprintf("compiler bug: @cImport generated invalid zig code"));
-
-        add_error_note(g, err, node, msg);
-
-        g->errors.append(err);
-        return err;
-    }
-
-    ErrorMsg *err = err_msg_create_with_line(node->owner->path, node->line, node->column,
-            node->owner->source_code, node->owner->line_offsets, msg);
-
-    g->errors.append(err);
-    return err;
-}
-
-ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, AstNode *node, Buf *msg) {
-    if (node->owner->c_import_node != nullptr) {
+static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ImportTableEntry *owner, Token *token,
+        Buf *msg)
+{
+    if (owner->c_import_node != nullptr) {
         // if this happens, then translate_c generated code that
         // failed semantic analysis, which isn't supposed to happen
 
@@ -64,13 +46,46 @@ ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, AstNode *node, Buf *m
         return note;
     }
 
-    ErrorMsg *err = err_msg_create_with_line(node->owner->path, node->line, node->column,
-            node->owner->source_code, node->owner->line_offsets, msg);
+    ErrorMsg *err = err_msg_create_with_line(owner->path, token->start_line, token->start_column,
+            owner->source_code, owner->line_offsets, msg);
 
     err_msg_add_note(parent_msg, err);
     return err;
 }
 
+ErrorMsg *add_token_error(CodeGen *g, ImportTableEntry *owner, Token *token, Buf *msg) {
+    if (owner->c_import_node != nullptr) {
+        // if this happens, then translate_c generated code that
+        // failed semantic analysis, which isn't supposed to happen
+        ErrorMsg *err = add_node_error(g, owner->c_import_node,
+            buf_sprintf("compiler bug: @cImport generated invalid zig code"));
+
+        add_error_note_token(g, err, owner, token, msg);
+
+        g->errors.append(err);
+        return err;
+    }
+    ErrorMsg *err = err_msg_create_with_line(owner->path, token->start_line, token->start_column,
+            owner->source_code, owner->line_offsets, msg);
+
+    g->errors.append(err);
+    return err;
+}
+
+ErrorMsg *add_node_error(CodeGen *g, AstNode *node, Buf *msg) {
+    Token fake_token;
+    fake_token.start_line = node->line;
+    fake_token.start_column = node->column;
+    return add_token_error(g, node->owner, &fake_token, msg);
+}
+
+ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, AstNode *node, Buf *msg) {
+    Token fake_token;
+    fake_token.start_line = node->line;
+    fake_token.start_column = node->column;
+    return add_error_note_token(g, parent_msg, node->owner, &fake_token, msg);
+}
+
 ZigType *new_type_table_entry(ZigTypeId id) {
     ZigType *entry = allocate<ZigType>(1);
     entry->id = id;
@@ -3668,6 +3683,7 @@ static void resolve_decl_var(CodeGen *g, TldVar *tld_var) {
     bool is_const = var_decl->is_const;
     bool is_extern = var_decl->is_extern;
     bool is_export = var_decl->is_export;
+    bool is_thread_local = var_decl->threadlocal_tok != nullptr;
 
     ZigType *explicit_type = nullptr;
     if (var_decl->type) {
@@ -3727,6 +3743,7 @@ static void resolve_decl_var(CodeGen *g, TldVar *tld_var) {
     tld_var->var = add_variable(g, source_node, tld_var->base.parent_scope, var_decl->symbol,
             is_const, init_val, &tld_var->base, type);
     tld_var->var->linkage = linkage;
+    tld_var->var->is_thread_local = is_thread_local;
 
     if (implicit_type != nullptr && type_is_invalid(implicit_type)) {
         tld_var->var->var_type = g->builtin_types.entry_invalid;
@@ -3747,6 +3764,10 @@ static void resolve_decl_var(CodeGen *g, TldVar *tld_var) {
         }
     }
 
+    if (is_thread_local && is_const) {
+        add_node_error(g, source_node, buf_sprintf("threadlocal variable cannot be constant"));
+    }
+
     g->global_vars.append(tld_var);
 }
 
diff --git a/src/analyze.hpp b/src/analyze.hpp
index f558fa44b0..9773782510 100644
--- a/src/analyze.hpp
+++ b/src/analyze.hpp
@@ -12,6 +12,7 @@
 
 void semantic_analyze(CodeGen *g);
 ErrorMsg *add_node_error(CodeGen *g, AstNode *node, Buf *msg);
+ErrorMsg *add_token_error(CodeGen *g, ImportTableEntry *owner, Token *token, Buf *msg);
 ErrorMsg *add_error_note(CodeGen *g, ErrorMsg *parent_msg, AstNode *node, Buf *msg);
 ZigType *new_type_table_entry(ZigTypeId id);
 ZigType *get_pointer_to_type(CodeGen *g, ZigType *child_type, bool is_const);
diff --git a/src/ast_render.cpp b/src/ast_render.cpp
index 994ba5f5b1..34a7faa2a5 100644
--- a/src/ast_render.cpp
+++ b/src/ast_render.cpp
@@ -132,6 +132,10 @@ static const char *const_or_var_string(bool is_const) {
     return is_const ? "const" : "var";
 }
 
+static const char *thread_local_string(Token *tok) {
+    return (tok == nullptr) ? "" : "threadlocal ";
+}
+
 const char *container_string(ContainerKind kind) {
     switch (kind) {
         case ContainerKindEnum: return "enum";
@@ -554,8 +558,9 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) {
             {
                 const char *pub_str = visib_mod_string(node->data.variable_declaration.visib_mod);
                 const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
+                const char *thread_local_str = thread_local_string(node->data.variable_declaration.threadlocal_tok);
                 const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
-                fprintf(ar->f, "%s%s%s ", pub_str, extern_str, const_or_var);
+                fprintf(ar->f, "%s%s%s%s ", pub_str, extern_str, thread_local_str, const_or_var);
                 print_symbol(ar, node->data.variable_declaration.symbol);
 
                 if (node->data.variable_declaration.type) {
diff --git a/src/codegen.cpp b/src/codegen.cpp
index 226a137796..c5de05e372 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -88,7 +88,7 @@ static const char *symbols_that_llvm_depends_on[] = {
 };
 
 CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode,
-    Buf *zig_lib_dir)
+    Buf *zig_lib_dir, Buf *override_std_dir)
 {
     CodeGen *g = allocate<CodeGen>(1);
 
@@ -96,8 +96,12 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out
 
     g->zig_lib_dir = zig_lib_dir;
 
-    g->zig_std_dir = buf_alloc();
-    os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
+    if (override_std_dir == nullptr) {
+        g->zig_std_dir = buf_alloc();
+        os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
+    } else {
+        g->zig_std_dir = override_std_dir;
+    }
 
     g->zig_c_headers_dir = buf_alloc();
     os_path_join(zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir);
@@ -6341,6 +6345,12 @@ static void validate_inline_fns(CodeGen *g) {
     report_errors_and_maybe_exit(g);
 }
 
+static void set_global_tls(CodeGen *g, ZigVar *var, LLVMValueRef global_value) {
+    if (var->is_thread_local && !g->is_single_threaded) {
+        LLVMSetThreadLocalMode(global_value, LLVMGeneralDynamicTLSModel);
+    }
+}
+
 static void do_code_gen(CodeGen *g) {
     assert(!g->errors.length);
 
@@ -6425,6 +6435,7 @@ static void do_code_gen(CodeGen *g) {
                 maybe_import_dll(g, global_value, GlobalLinkageIdStrong);
                 LLVMSetAlignment(global_value, var->align_bytes);
                 LLVMSetGlobalConstant(global_value, var->gen_is_const);
+                set_global_tls(g, var, global_value);
             }
         } else {
             bool exported = (var->linkage == VarLinkageExport);
@@ -6450,6 +6461,7 @@ static void do_code_gen(CodeGen *g) {
             }
 
             LLVMSetGlobalConstant(global_value, var->gen_is_const);
+            set_global_tls(g, var, global_value);
         }
 
         var->value_ref = global_value;
@@ -7500,6 +7512,7 @@ static Error define_builtin_compile_vars(CodeGen *g) {
     g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename);
     g->root_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
     g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
+    g->std_package->package_table.put(buf_create_from_str("std"), g->std_package);
     g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents);
     scan_import(g, g->compile_var_import);
 
@@ -8329,8 +8342,12 @@ static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) {
         if (!entry)
             break;
 
-        cache_buf(ch, entry->key);
-        add_cache_pkg(g, ch, entry->value);
+        // TODO: I think we need a more sophisticated detection of
+        // packages we have already seen
+        if (entry->value != pkg) {
+            cache_buf(ch, entry->key);
+            add_cache_pkg(g, ch, entry->value);
+        }
     }
 }
 
diff --git a/src/codegen.hpp b/src/codegen.hpp
index 6f1cdfb677..4bd8f2dcca 100644
--- a/src/codegen.hpp
+++ b/src/codegen.hpp
@@ -15,7 +15,7 @@
 #include <stdio.h>
 
 CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode,
-    Buf *zig_lib_dir);
+    Buf *zig_lib_dir, Buf *override_std_dir);
 
 void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len);
 void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len);
diff --git a/src/ir.cpp b/src/ir.cpp
index 3cbbdc8103..02b2b12230 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -5204,6 +5204,10 @@ static IrInstruction *ir_gen_var_decl(IrBuilder *irb, Scope *scope, AstNode *nod
         add_node_error(irb->codegen, variable_declaration->section_expr,
             buf_sprintf("cannot set section of local variable '%s'", buf_ptr(variable_declaration->symbol)));
     }
+    if (variable_declaration->threadlocal_tok != nullptr) {
+        add_token_error(irb->codegen, node->owner, variable_declaration->threadlocal_tok,
+            buf_sprintf("function-local variable '%s' cannot be threadlocal", buf_ptr(variable_declaration->symbol)));
+    }
 
     // Temporarily set the name of the IrExecutable to the VariableDeclaration
     // so that the struct or enum from the init expression inherits the name.
diff --git a/src/link.cpp b/src/link.cpp
index 593f7f309f..58221a99ea 100644
--- a/src/link.cpp
+++ b/src/link.cpp
@@ -42,7 +42,7 @@ static Buf *build_a_raw(CodeGen *parent_gen, const char *aname, Buf *full_path)
     }
 
     CodeGen *child_gen = codegen_create(full_path, child_target, child_out_type,
-        parent_gen->build_mode, parent_gen->zig_lib_dir);
+        parent_gen->build_mode, parent_gen->zig_lib_dir, parent_gen->zig_std_dir);
 
     child_gen->out_h_path = nullptr;
     child_gen->verbose_tokenize = parent_gen->verbose_tokenize;
diff --git a/src/main.cpp b/src/main.cpp
index 81f49089be..73a1d75d42 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -74,6 +74,7 @@ static int print_full_usage(const char *arg0) {
         "  -dirafter [dir]              same as -isystem but do it last\n"
         "  -isystem [dir]               add additional search path for other .h files\n"
         "  -mllvm [arg]                 forward an arg to LLVM's option processing\n"
+        "  --override-std-dir [arg]     use an alternate Zig standard library\n"
         "\n"
         "Link Options:\n"
         "  --dynamic-linker [path]      set the path to ld.so\n"
@@ -395,6 +396,7 @@ int main(int argc, char **argv) {
     bool system_linker_hack = false;
     TargetSubsystem subsystem = TargetSubsystemAuto;
     bool is_single_threaded = false;
+    Buf *override_std_dir = nullptr;
 
     if (argc >= 2 && strcmp(argv[1], "build") == 0) {
         Buf zig_exe_path_buf = BUF_INIT;
@@ -430,7 +432,8 @@ int main(int argc, char **argv) {
         Buf *build_runner_path = buf_alloc();
         os_path_join(get_zig_special_dir(), buf_create_from_str("build_runner.zig"), build_runner_path);
 
-        CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir());
+        CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir(),
+                override_std_dir);
         g->enable_time_report = timing_info;
         buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
         codegen_set_out_name(g, buf_create_from_str("build"));
@@ -645,6 +648,8 @@ int main(int argc, char **argv) {
                     clang_argv.append(argv[i]);
 
                     llvm_argv.append(argv[i]);
+                } else if (strcmp(arg, "--override-std-dir") == 0) {
+                    override_std_dir = buf_create_from_str(argv[i]);
                 } else if (strcmp(arg, "--library-path") == 0 || strcmp(arg, "-L") == 0) {
                     lib_dirs.append(argv[i]);
                 } else if (strcmp(arg, "--library") == 0) {
@@ -819,7 +824,7 @@ int main(int argc, char **argv) {
 
     switch (cmd) {
     case CmdBuiltin: {
-        CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir());
+        CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir(), override_std_dir);
         g->is_single_threaded = is_single_threaded;
         Buf *builtin_source = codegen_generate_builtin_source(g);
         if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) {
@@ -878,7 +883,8 @@ int main(int argc, char **argv) {
             if (cmd == CmdRun && buf_out_name == nullptr) {
                 buf_out_name = buf_create_from_str("run");
             }
-            CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir());
+            CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir(),
+                    override_std_dir);
             g->subsystem = subsystem;
 
             if (disable_pic) {
diff --git a/src/parser.cpp b/src/parser.cpp
index 81bd469d1c..1c1af87c51 100644
--- a/src/parser.cpp
+++ b/src/parser.cpp
@@ -844,12 +844,17 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc) {
 
 // VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON
 static AstNode *ast_parse_var_decl(ParseContext *pc) {
-    Token *first = eat_token_if(pc, TokenIdKeywordConst);
-    if (first == nullptr)
-        first = eat_token_if(pc, TokenIdKeywordVar);
-    if (first == nullptr)
-        return nullptr;
-
+    Token *thread_local_kw = eat_token_if(pc, TokenIdKeywordThreadLocal);
+    Token *mut_kw = eat_token_if(pc, TokenIdKeywordConst);
+    if (mut_kw == nullptr)
+        mut_kw = eat_token_if(pc, TokenIdKeywordVar);
+    if (mut_kw == nullptr) {
+        if (thread_local_kw == nullptr) {
+            return nullptr;
+        } else {
+            ast_invalid_token_error(pc, peek_token(pc));
+        }
+    }
     Token *identifier = expect_token(pc, TokenIdSymbol);
     AstNode *type_expr = nullptr;
     if (eat_token_if(pc, TokenIdColon) != nullptr)
@@ -863,8 +868,9 @@ static AstNode *ast_parse_var_decl(ParseContext *pc) {
 
     expect_token(pc, TokenIdSemicolon);
 
-    AstNode *res = ast_create_node(pc, NodeTypeVariableDeclaration, first);
-    res->data.variable_declaration.is_const = first->id == TokenIdKeywordConst;
+    AstNode *res = ast_create_node(pc, NodeTypeVariableDeclaration, mut_kw);
+    res->data.variable_declaration.threadlocal_tok = thread_local_kw;
+    res->data.variable_declaration.is_const = mut_kw->id == TokenIdKeywordConst;
     res->data.variable_declaration.symbol = token_buf(identifier);
     res->data.variable_declaration.type = type_expr;
     res->data.variable_declaration.align_expr = align_expr;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index d43bfabf6d..3acd605748 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -146,6 +146,7 @@ static const struct ZigKeyword zig_keywords[] = {
     {"suspend", TokenIdKeywordSuspend},
     {"switch", TokenIdKeywordSwitch},
     {"test", TokenIdKeywordTest},
+    {"threadlocal", TokenIdKeywordThreadLocal},
     {"true", TokenIdKeywordTrue},
     {"try", TokenIdKeywordTry},
     {"undefined", TokenIdKeywordUndefined},
@@ -1586,6 +1587,7 @@ const char * token_name(TokenId id) {
         case TokenIdKeywordStruct: return "struct";
         case TokenIdKeywordSwitch: return "switch";
         case TokenIdKeywordTest: return "test";
+        case TokenIdKeywordThreadLocal: return "threadlocal";
         case TokenIdKeywordTrue: return "true";
         case TokenIdKeywordTry: return "try";
         case TokenIdKeywordUndefined: return "undefined";
diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp
index 1574e95571..17f36699b3 100644
--- a/src/tokenizer.hpp
+++ b/src/tokenizer.hpp
@@ -88,6 +88,7 @@ enum TokenId {
     TokenIdKeywordSuspend,
     TokenIdKeywordSwitch,
     TokenIdKeywordTest,
+    TokenIdKeywordThreadLocal,
     TokenIdKeywordTrue,
     TokenIdKeywordTry,
     TokenIdKeywordUndefined,