From 8975fa5b38a791d318c0817afb636ba91ad8d038 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 16 Dec 2020 19:22:48 -0700
Subject: stage2: add test_evented_io to cache hash

---
 src/Compilation.zig | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 81a2bff37b..356a1cedf6 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2868,6 +2868,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
     man.hash.addOptionalEmitLoc(comp.emit_analysis);
     man.hash.addOptionalEmitLoc(comp.emit_docs);
+    man.hash.add(comp.test_evented_io);
     man.hash.addOptionalBytes(comp.test_filter);
     man.hash.addOptionalBytes(comp.test_name_prefix);
 
-- 
cgit v1.2.3


From 013efaf13987acfa6b41d40f07900c1ea77f5bda Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 17 Dec 2020 20:03:41 -0700
Subject: std: introduce a thread-local CSPRNG for general use

std.crypto.random

* cross platform, even freestanding
* can't fail. on initialization for some systems requires calling
  os.getrandom(), in which case there are rare but theoretically
  possible errors. The code panics in these cases, however the
  application may choose to override the default seed function and then
  handle the failure another way.
* thread-safe
* supports the full Random interface
* cryptographically secure
* no syscall required to initialize on Linux (AT_RANDOM)
* calls arc4random on systems that support it

`std.crypto.randomBytes` is removed in favor of `std.crypto.random.bytes`.

I moved some of the Random implementations into their own files in the
interest of organization.

stage2 no longer requires passing a RNG; instead it uses this API.

Closes #6704
---
 lib/std/crypto.zig                    |  11 +-
 lib/std/crypto/25519/ed25519.zig      |   8 +-
 lib/std/crypto/25519/edwards25519.zig |   4 +-
 lib/std/crypto/25519/x25519.zig       |   2 +-
 lib/std/crypto/bcrypt.zig             |   4 +-
 lib/std/crypto/salsa20.zig            |  18 +-
 lib/std/crypto/tlcsprng.zig           |  62 ++++
 lib/std/crypto/utils.zig              |   8 +-
 lib/std/fs.zig                        |   4 +-
 lib/std/rand.zig                      | 579 +---------------------------------
 lib/std/rand/Gimli.zig                |  40 +++
 lib/std/rand/Isaac64.zig              | 210 ++++++++++++
 lib/std/rand/Pcg.zig                  | 101 ++++++
 lib/std/rand/Sfc64.zig                | 108 +++++++
 lib/std/rand/Xoroshiro128.zig         | 133 ++++++++
 lib/std/start.zig                     |  26 ++
 lib/std/testing.zig                   |   3 +-
 src/Compilation.zig                   |   7 +-
 src/glibc.zig                         |   1 -
 src/libcxx.zig                        |   2 -
 src/libunwind.zig                     |   1 -
 src/main.zig                          |  15 -
 src/musl.zig                          |   1 -
 src/test.zig                          |  11 +-
 24 files changed, 730 insertions(+), 629 deletions(-)
 create mode 100644 lib/std/crypto/tlcsprng.zig
 create mode 100644 lib/std/rand/Gimli.zig
 create mode 100644 lib/std/rand/Isaac64.zig
 create mode 100644 lib/std/rand/Pcg.zig
 create mode 100644 lib/std/rand/Sfc64.zig
 create mode 100644 lib/std/rand/Xoroshiro128.zig

(limited to 'src/Compilation.zig')

diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig
index 6eb934473f..e3581cde96 100644
--- a/lib/std/crypto.zig
+++ b/lib/std/crypto.zig
@@ -134,8 +134,10 @@ pub const nacl = struct {
 
 pub const utils = @import("crypto/utils.zig");
 
+/// This is a thread-local, cryptographically secure pseudo random number generator.
+pub const random = &@import("crypto/tlcsprng.zig").interface;
+
 const std = @import("std.zig");
-pub const randomBytes = std.os.getrandom;
 
 test "crypto" {
     inline for (std.meta.declarations(@This())) |decl| {
@@ -178,6 +180,13 @@ test "crypto" {
     _ = @import("crypto/25519/ristretto255.zig");
 }
 
+test "CSPRNG" {
+    const a = random.int(u64);
+    const b = random.int(u64);
+    const c = random.int(u64);
+    std.testing.expect(a ^ b ^ c != 0);
+}
+
 test "issue #4532: no index out of bounds" {
     const types = [_]type{
         hash.Md5,
diff --git a/lib/std/crypto/25519/ed25519.zig b/lib/std/crypto/25519/ed25519.zig
index 842b08d706..7f90ba584c 100644
--- a/lib/std/crypto/25519/ed25519.zig
+++ b/lib/std/crypto/25519/ed25519.zig
@@ -43,7 +43,7 @@ pub const Ed25519 = struct {
         pub fn create(seed: ?[seed_length]u8) !KeyPair {
             const ss = seed orelse ss: {
                 var random_seed: [seed_length]u8 = undefined;
-                try crypto.randomBytes(&random_seed);
+                crypto.random.bytes(&random_seed);
                 break :ss random_seed;
             };
             var az: [Sha512.digest_length]u8 = undefined;
@@ -179,7 +179,7 @@ pub const Ed25519 = struct {
 
         var z_batch: [count]Curve.scalar.CompressedScalar = undefined;
         for (z_batch) |*z| {
-            try std.crypto.randomBytes(z[0..16]);
+            std.crypto.random.bytes(z[0..16]);
             mem.set(u8, z[16..], 0);
         }
 
@@ -232,8 +232,8 @@ test "ed25519 batch verification" {
         const key_pair = try Ed25519.KeyPair.create(null);
         var msg1: [32]u8 = undefined;
         var msg2: [32]u8 = undefined;
-        try std.crypto.randomBytes(&msg1);
-        try std.crypto.randomBytes(&msg2);
+        std.crypto.random.bytes(&msg1);
+        std.crypto.random.bytes(&msg2);
         const sig1 = try Ed25519.sign(&msg1, key_pair, null);
         const sig2 = try Ed25519.sign(&msg2, key_pair, null);
         var signature_batch = [_]Ed25519.BatchElement{
diff --git a/lib/std/crypto/25519/edwards25519.zig b/lib/std/crypto/25519/edwards25519.zig
index 008a4535b3..d64f06c421 100644
--- a/lib/std/crypto/25519/edwards25519.zig
+++ b/lib/std/crypto/25519/edwards25519.zig
@@ -484,8 +484,8 @@ test "edwards25519 packing/unpacking" {
 test "edwards25519 point addition/substraction" {
     var s1: [32]u8 = undefined;
     var s2: [32]u8 = undefined;
-    try std.crypto.randomBytes(&s1);
-    try std.crypto.randomBytes(&s2);
+    std.crypto.random.bytes(&s1);
+    std.crypto.random.bytes(&s2);
     const p = try Edwards25519.basePoint.clampedMul(s1);
     const q = try Edwards25519.basePoint.clampedMul(s2);
     const r = p.add(q).add(q).sub(q).sub(q);
diff --git a/lib/std/crypto/25519/x25519.zig b/lib/std/crypto/25519/x25519.zig
index 3b3ff551fe..0bf55d52fc 100644
--- a/lib/std/crypto/25519/x25519.zig
+++ b/lib/std/crypto/25519/x25519.zig
@@ -34,7 +34,7 @@ pub const X25519 = struct {
         pub fn create(seed: ?[seed_length]u8) !KeyPair {
             const sk = seed orelse sk: {
                 var random_seed: [seed_length]u8 = undefined;
-                try crypto.randomBytes(&random_seed);
+                crypto.random.bytes(&random_seed);
                 break :sk random_seed;
             };
             var kp: KeyPair = undefined;
diff --git a/lib/std/crypto/bcrypt.zig b/lib/std/crypto/bcrypt.zig
index 4cec59961b..6813495d25 100644
--- a/lib/std/crypto/bcrypt.zig
+++ b/lib/std/crypto/bcrypt.zig
@@ -262,7 +262,7 @@ fn strHashInternal(password: []const u8, rounds_log: u6, salt: [salt_length]u8)
 /// and then use the resulting hash as the password parameter for bcrypt.
 pub fn strHash(password: []const u8, rounds_log: u6) ![hash_length]u8 {
     var salt: [salt_length]u8 = undefined;
-    try crypto.randomBytes(&salt);
+    crypto.random.bytes(&salt);
     return strHashInternal(password, rounds_log, salt);
 }
 
@@ -283,7 +283,7 @@ pub fn strVerify(h: [hash_length]u8, password: []const u8) BcryptError!void {
 
 test "bcrypt codec" {
     var salt: [salt_length]u8 = undefined;
-    try crypto.randomBytes(&salt);
+    crypto.random.bytes(&salt);
     var salt_str: [salt_str_length]u8 = undefined;
     Codec.encode(salt_str[0..], salt[0..]);
     var salt2: [salt_length]u8 = undefined;
diff --git a/lib/std/crypto/salsa20.zig b/lib/std/crypto/salsa20.zig
index dd3e4fe99b..8122e9b25c 100644
--- a/lib/std/crypto/salsa20.zig
+++ b/lib/std/crypto/salsa20.zig
@@ -571,9 +571,9 @@ test "xsalsa20poly1305" {
     var key: [XSalsa20Poly1305.key_length]u8 = undefined;
     var nonce: [XSalsa20Poly1305.nonce_length]u8 = undefined;
     var tag: [XSalsa20Poly1305.tag_length]u8 = undefined;
-    try crypto.randomBytes(&msg);
-    try crypto.randomBytes(&key);
-    try crypto.randomBytes(&nonce);
+    crypto.random.bytes(&msg);
+    crypto.random.bytes(&key);
+    crypto.random.bytes(&nonce);
 
     XSalsa20Poly1305.encrypt(c[0..], &tag, msg[0..], "ad", nonce, key);
     try XSalsa20Poly1305.decrypt(msg2[0..], c[0..], tag, "ad", nonce, key);
@@ -585,9 +585,9 @@ test "xsalsa20poly1305 secretbox" {
     var key: [XSalsa20Poly1305.key_length]u8 = undefined;
     var nonce: [Box.nonce_length]u8 = undefined;
     var boxed: [msg.len + Box.tag_length]u8 = undefined;
-    try crypto.randomBytes(&msg);
-    try crypto.randomBytes(&key);
-    try crypto.randomBytes(&nonce);
+    crypto.random.bytes(&msg);
+    crypto.random.bytes(&key);
+    crypto.random.bytes(&nonce);
 
     SecretBox.seal(boxed[0..], msg[0..], nonce, key);
     try SecretBox.open(msg2[0..], boxed[0..], nonce, key);
@@ -598,8 +598,8 @@ test "xsalsa20poly1305 box" {
     var msg2: [msg.len]u8 = undefined;
     var nonce: [Box.nonce_length]u8 = undefined;
     var boxed: [msg.len + Box.tag_length]u8 = undefined;
-    try crypto.randomBytes(&msg);
-    try crypto.randomBytes(&nonce);
+    crypto.random.bytes(&msg);
+    crypto.random.bytes(&nonce);
 
     var kp1 = try Box.KeyPair.create(null);
     var kp2 = try Box.KeyPair.create(null);
@@ -611,7 +611,7 @@ test "xsalsa20poly1305 sealedbox" {
     var msg: [100]u8 = undefined;
     var msg2: [msg.len]u8 = undefined;
     var boxed: [msg.len + SealedBox.seal_length]u8 = undefined;
-    try crypto.randomBytes(&msg);
+    crypto.random.bytes(&msg);
 
     var kp = try Box.KeyPair.create(null);
     try SealedBox.seal(boxed[0..], msg[0..], kp.public_key);
diff --git a/lib/std/crypto/tlcsprng.zig b/lib/std/crypto/tlcsprng.zig
new file mode 100644
index 0000000000..0105cf0ce8
--- /dev/null
+++ b/lib/std/crypto/tlcsprng.zig
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! Thread-local cryptographically secure pseudo-random number generator.
+//! This file has public declarations that are intended to be used internally
+//! by the standard library; this namespace is not intended to be exposed
+//! directly to standard library users.
+
+const std = @import("std");
+const root = @import("root");
+const mem = std.mem;
+
+/// We use this as a layer of indirection because global const pointers cannot
+/// point to thread-local variables.
+pub var interface = std.rand.Random{ .fillFn = tlsCsprngFill };
+pub threadlocal var csprng_state: std.crypto.core.Gimli = undefined;
+pub threadlocal var csprng_state_initialized = false;
+fn tlsCsprngFill(r: *std.rand.Random, buf: []u8) void {
+    if (std.builtin.link_libc and @hasDecl(std.c, "arc4random_buf")) {
+        // arc4random is already a thread-local CSPRNG.
+        return std.c.arc4random_buf(buf.ptr, buf.len);
+    }
+    if (!csprng_state_initialized) {
+        var seed: [seed_len]u8 = undefined;
+        // Because we panic on getrandom() failing, we provide the opportunity
+        // to override the default seed function. This also makes
+        // `std.crypto.random` available on freestanding targets, provided that
+        // the `cryptoRandomSeed` function is provided.
+        if (@hasDecl(root, "cryptoRandomSeed")) {
+            root.cryptoRandomSeed(&seed);
+        } else {
+            defaultSeed(&seed);
+        }
+        init(seed);
+    }
+    if (buf.len != 0) {
+        csprng_state.squeeze(buf);
+    } else {
+        csprng_state.permute();
+    }
+    mem.set(u8, csprng_state.toSlice()[0..std.crypto.core.Gimli.RATE], 0);
+}
+
+fn defaultSeed(buffer: *[seed_len]u8) void {
+    std.os.getrandom(buffer) catch @panic("getrandom() failed to seed thread-local CSPRNG");
+}
+
+pub const seed_len = 32;
+
+pub fn init(seed: [seed_len]u8) void {
+    var initial_state: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined;
+    mem.copy(u8, initial_state[0..seed_len], &seed);
+    mem.set(u8, initial_state[seed_len..], 0);
+    csprng_state = std.crypto.core.Gimli.init(initial_state);
+
+    // This is at the end so that accidental recursive dependencies result
+    // in stack overflows instead of invalid random data.
+    csprng_state_initialized = true;
+}
diff --git a/lib/std/crypto/utils.zig b/lib/std/crypto/utils.zig
index 33ad6360f6..08271ac9f4 100644
--- a/lib/std/crypto/utils.zig
+++ b/lib/std/crypto/utils.zig
@@ -51,8 +51,8 @@ pub fn secureZero(comptime T: type, s: []T) void {
 test "crypto.utils.timingSafeEql" {
     var a: [100]u8 = undefined;
     var b: [100]u8 = undefined;
-    try std.crypto.randomBytes(a[0..]);
-    try std.crypto.randomBytes(b[0..]);
+    std.crypto.random.bytes(a[0..]);
+    std.crypto.random.bytes(b[0..]);
     testing.expect(!timingSafeEql([100]u8, a, b));
     mem.copy(u8, a[0..], b[0..]);
     testing.expect(timingSafeEql([100]u8, a, b));
@@ -61,8 +61,8 @@ test "crypto.utils.timingSafeEql" {
 test "crypto.utils.timingSafeEql (vectors)" {
     var a: [100]u8 = undefined;
     var b: [100]u8 = undefined;
-    try std.crypto.randomBytes(a[0..]);
-    try std.crypto.randomBytes(b[0..]);
+    std.crypto.random.bytes(a[0..]);
+    std.crypto.random.bytes(b[0..]);
     const v1: std.meta.Vector(100, u8) = a;
     const v2: std.meta.Vector(100, u8) = b;
     testing.expect(!timingSafeEql(std.meta.Vector(100, u8), v1, v2));
diff --git a/lib/std/fs.zig b/lib/std/fs.zig
index 8b949a57f1..4c346a2c89 100644
--- a/lib/std/fs.zig
+++ b/lib/std/fs.zig
@@ -82,7 +82,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path:
     mem.copy(u8, tmp_path[0..], dirname);
     tmp_path[dirname.len] = path.sep;
     while (true) {
-        try crypto.randomBytes(rand_buf[0..]);
+        crypto.random.bytes(rand_buf[0..]);
         base64_encoder.encode(tmp_path[dirname.len + 1 ..], &rand_buf);
 
         if (cwd().symLink(existing_path, tmp_path, .{})) {
@@ -157,7 +157,7 @@ pub const AtomicFile = struct {
         tmp_path_buf[base64.Base64Encoder.calcSize(RANDOM_BYTES)] = 0;
 
         while (true) {
-            try crypto.randomBytes(rand_buf[0..]);
+            crypto.random.bytes(rand_buf[0..]);
             base64_encoder.encode(&tmp_path_buf, &rand_buf);
 
             const file = dir.createFile(
diff --git a/lib/std/rand.zig b/lib/std/rand.zig
index a201968a5f..681c67f22d 100644
--- a/lib/std/rand.zig
+++ b/lib/std/rand.zig
@@ -4,19 +4,11 @@
 // The MIT license requires this copyright notice to be included in all copies
 // and substantial portions of the software.
 
-//! The engines provided here should be initialized from an external source. For now, randomBytes
-//! from the crypto package is the most suitable. Be sure to use a CSPRNG when required, otherwise using
-//! a normal PRNG will be faster and use substantially less stack space.
-//!
-//! ```
-//! var buf: [8]u8 = undefined;
-//! try std.crypto.randomBytes(buf[0..]);
-//! const seed = mem.readIntLittle(u64, buf[0..8]);
-//!
-//! var r = DefaultPrng.init(seed);
-//!
-//! const s = r.random.int(u64);
-//! ```
+//! The engines provided here should be initialized from an external source.
+//! For a thread-local cryptographically secure pseudo random number generator,
+//! use `std.crypto.random`.
+//! Be sure to use a CSPRNG when required, otherwise using a normal PRNG will
+//! be faster and use substantially less stack space.
 //!
 //! TODO(tiehuis): Benchmark these against other reference implementations.
 
@@ -36,6 +28,12 @@ pub const DefaultPrng = Xoroshiro128;
 /// Cryptographically secure random numbers.
 pub const DefaultCsprng = Gimli;
 
+pub const Isaac64 = @import("rand/Isaac64.zig");
+pub const Gimli = @import("rand/Gimli.zig");
+pub const Pcg = @import("rand/Pcg.zig");
+pub const Xoroshiro128 = @import("rand/Xoroshiro128.zig");
+pub const Sfc64 = @import("rand/Sfc64.zig");
+
 pub const Random = struct {
     fillFn: fn (r: *Random, buf: []u8) void,
 
@@ -491,7 +489,7 @@ test "Random Biased" {
 //
 // The number of cycles is thus limited to 64-bits regardless of the engine, but this
 // is still plenty for practical purposes.
-const SplitMix64 = struct {
+pub const SplitMix64 = struct {
     s: u64,
 
     pub fn init(seed: u64) SplitMix64 {
@@ -525,557 +523,6 @@ test "splitmix64 sequence" {
     }
 }
 
-// PCG32 - http://www.pcg-random.org/
-//
-// PRNG
-pub const Pcg = struct {
-    const default_multiplier = 6364136223846793005;
-
-    random: Random,
-
-    s: u64,
-    i: u64,
-
-    pub fn init(init_s: u64) Pcg {
-        var pcg = Pcg{
-            .random = Random{ .fillFn = fill },
-            .s = undefined,
-            .i = undefined,
-        };
-
-        pcg.seed(init_s);
-        return pcg;
-    }
-
-    fn next(self: *Pcg) u32 {
-        const l = self.s;
-        self.s = l *% default_multiplier +% (self.i | 1);
-
-        const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27);
-        const rot = @intCast(u32, l >> 59);
-
-        return (xor_s >> @intCast(u5, rot)) | (xor_s << @intCast(u5, (0 -% rot) & 31));
-    }
-
-    fn seed(self: *Pcg, init_s: u64) void {
-        // Pcg requires 128-bits of seed.
-        var gen = SplitMix64.init(init_s);
-        self.seedTwo(gen.next(), gen.next());
-    }
-
-    fn seedTwo(self: *Pcg, init_s: u64, init_i: u64) void {
-        self.s = 0;
-        self.i = (init_s << 1) | 1;
-        self.s = self.s *% default_multiplier +% self.i;
-        self.s +%= init_i;
-        self.s = self.s *% default_multiplier +% self.i;
-    }
-
-    fn fill(r: *Random, buf: []u8) void {
-        const self = @fieldParentPtr(Pcg, "random", r);
-
-        var i: usize = 0;
-        const aligned_len = buf.len - (buf.len & 7);
-
-        // Complete 4 byte segments.
-        while (i < aligned_len) : (i += 4) {
-            var n = self.next();
-            comptime var j: usize = 0;
-            inline while (j < 4) : (j += 1) {
-                buf[i + j] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-
-        // Remaining. (cuts the stream)
-        if (i != buf.len) {
-            var n = self.next();
-            while (i < buf.len) : (i += 1) {
-                buf[i] = @truncate(u8, n);
-                n >>= 4;
-            }
-        }
-    }
-};
-
-test "pcg sequence" {
-    var r = Pcg.init(0);
-    const s0: u64 = 0x9394bf54ce5d79de;
-    const s1: u64 = 0x84e9c579ef59bbf7;
-    r.seedTwo(s0, s1);
-
-    const seq = [_]u32{
-        2881561918,
-        3063928540,
-        1199791034,
-        2487695858,
-        1479648952,
-        3247963454,
-    };
-
-    for (seq) |s| {
-        expect(s == r.next());
-    }
-}
-
-// Xoroshiro128+ - http://xoroshiro.di.unimi.it/
-//
-// PRNG
-pub const Xoroshiro128 = struct {
-    random: Random,
-
-    s: [2]u64,
-
-    pub fn init(init_s: u64) Xoroshiro128 {
-        var x = Xoroshiro128{
-            .random = Random{ .fillFn = fill },
-            .s = undefined,
-        };
-
-        x.seed(init_s);
-        return x;
-    }
-
-    fn next(self: *Xoroshiro128) u64 {
-        const s0 = self.s[0];
-        var s1 = self.s[1];
-        const r = s0 +% s1;
-
-        s1 ^= s0;
-        self.s[0] = math.rotl(u64, s0, @as(u8, 55)) ^ s1 ^ (s1 << 14);
-        self.s[1] = math.rotl(u64, s1, @as(u8, 36));
-
-        return r;
-    }
-
-    // Skip 2^64 places ahead in the sequence
-    fn jump(self: *Xoroshiro128) void {
-        var s0: u64 = 0;
-        var s1: u64 = 0;
-
-        const table = [_]u64{
-            0xbeac0467eba5facb,
-            0xd86b048b86aa9922,
-        };
-
-        inline for (table) |entry| {
-            var b: usize = 0;
-            while (b < 64) : (b += 1) {
-                if ((entry & (@as(u64, 1) << @intCast(u6, b))) != 0) {
-                    s0 ^= self.s[0];
-                    s1 ^= self.s[1];
-                }
-                _ = self.next();
-            }
-        }
-
-        self.s[0] = s0;
-        self.s[1] = s1;
-    }
-
-    pub fn seed(self: *Xoroshiro128, init_s: u64) void {
-        // Xoroshiro requires 128-bits of seed.
-        var gen = SplitMix64.init(init_s);
-
-        self.s[0] = gen.next();
-        self.s[1] = gen.next();
-    }
-
-    fn fill(r: *Random, buf: []u8) void {
-        const self = @fieldParentPtr(Xoroshiro128, "random", r);
-
-        var i: usize = 0;
-        const aligned_len = buf.len - (buf.len & 7);
-
-        // Complete 8 byte segments.
-        while (i < aligned_len) : (i += 8) {
-            var n = self.next();
-            comptime var j: usize = 0;
-            inline while (j < 8) : (j += 1) {
-                buf[i + j] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-
-        // Remaining. (cuts the stream)
-        if (i != buf.len) {
-            var n = self.next();
-            while (i < buf.len) : (i += 1) {
-                buf[i] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-    }
-};
-
-test "xoroshiro sequence" {
-    var r = Xoroshiro128.init(0);
-    r.s[0] = 0xaeecf86f7878dd75;
-    r.s[1] = 0x01cd153642e72622;
-
-    const seq1 = [_]u64{
-        0xb0ba0da5bb600397,
-        0x18a08afde614dccc,
-        0xa2635b956a31b929,
-        0xabe633c971efa045,
-        0x9ac19f9706ca3cac,
-        0xf62b426578c1e3fb,
-    };
-
-    for (seq1) |s| {
-        expect(s == r.next());
-    }
-
-    r.jump();
-
-    const seq2 = [_]u64{
-        0x95344a13556d3e22,
-        0xb4fb32dafa4d00df,
-        0xb2011d9ccdcfe2dd,
-        0x05679a9b2119b908,
-        0xa860a1da7c9cd8a0,
-        0x658a96efe3f86550,
-    };
-
-    for (seq2) |s| {
-        expect(s == r.next());
-    }
-}
-
-// Gimli
-//
-// CSPRNG
-pub const Gimli = struct {
-    random: Random,
-    state: std.crypto.core.Gimli,
-
-    pub const secret_seed_length = 32;
-
-    /// The seed must be uniform, secret and `secret_seed_length` bytes long.
-    /// It can be generated using `std.crypto.randomBytes()`.
-    pub fn init(secret_seed: [secret_seed_length]u8) Gimli {
-        var initial_state: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined;
-        mem.copy(u8, initial_state[0..secret_seed_length], &secret_seed);
-        mem.set(u8, initial_state[secret_seed_length..], 0);
-        var self = Gimli{
-            .random = Random{ .fillFn = fill },
-            .state = std.crypto.core.Gimli.init(initial_state),
-        };
-        return self;
-    }
-
-    fn fill(r: *Random, buf: []u8) void {
-        const self = @fieldParentPtr(Gimli, "random", r);
-
-        if (buf.len != 0) {
-            self.state.squeeze(buf);
-        } else {
-            self.state.permute();
-        }
-        mem.set(u8, self.state.toSlice()[0..std.crypto.core.Gimli.RATE], 0);
-    }
-};
-
-// ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html
-//
-// Follows the general idea of the implementation from here with a few shortcuts.
-// https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html
-pub const Isaac64 = struct {
-    random: Random,
-
-    r: [256]u64,
-    m: [256]u64,
-    a: u64,
-    b: u64,
-    c: u64,
-    i: usize,
-
-    pub fn init(init_s: u64) Isaac64 {
-        var isaac = Isaac64{
-            .random = Random{ .fillFn = fill },
-            .r = undefined,
-            .m = undefined,
-            .a = undefined,
-            .b = undefined,
-            .c = undefined,
-            .i = undefined,
-        };
-
-        // seed == 0 => same result as the unseeded reference implementation
-        isaac.seed(init_s, 1);
-        return isaac;
-    }
-
-    fn step(self: *Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void {
-        const x = self.m[base + m1];
-        self.a = mix +% self.m[base + m2];
-
-        const y = self.a +% self.b +% self.m[@intCast(usize, (x >> 3) % self.m.len)];
-        self.m[base + m1] = y;
-
-        self.b = x +% self.m[@intCast(usize, (y >> 11) % self.m.len)];
-        self.r[self.r.len - 1 - base - m1] = self.b;
-    }
-
-    fn refill(self: *Isaac64) void {
-        const midpoint = self.r.len / 2;
-
-        self.c +%= 1;
-        self.b +%= self.c;
-
-        {
-            var i: usize = 0;
-            while (i < midpoint) : (i += 4) {
-                self.step(~(self.a ^ (self.a << 21)), i + 0, 0, midpoint);
-                self.step(self.a ^ (self.a >> 5), i + 1, 0, midpoint);
-                self.step(self.a ^ (self.a << 12), i + 2, 0, midpoint);
-                self.step(self.a ^ (self.a >> 33), i + 3, 0, midpoint);
-            }
-        }
-
-        {
-            var i: usize = 0;
-            while (i < midpoint) : (i += 4) {
-                self.step(~(self.a ^ (self.a << 21)), i + 0, midpoint, 0);
-                self.step(self.a ^ (self.a >> 5), i + 1, midpoint, 0);
-                self.step(self.a ^ (self.a << 12), i + 2, midpoint, 0);
-                self.step(self.a ^ (self.a >> 33), i + 3, midpoint, 0);
-            }
-        }
-
-        self.i = 0;
-    }
-
-    fn next(self: *Isaac64) u64 {
-        if (self.i >= self.r.len) {
-            self.refill();
-        }
-
-        const value = self.r[self.i];
-        self.i += 1;
-        return value;
-    }
-
-    fn seed(self: *Isaac64, init_s: u64, comptime rounds: usize) void {
-        // We ignore the multi-pass requirement since we don't currently expose full access to
-        // seeding the self.m array completely.
-        mem.set(u64, self.m[0..], 0);
-        self.m[0] = init_s;
-
-        // prescrambled golden ratio constants
-        var a = [_]u64{
-            0x647c4677a2884b7c,
-            0xb9f8b322c73ac862,
-            0x8c0ea5053d4712a0,
-            0xb29b2e824a595524,
-            0x82f053db8355e0ce,
-            0x48fe4a0fa5a09315,
-            0xae985bf2cbfc89ed,
-            0x98f5704f6c44c0ab,
-        };
-
-        comptime var i: usize = 0;
-        inline while (i < rounds) : (i += 1) {
-            var j: usize = 0;
-            while (j < self.m.len) : (j += 8) {
-                comptime var x1: usize = 0;
-                inline while (x1 < 8) : (x1 += 1) {
-                    a[x1] +%= self.m[j + x1];
-                }
-
-                a[0] -%= a[4];
-                a[5] ^= a[7] >> 9;
-                a[7] +%= a[0];
-                a[1] -%= a[5];
-                a[6] ^= a[0] << 9;
-                a[0] +%= a[1];
-                a[2] -%= a[6];
-                a[7] ^= a[1] >> 23;
-                a[1] +%= a[2];
-                a[3] -%= a[7];
-                a[0] ^= a[2] << 15;
-                a[2] +%= a[3];
-                a[4] -%= a[0];
-                a[1] ^= a[3] >> 14;
-                a[3] +%= a[4];
-                a[5] -%= a[1];
-                a[2] ^= a[4] << 20;
-                a[4] +%= a[5];
-                a[6] -%= a[2];
-                a[3] ^= a[5] >> 17;
-                a[5] +%= a[6];
-                a[7] -%= a[3];
-                a[4] ^= a[6] << 14;
-                a[6] +%= a[7];
-
-                comptime var x2: usize = 0;
-                inline while (x2 < 8) : (x2 += 1) {
-                    self.m[j + x2] = a[x2];
-                }
-            }
-        }
-
-        mem.set(u64, self.r[0..], 0);
-        self.a = 0;
-        self.b = 0;
-        self.c = 0;
-        self.i = self.r.len; // trigger refill on first value
-    }
-
-    fn fill(r: *Random, buf: []u8) void {
-        const self = @fieldParentPtr(Isaac64, "random", r);
-
-        var i: usize = 0;
-        const aligned_len = buf.len - (buf.len & 7);
-
-        // Fill complete 64-byte segments
-        while (i < aligned_len) : (i += 8) {
-            var n = self.next();
-            comptime var j: usize = 0;
-            inline while (j < 8) : (j += 1) {
-                buf[i + j] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-
-        // Fill trailing, ignoring excess (cut the stream).
-        if (i != buf.len) {
-            var n = self.next();
-            while (i < buf.len) : (i += 1) {
-                buf[i] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-    }
-};
-
-test "isaac64 sequence" {
-    var r = Isaac64.init(0);
-
-    // from reference implementation
-    const seq = [_]u64{
-        0xf67dfba498e4937c,
-        0x84a5066a9204f380,
-        0xfee34bd5f5514dbb,
-        0x4d1664739b8f80d6,
-        0x8607459ab52a14aa,
-        0x0e78bc5a98529e49,
-        0xfe5332822ad13777,
-        0x556c27525e33d01a,
-        0x08643ca615f3149f,
-        0xd0771faf3cb04714,
-        0x30e86f68a37b008d,
-        0x3074ebc0488a3adf,
-        0x270645ea7a2790bc,
-        0x5601a0a8d3763c6a,
-        0x2f83071f53f325dd,
-        0xb9090f3d42d2d2ea,
-    };
-
-    for (seq) |s| {
-        expect(s == r.next());
-    }
-}
-
-/// Sfc64 pseudo-random number generator from Practically Random.
-/// Fastest engine of pracrand and smallest footprint.
-/// See http://pracrand.sourceforge.net/
-pub const Sfc64 = struct {
-    random: Random,
-
-    a: u64 = undefined,
-    b: u64 = undefined,
-    c: u64 = undefined,
-    counter: u64 = undefined,
-
-    const Rotation = 24;
-    const RightShift = 11;
-    const LeftShift = 3;
-
-    pub fn init(init_s: u64) Sfc64 {
-        var x = Sfc64{
-            .random = Random{ .fillFn = fill },
-        };
-
-        x.seed(init_s);
-        return x;
-    }
-
-    fn next(self: *Sfc64) u64 {
-        const tmp = self.a +% self.b +% self.counter;
-        self.counter += 1;
-        self.a = self.b ^ (self.b >> RightShift);
-        self.b = self.c +% (self.c << LeftShift);
-        self.c = math.rotl(u64, self.c, Rotation) +% tmp;
-        return tmp;
-    }
-
-    fn seed(self: *Sfc64, init_s: u64) void {
-        self.a = init_s;
-        self.b = init_s;
-        self.c = init_s;
-        self.counter = 1;
-        var i: u32 = 0;
-        while (i < 12) : (i += 1) {
-            _ = self.next();
-        }
-    }
-
-    fn fill(r: *Random, buf: []u8) void {
-        const self = @fieldParentPtr(Sfc64, "random", r);
-
-        var i: usize = 0;
-        const aligned_len = buf.len - (buf.len & 7);
-
-        // Complete 8 byte segments.
-        while (i < aligned_len) : (i += 8) {
-            var n = self.next();
-            comptime var j: usize = 0;
-            inline while (j < 8) : (j += 1) {
-                buf[i + j] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-
-        // Remaining. (cuts the stream)
-        if (i != buf.len) {
-            var n = self.next();
-            while (i < buf.len) : (i += 1) {
-                buf[i] = @truncate(u8, n);
-                n >>= 8;
-            }
-        }
-    }
-};
-
-test "Sfc64 sequence" {
-    // Unfortunately there does not seem to be an official test sequence.
-    var r = Sfc64.init(0);
-
-    const seq = [_]u64{
-        0x3acfa029e3cc6041,
-        0xf5b6515bf2ee419c,
-        0x1259635894a29b61,
-        0xb6ae75395f8ebd6,
-        0x225622285ce302e2,
-        0x520d28611395cb21,
-        0xdb909c818901599d,
-        0x8ffd195365216f57,
-        0xe8c4ad5e258ac04a,
-        0x8f8ef2c89fdb63ca,
-        0xf9865b01d98d8e2f,
-        0x46555871a65d08ba,
-        0x66868677c6298fcd,
-        0x2ce15a7e6329f57d,
-        0xb2f1833ca91ca79,
-        0x4b0890ac9bf453ca,
-    };
-
-    for (seq) |s| {
-        expectEqual(s, r.next());
-    }
-}
-
 // Actual Random helper function tests, pcg engine is assumed correct.
 test "Random float" {
     var prng = DefaultPrng.init(0);
@@ -1147,7 +594,7 @@ fn testRangeBias(r: *Random, start: i8, end: i8, biased: bool) void {
 
 test "CSPRNG" {
     var secret_seed: [DefaultCsprng.secret_seed_length]u8 = undefined;
-    try std.crypto.randomBytes(&secret_seed);
+    std.crypto.random.bytes(&secret_seed);
     var csprng = DefaultCsprng.init(secret_seed);
     const a = csprng.random.int(u64);
     const b = csprng.random.int(u64);
diff --git a/lib/std/rand/Gimli.zig b/lib/std/rand/Gimli.zig
new file mode 100644
index 0000000000..32331e7153
--- /dev/null
+++ b/lib/std/rand/Gimli.zig
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! CSPRNG
+
+const std = @import("std");
+const Random = std.rand.Random;
+const mem = std.mem;
+const Gimli = @This();
+
+random: Random,
+state: std.crypto.core.Gimli,
+
+pub const secret_seed_length = 32;
+
+/// The seed must be uniform, secret and `secret_seed_length` bytes long.
+pub fn init(secret_seed: [secret_seed_length]u8) Gimli {
+    var initial_state: [std.crypto.core.Gimli.BLOCKBYTES]u8 = undefined;
+    mem.copy(u8, initial_state[0..secret_seed_length], &secret_seed);
+    mem.set(u8, initial_state[secret_seed_length..], 0);
+    var self = Gimli{
+        .random = Random{ .fillFn = fill },
+        .state = std.crypto.core.Gimli.init(initial_state),
+    };
+    return self;
+}
+
+fn fill(r: *Random, buf: []u8) void {
+    const self = @fieldParentPtr(Gimli, "random", r);
+
+    if (buf.len != 0) {
+        self.state.squeeze(buf);
+    } else {
+        self.state.permute();
+    }
+    mem.set(u8, self.state.toSlice()[0..std.crypto.core.Gimli.RATE], 0);
+}
diff --git a/lib/std/rand/Isaac64.zig b/lib/std/rand/Isaac64.zig
new file mode 100644
index 0000000000..a079f19b9e
--- /dev/null
+++ b/lib/std/rand/Isaac64.zig
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html
+//!
+//! Follows the general idea of the implementation from here with a few shortcuts.
+//! https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html
+
+const std = @import("std");
+const Random = std.rand.Random;
+const mem = std.mem;
+const Isaac64 = @This();
+
+random: Random,
+
+r: [256]u64,
+m: [256]u64,
+a: u64,
+b: u64,
+c: u64,
+i: usize,
+
+pub fn init(init_s: u64) Isaac64 {
+    var isaac = Isaac64{
+        .random = Random{ .fillFn = fill },
+        .r = undefined,
+        .m = undefined,
+        .a = undefined,
+        .b = undefined,
+        .c = undefined,
+        .i = undefined,
+    };
+
+    // seed == 0 => same result as the unseeded reference implementation
+    isaac.seed(init_s, 1);
+    return isaac;
+}
+
+fn step(self: *Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void {
+    const x = self.m[base + m1];
+    self.a = mix +% self.m[base + m2];
+
+    const y = self.a +% self.b +% self.m[@intCast(usize, (x >> 3) % self.m.len)];
+    self.m[base + m1] = y;
+
+    self.b = x +% self.m[@intCast(usize, (y >> 11) % self.m.len)];
+    self.r[self.r.len - 1 - base - m1] = self.b;
+}
+
+fn refill(self: *Isaac64) void {
+    const midpoint = self.r.len / 2;
+
+    self.c +%= 1;
+    self.b +%= self.c;
+
+    {
+        var i: usize = 0;
+        while (i < midpoint) : (i += 4) {
+            self.step(~(self.a ^ (self.a << 21)), i + 0, 0, midpoint);
+            self.step(self.a ^ (self.a >> 5), i + 1, 0, midpoint);
+            self.step(self.a ^ (self.a << 12), i + 2, 0, midpoint);
+            self.step(self.a ^ (self.a >> 33), i + 3, 0, midpoint);
+        }
+    }
+
+    {
+        var i: usize = 0;
+        while (i < midpoint) : (i += 4) {
+            self.step(~(self.a ^ (self.a << 21)), i + 0, midpoint, 0);
+            self.step(self.a ^ (self.a >> 5), i + 1, midpoint, 0);
+            self.step(self.a ^ (self.a << 12), i + 2, midpoint, 0);
+            self.step(self.a ^ (self.a >> 33), i + 3, midpoint, 0);
+        }
+    }
+
+    self.i = 0;
+}
+
+fn next(self: *Isaac64) u64 {
+    if (self.i >= self.r.len) {
+        self.refill();
+    }
+
+    const value = self.r[self.i];
+    self.i += 1;
+    return value;
+}
+
+fn seed(self: *Isaac64, init_s: u64, comptime rounds: usize) void {
+    // We ignore the multi-pass requirement since we don't currently expose full access to
+    // seeding the self.m array completely.
+    mem.set(u64, self.m[0..], 0);
+    self.m[0] = init_s;
+
+    // prescrambled golden ratio constants
+    var a = [_]u64{
+        0x647c4677a2884b7c,
+        0xb9f8b322c73ac862,
+        0x8c0ea5053d4712a0,
+        0xb29b2e824a595524,
+        0x82f053db8355e0ce,
+        0x48fe4a0fa5a09315,
+        0xae985bf2cbfc89ed,
+        0x98f5704f6c44c0ab,
+    };
+
+    comptime var i: usize = 0;
+    inline while (i < rounds) : (i += 1) {
+        var j: usize = 0;
+        while (j < self.m.len) : (j += 8) {
+            comptime var x1: usize = 0;
+            inline while (x1 < 8) : (x1 += 1) {
+                a[x1] +%= self.m[j + x1];
+            }
+
+            a[0] -%= a[4];
+            a[5] ^= a[7] >> 9;
+            a[7] +%= a[0];
+            a[1] -%= a[5];
+            a[6] ^= a[0] << 9;
+            a[0] +%= a[1];
+            a[2] -%= a[6];
+            a[7] ^= a[1] >> 23;
+            a[1] +%= a[2];
+            a[3] -%= a[7];
+            a[0] ^= a[2] << 15;
+            a[2] +%= a[3];
+            a[4] -%= a[0];
+            a[1] ^= a[3] >> 14;
+            a[3] +%= a[4];
+            a[5] -%= a[1];
+            a[2] ^= a[4] << 20;
+            a[4] +%= a[5];
+            a[6] -%= a[2];
+            a[3] ^= a[5] >> 17;
+            a[5] +%= a[6];
+            a[7] -%= a[3];
+            a[4] ^= a[6] << 14;
+            a[6] +%= a[7];
+
+            comptime var x2: usize = 0;
+            inline while (x2 < 8) : (x2 += 1) {
+                self.m[j + x2] = a[x2];
+            }
+        }
+    }
+
+    mem.set(u64, self.r[0..], 0);
+    self.a = 0;
+    self.b = 0;
+    self.c = 0;
+    self.i = self.r.len; // trigger refill on first value
+}
+
+fn fill(r: *Random, buf: []u8) void {
+    const self = @fieldParentPtr(Isaac64, "random", r);
+
+    var i: usize = 0;
+    const aligned_len = buf.len - (buf.len & 7);
+
+    // Fill complete 64-byte segments
+    while (i < aligned_len) : (i += 8) {
+        var n = self.next();
+        comptime var j: usize = 0;
+        inline while (j < 8) : (j += 1) {
+            buf[i + j] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+
+    // Fill trailing, ignoring excess (cut the stream).
+    if (i != buf.len) {
+        var n = self.next();
+        while (i < buf.len) : (i += 1) {
+            buf[i] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+}
+
+test "isaac64 sequence" {
+    var r = Isaac64.init(0);
+
+    // from reference implementation
+    const seq = [_]u64{
+        0xf67dfba498e4937c,
+        0x84a5066a9204f380,
+        0xfee34bd5f5514dbb,
+        0x4d1664739b8f80d6,
+        0x8607459ab52a14aa,
+        0x0e78bc5a98529e49,
+        0xfe5332822ad13777,
+        0x556c27525e33d01a,
+        0x08643ca615f3149f,
+        0xd0771faf3cb04714,
+        0x30e86f68a37b008d,
+        0x3074ebc0488a3adf,
+        0x270645ea7a2790bc,
+        0x5601a0a8d3763c6a,
+        0x2f83071f53f325dd,
+        0xb9090f3d42d2d2ea,
+    };
+
+    for (seq) |s| {
+        std.testing.expect(s == r.next());
+    }
+}
diff --git a/lib/std/rand/Pcg.zig b/lib/std/rand/Pcg.zig
new file mode 100644
index 0000000000..1bbe8beb63
--- /dev/null
+++ b/lib/std/rand/Pcg.zig
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! PCG32 - http://www.pcg-random.org/
+//!
+//! PRNG
+
+const std = @import("std");
+const Random = std.rand.Random;
+const Pcg = @This();
+
+const default_multiplier = 6364136223846793005;
+
+random: Random,
+
+s: u64,
+i: u64,
+
+pub fn init(init_s: u64) Pcg {
+    var pcg = Pcg{
+        .random = Random{ .fillFn = fill },
+        .s = undefined,
+        .i = undefined,
+    };
+
+    pcg.seed(init_s);
+    return pcg;
+}
+
+fn next(self: *Pcg) u32 {
+    const l = self.s;
+    self.s = l *% default_multiplier +% (self.i | 1);
+
+    const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27);
+    const rot = @intCast(u32, l >> 59);
+
+    return (xor_s >> @intCast(u5, rot)) | (xor_s << @intCast(u5, (0 -% rot) & 31));
+}
+
+fn seed(self: *Pcg, init_s: u64) void {
+    // Pcg requires 128-bits of seed.
+    var gen = std.rand.SplitMix64.init(init_s);
+    self.seedTwo(gen.next(), gen.next());
+}
+
+fn seedTwo(self: *Pcg, init_s: u64, init_i: u64) void {
+    self.s = 0;
+    self.i = (init_s << 1) | 1;
+    self.s = self.s *% default_multiplier +% self.i;
+    self.s +%= init_i;
+    self.s = self.s *% default_multiplier +% self.i;
+}
+
+fn fill(r: *Random, buf: []u8) void {
+    const self = @fieldParentPtr(Pcg, "random", r);
+
+    var i: usize = 0;
+    const aligned_len = buf.len - (buf.len & 7);
+
+    // Complete 4 byte segments.
+    while (i < aligned_len) : (i += 4) {
+        var n = self.next();
+        comptime var j: usize = 0;
+        inline while (j < 4) : (j += 1) {
+            buf[i + j] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+
+    // Remaining. (cuts the stream)
+    if (i != buf.len) {
+        var n = self.next();
+        while (i < buf.len) : (i += 1) {
+            buf[i] = @truncate(u8, n);
+            n >>= 4;
+        }
+    }
+}
+
+test "pcg sequence" {
+    var r = Pcg.init(0);
+    const s0: u64 = 0x9394bf54ce5d79de;
+    const s1: u64 = 0x84e9c579ef59bbf7;
+    r.seedTwo(s0, s1);
+
+    const seq = [_]u32{
+        2881561918,
+        3063928540,
+        1199791034,
+        2487695858,
+        1479648952,
+        3247963454,
+    };
+
+    for (seq) |s| {
+        std.testing.expect(s == r.next());
+    }
+}
diff --git a/lib/std/rand/Sfc64.zig b/lib/std/rand/Sfc64.zig
new file mode 100644
index 0000000000..feba5d884c
--- /dev/null
+++ b/lib/std/rand/Sfc64.zig
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! Sfc64 pseudo-random number generator from Practically Random.
+//! Fastest engine of pracrand and smallest footprint.
+//! See http://pracrand.sourceforge.net/
+
+const std = @import("std");
+const Random = std.rand.Random;
+const math = std.math;
+const Sfc64 = @This();
+
+random: Random,
+
+a: u64 = undefined,
+b: u64 = undefined,
+c: u64 = undefined,
+counter: u64 = undefined,
+
+const Rotation = 24;
+const RightShift = 11;
+const LeftShift = 3;
+
+pub fn init(init_s: u64) Sfc64 {
+    var x = Sfc64{
+        .random = Random{ .fillFn = fill },
+    };
+
+    x.seed(init_s);
+    return x;
+}
+
+fn next(self: *Sfc64) u64 {
+    const tmp = self.a +% self.b +% self.counter;
+    self.counter += 1;
+    self.a = self.b ^ (self.b >> RightShift);
+    self.b = self.c +% (self.c << LeftShift);
+    self.c = math.rotl(u64, self.c, Rotation) +% tmp;
+    return tmp;
+}
+
+fn seed(self: *Sfc64, init_s: u64) void {
+    self.a = init_s;
+    self.b = init_s;
+    self.c = init_s;
+    self.counter = 1;
+    var i: u32 = 0;
+    while (i < 12) : (i += 1) {
+        _ = self.next();
+    }
+}
+
+fn fill(r: *Random, buf: []u8) void {
+    const self = @fieldParentPtr(Sfc64, "random", r);
+
+    var i: usize = 0;
+    const aligned_len = buf.len - (buf.len & 7);
+
+    // Complete 8 byte segments.
+    while (i < aligned_len) : (i += 8) {
+        var n = self.next();
+        comptime var j: usize = 0;
+        inline while (j < 8) : (j += 1) {
+            buf[i + j] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+
+    // Remaining. (cuts the stream)
+    if (i != buf.len) {
+        var n = self.next();
+        while (i < buf.len) : (i += 1) {
+            buf[i] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+}
+
+test "Sfc64 sequence" {
+    // Unfortunately there does not seem to be an official test sequence.
+    var r = Sfc64.init(0);
+
+    const seq = [_]u64{
+        0x3acfa029e3cc6041,
+        0xf5b6515bf2ee419c,
+        0x1259635894a29b61,
+        0xb6ae75395f8ebd6,
+        0x225622285ce302e2,
+        0x520d28611395cb21,
+        0xdb909c818901599d,
+        0x8ffd195365216f57,
+        0xe8c4ad5e258ac04a,
+        0x8f8ef2c89fdb63ca,
+        0xf9865b01d98d8e2f,
+        0x46555871a65d08ba,
+        0x66868677c6298fcd,
+        0x2ce15a7e6329f57d,
+        0xb2f1833ca91ca79,
+        0x4b0890ac9bf453ca,
+    };
+
+    for (seq) |s| {
+        std.testing.expectEqual(s, r.next());
+    }
+}
diff --git a/lib/std/rand/Xoroshiro128.zig b/lib/std/rand/Xoroshiro128.zig
new file mode 100644
index 0000000000..2cc9bf9070
--- /dev/null
+++ b/lib/std/rand/Xoroshiro128.zig
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! Xoroshiro128+ - http://xoroshiro.di.unimi.it/
+//!
+//! PRNG
+
+const std = @import("std");
+const Random = std.rand.Random;
+const math = std.math;
+const Xoroshiro128 = @This();
+
+random: Random,
+
+s: [2]u64,
+
+pub fn init(init_s: u64) Xoroshiro128 {
+    var x = Xoroshiro128{
+        .random = Random{ .fillFn = fill },
+        .s = undefined,
+    };
+
+    x.seed(init_s);
+    return x;
+}
+
+fn next(self: *Xoroshiro128) u64 {
+    const s0 = self.s[0];
+    var s1 = self.s[1];
+    const r = s0 +% s1;
+
+    s1 ^= s0;
+    self.s[0] = math.rotl(u64, s0, @as(u8, 55)) ^ s1 ^ (s1 << 14);
+    self.s[1] = math.rotl(u64, s1, @as(u8, 36));
+
+    return r;
+}
+
+// Skip 2^64 places ahead in the sequence
+fn jump(self: *Xoroshiro128) void {
+    var s0: u64 = 0;
+    var s1: u64 = 0;
+
+    const table = [_]u64{
+        0xbeac0467eba5facb,
+        0xd86b048b86aa9922,
+    };
+
+    inline for (table) |entry| {
+        var b: usize = 0;
+        while (b < 64) : (b += 1) {
+            if ((entry & (@as(u64, 1) << @intCast(u6, b))) != 0) {
+                s0 ^= self.s[0];
+                s1 ^= self.s[1];
+            }
+            _ = self.next();
+        }
+    }
+
+    self.s[0] = s0;
+    self.s[1] = s1;
+}
+
+pub fn seed(self: *Xoroshiro128, init_s: u64) void {
+    // Xoroshiro requires 128-bits of seed.
+    var gen = std.rand.SplitMix64.init(init_s);
+
+    self.s[0] = gen.next();
+    self.s[1] = gen.next();
+}
+
+fn fill(r: *Random, buf: []u8) void {
+    const self = @fieldParentPtr(Xoroshiro128, "random", r);
+
+    var i: usize = 0;
+    const aligned_len = buf.len - (buf.len & 7);
+
+    // Complete 8 byte segments.
+    while (i < aligned_len) : (i += 8) {
+        var n = self.next();
+        comptime var j: usize = 0;
+        inline while (j < 8) : (j += 1) {
+            buf[i + j] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+
+    // Remaining. (cuts the stream)
+    if (i != buf.len) {
+        var n = self.next();
+        while (i < buf.len) : (i += 1) {
+            buf[i] = @truncate(u8, n);
+            n >>= 8;
+        }
+    }
+}
+
+test "xoroshiro sequence" {
+    var r = Xoroshiro128.init(0);
+    r.s[0] = 0xaeecf86f7878dd75;
+    r.s[1] = 0x01cd153642e72622;
+
+    const seq1 = [_]u64{
+        0xb0ba0da5bb600397,
+        0x18a08afde614dccc,
+        0xa2635b956a31b929,
+        0xabe633c971efa045,
+        0x9ac19f9706ca3cac,
+        0xf62b426578c1e3fb,
+    };
+
+    for (seq1) |s| {
+        std.testing.expect(s == r.next());
+    }
+
+    r.jump();
+
+    const seq2 = [_]u64{
+        0x95344a13556d3e22,
+        0xb4fb32dafa4d00df,
+        0xb2011d9ccdcfe2dd,
+        0x05679a9b2119b908,
+        0xa860a1da7c9cd8a0,
+        0x658a96efe3f86550,
+    };
+
+    for (seq2) |s| {
+        std.testing.expect(s == r.next());
+    }
+}
diff --git a/lib/std/start.zig b/lib/std/start.zig
index 7d3a9c45c7..b6fb7e4dfd 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -10,6 +10,7 @@ const std = @import("std.zig");
 const builtin = std.builtin;
 const assert = std.debug.assert;
 const uefi = std.os.uefi;
+const tlcsprng = @import("crypto/tlcsprng.zig");
 
 var argc_argv_ptr: [*]usize = undefined;
 
@@ -215,6 +216,28 @@ fn posixCallMainAndExit() noreturn {
             std.os.linux.tls.initStaticTLS();
         }
 
+        {
+            // Initialize the per-thread CSPRNG since Linux gave us the handy-dandy
+            // AT_RANDOM. This depends on the TLS initialization above.
+            var i: usize = 0;
+            while (auxv[i].a_type != std.elf.AT_NULL) : (i += 1) {
+                switch (auxv[i].a_type) {
+                    std.elf.AT_RANDOM => {
+                        // "The address of sixteen bytes containing a random value."
+                        const addr = auxv[i].a_un.a_val;
+                        if (addr == 0) break;
+                        const ptr = @intToPtr(*const [16]u8, addr);
+                        var seed: [32]u8 = undefined;
+                        seed[0..16].* = ptr.*;
+                        seed[16..].* = ptr.*;
+                        tlcsprng.init(seed);
+                        break;
+                    },
+                    else => continue,
+                }
+            }
+        }
+
         // TODO This is disabled because what should we do when linking libc and this code
         // does not execute? And also it's causing a test failure in stack traces in release modes.
 
@@ -250,6 +273,9 @@ fn callMainWithArgs(argc: usize, argv: [*][*:0]u8, envp: [][*:0]u8) u8 {
 }
 
 fn main(c_argc: i32, c_argv: [*][*:0]u8, c_envp: [*:null]?[*:0]u8) callconv(.C) i32 {
+    // We do not attempt to initialize tlcsprng from AT_RANDOM here because
+    // libc owns the start code, not us, and therefore libc ows the random bytes
+    // from AT_RANDOM.
     var env_count: usize = 0;
     while (c_envp[env_count] != null) : (env_count += 1) {}
     const envp = @ptrCast([*][*:0]u8, c_envp)[0..env_count];
diff --git a/lib/std/testing.zig b/lib/std/testing.zig
index 69df01190d..63054892db 100644
--- a/lib/std/testing.zig
+++ b/lib/std/testing.zig
@@ -303,8 +303,7 @@ fn getCwdOrWasiPreopen() std.fs.Dir {
 
 pub fn tmpDir(opts: std.fs.Dir.OpenDirOptions) TmpDir {
     var random_bytes: [TmpDir.random_bytes_count]u8 = undefined;
-    std.crypto.randomBytes(&random_bytes) catch
-        @panic("unable to make tmp dir for testing: unable to get random bytes");
+    std.crypto.random.bytes(&random_bytes);
     var sub_path: [TmpDir.sub_path_len]u8 = undefined;
     std.fs.base64_encoder.encode(&sub_path, &random_bytes);
 
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 356a1cedf6..5e95575642 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -74,7 +74,6 @@ zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
 libc_include_dir_list: []const []const u8,
-rand: *std.rand.Random,
 
 /// Populated when we build the libc++ static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
@@ -331,7 +330,6 @@ pub const InitOptions = struct {
     root_name: []const u8,
     root_pkg: ?*Package,
     output_mode: std.builtin.OutputMode,
-    rand: *std.rand.Random,
     dynamic_linker: ?[]const u8 = null,
     /// `null` means to not emit a binary file.
     emit_bin: ?EmitLoc,
@@ -981,7 +979,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .self_exe_path = options.self_exe_path,
             .libc_include_dir_list = libc_dirs.libc_include_dir_list,
             .sanitize_c = sanitize_c,
-            .rand = options.rand,
             .clang_passthrough_mode = options.clang_passthrough_mode,
             .clang_preprocessor_mode = options.clang_preprocessor_mode,
             .verbose_cc = options.verbose_cc,
@@ -1909,7 +1906,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
 
 pub fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 {
     const s = std.fs.path.sep_str;
-    const rand_int = comp.rand.int(u64);
+    const rand_int = std.crypto.random.int(u64);
     if (comp.local_cache_directory.path) |p| {
         return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
     } else {
@@ -2778,7 +2775,6 @@ fn buildOutputFromZig(
         .root_name = root_name,
         .root_pkg = &root_pkg,
         .output_mode = fixed_output_mode,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = optimize_mode,
@@ -3152,7 +3148,6 @@ pub fn build_crt_file(
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/glibc.zig b/src/glibc.zig
index 901054176e..15c9d743f9 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -936,7 +936,6 @@ fn buildSharedLib(
         .root_pkg = null,
         .output_mode = .Lib,
         .link_mode = .Dynamic,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index 39ec776659..8de45a49b2 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -162,7 +162,6 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
@@ -281,7 +280,6 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index ec8fe990e9..9822016ae1 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -95,7 +95,6 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
-        .rand = comp.rand,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/main.zig b/src/main.zig
index 0222d499fa..c54cc6515b 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1632,13 +1632,6 @@ fn buildOutputType(
         };
     defer zig_lib_directory.handle.close();
 
-    const random_seed = blk: {
-        var random_seed: u64 = undefined;
-        try std.crypto.randomBytes(mem.asBytes(&random_seed));
-        break :blk random_seed;
-    };
-    var default_prng = std.rand.DefaultPrng.init(random_seed);
-
     var libc_installation: ?LibCInstallation = null;
     defer if (libc_installation) |*l| l.deinit(gpa);
 
@@ -1754,7 +1747,6 @@ fn buildOutputType(
         .single_threaded = single_threaded,
         .function_sections = function_sections,
         .self_exe_path = self_exe_path,
-        .rand = &default_prng.random,
         .clang_passthrough_mode = arg_mode != .build,
         .clang_preprocessor_mode = clang_preprocessor_mode,
         .version = optional_version,
@@ -2420,12 +2412,6 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             .directory = null, // Use the local zig-cache.
             .basename = exe_basename,
         };
-        const random_seed = blk: {
-            var random_seed: u64 = undefined;
-            try std.crypto.randomBytes(mem.asBytes(&random_seed));
-            break :blk random_seed;
-        };
-        var default_prng = std.rand.DefaultPrng.init(random_seed);
         const comp = Compilation.create(gpa, .{
             .zig_lib_directory = zig_lib_directory,
             .local_cache_directory = local_cache_directory,
@@ -2441,7 +2427,6 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             .emit_h = null,
             .optimize_mode = .Debug,
             .self_exe_path = self_exe_path,
-            .rand = &default_prng.random,
         }) catch |err| {
             fatal("unable to create compilation: {}", .{@errorName(err)});
         };
diff --git a/src/musl.zig b/src/musl.zig
index ac39eb7d74..1a30a6e2b9 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -200,7 +200,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .root_pkg = null,
                 .output_mode = .Lib,
                 .link_mode = .Dynamic,
-                .rand = comp.rand,
                 .libc_installation = comp.bin_file.options.libc_installation,
                 .emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
                 .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/test.zig b/src/test.zig
index 251edc2d34..e0497a8cd5 100644
--- a/src/test.zig
+++ b/src/test.zig
@@ -467,13 +467,6 @@ pub const TestContext = struct {
         defer zig_lib_directory.handle.close();
         defer std.testing.allocator.free(zig_lib_directory.path.?);
 
-        const random_seed = blk: {
-            var random_seed: u64 = undefined;
-            try std.crypto.randomBytes(std.mem.asBytes(&random_seed));
-            break :blk random_seed;
-        };
-        var default_prng = std.rand.DefaultPrng.init(random_seed);
-
         for (self.cases.items) |case| {
             if (build_options.skip_non_native and case.target.getCpuArch() != std.Target.current.cpu.arch)
                 continue;
@@ -487,7 +480,7 @@ pub const TestContext = struct {
             progress.initial_delay_ns = 0;
             progress.refresh_rate_ns = 0;
 
-            try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory, &default_prng.random);
+            try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory);
         }
     }
 
@@ -497,7 +490,6 @@ pub const TestContext = struct {
         root_node: *std.Progress.Node,
         case: Case,
         zig_lib_directory: Compilation.Directory,
-        rand: *std.rand.Random,
     ) !void {
         const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target);
         const target = target_info.target;
@@ -547,7 +539,6 @@ pub const TestContext = struct {
             .local_cache_directory = zig_cache_directory,
             .global_cache_directory = zig_cache_directory,
             .zig_lib_directory = zig_lib_directory,
-            .rand = rand,
             .root_name = "test_case",
             .target = target,
             // TODO: support tests for object file building, and library builds
-- 
cgit v1.2.3


From 01d33855c736b04160d9616f138442aa4e41a738 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 15 Dec 2020 16:55:56 -0700
Subject: stage2: protect mutable state from data races in updateCObject

---
 src/Compilation.zig | 81 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 53 insertions(+), 28 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 5e95575642..a7fd75fe56 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -41,7 +41,12 @@ link_error_flags: link.File.ErrorFlags = .{},
 
 work_queue: std.fifo.LinearFifo(Job, .Dynamic),
 
+/// These jobs are to invoke the Clang compiler to create an object file, which
+/// gets linked with the Compilation.
+c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
+
 /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
+/// This data is accessed by multiple threads and is protected by `mutex`.
 failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{},
 
 keep_source_files_loaded: bool,
@@ -111,6 +116,9 @@ owned_link_dir: ?std.fs.Dir,
 /// Don't use this for anything other than stage1 compatibility.
 color: @import("main.zig").Color = .auto,
 
+/// This mutex guards all `Compilation` mutable state.
+mutex: std.Mutex = .{},
+
 test_filter: ?[]const u8,
 test_name_prefix: ?[]const u8,
 test_evented_io: bool,
@@ -150,9 +158,6 @@ const Job = union(enum) {
     /// The source file containing the Decl has been updated, and so the
     /// Decl may need its line number information updated in the debug info.
     update_line_number: *Module.Decl,
-    /// Invoke the Clang compiler to create an object file, which gets linked
-    /// with the Compilation.
-    c_object: *CObject,
 
     /// one of the glibc static objects
     glibc_crt_file: glibc.CRTFile,
@@ -971,6 +976,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .emit_analysis = options.emit_analysis,
             .emit_docs = options.emit_docs,
             .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
+            .c_object_work_queue = std.fifo.LinearFifo(*CObject, .Dynamic).init(gpa),
             .keep_source_files_loaded = options.keep_source_files_loaded,
             .use_clang = use_clang,
             .clang_argv = options.clang_argv,
@@ -1190,11 +1196,13 @@ pub fn update(self: *Compilation) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
+    self.c_object_cache_digest_set.clearRetainingCapacity();
+
     // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
     // Add a Job for each C object.
-    try self.work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
+    try self.c_object_work_queue.ensureUnusedCapacity(self.c_object_table.items().len);
     for (self.c_object_table.items()) |entry| {
-        self.work_queue.writeItemAssumeCapacity(.{ .c_object = entry.key });
+        self.c_object_work_queue.writeItemAssumeCapacity(entry.key);
     }
 
     const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
@@ -1372,6 +1380,26 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_comp_progress_node.end();
 
+    while (self.c_object_work_queue.readItem()) |c_object| {
+        self.updateCObject(c_object, &c_comp_progress_node) catch |err| switch (err) {
+            error.AnalysisFail => continue,
+            else => {
+                {
+                    var lock = self.mutex.acquire();
+                    defer lock.release();
+                    try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
+                    self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
+                        self.gpa,
+                        0,
+                        "unable to build C object: {s}",
+                        .{@errorName(err)},
+                    ));
+                }
+                c_object.status = .{ .failure = {} };
+            },
+        };
+    }
+
     while (self.work_queue.readItem()) |work_item| switch (work_item) {
         .codegen_decl => |decl| switch (decl.analysis) {
             .unreferenced => unreachable,
@@ -1447,21 +1475,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 decl.analysis = .codegen_failure_retryable;
             };
         },
-        .c_object => |c_object| {
-            self.updateCObject(c_object, &c_comp_progress_node) catch |err| switch (err) {
-                error.AnalysisFail => continue,
-                else => {
-                    try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
-                    self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
-                        self.gpa,
-                        0,
-                        "unable to build C object: {s}",
-                        .{@errorName(err)},
-                    ));
-                    c_object.status = .{ .failure = {} };
-                },
-            };
-        },
         .glibc_crt_file => |crt_file| {
             glibc.buildCRTFile(self, crt_file) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
@@ -1553,7 +1566,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     };
 }
 
-pub fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest {
+pub fn obtainCObjectCacheManifest(comp: *const Compilation) Cache.Manifest {
     var man = comp.cache_parent.obtain();
 
     // Only things that need to be added on top of the base hash, and only things
@@ -1720,6 +1733,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
 
     if (c_object.clearStatus(comp.gpa)) {
         // There was previous failure.
+        var lock = comp.mutex.acquire();
+        defer lock.release();
         comp.failed_c_objects.removeAssertDiscard(c_object);
     }
 
@@ -1747,8 +1762,14 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
     }
 
     {
-        const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
-        if (gop.found_existing) {
+        const is_collision = blk: {
+            var lock = comp.mutex.acquire();
+            defer lock.release();
+
+            const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
+            break :blk gop.found_existing;
+        };
+        if (is_collision) {
             return comp.failCObj(
                 c_object,
                 "the same source file was already added to the same compilation with the same flags",
@@ -1929,7 +1950,7 @@ pub fn addTranslateCCArgs(
 
 /// Add common C compiler args between translate-c and C object compilation.
 pub fn addCCArgs(
-    comp: *Compilation,
+    comp: *const Compilation,
     arena: *Allocator,
     argv: *std.ArrayList([]const u8),
     ext: FileExt,
@@ -2164,10 +2185,14 @@ fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8,
 
 fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError {
     {
-        errdefer err_msg.destroy(comp.gpa);
-        try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
+        var lock = comp.mutex.acquire();
+        defer lock.release();
+        {
+            errdefer err_msg.destroy(comp.gpa);
+            try comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1);
+        }
+        comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
     }
-    comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, err_msg);
     c_object.status = .failure;
     return error.AnalysisFail;
 }
@@ -2324,7 +2349,7 @@ test "classifyFileExt" {
     std.testing.expectEqual(FileExt.zir, classifyFileExt("foo.zir"));
 }
 
-fn haveFramePointer(comp: *Compilation) bool {
+fn haveFramePointer(comp: *const Compilation) bool {
     // If you complicate this logic make sure you update the parent cache hash.
     // Right now it's not in the cache hash because the value depends on optimize_mode
     // and strip which are both already part of the hash.
-- 
cgit v1.2.3


From 0d1cd0d4822628c104890af4c31cdf38c6f96d35 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 15 Dec 2020 18:30:29 -0700
Subject: use kprotty's ThreadPool implementation (v5)

---
 src/Compilation.zig |  62 ++++++++++++++++++++--------
 src/ThreadPool.zig  | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/WaitGroup.zig   |  22 ++++++++++
 src/glibc.zig       |   1 +
 src/libcxx.zig      |   2 +
 src/libunwind.zig   |   1 +
 src/main.zig        |  10 +++++
 src/musl.zig        |   1 +
 src/test.zig        |  15 ++++++-
 9 files changed, 212 insertions(+), 18 deletions(-)
 create mode 100644 src/ThreadPool.zig
 create mode 100644 src/WaitGroup.zig

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index a7fd75fe56..ef07de7b17 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -26,6 +26,8 @@ const Module = @import("Module.zig");
 const Cache = @import("Cache.zig");
 const stage1 = @import("stage1.zig");
 const translate_c = @import("translate_c.zig");
+const ThreadPool = @import("ThreadPool.zig");
+const WaitGroup = @import("WaitGroup.zig");
 
 /// General-purpose allocator. Used for both temporary and long-term storage.
 gpa: *Allocator,
@@ -79,6 +81,7 @@ zig_lib_directory: Directory,
 local_cache_directory: Directory,
 global_cache_directory: Directory,
 libc_include_dir_list: []const []const u8,
+thread_pool: *ThreadPool,
 
 /// Populated when we build the libc++ static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
@@ -335,6 +338,7 @@ pub const InitOptions = struct {
     root_name: []const u8,
     root_pkg: ?*Package,
     output_mode: std.builtin.OutputMode,
+    thread_pool: *ThreadPool,
     dynamic_linker: ?[]const u8 = null,
     /// `null` means to not emit a binary file.
     emit_bin: ?EmitLoc,
@@ -985,6 +989,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .self_exe_path = options.self_exe_path,
             .libc_include_dir_list = libc_dirs.libc_include_dir_list,
             .sanitize_c = sanitize_c,
+            .thread_pool = options.thread_pool,
             .clang_passthrough_mode = options.clang_passthrough_mode,
             .clang_preprocessor_mode = options.clang_preprocessor_mode,
             .verbose_cc = options.verbose_cc,
@@ -1380,24 +1385,14 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_comp_progress_node.end();
 
+    var wg = WaitGroup{};
+    defer wg.wait();
+
     while (self.c_object_work_queue.readItem()) |c_object| {
-        self.updateCObject(c_object, &c_comp_progress_node) catch |err| switch (err) {
-            error.AnalysisFail => continue,
-            else => {
-                {
-                    var lock = self.mutex.acquire();
-                    defer lock.release();
-                    try self.failed_c_objects.ensureCapacity(self.gpa, self.failed_c_objects.items().len + 1);
-                    self.failed_c_objects.putAssumeCapacityNoClobber(c_object, try ErrorMsg.create(
-                        self.gpa,
-                        0,
-                        "unable to build C object: {s}",
-                        .{@errorName(err)},
-                    ));
-                }
-                c_object.status = .{ .failure = {} };
-            },
-        };
+        wg.start();
+        try self.thread_pool.spawn(workerUpdateCObject, .{
+            self, c_object, &c_comp_progress_node, &wg,
+        });
     }
 
     while (self.work_queue.readItem()) |work_item| switch (work_item) {
@@ -1721,6 +1716,37 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
     };
 }
 
+fn workerUpdateCObject(
+    comp: *Compilation,
+    c_object: *CObject,
+    progress_node: *std.Progress.Node,
+    wg: *WaitGroup,
+) void {
+    defer wg.stop();
+
+    comp.updateCObject(c_object, progress_node) catch |err| switch (err) {
+        error.AnalysisFail => return,
+        else => {
+            {
+                var lock = comp.mutex.acquire();
+                defer lock.release();
+                comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
+                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
+                };
+                comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
+                    comp.gpa,
+                    0,
+                    "unable to build C object: {s}",
+                    .{@errorName(err)},
+                ) catch {
+                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
+                });
+            }
+            c_object.status = .{ .failure = {} };
+        },
+    };
+}
+
 fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *std.Progress.Node) !void {
     if (!build_options.have_llvm) {
         return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
@@ -2800,6 +2826,7 @@ fn buildOutputFromZig(
         .root_name = root_name,
         .root_pkg = &root_pkg,
         .output_mode = fixed_output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = optimize_mode,
@@ -3173,6 +3200,7 @@ pub fn build_crt_file(
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/ThreadPool.zig b/src/ThreadPool.zig
new file mode 100644
index 0000000000..a5f59a30e7
--- /dev/null
+++ b/src/ThreadPool.zig
@@ -0,0 +1,116 @@
+const std = @import("std");
+const ThreadPool = @This();
+
+lock: std.Mutex = .{},
+is_running: bool = true,
+allocator: *std.mem.Allocator,
+running: usize = 0,
+threads: []*std.Thread,
+run_queue: RunQueue = .{},
+idle_queue: IdleQueue = .{},
+
+const IdleQueue = std.SinglyLinkedList(std.AutoResetEvent);
+const RunQueue = std.SinglyLinkedList(Runnable);
+const Runnable = struct {
+    runFn: fn (*Runnable) void,
+};
+
+pub fn init(self: *ThreadPool, allocator: *std.mem.Allocator) !void {
+    self.* = .{
+        .allocator = allocator,
+        .threads = &[_]*std.Thread{},
+    };
+
+    errdefer self.deinit();
+
+    var num_threads = std.Thread.cpuCount() catch 1;
+    if (num_threads > 0)
+        self.threads = try allocator.alloc(*std.Thread, num_threads);
+
+    while (num_threads > 0) : (num_threads -= 1) {
+        const thread = try std.Thread.spawn(self, runWorker);
+        self.threads[self.running] = thread;
+        self.running += 1;
+    }
+}
+
+pub fn deinit(self: *ThreadPool) void {
+    self.shutdown();
+
+    std.debug.assert(!self.is_running);
+    for (self.threads[0..self.running]) |thread|
+        thread.wait();
+
+    defer self.threads = &[_]*std.Thread{};
+    if (self.running > 0)
+        self.allocator.free(self.threads);
+}
+
+pub fn shutdown(self: *ThreadPool) void {
+    const held = self.lock.acquire();
+
+    if (!self.is_running)
+        return held.release();
+
+    var idle_queue = self.idle_queue;
+    self.idle_queue = .{};
+    self.is_running = false;
+    held.release();
+
+    while (idle_queue.popFirst()) |idle_node|
+        idle_node.data.set();
+}
+
+pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
+    const Args = @TypeOf(args);
+    const Closure = struct {
+        arguments: Args,
+        pool: *ThreadPool,
+        run_node: RunQueue.Node = .{ .data = .{ .runFn = runFn } },
+
+        fn runFn(runnable: *Runnable) void {
+            const run_node = @fieldParentPtr(RunQueue.Node, "data", runnable);
+            const closure = @fieldParentPtr(@This(), "run_node", run_node);
+            const result = @call(.{}, func, closure.arguments);
+            closure.pool.allocator.destroy(closure);
+        }
+    };
+
+    const closure = try self.allocator.create(Closure);
+    errdefer self.allocator.destroy(closure);
+    closure.* = .{
+        .arguments = args,
+        .pool = self,
+    };
+
+    const held = self.lock.acquire();
+    self.run_queue.prepend(&closure.run_node);
+
+    const idle_node = self.idle_queue.popFirst();
+    held.release();
+
+    if (idle_node) |node|
+        node.data.set();
+}
+
+fn runWorker(self: *ThreadPool) void {
+    while (true) {
+        const held = self.lock.acquire();
+
+        if (self.run_queue.popFirst()) |run_node| {
+            held.release();
+            (run_node.data.runFn)(&run_node.data);
+            continue;
+        }
+
+        if (!self.is_running) {
+            held.release();
+            return;
+        }
+
+        var idle_node = IdleQueue.Node{ .data = .{} };
+        self.idle_queue.prepend(&idle_node);
+        held.release();
+        idle_node.data.wait();
+    }
+}
diff --git a/src/WaitGroup.zig b/src/WaitGroup.zig
new file mode 100644
index 0000000000..295dfd39dc
--- /dev/null
+++ b/src/WaitGroup.zig
@@ -0,0 +1,22 @@
+const std = @import("std");
+const WaitGroup = @This();
+
+counter: usize = 0,
+event: ?*std.AutoResetEvent = null,
+
+pub fn start(self: *WaitGroup) void {
+    _ = @atomicRmw(usize, &self.counter, .Add, 1, .SeqCst);
+}
+
+pub fn stop(self: *WaitGroup) void {
+    if (@atomicRmw(usize, &self.counter, .Sub, 1, .SeqCst) == 1)
+        if (@atomicRmw(?*std.AutoResetEvent, &self.event, .Xchg, null, .SeqCst)) |event|
+            event.set();
+}
+
+pub fn wait(self: *WaitGroup) void {
+    var event = std.AutoResetEvent{};
+    @atomicStore(?*std.AutoResetEvent, &self.event, &event, .SeqCst);
+    if (@atomicLoad(usize, &self.counter, .SeqCst) != 0)
+        event.wait();
+}
diff --git a/src/glibc.zig b/src/glibc.zig
index 15c9d743f9..413e38637b 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -936,6 +936,7 @@ fn buildSharedLib(
         .root_pkg = null,
         .output_mode = .Lib,
         .link_mode = .Dynamic,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index 8de45a49b2..142d014b2f 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -162,6 +162,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
@@ -280,6 +281,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 9822016ae1..13a4fdf7c7 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -95,6 +95,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .root_name = root_name,
         .root_pkg = null,
         .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
         .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/main.zig b/src/main.zig
index c54cc6515b..c25bc20d85 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -19,6 +19,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
 const translate_c = @import("translate_c.zig");
 const Cache = @import("Cache.zig");
 const target_util = @import("target.zig");
+const ThreadPool = @import("ThreadPool.zig");
 
 pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
     std.log.emerg(format, args);
@@ -1632,6 +1633,10 @@ fn buildOutputType(
         };
     defer zig_lib_directory.handle.close();
 
+    var thread_pool: ThreadPool = undefined;
+    try thread_pool.init(gpa);
+    defer thread_pool.deinit();
+
     var libc_installation: ?LibCInstallation = null;
     defer if (libc_installation) |*l| l.deinit(gpa);
 
@@ -1747,6 +1752,7 @@ fn buildOutputType(
         .single_threaded = single_threaded,
         .function_sections = function_sections,
         .self_exe_path = self_exe_path,
+        .thread_pool = &thread_pool,
         .clang_passthrough_mode = arg_mode != .build,
         .clang_preprocessor_mode = clang_preprocessor_mode,
         .version = optional_version,
@@ -2412,6 +2418,9 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             .directory = null, // Use the local zig-cache.
             .basename = exe_basename,
         };
+        var thread_pool: ThreadPool = undefined;
+        try thread_pool.init(gpa);
+        defer thread_pool.deinit();
         const comp = Compilation.create(gpa, .{
             .zig_lib_directory = zig_lib_directory,
             .local_cache_directory = local_cache_directory,
@@ -2427,6 +2436,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             .emit_h = null,
             .optimize_mode = .Debug,
             .self_exe_path = self_exe_path,
+            .thread_pool = &thread_pool,
         }) catch |err| {
             fatal("unable to create compilation: {}", .{@errorName(err)});
         };
diff --git a/src/musl.zig b/src/musl.zig
index 1a30a6e2b9..80943ab1f7 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -200,6 +200,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .root_pkg = null,
                 .output_mode = .Lib,
                 .link_mode = .Dynamic,
+                .thread_pool = comp.thread_pool,
                 .libc_installation = comp.bin_file.options.libc_installation,
                 .emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
                 .optimize_mode = comp.bin_file.options.optimize_mode,
diff --git a/src/test.zig b/src/test.zig
index e0497a8cd5..795d4f0d65 100644
--- a/src/test.zig
+++ b/src/test.zig
@@ -10,6 +10,7 @@ const enable_qemu: bool = build_options.enable_qemu;
 const enable_wine: bool = build_options.enable_wine;
 const enable_wasmtime: bool = build_options.enable_wasmtime;
 const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
+const ThreadPool = @import("ThreadPool.zig");
 
 const cheader = @embedFile("link/cbe.h");
 
@@ -467,6 +468,10 @@ pub const TestContext = struct {
         defer zig_lib_directory.handle.close();
         defer std.testing.allocator.free(zig_lib_directory.path.?);
 
+        var thread_pool: ThreadPool = undefined;
+        try thread_pool.init(std.testing.allocator);
+        defer thread_pool.deinit();
+
         for (self.cases.items) |case| {
             if (build_options.skip_non_native and case.target.getCpuArch() != std.Target.current.cpu.arch)
                 continue;
@@ -480,7 +485,13 @@ pub const TestContext = struct {
             progress.initial_delay_ns = 0;
             progress.refresh_rate_ns = 0;
 
-            try self.runOneCase(std.testing.allocator, &prg_node, case, zig_lib_directory);
+            try self.runOneCase(
+                std.testing.allocator,
+                &prg_node,
+                case,
+                zig_lib_directory,
+                &thread_pool,
+            );
         }
     }
 
@@ -490,6 +501,7 @@ pub const TestContext = struct {
         root_node: *std.Progress.Node,
         case: Case,
         zig_lib_directory: Compilation.Directory,
+        thread_pool: *ThreadPool,
     ) !void {
         const target_info = try std.zig.system.NativeTargetInfo.detect(allocator, case.target);
         const target = target_info.target;
@@ -539,6 +551,7 @@ pub const TestContext = struct {
             .local_cache_directory = zig_cache_directory,
             .global_cache_directory = zig_cache_directory,
             .zig_lib_directory = zig_lib_directory,
+            .thread_pool = thread_pool,
             .root_name = "test_case",
             .target = target,
             // TODO: support tests for object file building, and library builds
-- 
cgit v1.2.3


From aa6ef10cc657e2bbe59c362f27d7a557c43d7fae Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Fri, 18 Dec 2020 21:51:18 -0700
Subject: std.Progress: make the API thread-safe

We generally get away with atomic primitives, however a lock is required
around the refresh function since it traverses the Node graph, and we
need to be sure no references to Nodes remain after end() is called.
---
 CMakeLists.txt                  |   2 +-
 lib/std/Progress.zig            | 493 +++++++++++++++++++++-------------------
 lib/std/special/test_runner.zig |   2 +-
 lib/std/std.zig                 |   2 +-
 src/Compilation.zig             |   4 +-
 src/stage1.zig                  |   8 +-
 6 files changed, 273 insertions(+), 238 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ea61fd95c0..2580c06066 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -410,7 +410,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/win32error.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/pdb.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/process.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/progress.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/rand.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/reset_event.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/sort.zig"
diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig
index 82f2801fa1..cd5d5ea79b 100644
--- a/lib/std/Progress.zig
+++ b/lib/std/Progress.zig
@@ -3,263 +3,298 @@
 // This file is part of [zig](https://ziglang.org/), which is MIT licensed.
 // The MIT license requires this copyright notice to be included in all copies
 // and substantial portions of the software.
+
+//! This API non-allocating, non-fallible, and thread-safe.
+//! The tradeoff is that users of this API must provide the storage
+//! for each `Progress.Node`.
+//!
+//! Initialize the struct directly, overriding these fields as desired:
+//! * `refresh_rate_ms`
+//! * `initial_delay_ms`
+
 const std = @import("std");
 const windows = std.os.windows;
 const testing = std.testing;
 const assert = std.debug.assert;
+const Progress = @This();
 
-/// This API is non-allocating and non-fallible. The tradeoff is that users of
-/// this API must provide the storage for each `Progress.Node`.
-/// Initialize the struct directly, overriding these fields as desired:
-/// * `refresh_rate_ms`
-/// * `initial_delay_ms`
-pub const Progress = struct {
-    /// `null` if the current node (and its children) should
-    /// not print on update()
-    terminal: ?std.fs.File = undefined,
-
-    /// Whether the terminal supports ANSI escape codes.
-    supports_ansi_escape_codes: bool = false,
-
-    root: Node = undefined,
-
-    /// Keeps track of how much time has passed since the beginning.
-    /// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
-    timer: std.time.Timer = undefined,
-
-    /// When the previous refresh was written to the terminal.
-    /// Used to compare with `refresh_rate_ms`.
-    prev_refresh_timestamp: u64 = undefined,
-
-    /// This buffer represents the maximum number of bytes written to the terminal
-    /// with each refresh.
-    output_buffer: [100]u8 = undefined,
-
-    /// How many nanoseconds between writing updates to the terminal.
-    refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
-
-    /// How many nanoseconds to keep the output hidden
-    initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
-
-    done: bool = true,
-
-    /// Keeps track of how many columns in the terminal have been output, so that
-    /// we can move the cursor back later.
-    columns_written: usize = undefined,
-
-    /// Represents one unit of progress. Each node can have children nodes, or
-    /// one can use integers with `update`.
-    pub const Node = struct {
-        context: *Progress,
-        parent: ?*Node,
-        completed_items: usize,
-        name: []const u8,
-        recently_updated_child: ?*Node = null,
-
-        /// This field may be updated freely.
-        estimated_total_items: ?usize,
-
-        /// Create a new child progress node.
-        /// Call `Node.end` when done.
-        /// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
-        /// API to set `self.parent.recently_updated_child` with the return value.
-        /// Until that is fixed you probably want to call `activate` on the return value.
-        pub fn start(self: *Node, name: []const u8, estimated_total_items: ?usize) Node {
-            return Node{
-                .context = self.context,
-                .parent = self,
-                .completed_items = 0,
-                .name = name,
-                .estimated_total_items = estimated_total_items,
-            };
-        }
+/// `null` if the current node (and its children) should
+/// not print on update()
+terminal: ?std.fs.File = undefined,
 
-        /// This is the same as calling `start` and then `end` on the returned `Node`.
-        pub fn completeOne(self: *Node) void {
-            if (self.parent) |parent| parent.recently_updated_child = self;
-            self.completed_items += 1;
-            self.context.maybeRefresh();
-        }
+/// Whether the terminal supports ANSI escape codes.
+supports_ansi_escape_codes: bool = false,
 
-        pub fn end(self: *Node) void {
-            self.context.maybeRefresh();
-            if (self.parent) |parent| {
-                if (parent.recently_updated_child) |parent_child| {
-                    if (parent_child == self) {
-                        parent.recently_updated_child = null;
-                    }
-                }
-                parent.completeOne();
-            } else {
-                self.context.done = true;
-                self.context.refresh();
-            }
-        }
+root: Node = undefined,
 
-        /// Tell the parent node that this node is actively being worked on.
-        pub fn activate(self: *Node) void {
-            if (self.parent) |parent| parent.recently_updated_child = self;
-        }
-    };
+/// Keeps track of how much time has passed since the beginning.
+/// Used to compare with `initial_delay_ms` and `refresh_rate_ms`.
+timer: std.time.Timer = undefined,
+
+/// When the previous refresh was written to the terminal.
+/// Used to compare with `refresh_rate_ms`.
+prev_refresh_timestamp: u64 = undefined,
+
+/// This buffer represents the maximum number of bytes written to the terminal
+/// with each refresh.
+output_buffer: [100]u8 = undefined,
+
+/// How many nanoseconds between writing updates to the terminal.
+refresh_rate_ns: u64 = 50 * std.time.ns_per_ms,
+
+/// How many nanoseconds to keep the output hidden
+initial_delay_ns: u64 = 500 * std.time.ns_per_ms,
+
+done: bool = true,
+
+/// Protects the `refresh` function, as well as `node.recently_updated_child`.
+/// Without this, callsites would call `Node.end` and then free `Node` memory
+/// while it was still being accessed by the `refresh` function.
+update_lock: std.Mutex = .{},
+
+/// Keeps track of how many columns in the terminal have been output, so that
+/// we can move the cursor back later.
+columns_written: usize = undefined,
 
-    /// Create a new progress node.
+/// Represents one unit of progress. Each node can have children nodes, or
+/// one can use integers with `update`.
+pub const Node = struct {
+    context: *Progress,
+    parent: ?*Node,
+    name: []const u8,
+    /// Must be handled atomically to be thread-safe.
+    recently_updated_child: ?*Node = null,
+    /// Must be handled atomically to be thread-safe. 0 means null.
+    unprotected_estimated_total_items: usize,
+    /// Must be handled atomically to be thread-safe.
+    unprotected_completed_items: usize,
+
+    /// Create a new child progress node. Thread-safe.
     /// Call `Node.end` when done.
     /// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
-    /// API to return Progress rather than accept it as a parameter.
-    pub fn start(self: *Progress, name: []const u8, estimated_total_items: ?usize) !*Node {
-        const stderr = std.io.getStdErr();
-        self.terminal = null;
-        if (stderr.supportsAnsiEscapeCodes()) {
-            self.terminal = stderr;
-            self.supports_ansi_escape_codes = true;
-        } else if (std.builtin.os.tag == .windows and stderr.isTty()) {
-            self.terminal = stderr;
-        }
-        self.root = Node{
-            .context = self,
-            .parent = null,
-            .completed_items = 0,
+    /// API to set `self.parent.recently_updated_child` with the return value.
+    /// Until that is fixed you probably want to call `activate` on the return value.
+    /// Passing 0 for `estimated_total_items` means unknown.
+    pub fn start(self: *Node, name: []const u8, estimated_total_items: usize) Node {
+        return Node{
+            .context = self.context,
+            .parent = self,
             .name = name,
-            .estimated_total_items = estimated_total_items,
+            .unprotected_estimated_total_items = estimated_total_items,
+            .unprotected_completed_items = 0,
         };
-        self.columns_written = 0;
-        self.prev_refresh_timestamp = 0;
-        self.timer = try std.time.Timer.start();
-        self.done = false;
-        return &self.root;
     }
 
-    /// Updates the terminal if enough time has passed since last update.
-    pub fn maybeRefresh(self: *Progress) void {
-        const now = self.timer.read();
-        if (now < self.initial_delay_ns) return;
-        if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
-        self.refresh();
+    /// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe.
+    pub fn completeOne(self: *Node) void {
+        self.activate();
+        _ = @atomicRmw(usize, &self.unprotected_completed_items, .Add, 1, .Monotonic);
+        self.context.maybeRefresh();
     }
 
-    /// Updates the terminal and resets `self.next_refresh_timestamp`.
-    pub fn refresh(self: *Progress) void {
-        const file = self.terminal orelse return;
-
-        const prev_columns_written = self.columns_written;
-        var end: usize = 0;
-        if (self.columns_written > 0) {
-            // restore the cursor position by moving the cursor
-            // `columns_written` cells to the left, then clear the rest of the
-            // line
-            if (self.supports_ansi_escape_codes) {
-                end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{}D", .{self.columns_written}) catch unreachable).len;
-                end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
-            } else if (std.builtin.os.tag == .windows) winapi: {
-                var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
-                if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
-                    unreachable;
-
-                var cursor_pos = windows.COORD{
-                    .X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
-                    .Y = info.dwCursorPosition.Y,
-                };
-
-                if (cursor_pos.X < 0)
-                    cursor_pos.X = 0;
-
-                const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
-
-                var written: windows.DWORD = undefined;
-                if (windows.kernel32.FillConsoleOutputAttribute(
-                    file.handle,
-                    info.wAttributes,
-                    fill_chars,
-                    cursor_pos,
-                    &written,
-                ) != windows.TRUE) {
-                    // Stop trying to write to this file.
-                    self.terminal = null;
-                    break :winapi;
-                }
-                if (windows.kernel32.FillConsoleOutputCharacterA(
-                    file.handle,
-                    ' ',
-                    fill_chars,
-                    cursor_pos,
-                    &written,
-                ) != windows.TRUE) unreachable;
-
-                if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
-                    unreachable;
-            } else unreachable;
-
-            self.columns_written = 0;
+    /// Finish a started `Node`. Thread-safe.
+    pub fn end(self: *Node) void {
+        self.context.maybeRefresh();
+        if (self.parent) |parent| {
+            {
+                const held = self.context.update_lock.acquire();
+                defer held.release();
+                _ = @cmpxchgStrong(?*Node, &parent.recently_updated_child, self, null, .Monotonic, .Monotonic);
+            }
+            parent.completeOne();
+        } else {
+            self.context.done = true;
+            self.context.refresh();
         }
+    }
 
-        if (!self.done) {
-            var need_ellipse = false;
-            var maybe_node: ?*Node = &self.root;
-            while (maybe_node) |node| {
-                if (need_ellipse) {
-                    self.bufWrite(&end, "... ", .{});
-                }
-                need_ellipse = false;
-                if (node.name.len != 0 or node.estimated_total_items != null) {
-                    if (node.name.len != 0) {
-                        self.bufWrite(&end, "{}", .{node.name});
-                        need_ellipse = true;
-                    }
-                    if (node.estimated_total_items) |total| {
-                        if (need_ellipse) self.bufWrite(&end, " ", .{});
-                        self.bufWrite(&end, "[{}/{}] ", .{ node.completed_items + 1, total });
-                        need_ellipse = false;
-                    } else if (node.completed_items != 0) {
-                        if (need_ellipse) self.bufWrite(&end, " ", .{});
-                        self.bufWrite(&end, "[{}] ", .{node.completed_items + 1});
-                        need_ellipse = false;
-                    }
-                }
-                maybe_node = node.recently_updated_child;
-            }
-            if (need_ellipse) {
-                self.bufWrite(&end, "... ", .{});
-            }
+    /// Tell the parent node that this node is actively being worked on. Thread-safe.
+    pub fn activate(self: *Node) void {
+        if (self.parent) |parent| {
+            @atomicStore(?*Node, &parent.recently_updated_child, self, .Monotonic);
         }
+    }
 
-        _ = file.write(self.output_buffer[0..end]) catch |e| {
-            // Stop trying to write to this file once it errors.
-            self.terminal = null;
-        };
-        self.prev_refresh_timestamp = self.timer.read();
+    /// Thread-safe. 0 means unknown.
+    pub fn setEstimatedTotalItems(self: *Node, count: usize) void {
+        @atomicStore(usize, &self.unprotected_estimated_total_items, count, .Monotonic);
     }
 
-    pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
-        const file = self.terminal orelse return;
-        self.refresh();
-        file.outStream().print(format, args) catch {
-            self.terminal = null;
-            return;
-        };
+    /// Thread-safe.
+    pub fn setCompletedItems(self: *Node, completed_items: usize) void {
+        @atomicStore(usize, &self.unprotected_completed_items, completed_items, .Monotonic);
+    }
+};
+
+/// Create a new progress node.
+/// Call `Node.end` when done.
+/// TODO solve https://github.com/ziglang/zig/issues/2765 and then change this
+/// API to return Progress rather than accept it as a parameter.
+/// `estimated_total_items` value of 0 means unknown.
+pub fn start(self: *Progress, name: []const u8, estimated_total_items: usize) !*Node {
+    const stderr = std.io.getStdErr();
+    self.terminal = null;
+    if (stderr.supportsAnsiEscapeCodes()) {
+        self.terminal = stderr;
+        self.supports_ansi_escape_codes = true;
+    } else if (std.builtin.os.tag == .windows and stderr.isTty()) {
+        self.terminal = stderr;
+    }
+    self.root = Node{
+        .context = self,
+        .parent = null,
+        .name = name,
+        .unprotected_estimated_total_items = estimated_total_items,
+        .unprotected_completed_items = 0,
+    };
+    self.columns_written = 0;
+    self.prev_refresh_timestamp = 0;
+    self.timer = try std.time.Timer.start();
+    self.done = false;
+    return &self.root;
+}
+
+/// Updates the terminal if enough time has passed since last update. Thread-safe.
+pub fn maybeRefresh(self: *Progress) void {
+    const now = self.timer.read();
+    if (now < self.initial_delay_ns) return;
+    const held = self.update_lock.tryAcquire() orelse return;
+    defer held.release();
+    if (now - self.prev_refresh_timestamp < self.refresh_rate_ns) return;
+    return self.refreshWithHeldLock();
+}
+
+/// Updates the terminal and resets `self.next_refresh_timestamp`. Thread-safe.
+pub fn refresh(self: *Progress) void {
+    const held = self.update_lock.tryAcquire() orelse return;
+    defer held.release();
+
+    return self.refreshWithHeldLock();
+}
+
+fn refreshWithHeldLock(self: *Progress) void {
+    const file = self.terminal orelse return;
+
+    const prev_columns_written = self.columns_written;
+    var end: usize = 0;
+    if (self.columns_written > 0) {
+        // restore the cursor position by moving the cursor
+        // `columns_written` cells to the left, then clear the rest of the
+        // line
+        if (self.supports_ansi_escape_codes) {
+            end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[{d}D", .{self.columns_written}) catch unreachable).len;
+            end += (std.fmt.bufPrint(self.output_buffer[end..], "\x1b[0K", .{}) catch unreachable).len;
+        } else if (std.builtin.os.tag == .windows) winapi: {
+            var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;
+            if (windows.kernel32.GetConsoleScreenBufferInfo(file.handle, &info) != windows.TRUE)
+                unreachable;
+
+            var cursor_pos = windows.COORD{
+                .X = info.dwCursorPosition.X - @intCast(windows.SHORT, self.columns_written),
+                .Y = info.dwCursorPosition.Y,
+            };
+
+            if (cursor_pos.X < 0)
+                cursor_pos.X = 0;
+
+            const fill_chars = @intCast(windows.DWORD, info.dwSize.X - cursor_pos.X);
+
+            var written: windows.DWORD = undefined;
+            if (windows.kernel32.FillConsoleOutputAttribute(
+                file.handle,
+                info.wAttributes,
+                fill_chars,
+                cursor_pos,
+                &written,
+            ) != windows.TRUE) {
+                // Stop trying to write to this file.
+                self.terminal = null;
+                break :winapi;
+            }
+            if (windows.kernel32.FillConsoleOutputCharacterA(
+                file.handle,
+                ' ',
+                fill_chars,
+                cursor_pos,
+                &written,
+            ) != windows.TRUE) unreachable;
+
+            if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
+                unreachable;
+        } else unreachable;
+
         self.columns_written = 0;
     }
 
-    fn bufWrite(self: *Progress, end: *usize, comptime format: []const u8, args: anytype) void {
-        if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) |written| {
-            const amt = written.len;
-            end.* += amt;
-            self.columns_written += amt;
-        } else |err| switch (err) {
-            error.NoSpaceLeft => {
-                self.columns_written += self.output_buffer.len - end.*;
-                end.* = self.output_buffer.len;
-            },
+    if (!self.done) {
+        var need_ellipse = false;
+        var maybe_node: ?*Node = &self.root;
+        while (maybe_node) |node| {
+            if (need_ellipse) {
+                self.bufWrite(&end, "... ", .{});
+            }
+            need_ellipse = false;
+            const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic);
+            const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic);
+            if (node.name.len != 0 or eti > 0) {
+                if (node.name.len != 0) {
+                    self.bufWrite(&end, "{s}", .{node.name});
+                    need_ellipse = true;
+                }
+                if (eti > 0) {
+                    if (need_ellipse) self.bufWrite(&end, " ", .{});
+                    self.bufWrite(&end, "[{d}/{d}] ", .{ completed_items + 1, eti });
+                    need_ellipse = false;
+                } else if (completed_items != 0) {
+                    if (need_ellipse) self.bufWrite(&end, " ", .{});
+                    self.bufWrite(&end, "[{d}] ", .{completed_items + 1});
+                    need_ellipse = false;
+                }
+            }
+            maybe_node = @atomicLoad(?*Node, &node.recently_updated_child, .Monotonic);
         }
-        const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
-        const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
-        if (end.* > max_end) {
-            const suffix = "... ";
-            self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
-            std.mem.copy(u8, self.output_buffer[max_end..], suffix);
-            end.* = max_end + suffix.len;
+        if (need_ellipse) {
+            self.bufWrite(&end, "... ", .{});
         }
     }
-};
+
+    _ = file.write(self.output_buffer[0..end]) catch |e| {
+        // Stop trying to write to this file once it errors.
+        self.terminal = null;
+    };
+    self.prev_refresh_timestamp = self.timer.read();
+}
+
+pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
+    const file = self.terminal orelse return;
+    self.refresh();
+    file.outStream().print(format, args) catch {
+        self.terminal = null;
+        return;
+    };
+    self.columns_written = 0;
+}
+
+fn bufWrite(self: *Progress, end: *usize, comptime format: []const u8, args: anytype) void {
+    if (std.fmt.bufPrint(self.output_buffer[end.*..], format, args)) |written| {
+        const amt = written.len;
+        end.* += amt;
+        self.columns_written += amt;
+    } else |err| switch (err) {
+        error.NoSpaceLeft => {
+            self.columns_written += self.output_buffer.len - end.*;
+            end.* = self.output_buffer.len;
+        },
+    }
+    const bytes_needed_for_esc_codes_at_end = if (std.builtin.os.tag == .windows) 0 else 11;
+    const max_end = self.output_buffer.len - bytes_needed_for_esc_codes_at_end;
+    if (end.* > max_end) {
+        const suffix = "... ";
+        self.columns_written = self.columns_written - (end.* - max_end) + suffix.len;
+        std.mem.copy(u8, self.output_buffer[max_end..], suffix);
+        end.* = max_end + suffix.len;
+    }
+}
 
 test "basic functionality" {
     var disable = true;
@@ -300,7 +335,7 @@ test "basic functionality" {
         std.time.sleep(5 * std.time.ns_per_ms);
     }
     {
-        var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", null);
+        var node = root_node.start("this is a really long name designed to activate the truncation code. let's find out if it works", 0);
         node.activate();
         std.time.sleep(10 * std.time.ns_per_ms);
         progress.refresh();
diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig
index 2b2fe78262..4bb9202858 100644
--- a/lib/std/special/test_runner.zig
+++ b/lib/std/special/test_runner.zig
@@ -36,7 +36,7 @@ pub fn main() anyerror!void {
         }
         std.testing.log_level = .warn;
 
-        var test_node = root_node.start(test_fn.name, null);
+        var test_node = root_node.start(test_fn.name, 0);
         test_node.activate();
         progress.refresh();
         if (progress.terminal == null) {
diff --git a/lib/std/std.zig b/lib/std/std.zig
index f6da7afc55..5fbf2662b9 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -29,7 +29,7 @@ pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayE
 pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
 pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
 pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
-pub const Progress = @import("progress.zig").Progress;
+pub const Progress = @import("Progress.zig");
 pub const ResetEvent = @import("reset_event.zig").ResetEvent;
 pub const SemanticVersion = @import("SemanticVersion.zig");
 pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
diff --git a/src/Compilation.zig b/src/Compilation.zig
index ef07de7b17..1c27a589ee 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1378,7 +1378,7 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
 
 pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
     var progress: std.Progress = .{};
-    var main_progress_node = try progress.start("", null);
+    var main_progress_node = try progress.start("", 0);
     defer main_progress_node.end();
     if (self.color == .off) progress.terminal = null;
 
@@ -1811,7 +1811,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
     const c_source_basename = std.fs.path.basename(c_object.src.src_path);
 
     c_comp_progress_node.activate();
-    var child_progress_node = c_comp_progress_node.start(c_source_basename, null);
+    var child_progress_node = c_comp_progress_node.start(c_source_basename, 0);
     child_progress_node.activate();
     defer child_progress_node.end();
 
diff --git a/src/stage1.zig b/src/stage1.zig
index 8576459aa8..7f2d3d732d 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -293,7 +293,7 @@ export fn stage2_progress_start_root(
 ) *std.Progress.Node {
     return progress.start(
         name_ptr[0..name_len],
-        if (estimated_total_items == 0) null else estimated_total_items,
+        estimated_total_items,
     ) catch @panic("timer unsupported");
 }
 
@@ -312,7 +312,7 @@ export fn stage2_progress_start(
     const child_node = std.heap.c_allocator.create(std.Progress.Node) catch @panic("out of memory");
     child_node.* = node.start(
         name_ptr[0..name_len],
-        if (estimated_total_items == 0) null else estimated_total_items,
+        estimated_total_items,
     );
     child_node.activate();
     return child_node;
@@ -333,8 +333,8 @@ export fn stage2_progress_complete_one(node: *std.Progress.Node) void {
 
 // ABI warning
 export fn stage2_progress_update_node(node: *std.Progress.Node, done_count: usize, total_count: usize) void {
-    node.completed_items = done_count;
-    node.estimated_total_items = total_count;
+    node.setCompletedItems(done_count);
+    node.setEstimatedTotalItems(total_count);
     node.activate();
     node.context.maybeRefresh();
 }
-- 
cgit v1.2.3


From 1d94a6893689ad1fb8e06308ae51603a6c8708a8 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 20 Dec 2020 15:37:58 -0700
Subject: add an option to compile zig in single-threaded mode

And enable it for Drone CI. I hate to do this, but I need to make
progress on other fronts.
---
 CMakeLists.txt        | 10 ++++++++++
 ci/drone/linux_script |  3 ++-
 src/Compilation.zig   | 12 +++++++-----
 src/ThreadPool.zig    |  6 ++++++
 src/stage1/zig0.cpp   |  4 ++++
 5 files changed, 29 insertions(+), 6 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2580c06066..e3c4e67c5a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -75,6 +75,7 @@ set(ZIG_TARGET_TRIPLE "native" CACHE STRING "arch-os-abi to output binaries for"
 set(ZIG_TARGET_MCPU "baseline" CACHE STRING "-mcpu parameter to output binaries for")
 set(ZIG_EXECUTABLE "" CACHE STRING "(when cross compiling) path to already-built zig binary")
 set(ZIG_PREFER_LLVM_CONFIG off CACHE BOOL "(when cross compiling) use llvm-config to find target llvm dependencies if needed")
+set(ZIG_SINGLE_THREADED off CACHE BOOL "limit the zig compiler to use only 1 thread")
 
 find_package(llvm)
 find_package(clang)
@@ -510,10 +511,13 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/Cache.zig"
     "${CMAKE_SOURCE_DIR}/src/Compilation.zig"
     "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
+    "${CMAKE_SOURCE_DIR}/src/Event.zig"
     "${CMAKE_SOURCE_DIR}/src/Module.zig"
     "${CMAKE_SOURCE_DIR}/src/Package.zig"
     "${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
+    "${CMAKE_SOURCE_DIR}/src/ThreadPool.zig"
     "${CMAKE_SOURCE_DIR}/src/TypedValue.zig"
+    "${CMAKE_SOURCE_DIR}/src/WaitGroup.zig"
     "${CMAKE_SOURCE_DIR}/src/astgen.zig"
     "${CMAKE_SOURCE_DIR}/src/clang.zig"
     "${CMAKE_SOURCE_DIR}/src/clang_options.zig"
@@ -713,6 +717,11 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
 else()
     set(ZIG1_RELEASE_ARG -OReleaseFast --strip)
 endif()
+if(ZIG_SINGLE_THREADED)
+  set(ZIG1_SINGLE_THREADED_ARG "--single-threaded")
+else()
+  set(ZIG1_SINGLE_THREADED_ARG "")
+endif()
 
 set(BUILD_ZIG1_ARGS
     "src/stage1.zig"
@@ -722,6 +731,7 @@ set(BUILD_ZIG1_ARGS
     --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
     "-femit-bin=${ZIG1_OBJECT}"
     "${ZIG1_RELEASE_ARG}"
+    "${ZIG1_SINGLE_THREADED_ARG}"
     -lc
     --pkg-begin build_options "${ZIG_CONFIG_ZIG_OUT}"
     --pkg-end
diff --git a/ci/drone/linux_script b/ci/drone/linux_script
index fdc1704fb7..8c5dc1be2a 100755
--- a/ci/drone/linux_script
+++ b/ci/drone/linux_script
@@ -17,7 +17,8 @@ git config core.abbrev 9
 
 mkdir build
 cd build
-cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja
+# TODO figure out why Drone CI is deadlocking and stop passing -DZIG_SINGLE_THREADED=ON
+cmake .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=$DISTDIR" -DZIG_STATIC=ON -DCMAKE_PREFIX_PATH=/deps/local -GNinja -DZIG_SINGLE_THREADED=ON
 
 samu install
 ./zig build test -Dskip-release -Dskip-non-native
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 1c27a589ee..23f67f5b37 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1728,7 +1728,7 @@ fn workerUpdateCObject(
         error.AnalysisFail => return,
         else => {
             {
-                var lock = comp.mutex.acquire();
+                const lock = comp.mutex.acquire();
                 defer lock.release();
                 comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
                     fatal("TODO handle this by setting c_object.status = oom failure", .{});
@@ -1759,7 +1759,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
 
     if (c_object.clearStatus(comp.gpa)) {
         // There was previous failure.
-        var lock = comp.mutex.acquire();
+        const lock = comp.mutex.acquire();
         defer lock.release();
         comp.failed_c_objects.removeAssertDiscard(c_object);
     }
@@ -1789,10 +1789,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
 
     {
         const is_collision = blk: {
-            var lock = comp.mutex.acquire();
+            const bin_digest = man.hash.peekBin();
+
+            const lock = comp.mutex.acquire();
             defer lock.release();
 
-            const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
+            const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
             break :blk gop.found_existing;
         };
         if (is_collision) {
@@ -2211,7 +2213,7 @@ fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8,
 
 fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError {
     {
-        var lock = comp.mutex.acquire();
+        const lock = comp.mutex.acquire();
         defer lock.release();
         {
             errdefer err_msg.destroy(comp.gpa);
diff --git a/src/ThreadPool.zig b/src/ThreadPool.zig
index 7d6af3d24c..00cb26772a 100644
--- a/src/ThreadPool.zig
+++ b/src/ThreadPool.zig
@@ -25,6 +25,8 @@ pub fn init(self: *ThreadPool, allocator: *std.mem.Allocator) !void {
         .allocator = allocator,
         .threads = &[_]*std.Thread{},
     };
+    if (std.builtin.single_threaded)
+        return;
 
     errdefer self.deinit();
 
@@ -67,6 +69,10 @@ pub fn shutdown(self: *ThreadPool) void {
 }
 
 pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
+    if (std.builtin.single_threaded) {
+        @call(.{}, func, args);
+        return;
+    }
     const Args = @TypeOf(args);
     const Closure = struct {
         arguments: Args,
diff --git a/src/stage1/zig0.cpp b/src/stage1/zig0.cpp
index 73f5b4f685..bcc9dbc00a 100644
--- a/src/stage1/zig0.cpp
+++ b/src/stage1/zig0.cpp
@@ -266,6 +266,7 @@ int main(int argc, char **argv) {
     TargetSubsystem subsystem = TargetSubsystemAuto;
     const char *override_lib_dir = nullptr;
     const char *mcpu = nullptr;
+    bool single_threaded = false;
 
     for (int i = 1; i < argc; i += 1) {
         char *arg = argv[i];
@@ -281,6 +282,8 @@ int main(int argc, char **argv) {
                 optimize_mode = BuildModeSafeRelease;
             } else if (strcmp(arg, "-OReleaseSmall") == 0) {
                 optimize_mode = BuildModeSmallRelease;
+            } else if (strcmp(arg, "--single-threaded") == 0) {
+                single_threaded = true;
             } else if (strcmp(arg, "--help") == 0) {
                 return print_full_usage(arg0, stdout, EXIT_SUCCESS);
             } else if (strcmp(arg, "--strip") == 0) {
@@ -469,6 +472,7 @@ int main(int argc, char **argv) {
     stage1->link_libcpp = link_libcpp;
     stage1->subsystem = subsystem;
     stage1->pic = true;
+    stage1->is_single_threaded = single_threaded;
 
     zig_stage1_build_object(stage1);
 
-- 
cgit v1.2.3


From 40f0275e7cb7f3f4b2c382e5b457e714080c4cf2 Mon Sep 17 00:00:00 2001
From: Alex Cameron <ascottcameron@gmail.com>
Date: Fri, 13 Nov 2020 00:50:02 +1100
Subject: Implement emit-h

---
 src/Compilation.zig |  77 +++++++++++++++++++++++--------
 src/codegen/c.zig   | 130 +++++++++++++++++++++++++++++++++-------------------
 src/link/C.zig      |  62 +++++++++++++++++--------
 3 files changed, 183 insertions(+), 86 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 23f67f5b37..5b06bd6370 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -26,6 +26,8 @@ const Module = @import("Module.zig");
 const Cache = @import("Cache.zig");
 const stage1 = @import("stage1.zig");
 const translate_c = @import("translate_c.zig");
+const c_codegen = @import("codegen/c.zig");
+const c_link = @import("link/C.zig");
 const ThreadPool = @import("ThreadPool.zig");
 const WaitGroup = @import("WaitGroup.zig");
 
@@ -131,6 +133,7 @@ emit_asm: ?EmitLoc,
 emit_llvm_ir: ?EmitLoc,
 emit_analysis: ?EmitLoc,
 emit_docs: ?EmitLoc,
+c_header: ?c_link.Header,
 
 pub const InnerError = Module.InnerError;
 
@@ -895,10 +898,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             };
         };
 
-        if (!use_llvm and options.emit_h != null) {
-            fatal("TODO implement support for -femit-h in the self-hosted backend", .{});
-        }
-
         var system_libs: std.StringArrayHashMapUnmanaged(void) = .{};
         errdefer system_libs.deinit(gpa);
         try system_libs.ensureCapacity(gpa, options.system_libs.len);
@@ -975,6 +974,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
             .emit_h = options.emit_h,
+            .c_header = if (!use_llvm and options.emit_h != null) c_link.Header.init(gpa) else null,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_analysis = options.emit_analysis,
@@ -1286,6 +1286,20 @@ pub fn update(self: *Compilation) !void {
             module.root_scope.unload(self.gpa);
         }
     }
+
+    // If we've chosen to emit a C header, flush the header to the disk.
+    if (self.c_header) |header| {
+        const header_path = self.emit_h.?;
+        // If a directory has been provided, write the header there. Otherwise, just write it to the
+        // cache directory.
+        const header_dir = if (header_path.directory) |dir|
+            dir.handle
+        else
+            self.local_cache_directory.handle;
+        const header_file = try header_dir.createFile(header_path.basename, .{});
+        defer header_file.close();
+        try header.flush(header_file.writer());
+    }
 }
 
 /// Having the file open for writing is problematic as far as executing the
@@ -1385,6 +1399,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_comp_progress_node.end();
 
+    var arena = std.heap.ArenaAllocator.init(self.gpa);
+    defer arena.deinit();
+
     var wg = WaitGroup{};
     defer wg.wait();
 
@@ -1432,22 +1449,44 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
 
-                self.bin_file.updateDecl(module, decl) catch |err| switch (err) {
-                    error.OutOfMemory => return error.OutOfMemory,
-                    error.AnalysisFail => {
-                        decl.analysis = .dependency_failure;
-                    },
-                    else => {
-                        try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                        module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
-                            module.gpa,
-                            decl.src(),
-                            "unable to codegen: {}",
-                            .{@errorName(err)},
-                        ));
-                        decl.analysis = .codegen_failure_retryable;
-                    },
+                self.bin_file.updateDecl(module, decl) catch |err| {
+                    switch (err) {
+                        error.OutOfMemory => return error.OutOfMemory,
+                        error.AnalysisFail => {
+                            decl.analysis = .dependency_failure;
+                        },
+                        else => {
+                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                                module.gpa,
+                                decl.src(),
+                                "unable to codegen: {}",
+                                .{@errorName(err)},
+                            ));
+                            decl.analysis = .codegen_failure_retryable;
+                        },
+                    }
+                    return;
                 };
+
+                if (self.c_header) |*header| {
+                    c_codegen.generateHeader(&arena, module, &header.*, decl) catch |err| switch (err) {
+                        error.OutOfMemory => return error.OutOfMemory,
+                        error.AnalysisFail => {
+                            decl.analysis = .dependency_failure;
+                        },
+                        else => {
+                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                                module.gpa,
+                                decl.src(),
+                                "unable to generate C header: {}",
+                                .{@errorName(err)},
+                            ));
+                            decl.analysis = .codegen_failure_retryable;
+                        },
+                    };
+                }
             },
         },
         .analyze_decl => |decl| {
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index e562ce320e..425663ad9c 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -2,6 +2,7 @@ const std = @import("std");
 
 const link = @import("../link.zig");
 const Module = @import("../Module.zig");
+const Compilation = @import("../Compilation.zig");
 
 const Inst = @import("../ir.zig").Inst;
 const Value = @import("../value.zig").Value;
@@ -19,7 +20,7 @@ fn map(allocator: *std.mem.Allocator, name: []const u8) ![]const u8 {
     return allocator.dupe(u8, name);
 }
 
-fn renderType(ctx: *Context, writer: std.ArrayList(u8).Writer, T: Type) !void {
+fn renderType(ctx: *Context, header: *C.Header, writer: std.ArrayList(u8).Writer, T: Type) !void {
     switch (T.zigTypeTag()) {
         .NoReturn => {
             try writer.writeAll("zig_noreturn void");
@@ -27,16 +28,16 @@ fn renderType(ctx: *Context, writer: std.ArrayList(u8).Writer, T: Type) !void {
         .Void => try writer.writeAll("void"),
         .Int => {
             if (T.tag() == .u8) {
-                ctx.file.need_stdint = true;
+                header.need_stdint = true;
                 try writer.writeAll("uint8_t");
             } else if (T.tag() == .usize) {
-                ctx.file.need_stddef = true;
+                header.need_stddef = true;
                 try writer.writeAll("size_t");
             } else {
-                return ctx.file.fail(ctx.decl.src(), "TODO implement int types", .{});
+                return ctx.fail(ctx.decl.src(), "TODO implement int types", .{});
             }
         },
-        else => |e| return ctx.file.fail(ctx.decl.src(), "TODO implement type {}", .{e}),
+        else => |e| return ctx.fail(ctx.decl.src(), "TODO implement type {}", .{e}),
     }
 }
 
@@ -47,13 +48,13 @@ fn renderValue(ctx: *Context, writer: std.ArrayList(u8).Writer, T: Type, val: Va
                 return writer.print("{}", .{val.toSignedInt()});
             return writer.print("{}", .{val.toUnsignedInt()});
         },
-        else => |e| return ctx.file.fail(ctx.decl.src(), "TODO implement value {}", .{e}),
+        else => |e| return ctx.fail(ctx.decl.src(), "TODO implement value {}", .{e}),
     }
 }
 
-fn renderFunctionSignature(ctx: *Context, writer: std.ArrayList(u8).Writer, decl: *Decl) !void {
+fn renderFunctionSignature(ctx: *Context, header: *C.Header, writer: std.ArrayList(u8).Writer, decl: *Decl) !void {
     const tv = decl.typed_value.most_recent.typed_value;
-    try renderType(ctx, writer, tv.ty.fnReturnType());
+    try renderType(ctx, header, writer, tv.ty.fnReturnType());
     // Use the child allocator directly, as we know the name can be freed before
     // the rest of the arena.
     const name = try map(ctx.arena.child_allocator, mem.spanZ(decl.name));
@@ -68,7 +69,7 @@ fn renderFunctionSignature(ctx: *Context, writer: std.ArrayList(u8).Writer, decl
             if (index > 0) {
                 try writer.writeAll(", ");
             }
-            try renderType(ctx, writer, tv.ty.fnParamType(index));
+            try renderType(ctx, header, writer, tv.ty.fnParamType(index));
             try writer.print(" arg{}", .{index});
         }
     }
@@ -83,6 +84,34 @@ pub fn generate(file: *C, decl: *Decl) !void {
     }
 }
 
+pub fn generateHeader(
+    arena: *std.heap.ArenaAllocator,
+    module: *Module,
+    header: *C.Header,
+    decl: *Decl,
+) error{ AnalysisFail, OutOfMemory }!void {
+    switch (decl.typed_value.most_recent.typed_value.ty.zigTypeTag()) {
+        .Fn => {
+            var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
+            defer inst_map.deinit();
+            var ctx = Context{
+                .decl = decl,
+                .arena = arena,
+                .inst_map = &inst_map,
+            };
+            const writer = header.buf.writer();
+            renderFunctionSignature(&ctx, header, writer, decl) catch |err| {
+                if (err == error.AnalysisFail) {
+                    try module.failed_decls.put(module.gpa, decl, ctx.error_msg);
+                }
+                return err;
+            };
+            try writer.writeAll(";\n");
+        },
+        else => {},
+    }
+}
+
 fn genArray(file: *C, decl: *Decl) !void {
     const tv = decl.typed_value.most_recent.typed_value;
     // TODO: prevent inline asm constants from being emitted
@@ -102,12 +131,12 @@ fn genArray(file: *C, decl: *Decl) !void {
 }
 
 const Context = struct {
-    file: *C,
     decl: *Decl,
     inst_map: *std.AutoHashMap(*Inst, []u8),
     arena: *std.heap.ArenaAllocator,
     argdex: usize = 0,
     unnamed_index: usize = 0,
+    error_msg: *Compilation.ErrorMsg = undefined,
 
     fn resolveInst(self: *Context, inst: *Inst) ![]u8 {
         if (inst.cast(Inst.Constant)) |const_inst| {
@@ -127,6 +156,11 @@ const Context = struct {
         return val;
     }
 
+    fn fail(self: *Context, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } {
+        self.error_msg = try Compilation.ErrorMsg.create(self.arena.child_allocator, src, format, args);
+        return error.AnalysisFail;
+    }
+
     fn deinit(self: *Context) void {
         self.* = undefined;
     }
@@ -141,14 +175,16 @@ fn genFn(file: *C, decl: *Decl) !void {
     var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
     defer inst_map.deinit();
     var ctx = Context{
-        .file = file,
         .decl = decl,
         .arena = &arena,
         .inst_map = &inst_map,
     };
-    defer ctx.deinit();
+    defer {
+        file.error_msg = ctx.error_msg;
+        ctx.deinit();
+    }
 
-    try renderFunctionSignature(&ctx, writer, decl);
+    try renderFunctionSignature(&ctx, &file.header, writer, decl);
 
     try writer.writeAll(" {");
 
@@ -158,18 +194,18 @@ fn genFn(file: *C, decl: *Decl) !void {
         try writer.writeAll("\n");
         for (instructions) |inst| {
             if (switch (inst.tag) {
-                .assembly => try genAsm(&ctx, inst.castTag(.assembly).?),
-                .call => try genCall(&ctx, inst.castTag(.call).?),
-                .add => try genBinOp(&ctx, inst.cast(Inst.BinOp).?, "+"),
-                .sub => try genBinOp(&ctx, inst.cast(Inst.BinOp).?, "-"),
+                .assembly => try genAsm(&ctx, file, inst.castTag(.assembly).?),
+                .call => try genCall(&ctx, file, inst.castTag(.call).?),
+                .add => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "+"),
+                .sub => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "-"),
                 .ret => try genRet(&ctx, inst.castTag(.ret).?),
-                .retvoid => try genRetVoid(&ctx),
+                .retvoid => try genRetVoid(file),
                 .arg => try genArg(&ctx),
                 .dbg_stmt => try genDbgStmt(&ctx, inst.castTag(.dbg_stmt).?),
                 .breakpoint => try genBreak(&ctx, inst.castTag(.breakpoint).?),
-                .unreach => try genUnreach(&ctx, inst.castTag(.unreach).?),
-                .intcast => try genIntCast(&ctx, inst.castTag(.intcast).?),
-                else => |e| return file.fail(decl.src(), "TODO implement C codegen for {}", .{e}),
+                .unreach => try genUnreach(file, inst.castTag(.unreach).?),
+                .intcast => try genIntCast(&ctx, file, inst.castTag(.intcast).?),
+                else => |e| return ctx.fail(decl.src(), "TODO implement C codegen for {}", .{e}),
             }) |name| {
                 try ctx.inst_map.putNoClobber(inst, name);
             }
@@ -185,46 +221,46 @@ fn genArg(ctx: *Context) !?[]u8 {
     return name;
 }
 
-fn genRetVoid(ctx: *Context) !?[]u8 {
-    try ctx.file.main.writer().print(indentation ++ "return;\n", .{});
+fn genRetVoid(file: *C) !?[]u8 {
+    try file.main.writer().print(indentation ++ "return;\n", .{});
     return null;
 }
 
 fn genRet(ctx: *Context, inst: *Inst.UnOp) !?[]u8 {
-    return ctx.file.fail(ctx.decl.src(), "TODO return", .{});
+    return ctx.fail(ctx.decl.src(), "TODO return", .{});
 }
 
-fn genIntCast(ctx: *Context, inst: *Inst.UnOp) !?[]u8 {
+fn genIntCast(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
     if (inst.base.isUnused())
         return null;
     const op = inst.operand;
-    const writer = ctx.file.main.writer();
+    const writer = file.main.writer();
     const name = try ctx.name();
     const from = try ctx.resolveInst(inst.operand);
     try writer.writeAll(indentation ++ "const ");
-    try renderType(ctx, writer, inst.base.ty);
+    try renderType(ctx, &file.header, writer, inst.base.ty);
     try writer.print(" {} = (", .{name});
-    try renderType(ctx, writer, inst.base.ty);
+    try renderType(ctx, &file.header, writer, inst.base.ty);
     try writer.print("){};\n", .{from});
     return name;
 }
 
-fn genBinOp(ctx: *Context, inst: *Inst.BinOp, comptime operator: []const u8) !?[]u8 {
+fn genBinOp(ctx: *Context, file: *C, inst: *Inst.BinOp, comptime operator: []const u8) !?[]u8 {
     if (inst.base.isUnused())
         return null;
     const lhs = ctx.resolveInst(inst.lhs);
     const rhs = ctx.resolveInst(inst.rhs);
-    const writer = ctx.file.main.writer();
+    const writer = file.main.writer();
     const name = try ctx.name();
     try writer.writeAll(indentation ++ "const ");
-    try renderType(ctx, writer, inst.base.ty);
+    try renderType(ctx, &file.header, writer, inst.base.ty);
     try writer.print(" {} = {} " ++ operator ++ " {};\n", .{ name, lhs, rhs });
     return name;
 }
 
-fn genCall(ctx: *Context, inst: *Inst.Call) !?[]u8 {
-    const writer = ctx.file.main.writer();
-    const header = ctx.file.header.writer();
+fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
+    const writer = file.main.writer();
+    const header = file.header.buf.writer();
     try writer.writeAll(indentation);
     if (inst.func.castTag(.constant)) |func_inst| {
         if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
@@ -235,9 +271,9 @@ fn genCall(ctx: *Context, inst: *Inst.Call) !?[]u8 {
                 try writer.print("(void)", .{});
             }
             const tname = mem.spanZ(target.name);
-            if (ctx.file.called.get(tname) == null) {
-                try ctx.file.called.put(tname, void{});
-                try renderFunctionSignature(ctx, header, target);
+            if (file.called.get(tname) == null) {
+                try file.called.put(tname, void{});
+                try renderFunctionSignature(ctx, &file.header, header, target);
                 try header.writeAll(";\n");
             }
             try writer.print("{}(", .{tname});
@@ -256,10 +292,10 @@ fn genCall(ctx: *Context, inst: *Inst.Call) !?[]u8 {
             }
             try writer.writeAll(");\n");
         } else {
-            return ctx.file.fail(ctx.decl.src(), "TODO non-function call target?", .{});
+            return ctx.fail(ctx.decl.src(), "TODO non-function call target?", .{});
         }
     } else {
-        return ctx.file.fail(ctx.decl.src(), "TODO non-constant call inst?", .{});
+        return ctx.fail(ctx.decl.src(), "TODO non-constant call inst?", .{});
     }
     return null;
 }
@@ -274,20 +310,20 @@ fn genBreak(ctx: *Context, inst: *Inst.NoOp) !?[]u8 {
     return null;
 }
 
-fn genUnreach(ctx: *Context, inst: *Inst.NoOp) !?[]u8 {
-    try ctx.file.main.writer().writeAll(indentation ++ "zig_unreachable();\n");
+fn genUnreach(file: *C, inst: *Inst.NoOp) !?[]u8 {
+    try file.main.writer().writeAll(indentation ++ "zig_unreachable();\n");
     return null;
 }
 
-fn genAsm(ctx: *Context, as: *Inst.Assembly) !?[]u8 {
-    const writer = ctx.file.main.writer();
+fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
+    const writer = file.main.writer();
     try writer.writeAll(indentation);
     for (as.inputs) |i, index| {
         if (i[0] == '{' and i[i.len - 1] == '}') {
             const reg = i[1 .. i.len - 1];
             const arg = as.args[index];
             try writer.writeAll("register ");
-            try renderType(ctx, writer, arg.ty);
+            try renderType(ctx, &file.header, writer, arg.ty);
             try writer.print(" {}_constant __asm__(\"{}\") = ", .{ reg, reg });
             // TODO merge constant handling into inst_map as well
             if (arg.castTag(.constant)) |c| {
@@ -296,17 +332,17 @@ fn genAsm(ctx: *Context, as: *Inst.Assembly) !?[]u8 {
             } else {
                 const gop = try ctx.inst_map.getOrPut(arg);
                 if (!gop.found_existing) {
-                    return ctx.file.fail(ctx.decl.src(), "Internal error in C backend: asm argument not found in inst_map", .{});
+                    return ctx.fail(ctx.decl.src(), "Internal error in C backend: asm argument not found in inst_map", .{});
                 }
                 try writer.print("{};\n    ", .{gop.entry.value});
             }
         } else {
-            return ctx.file.fail(ctx.decl.src(), "TODO non-explicit inline asm regs", .{});
+            return ctx.fail(ctx.decl.src(), "TODO non-explicit inline asm regs", .{});
         }
     }
     try writer.print("__asm {} (\"{}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source });
     if (as.output) |o| {
-        return ctx.file.fail(ctx.decl.src(), "TODO inline asm output", .{});
+        return ctx.fail(ctx.decl.src(), "TODO inline asm output", .{});
     }
     if (as.inputs.len > 0) {
         if (as.output == null) {
diff --git a/src/link/C.zig b/src/link/C.zig
index 467e10998e..0906482dd1 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -13,15 +13,52 @@ const C = @This();
 
 pub const base_tag: File.Tag = .c;
 
+pub const Header = struct {
+    buf: std.ArrayList(u8),
+    need_stddef: bool = false,
+    need_stdint: bool = false,
+
+    pub fn init(allocator: *Allocator) Header {
+        return .{
+            .buf = std.ArrayList(u8).init(allocator),
+        };
+    }
+
+    pub fn flush(self: *const Header, writer: anytype) !void {
+        const tracy = trace(@src());
+        defer tracy.end();
+
+        try writer.writeAll(@embedFile("cbe.h"));
+        var includes = false;
+        if (self.need_stddef) {
+            try writer.writeAll("#include <stddef.h>\n");
+            includes = true;
+        }
+        if (self.need_stdint) {
+            try writer.writeAll("#include <stdint.h>\n");
+            includes = true;
+        }
+        if (includes) {
+            try writer.writeByte('\n');
+        }
+        if (self.buf.items.len > 0) {
+            try writer.print("{}\n", .{self.buf.items});
+        }
+    }
+
+    pub fn deinit(self: *Header) void {
+        self.buf.deinit();
+        self.* = undefined;
+    }
+};
+
 base: File,
 
-header: std.ArrayList(u8),
+header: Header,
 constants: std.ArrayList(u8),
 main: std.ArrayList(u8),
 
 called: std.StringHashMap(void),
-need_stddef: bool = false,
-need_stdint: bool = false,
 error_msg: *Compilation.ErrorMsg = undefined,
 
 pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*C {
@@ -44,7 +81,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
             .allocator = allocator,
         },
         .main = std.ArrayList(u8).init(allocator),
-        .header = std.ArrayList(u8).init(allocator),
+        .header = Header.init(allocator),
         .constants = std.ArrayList(u8).init(allocator),
         .called = std.StringHashMap(void).init(allocator),
     };
@@ -82,22 +119,7 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     defer tracy.end();
 
     const writer = self.base.file.?.writer();
-    try writer.writeAll(@embedFile("cbe.h"));
-    var includes = false;
-    if (self.need_stddef) {
-        try writer.writeAll("#include <stddef.h>\n");
-        includes = true;
-    }
-    if (self.need_stdint) {
-        try writer.writeAll("#include <stdint.h>\n");
-        includes = true;
-    }
-    if (includes) {
-        try writer.writeByte('\n');
-    }
-    if (self.header.items.len > 0) {
-        try writer.print("{}\n", .{self.header.items});
-    }
+    try self.header.flush(writer);
     if (self.constants.items.len > 0) {
         try writer.print("{}\n", .{self.constants.items});
     }
-- 
cgit v1.2.3


From c87da2f45afedb33b5d1531ee0e4f67675aa9f78 Mon Sep 17 00:00:00 2001
From: Alex Cameron <ascottcameron@gmail.com>
Date: Thu, 19 Nov 2020 19:57:22 +1100
Subject: Remove redundant emit_h member in Compilation struct.

---
 src/Compilation.zig | 13 ++++++-------
 src/link/C.zig      |  6 ++++--
 2 files changed, 10 insertions(+), 9 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 5b06bd6370..2754141370 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -128,11 +128,11 @@ test_filter: ?[]const u8,
 test_name_prefix: ?[]const u8,
 test_evented_io: bool,
 
-emit_h: ?EmitLoc,
 emit_asm: ?EmitLoc,
 emit_llvm_ir: ?EmitLoc,
 emit_analysis: ?EmitLoc,
 emit_docs: ?EmitLoc,
+
 c_header: ?c_link.Header,
 
 pub const InnerError = Module.InnerError;
@@ -973,8 +973,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .local_cache_directory = options.local_cache_directory,
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
-            .emit_h = options.emit_h,
-            .c_header = if (!use_llvm and options.emit_h != null) c_link.Header.init(gpa) else null,
+            .c_header = if (!use_llvm and options.emit_h != null) c_link.Header.init(gpa, options.emit_h) else null,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_analysis = options.emit_analysis,
@@ -1289,7 +1288,7 @@ pub fn update(self: *Compilation) !void {
 
     // If we've chosen to emit a C header, flush the header to the disk.
     if (self.c_header) |header| {
-        const header_path = self.emit_h.?;
+        const header_path = header.emit_loc.?;
         // If a directory has been provided, write the header there. Otherwise, just write it to the
         // cache directory.
         const header_dir = if (header_path.directory) |dir|
@@ -2952,7 +2951,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     man.hash.add(comp.bin_file.options.function_sections);
     man.hash.add(comp.bin_file.options.is_test);
     man.hash.add(comp.bin_file.options.emit != null);
-    man.hash.addOptionalEmitLoc(comp.emit_h);
+    man.hash.add(comp.c_header != null);
     man.hash.addOptionalEmitLoc(comp.emit_asm);
     man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
     man.hash.addOptionalEmitLoc(comp.emit_analysis);
@@ -3051,10 +3050,10 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         });
         break :blk try directory.join(arena, &[_][]const u8{bin_basename});
     } else "";
-    if (comp.emit_h != null) {
+    if (comp.c_header != null) {
         log.warn("-femit-h is not available in the stage1 backend; no .h file will be produced", .{});
     }
-    const emit_h_path = try stage1LocPath(arena, comp.emit_h, directory);
+    const emit_h_path = try stage1LocPath(arena, if (comp.c_header) |header| header.emit_loc else null, directory);
     const emit_asm_path = try stage1LocPath(arena, comp.emit_asm, directory);
     const emit_llvm_ir_path = try stage1LocPath(arena, comp.emit_llvm_ir, directory);
     const emit_analysis_path = try stage1LocPath(arena, comp.emit_analysis, directory);
diff --git a/src/link/C.zig b/src/link/C.zig
index 0906482dd1..839c8cabfe 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -17,10 +17,12 @@ pub const Header = struct {
     buf: std.ArrayList(u8),
     need_stddef: bool = false,
     need_stdint: bool = false,
+    emit_loc: ?Compilation.EmitLoc,
 
-    pub fn init(allocator: *Allocator) Header {
+    pub fn init(allocator: *Allocator, emit_loc: ?Compilation.EmitLoc) Header {
         return .{
             .buf = std.ArrayList(u8).init(allocator),
+            .emit_loc = emit_loc,
         };
     }
 
@@ -81,7 +83,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
             .allocator = allocator,
         },
         .main = std.ArrayList(u8).init(allocator),
-        .header = Header.init(allocator),
+        .header = Header.init(allocator, null),
         .constants = std.ArrayList(u8).init(allocator),
         .called = std.StringHashMap(void).init(allocator),
     };
-- 
cgit v1.2.3


From 15a148db01896d8ef7290593755d185292584510 Mon Sep 17 00:00:00 2001
From: Noam Preil <noam@pixelhero.dev>
Date: Wed, 18 Nov 2020 20:50:12 -0500
Subject: Fix memory leak

---
 src/Compilation.zig | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 2754141370..8f1c9ac29d 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1184,6 +1184,10 @@ pub fn destroy(self: *Compilation) void {
     }
     self.failed_c_objects.deinit(gpa);
 
+    if (self.c_header) |*header| {
+        header.deinit();
+    }
+
     self.cache_parent.manifest_dir.close();
     if (self.owned_link_dir) |*dir| dir.close();
 
-- 
cgit v1.2.3


From 9849e894d52dd7b9aedc149011cb19d2bccaab70 Mon Sep 17 00:00:00 2001
From: Alex Cameron <ascottcameron@gmail.com>
Date: Sat, 19 Dec 2020 05:31:00 +0000
Subject: Add emit_h path to compilation hash.

---
 src/Compilation.zig | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 8f1c9ac29d..11521e5a52 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2956,6 +2956,9 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     man.hash.add(comp.bin_file.options.is_test);
     man.hash.add(comp.bin_file.options.emit != null);
     man.hash.add(comp.c_header != null);
+    if (comp.c_header) |header| {
+        man.hash.addEmitLoc(header.emit_loc.?);
+    }
     man.hash.addOptionalEmitLoc(comp.emit_asm);
     man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
     man.hash.addOptionalEmitLoc(comp.emit_analysis);
-- 
cgit v1.2.3


From 177377b6e356b34bbed40cadca596658d158af6b Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 23 Dec 2020 16:57:18 -0800
Subject: rework std.ResetEvent, improve std lib Darwin integration

 * split std.ResetEvent into:
   - ResetEvent - requires init() at runtime and it can fail. Also
     requires deinit().
   - StaticResetEvent - can be statically initialized and requires no
     deinitialization. Initialization cannot fail.
 * the POSIX sem_t implementation can in fact fail on initialization
   because it is allowed to be implemented as a file descriptor.
 * Completely define, clarify, and explain in detail the semantics of
   these APIs. Remove the `isSet` function.
 * `ResetEvent.timedWait` returns an enum instead of a possible error.
 * `ResetEvent.init` takes a pointer to the ResetEvent instead of
   returning a copy.
 * On Darwin, `ResetEvent` is implemented using Grand Central Dispatch,
   which is exposed by libSystem.

stage2 changes:
 * ThreadPool: use a single, pre-initialized `ResetEvent` per worker.
 * WaitGroup: now requires init() and deinit() and init() can fail.
   - Add a `reset` function.
   - Compilation initializes one for the work queue in creation and
     re-uses it for every update.
   - Rename `stop` to `finish`.
   - Simplify the implementation based on the usage pattern.
---
 CMakeLists.txt               |   5 +-
 lib/std/ResetEvent.zig       | 297 +++++++++++++++++++++++++
 lib/std/StaticResetEvent.zig | 396 ++++++++++++++++++++++++++++++++++
 lib/std/c/darwin.zig         |  12 ++
 lib/std/debug.zig            |   3 +-
 lib/std/fs/test.zig          |   2 -
 lib/std/mutex.zig            |   4 +-
 lib/std/reset_event.zig      | 501 -------------------------------------------
 lib/std/std.zig              |   3 +-
 src/Compilation.zig          |  18 +-
 src/ThreadPool.zig           |  96 +++++----
 src/WaitGroup.zig            |  35 +--
 12 files changed, 805 insertions(+), 567 deletions(-)
 create mode 100644 lib/std/ResetEvent.zig
 create mode 100644 lib/std/StaticResetEvent.zig
 delete mode 100644 lib/std/reset_event.zig

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 272cdc6921..b9b9310946 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -410,11 +410,12 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/bits.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/ntstatus.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/win32error.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/ResetEvent.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/StaticResetEvent.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/pdb.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/process.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/rand.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/reset_event.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/sort.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt/addXf3.zig"
diff --git a/lib/std/ResetEvent.zig b/lib/std/ResetEvent.zig
new file mode 100644
index 0000000000..cd62eb6e21
--- /dev/null
+++ b/lib/std/ResetEvent.zig
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A thread-safe resource which supports blocking until signaled.
+//! This API is for kernel threads, not evented I/O.
+//! This API requires being initialized at runtime, and initialization
+//! can fail. Once initialized, the core operations cannot fail.
+//! If you need an abstraction that cannot fail to be initialized, see
+//! `std.StaticResetEvent`. However if you can handle initialization failure,
+//! it is preferred to use `ResetEvent`.
+
+const ResetEvent = @This();
+const std = @import("std.zig");
+const builtin = std.builtin;
+const testing = std.testing;
+const assert = std.debug.assert;
+const c = std.c;
+const os = std.os;
+const time = std.time;
+
+impl: Impl,
+
+pub const Impl = if (builtin.single_threaded)
+    std.StaticResetEvent.DebugEvent
+else if (std.Target.current.isDarwin())
+    DarwinEvent
+else if (std.Thread.use_pthreads)
+    PosixEvent
+else
+    std.StaticResetEvent.AtomicEvent;
+
+pub const InitError = error{SystemResources};
+
+/// After `init`, it is legal to call any other function.
+pub fn init(ev: *ResetEvent) InitError!void {
+    return ev.impl.init();
+}
+
+/// This function is not thread-safe.
+/// After `deinit`, the only legal function to call is `init`.
+pub fn deinit(ev: *ResetEvent) void {
+    return ev.impl.deinit();
+}
+
+/// Sets the event if not already set and wakes up all the threads waiting on
+/// the event. It is safe to call `set` multiple times before calling `wait`.
+/// However it is illegal to call `set` after `wait` is called until the event
+/// is `reset`. This function is thread-safe.
+pub fn set(ev: *ResetEvent) void {
+    return ev.impl.set();
+}
+
+/// Resets the event to its original, unset state.
+/// This function is *not* thread-safe. It is equivalent to calling
+/// `deinit` followed by `init` but without the possibility of failure.
+pub fn reset(ev: *ResetEvent) void {
+    return ev.impl.reset();
+}
+
+/// Wait for the event to be set by blocking the current thread.
+/// Thread-safe. No spurious wakeups.
+/// Upon return from `wait`, the only functions available to be called
+/// in `ResetEvent` are `reset` and `deinit`.
+pub fn wait(ev: *ResetEvent) void {
+    return ev.impl.wait();
+}
+
+pub const TimedWaitResult = enum { event_set, timed_out };
+
+/// Wait for the event to be set by blocking the current thread.
+/// A timeout in nanoseconds can be provided as a hint for how
+/// long the thread should block on the unset event before returning
+/// `TimedWaitResult.timed_out`.
+/// Thread-safe. No precision of timing is guaranteed.
+/// Upon return from `wait`, the only functions available to be called
+/// in `ResetEvent` are `reset` and `deinit`.
+pub fn timedWait(ev: *ResetEvent, timeout_ns: u64) TimedWaitResult {
+    return ev.impl.timedWait(timeout_ns);
+}
+
+/// Apple has decided to not support POSIX semaphores, so we go with a
+/// different approach using Grand Central Dispatch. This API is exposed
+/// by libSystem so it is guaranteed to be available on all Darwin platforms.
+pub const DarwinEvent = struct {
+    sem: c.dispatch_semaphore_t = undefined,
+
+    pub fn init(ev: *DarwinEvent) !void {
+        ev.* = .{
+            .sem = c.dispatch_semaphore_create(0) orelse return error.SystemResources,
+        };
+    }
+
+    pub fn deinit(ev: *DarwinEvent) void {
+        c.dispatch_release(ev.sem);
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *DarwinEvent) void {
+        // Empirically this returns the numerical value of the semaphore.
+        _ = c.dispatch_semaphore_signal(ev.sem);
+    }
+
+    pub fn wait(ev: *DarwinEvent) void {
+        assert(c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_FOREVER) == 0);
+    }
+
+    pub fn timedWait(ev: *DarwinEvent, timeout_ns: u64) TimedWaitResult {
+        const t = c.dispatch_time(c.DISPATCH_TIME_NOW, @intCast(i64, timeout_ns));
+        if (c.dispatch_semaphore_wait(ev.sem, t) != 0) {
+            return .timed_out;
+        } else {
+            return .event_set;
+        }
+    }
+
+    pub fn reset(ev: *DarwinEvent) void {
+        // Keep calling until the semaphore goes back down to 0.
+        while (c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_NOW) == 0) {}
+    }
+};
+
+/// POSIX semaphores must be initialized at runtime because they are allowed to
+/// be implemented as file descriptors, in which case initialization would require
+/// a syscall to open the fd.
+pub const PosixEvent = struct {
+    sem: c.sem_t = undefined,
+
+    pub fn init(ev: *PosixEvent) !void {
+        switch (c.getErrno(c.sem_init(&ev.sem, 0, 0))) {
+            0 => return,
+            else => return error.SystemResources,
+        }
+    }
+
+    pub fn deinit(ev: *PosixEvent) void {
+        assert(c.sem_destroy(&ev.sem) == 0);
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *PosixEvent) void {
+        assert(c.sem_post(&ev.sem) == 0);
+    }
+
+    pub fn wait(ev: *PosixEvent) void {
+        while (true) {
+            switch (c.getErrno(c.sem_wait(&ev.sem))) {
+                0 => return,
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                else => unreachable,
+            }
+        }
+    }
+
+    pub fn timedWait(ev: *PosixEvent, timeout_ns: u64) TimedWaitResult {
+        var ts: os.timespec = undefined;
+        var timeout_abs = timeout_ns;
+        os.clock_gettime(os.CLOCK_REALTIME, &ts) catch return .timed_out;
+        timeout_abs += @intCast(u64, ts.tv_sec) * time.ns_per_s;
+        timeout_abs += @intCast(u64, ts.tv_nsec);
+        ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), @divFloor(timeout_abs, time.ns_per_s));
+        ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), @mod(timeout_abs, time.ns_per_s));
+        while (true) {
+            switch (c.getErrno(c.sem_timedwait(&ev.sem, &ts))) {
+                0 => return .event_set,
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                c.ETIMEDOUT => return .timed_out,
+                else => unreachable,
+            }
+        }
+    }
+
+    pub fn reset(ev: *PosixEvent) void {
+        while (true) {
+            switch (c.getErrno(c.sem_trywait(&ev.sem))) {
+                0 => continue, // Need to make it go to zero.
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                c.EAGAIN => return, // The semaphore currently has the value zero.
+                else => unreachable,
+            }
+        }
+    }
+};
+
+test "basic usage" {
+    var event: ResetEvent = undefined;
+    try event.init();
+    defer event.deinit();
+
+    // test event setting
+    event.set();
+
+    // test event resetting
+    event.reset();
+
+    // test event waiting (non-blocking)
+    event.set();
+    event.wait();
+    event.reset();
+
+    event.set();
+    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
+
+    // test cross-thread signaling
+    if (builtin.single_threaded)
+        return;
+
+    const Context = struct {
+        const Self = @This();
+
+        value: u128,
+        in: ResetEvent,
+        out: ResetEvent,
+
+        fn init(self: *Self) !void {
+            self.* = .{
+                .value = 0,
+                .in = undefined,
+                .out = undefined,
+            };
+            try self.in.init();
+            try self.out.init();
+        }
+
+        fn deinit(self: *Self) void {
+            self.in.deinit();
+            self.out.deinit();
+            self.* = undefined;
+        }
+
+        fn sender(self: *Self) void {
+            // update value and signal input
+            testing.expect(self.value == 0);
+            self.value = 1;
+            self.in.set();
+
+            // wait for receiver to update value and signal output
+            self.out.wait();
+            testing.expect(self.value == 2);
+
+            // update value and signal final input
+            self.value = 3;
+            self.in.set();
+        }
+
+        fn receiver(self: *Self) void {
+            // wait for sender to update value and signal input
+            self.in.wait();
+            assert(self.value == 1);
+
+            // update value and signal output
+            self.in.reset();
+            self.value = 2;
+            self.out.set();
+
+            // wait for sender to update value and signal final input
+            self.in.wait();
+            assert(self.value == 3);
+        }
+
+        fn sleeper(self: *Self) void {
+            self.in.set();
+            time.sleep(time.ns_per_ms * 2);
+            self.value = 5;
+            self.out.set();
+        }
+
+        fn timedWaiter(self: *Self) !void {
+            self.in.wait();
+            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
+            try self.out.timedWait(time.ns_per_ms * 100);
+            testing.expect(self.value == 5);
+        }
+    };
+
+    var context: Context = undefined;
+    try context.init();
+    defer context.deinit();
+    const receiver = try std.Thread.spawn(&context, Context.receiver);
+    defer receiver.wait();
+    context.sender();
+
+    if (false) {
+        // I have now observed this fail on macOS, Windows, and Linux.
+        // https://github.com/ziglang/zig/issues/7009
+        var timed = Context.init();
+        defer timed.deinit();
+        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
+        defer sleeper.wait();
+        try timed.timedWaiter();
+    }
+}
diff --git a/lib/std/StaticResetEvent.zig b/lib/std/StaticResetEvent.zig
new file mode 100644
index 0000000000..b41e7666ac
--- /dev/null
+++ b/lib/std/StaticResetEvent.zig
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A thread-safe resource which supports blocking until signaled.
+//! This API is for kernel threads, not evented I/O.
+//! This API is statically initializable. It cannot fail to be initialized
+//! and it requires no deinitialization. The downside is that it may not
+//! integrate as cleanly into other synchronization APIs, or, in a worst case,
+//! may be forced to fall back on spin locking. As a rule of thumb, prefer
+//! to use `std.ResetEvent` when possible, and use `StaticResetEvent` when
+//! the logic needs stronger API guarantees.
+
+const std = @import("std.zig");
+const StaticResetEvent = @This();
+const SpinLock = std.SpinLock;
+const assert = std.debug.assert;
+const os = std.os;
+const time = std.time;
+const linux = std.os.linux;
+const windows = std.os.windows;
+const testing = std.testing;
+
+impl: Impl = .{},
+
+pub const Impl = if (std.builtin.single_threaded)
+    DebugEvent
+else
+    AtomicEvent;
+
+/// Sets the event if not already set and wakes up all the threads waiting on
+/// the event. It is safe to call `set` multiple times before calling `wait`.
+/// However it is illegal to call `set` after `wait` is called until the event
+/// is `reset`. This function is thread-safe.
+pub fn set(ev: *StaticResetEvent) void {
+    return ev.impl.set();
+}
+
+/// Wait for the event to be set by blocking the current thread.
+/// Thread-safe. No spurious wakeups.
+/// Upon return from `wait`, the only function available to be called
+/// in `StaticResetEvent` is `reset`.
+pub fn wait(ev: *StaticResetEvent) void {
+    return ev.impl.wait();
+}
+
+/// Resets the event to its original, unset state.
+/// This function is *not* thread-safe. It is equivalent to calling
+/// `deinit` followed by `init` but without the possibility of failure.
+pub fn reset(ev: *StaticResetEvent) void {
+    return ev.impl.reset();
+}
+
+pub const TimedWaitResult = std.ResetEvent.TimedWaitResult;
+
+/// Wait for the event to be set by blocking the current thread.
+/// A timeout in nanoseconds can be provided as a hint for how
+/// long the thread should block on the unset event before returning
+/// `TimedWaitResult.timed_out`.
+/// Thread-safe. No precision of timing is guaranteed.
+/// Upon return from `timedWait`, the only function available to be called
+/// in `StaticResetEvent` is `reset`.
+pub fn timedWait(ev: *StaticResetEvent, timeout_ns: u64) TimedWaitResult {
+    return ev.impl.timedWait(timeout_ns);
+}
+
+/// For single-threaded builds, we use this to detect deadlocks.
+/// In unsafe modes this ends up being no-ops.
+pub const DebugEvent = struct {
+    state: State = State.unset,
+
+    const State = enum {
+        unset,
+        set,
+        waited,
+    };
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.ResetEvent`.
+    pub fn init(ev: *DebugEvent) void {
+        ev.* = .{};
+    }
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.ResetEvent`.
+    pub fn deinit(ev: *DebugEvent) void {
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *DebugEvent) void {
+        switch (ev.state) {
+            .unset => ev.state = .set,
+            .set => {},
+            .waited => unreachable, // Not allowed to call `set` until `reset`.
+        }
+    }
+
+    pub fn wait(ev: *DebugEvent) void {
+        switch (ev.state) {
+            .unset => unreachable, // Deadlock detected.
+            .set => return,
+            .waited => unreachable, // Not allowed to call `wait` until `reset`.
+        }
+    }
+
+    fn timedWait(ev: *DebugEvent, timeout: u64) TimedWaitResult {
+        switch (ev.state) {
+            .unset => return .timed_out,
+            .set => return .event_set,
+            .waited => unreachable, // Not allowed to call `wait` until `reset`.
+        }
+    }
+
+    pub fn reset(ev: *DebugEvent) void {
+        ev.state = .unset;
+    }
+};
+
+pub const AtomicEvent = struct {
+    waiters: u32 = 0,
+
+    const WAKE = 1 << 0;
+    const WAIT = 1 << 1;
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.ResetEvent`.
+    pub fn init(ev: *AtomicEvent) void {
+        ev.* = .{};
+    }
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.ResetEvent`.
+    pub fn deinit(ev: *AtomicEvent) void {
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *AtomicEvent) void {
+        const waiters = @atomicRmw(u32, &ev.waiters, .Xchg, WAKE, .Release);
+        if (waiters >= WAIT) {
+            return Futex.wake(&ev.waiters, waiters >> 1);
+        }
+    }
+
+    pub fn wait(ev: *AtomicEvent) void {
+        switch (ev.timedWait(null)) {
+            .timed_out => unreachable,
+            .event_set => return,
+        }
+    }
+
+    pub fn timedWait(ev: *AtomicEvent, timeout: ?u64) TimedWaitResult {
+        var waiters = @atomicLoad(u32, &ev.waiters, .Acquire);
+        while (waiters != WAKE) {
+            waiters = @cmpxchgWeak(u32, &ev.waiters, waiters, waiters + WAIT, .Acquire, .Acquire) orelse {
+                if (Futex.wait(&ev.waiters, timeout)) |_| {
+                    return .event_set;
+                } else |_| {
+                    return .timed_out;
+                }
+            };
+        }
+        return .event_set;
+    }
+
+    pub fn reset(ev: *AtomicEvent) void {
+        @atomicStore(u32, &ev.waiters, 0, .Monotonic);
+    }
+
+    pub const Futex = switch (std.Target.current.os.tag) {
+        .windows => WindowsFutex,
+        .linux => LinuxFutex,
+        else => SpinFutex,
+    };
+
+    pub const SpinFutex = struct {
+        fn wake(waiters: *u32, wake_count: u32) void {}
+
+        fn wait(waiters: *u32, timeout: ?u64) !void {
+            var timer: time.Timer = undefined;
+            if (timeout != null)
+                timer = time.Timer.start() catch return error.TimedOut;
+
+            while (@atomicLoad(u32, waiters, .Acquire) != WAKE) {
+                SpinLock.yield();
+                if (timeout) |timeout_ns| {
+                    if (timer.read() >= timeout_ns)
+                        return error.TimedOut;
+                }
+            }
+        }
+    };
+
+    pub const LinuxFutex = struct {
+        fn wake(waiters: *u32, wake_count: u32) void {
+            const waiting = std.math.maxInt(i32); // wake_count
+            const ptr = @ptrCast(*const i32, waiters);
+            const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, waiting);
+            assert(linux.getErrno(rc) == 0);
+        }
+
+        fn wait(waiters: *u32, timeout: ?u64) !void {
+            var ts: linux.timespec = undefined;
+            var ts_ptr: ?*linux.timespec = null;
+            if (timeout) |timeout_ns| {
+                ts_ptr = &ts;
+                ts.tv_sec = @intCast(isize, timeout_ns / time.ns_per_s);
+                ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);
+            }
+
+            while (true) {
+                const waiting = @atomicLoad(u32, waiters, .Acquire);
+                if (waiting == WAKE)
+                    return;
+                const expected = @intCast(i32, waiting);
+                const ptr = @ptrCast(*const i32, waiters);
+                const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);
+                switch (linux.getErrno(rc)) {
+                    0 => continue,
+                    os.ETIMEDOUT => return error.TimedOut,
+                    os.EINTR => continue,
+                    os.EAGAIN => return,
+                    else => unreachable,
+                }
+            }
+        }
+    };
+
+    pub const WindowsFutex = struct {
+        pub fn wake(waiters: *u32, wake_count: u32) void {
+            const handle = getEventHandle() orelse return SpinFutex.wake(waiters, wake_count);
+            const key = @ptrCast(*const c_void, waiters);
+
+            var waiting = wake_count;
+            while (waiting != 0) : (waiting -= 1) {
+                const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
+                assert(rc == .SUCCESS);
+            }
+        }
+
+        pub fn wait(waiters: *u32, timeout: ?u64) !void {
+            const handle = getEventHandle() orelse return SpinFutex.wait(waiters, timeout);
+            const key = @ptrCast(*const c_void, waiters);
+
+            // NT uses timeouts in units of 100ns with negative value being relative
+            var timeout_ptr: ?*windows.LARGE_INTEGER = null;
+            var timeout_value: windows.LARGE_INTEGER = undefined;
+            if (timeout) |timeout_ns| {
+                timeout_ptr = &timeout_value;
+                timeout_value = -@intCast(windows.LARGE_INTEGER, timeout_ns / 100);
+            }
+
+            // NtWaitForKeyedEvent doesnt have spurious wake-ups
+            var rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
+            switch (rc) {
+                .TIMEOUT => {
+                    // update the wait count to signal that we're not waiting anymore.
+                    // if the .set() thread already observed that we are, perform a
+                    // matching NtWaitForKeyedEvent so that the .set() thread doesn't
+                    // deadlock trying to run NtReleaseKeyedEvent above.
+                    var waiting = @atomicLoad(u32, waiters, .Monotonic);
+                    while (true) {
+                        if (waiting == WAKE) {
+                            rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
+                            assert(rc == .WAIT_0);
+                            break;
+                        } else {
+                            waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - WAIT, .Acquire, .Monotonic) orelse break;
+                            continue;
+                        }
+                    }
+                    return error.TimedOut;
+                },
+                .WAIT_0 => {},
+                else => unreachable,
+            }
+        }
+
+        var event_handle: usize = EMPTY;
+        const EMPTY = ~@as(usize, 0);
+        const LOADING = EMPTY - 1;
+
+        pub fn getEventHandle() ?windows.HANDLE {
+            var handle = @atomicLoad(usize, &event_handle, .Monotonic);
+            while (true) {
+                switch (handle) {
+                    EMPTY => handle = @cmpxchgWeak(usize, &event_handle, EMPTY, LOADING, .Acquire, .Monotonic) orelse {
+                        const handle_ptr = @ptrCast(*windows.HANDLE, &handle);
+                        const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
+                        if (windows.ntdll.NtCreateKeyedEvent(handle_ptr, access_mask, null, 0) != .SUCCESS)
+                            handle = 0;
+                        @atomicStore(usize, &event_handle, handle, .Monotonic);
+                        return @intToPtr(?windows.HANDLE, handle);
+                    },
+                    LOADING => {
+                        SpinLock.yield();
+                        handle = @atomicLoad(usize, &event_handle, .Monotonic);
+                    },
+                    else => {
+                        return @intToPtr(?windows.HANDLE, handle);
+                    },
+                }
+            }
+        }
+    };
+};
+
+test "basic usage" {
+    var event = StaticResetEvent{};
+
+    // test event setting
+    event.set();
+
+    // test event resetting
+    event.reset();
+
+    // test event waiting (non-blocking)
+    event.set();
+    event.wait();
+    event.reset();
+
+    event.set();
+    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
+
+    // test cross-thread signaling
+    if (std.builtin.single_threaded)
+        return;
+
+    const Context = struct {
+        const Self = @This();
+
+        value: u128 = 0,
+        in: StaticResetEvent = .{},
+        out: StaticResetEvent = .{},
+
+        fn sender(self: *Self) void {
+            // update value and signal input
+            testing.expect(self.value == 0);
+            self.value = 1;
+            self.in.set();
+
+            // wait for receiver to update value and signal output
+            self.out.wait();
+            testing.expect(self.value == 2);
+
+            // update value and signal final input
+            self.value = 3;
+            self.in.set();
+        }
+
+        fn receiver(self: *Self) void {
+            // wait for sender to update value and signal input
+            self.in.wait();
+            assert(self.value == 1);
+
+            // update value and signal output
+            self.in.reset();
+            self.value = 2;
+            self.out.set();
+
+            // wait for sender to update value and signal final input
+            self.in.wait();
+            assert(self.value == 3);
+        }
+
+        fn sleeper(self: *Self) void {
+            self.in.set();
+            time.sleep(time.ns_per_ms * 2);
+            self.value = 5;
+            self.out.set();
+        }
+
+        fn timedWaiter(self: *Self) !void {
+            self.in.wait();
+            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
+            try self.out.timedWait(time.ns_per_ms * 100);
+            testing.expect(self.value == 5);
+        }
+    };
+
+    var context = Context{};
+    const receiver = try std.Thread.spawn(&context, Context.receiver);
+    defer receiver.wait();
+    context.sender();
+
+    if (false) {
+        // I have now observed this fail on macOS, Windows, and Linux.
+        // https://github.com/ziglang/zig/issues/7009
+        var timed = Context.init();
+        defer timed.deinit();
+        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
+        defer sleeper.wait();
+        try timed.timedWaiter();
+    }
+}
diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index 9c5056d9c4..0850464567 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -187,3 +187,15 @@ pub const pthread_attr_t = extern struct {
 };
 
 pub extern "c" fn arc4random_buf(buf: [*]u8, len: usize) void;
+
+// Grand Central Dispatch is exposed by libSystem.
+pub const dispatch_semaphore_t = *opaque{};
+pub const dispatch_time_t = u64;
+pub const DISPATCH_TIME_NOW = @as(dispatch_time_t, 0);
+pub const DISPATCH_TIME_FOREVER = ~@as(dispatch_time_t, 0);
+pub extern "c" fn dispatch_semaphore_create(value: isize) ?dispatch_semaphore_t;
+pub extern "c" fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) isize;
+pub extern "c" fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) isize;
+
+pub extern "c" fn dispatch_release(object: *c_void) void;
+pub extern "c" fn dispatch_time(when: dispatch_time_t, delta: i64) dispatch_time_t;
diff --git a/lib/std/debug.zig b/lib/std/debug.zig
index 7284237cb2..56428075bf 100644
--- a/lib/std/debug.zig
+++ b/lib/std/debug.zig
@@ -274,9 +274,8 @@ pub fn panicExtra(trace: ?*const builtin.StackTrace, first_trace_addr: ?usize, c
                 // and call abort()
 
                 // Sleep forever without hammering the CPU
-                var event = std.ResetEvent.init();
+                var event: std.StaticResetEvent = .{};
                 event.wait();
-
                 unreachable;
             }
         },
diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
index 61df39be0c..06e5bfe66b 100644
--- a/lib/std/fs/test.zig
+++ b/lib/std/fs/test.zig
@@ -771,8 +771,6 @@ test "open file with exclusive lock twice, make sure it waits" {
         std.time.sleep(SLEEP_TIMEOUT_NS);
         if (timer.read() >= SLEEP_TIMEOUT_NS) break;
     }
-    // Check that createFile is still waiting for the lock to be released.
-    testing.expect(!evt.isSet());
     file.close();
     // No timeout to avoid failures on heavily loaded systems.
     evt.wait();
diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig
index 349a250fea..6fa8fb6a62 100644
--- a/lib/std/mutex.zig
+++ b/lib/std/mutex.zig
@@ -284,7 +284,7 @@ const WindowsMutex = struct {
     fn acquireSlow(self: *WindowsMutex) Held {
         // try to use NT keyed events for blocking, falling back to spinlock if unavailable
         @setCold(true);
-        const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return self.acquireSpinning();
+        const handle = ResetEvent.Impl.Futex.getEventHandle() orelse return self.acquireSpinning();
         const key = @ptrCast(*const c_void, &self.state.waiters);
 
         while (true) : (SpinLock.loopHint(1)) {
@@ -312,7 +312,7 @@ const WindowsMutex = struct {
         pub fn release(self: Held) void {
             // unlock without a rmw/cmpxchg instruction
             @atomicStore(u8, @ptrCast(*u8, &self.mutex.state.locked), 0, .Release);
-            const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return;
+            const handle = ResetEvent.Impl.Futex.getEventHandle() orelse return;
             const key = @ptrCast(*const c_void, &self.mutex.state.waiters);
 
             while (true) : (SpinLock.loopHint(1)) {
diff --git a/lib/std/reset_event.zig b/lib/std/reset_event.zig
deleted file mode 100644
index 58dc40994d..0000000000
--- a/lib/std/reset_event.zig
+++ /dev/null
@@ -1,501 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2020 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const std = @import("std.zig");
-const builtin = @import("builtin");
-const testing = std.testing;
-const SpinLock = std.SpinLock;
-const assert = std.debug.assert;
-const c = std.c;
-const os = std.os;
-const time = std.time;
-const linux = os.linux;
-const windows = os.windows;
-
-/// A resource object which supports blocking until signaled.
-/// Once finished, the `deinit()` method should be called for correctness.
-pub const ResetEvent = struct {
-    os_event: OsEvent,
-
-    pub const OsEvent = if (builtin.single_threaded)
-        DebugEvent
-    else if (std.Thread.use_pthreads)
-        PosixEvent
-    else
-        AtomicEvent;
-
-    pub fn init() ResetEvent {
-        return ResetEvent{ .os_event = OsEvent.init() };
-    }
-
-    pub fn deinit(self: *ResetEvent) void {
-        self.os_event.deinit();
-    }
-
-    /// When `wait` would return without blocking, this returns `true`.
-    /// Note that the value may be immediately invalid upon this function's
-    /// return, because another thread may call `wait` in between, changing
-    /// the event's set/cleared status.
-    pub fn isSet(self: *ResetEvent) bool {
-        return self.os_event.isSet();
-    }
-
-    /// Sets the event if not already set and
-    /// wakes up all the threads waiting on the event.
-    pub fn set(self: *ResetEvent) void {
-        return self.os_event.set();
-    }
-
-    /// Resets the event to its original, unset state.
-    /// TODO improve these docs:
-    ///  * under what circumstances does it make sense to call this function?
-    pub fn reset(self: *ResetEvent) void {
-        return self.os_event.reset();
-    }
-
-    /// Wait for the event to be set by blocking the current thread.
-    /// TODO improve these docs:
-    ///  * is the function thread-safe?
-    ///  * does it have suprious wakeups?
-    pub fn wait(self: *ResetEvent) void {
-        return self.os_event.wait();
-    }
-
-    /// Wait for the event to be set by blocking the current thread.
-    /// A timeout in nanoseconds can be provided as a hint for how
-    /// long the thread should block on the unset event before throwing error.TimedOut.
-    /// TODO improve these docs:
-    ///  * is the function thread-safe?
-    ///  * does it have suprious wakeups?
-    pub fn timedWait(self: *ResetEvent, timeout_ns: u64) !void {
-        return self.os_event.timedWait(timeout_ns);
-    }
-};
-
-const DebugEvent = struct {
-    is_set: bool,
-
-    fn init() DebugEvent {
-        return DebugEvent{ .is_set = false };
-    }
-
-    fn deinit(self: *DebugEvent) void {
-        self.* = undefined;
-    }
-
-    fn isSet(self: *DebugEvent) bool {
-        return self.is_set;
-    }
-
-    fn reset(self: *DebugEvent) void {
-        self.is_set = false;
-    }
-
-    fn set(self: *DebugEvent) void {
-        self.is_set = true;
-    }
-
-    fn wait(self: *DebugEvent) void {
-        if (self.is_set)
-            return;
-
-        @panic("deadlock detected");
-    }
-
-    fn timedWait(self: *DebugEvent, timeout: u64) !void {
-        if (self.is_set)
-            return;
-
-        return error.TimedOut;
-    }
-};
-
-const PosixEvent = struct {
-    sem: c.sem_t = undefined,
-    /// Sadly this is needed because pthreads semaphore API does not
-    /// support static initialization.
-    init_mutex: std.mutex.PthreadMutex = .{},
-    state: enum { uninit, init } = .uninit,
-
-    fn init() PosixEvent {
-        return .{};
-    }
-
-    /// Not thread-safe.
-    fn deinit(self: *PosixEvent) void {
-        switch (self.state) {
-            .uninit => {},
-            .init => {
-                assert(c.sem_destroy(&self.sem) == 0);
-            },
-        }
-        self.* = undefined;
-    }
-
-    fn isSet(self: *PosixEvent) bool {
-        const sem = self.getInitializedSem();
-        var val: c_int = undefined;
-        assert(c.sem_getvalue(sem, &val) == 0);
-        return val > 0;
-    }
-
-    fn reset(self: *PosixEvent) void {
-        const sem = self.getInitializedSem();
-        while (true) {
-            switch (c.getErrno(c.sem_trywait(sem))) {
-                0 => continue, // Need to make it go to zero.
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                c.EAGAIN => return, // The semaphore currently has the value zero.
-                else => unreachable,
-            }
-        }
-    }
-
-    fn set(self: *PosixEvent) void {
-        const sem = self.getInitializedSem();
-        assert(c.sem_post(sem) == 0);
-    }
-
-    fn wait(self: *PosixEvent) void {
-        const sem = self.getInitializedSem();
-        while (true) {
-            switch (c.getErrno(c.sem_wait(sem))) {
-                0 => return,
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                else => unreachable,
-            }
-        }
-    }
-
-    fn timedWait(self: *PosixEvent, timeout_ns: u64) !void {
-        var ts: os.timespec = undefined;
-        var timeout_abs = timeout_ns;
-        if (comptime std.Target.current.isDarwin()) {
-            var tv: os.darwin.timeval = undefined;
-            assert(os.darwin.gettimeofday(&tv, null) == 0);
-            timeout_abs += @intCast(u64, tv.tv_sec) * time.ns_per_s;
-            timeout_abs += @intCast(u64, tv.tv_usec) * time.ns_per_us;
-        } else {
-            os.clock_gettime(os.CLOCK_REALTIME, &ts) catch return error.TimedOut;
-            timeout_abs += @intCast(u64, ts.tv_sec) * time.ns_per_s;
-            timeout_abs += @intCast(u64, ts.tv_nsec);
-        }
-        ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), @divFloor(timeout_abs, time.ns_per_s));
-        ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), @mod(timeout_abs, time.ns_per_s));
-        const sem = self.getInitializedSem();
-        while (true) {
-            switch (c.getErrno(c.sem_timedwait(&self.sem, &ts))) {
-                0 => return,
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                c.ETIMEDOUT => return error.TimedOut,
-                else => unreachable,
-            }
-        }
-    }
-
-    fn getInitializedSem(self: *PosixEvent) *c.sem_t {
-        const held = self.init_mutex.acquire();
-        defer held.release();
-
-        switch (self.state) {
-            .init => return &self.sem,
-            .uninit => {
-                self.state = .init;
-                assert(c.sem_init(&self.sem, 0, 0) == 0);
-                return &self.sem;
-            },
-        }
-    }
-};
-
-const AtomicEvent = struct {
-    waiters: u32,
-
-    const WAKE = 1 << 0;
-    const WAIT = 1 << 1;
-
-    fn init() AtomicEvent {
-        return AtomicEvent{ .waiters = 0 };
-    }
-
-    fn deinit(self: *AtomicEvent) void {
-        self.* = undefined;
-    }
-
-    fn isSet(self: *const AtomicEvent) bool {
-        return @atomicLoad(u32, &self.waiters, .Acquire) == WAKE;
-    }
-
-    fn reset(self: *AtomicEvent) void {
-        @atomicStore(u32, &self.waiters, 0, .Monotonic);
-    }
-
-    fn set(self: *AtomicEvent) void {
-        const waiters = @atomicRmw(u32, &self.waiters, .Xchg, WAKE, .Release);
-        if (waiters >= WAIT) {
-            return Futex.wake(&self.waiters, waiters >> 1);
-        }
-    }
-
-    fn wait(self: *AtomicEvent) void {
-        return self.timedWait(null) catch unreachable;
-    }
-
-    fn timedWait(self: *AtomicEvent, timeout: ?u64) !void {
-        var waiters = @atomicLoad(u32, &self.waiters, .Acquire);
-        while (waiters != WAKE) {
-            waiters = @cmpxchgWeak(u32, &self.waiters, waiters, waiters + WAIT, .Acquire, .Acquire) orelse return Futex.wait(&self.waiters, timeout);
-        }
-    }
-
-    pub const Futex = switch (builtin.os.tag) {
-        .windows => WindowsFutex,
-        .linux => LinuxFutex,
-        else => SpinFutex,
-    };
-
-    const SpinFutex = struct {
-        fn wake(waiters: *u32, wake_count: u32) void {}
-
-        fn wait(waiters: *u32, timeout: ?u64) !void {
-            // TODO: handle platforms where a monotonic timer isnt available
-            var timer: time.Timer = undefined;
-            if (timeout != null)
-                timer = time.Timer.start() catch unreachable;
-
-            while (@atomicLoad(u32, waiters, .Acquire) != WAKE) {
-                SpinLock.yield();
-                if (timeout) |timeout_ns| {
-                    if (timer.read() >= timeout_ns)
-                        return error.TimedOut;
-                }
-            }
-        }
-    };
-
-    const LinuxFutex = struct {
-        fn wake(waiters: *u32, wake_count: u32) void {
-            const waiting = std.math.maxInt(i32); // wake_count
-            const ptr = @ptrCast(*const i32, waiters);
-            const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, waiting);
-            assert(linux.getErrno(rc) == 0);
-        }
-
-        fn wait(waiters: *u32, timeout: ?u64) !void {
-            var ts: linux.timespec = undefined;
-            var ts_ptr: ?*linux.timespec = null;
-            if (timeout) |timeout_ns| {
-                ts_ptr = &ts;
-                ts.tv_sec = @intCast(isize, timeout_ns / time.ns_per_s);
-                ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);
-            }
-
-            while (true) {
-                const waiting = @atomicLoad(u32, waiters, .Acquire);
-                if (waiting == WAKE)
-                    return;
-                const expected = @intCast(i32, waiting);
-                const ptr = @ptrCast(*const i32, waiters);
-                const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);
-                switch (linux.getErrno(rc)) {
-                    0 => continue,
-                    os.ETIMEDOUT => return error.TimedOut,
-                    os.EINTR => continue,
-                    os.EAGAIN => return,
-                    else => unreachable,
-                }
-            }
-        }
-    };
-
-    const WindowsFutex = struct {
-        pub fn wake(waiters: *u32, wake_count: u32) void {
-            const handle = getEventHandle() orelse return SpinFutex.wake(waiters, wake_count);
-            const key = @ptrCast(*const c_void, waiters);
-
-            var waiting = wake_count;
-            while (waiting != 0) : (waiting -= 1) {
-                const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
-                assert(rc == .SUCCESS);
-            }
-        }
-
-        pub fn wait(waiters: *u32, timeout: ?u64) !void {
-            const handle = getEventHandle() orelse return SpinFutex.wait(waiters, timeout);
-            const key = @ptrCast(*const c_void, waiters);
-
-            // NT uses timeouts in units of 100ns with negative value being relative
-            var timeout_ptr: ?*windows.LARGE_INTEGER = null;
-            var timeout_value: windows.LARGE_INTEGER = undefined;
-            if (timeout) |timeout_ns| {
-                timeout_ptr = &timeout_value;
-                timeout_value = -@intCast(windows.LARGE_INTEGER, timeout_ns / 100);
-            }
-
-            // NtWaitForKeyedEvent doesnt have spurious wake-ups
-            var rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
-            switch (rc) {
-                .TIMEOUT => {
-                    // update the wait count to signal that we're not waiting anymore.
-                    // if the .set() thread already observed that we are, perform a
-                    // matching NtWaitForKeyedEvent so that the .set() thread doesn't
-                    // deadlock trying to run NtReleaseKeyedEvent above.
-                    var waiting = @atomicLoad(u32, waiters, .Monotonic);
-                    while (true) {
-                        if (waiting == WAKE) {
-                            rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
-                            assert(rc == .WAIT_0);
-                            break;
-                        } else {
-                            waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - WAIT, .Acquire, .Monotonic) orelse break;
-                            continue;
-                        }
-                    }
-                    return error.TimedOut;
-                },
-                .WAIT_0 => {},
-                else => unreachable,
-            }
-        }
-
-        var event_handle: usize = EMPTY;
-        const EMPTY = ~@as(usize, 0);
-        const LOADING = EMPTY - 1;
-
-        pub fn getEventHandle() ?windows.HANDLE {
-            var handle = @atomicLoad(usize, &event_handle, .Monotonic);
-            while (true) {
-                switch (handle) {
-                    EMPTY => handle = @cmpxchgWeak(usize, &event_handle, EMPTY, LOADING, .Acquire, .Monotonic) orelse {
-                        const handle_ptr = @ptrCast(*windows.HANDLE, &handle);
-                        const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
-                        if (windows.ntdll.NtCreateKeyedEvent(handle_ptr, access_mask, null, 0) != .SUCCESS)
-                            handle = 0;
-                        @atomicStore(usize, &event_handle, handle, .Monotonic);
-                        return @intToPtr(?windows.HANDLE, handle);
-                    },
-                    LOADING => {
-                        SpinLock.yield();
-                        handle = @atomicLoad(usize, &event_handle, .Monotonic);
-                    },
-                    else => {
-                        return @intToPtr(?windows.HANDLE, handle);
-                    },
-                }
-            }
-        }
-    };
-};
-
-test "ResetEvent" {
-    var event = ResetEvent.init();
-    defer event.deinit();
-
-    // test event setting
-    testing.expect(!event.isSet());
-    event.set();
-    testing.expect(event.isSet());
-
-    // test event resetting
-    event.reset();
-    testing.expect(!event.isSet());
-
-    // test event waiting (non-blocking)
-    event.set();
-    event.wait();
-    event.reset();
-
-    event.set();
-    try event.timedWait(1);
-
-    // test cross-thread signaling
-    if (builtin.single_threaded)
-        return;
-
-    const Context = struct {
-        const Self = @This();
-
-        value: u128,
-        in: ResetEvent,
-        out: ResetEvent,
-
-        fn init() Self {
-            return Self{
-                .value = 0,
-                .in = ResetEvent.init(),
-                .out = ResetEvent.init(),
-            };
-        }
-
-        fn deinit(self: *Self) void {
-            self.in.deinit();
-            self.out.deinit();
-            self.* = undefined;
-        }
-
-        fn sender(self: *Self) void {
-            // update value and signal input
-            testing.expect(self.value == 0);
-            self.value = 1;
-            self.in.set();
-
-            // wait for receiver to update value and signal output
-            self.out.wait();
-            testing.expect(self.value == 2);
-
-            // update value and signal final input
-            self.value = 3;
-            self.in.set();
-        }
-
-        fn receiver(self: *Self) void {
-            // wait for sender to update value and signal input
-            self.in.wait();
-            assert(self.value == 1);
-
-            // update value and signal output
-            self.in.reset();
-            self.value = 2;
-            self.out.set();
-
-            // wait for sender to update value and signal final input
-            self.in.wait();
-            assert(self.value == 3);
-        }
-
-        fn sleeper(self: *Self) void {
-            self.in.set();
-            time.sleep(time.ns_per_ms * 2);
-            self.value = 5;
-            self.out.set();
-        }
-
-        fn timedWaiter(self: *Self) !void {
-            self.in.wait();
-            testing.expectError(error.TimedOut, self.out.timedWait(time.ns_per_us));
-            try self.out.timedWait(time.ns_per_ms * 100);
-            testing.expect(self.value == 5);
-        }
-    };
-
-    var context = Context.init();
-    defer context.deinit();
-    const receiver = try std.Thread.spawn(&context, Context.receiver);
-    defer receiver.wait();
-    context.sender();
-
-    if (false) {
-        // I have now observed this fail on macOS, Windows, and Linux.
-        // https://github.com/ziglang/zig/issues/7009
-        var timed = Context.init();
-        defer timed.deinit();
-        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
-        defer sleeper.wait();
-        try timed.timedWaiter();
-    }
-}
diff --git a/lib/std/std.zig b/lib/std/std.zig
index 5fbf2662b9..69f4ea671b 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -30,10 +30,11 @@ pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
 pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
 pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
 pub const Progress = @import("Progress.zig");
-pub const ResetEvent = @import("reset_event.zig").ResetEvent;
+pub const ResetEvent = @import("ResetEvent.zig");
 pub const SemanticVersion = @import("SemanticVersion.zig");
 pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
 pub const SpinLock = @import("spinlock.zig").SpinLock;
+pub const StaticResetEvent = @import("StaticResetEvent.zig");
 pub const StringHashMap = hash_map.StringHashMap;
 pub const StringHashMapUnmanaged = hash_map.StringHashMapUnmanaged;
 pub const StringArrayHashMap = array_hash_map.StringArrayHashMap;
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 11521e5a52..d172cbadcc 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -135,6 +135,8 @@ emit_docs: ?EmitLoc,
 
 c_header: ?c_link.Header,
 
+work_queue_wait_group: WaitGroup,
+
 pub const InnerError = Module.InnerError;
 
 pub const CRTFile = struct {
@@ -1006,11 +1008,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .test_filter = options.test_filter,
             .test_name_prefix = options.test_name_prefix,
             .test_evented_io = options.test_evented_io,
+            .work_queue_wait_group = undefined,
         };
         break :comp comp;
     };
     errdefer comp.destroy();
 
+    try comp.work_queue_wait_group.init();
+    errdefer comp.work_queue_wait_group.deinit();
+
     if (comp.bin_file.options.module) |mod| {
         try comp.work_queue.writeItem(.{ .generate_builtin_zig = {} });
     }
@@ -1191,6 +1197,8 @@ pub fn destroy(self: *Compilation) void {
     self.cache_parent.manifest_dir.close();
     if (self.owned_link_dir) |*dir| dir.close();
 
+    self.work_queue_wait_group.deinit();
+
     // This destroys `self`.
     self.arena_state.promote(gpa).deinit();
 }
@@ -1405,13 +1413,13 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var arena = std.heap.ArenaAllocator.init(self.gpa);
     defer arena.deinit();
 
-    var wg = WaitGroup{};
-    defer wg.wait();
+    self.work_queue_wait_group.reset();
+    defer self.work_queue_wait_group.wait();
 
     while (self.c_object_work_queue.readItem()) |c_object| {
-        wg.start();
+        self.work_queue_wait_group.start();
         try self.thread_pool.spawn(workerUpdateCObject, .{
-            self, c_object, &c_comp_progress_node, &wg,
+            self, c_object, &c_comp_progress_node, &self.work_queue_wait_group,
         });
     }
 
@@ -1764,7 +1772,7 @@ fn workerUpdateCObject(
     progress_node: *std.Progress.Node,
     wg: *WaitGroup,
 ) void {
-    defer wg.stop();
+    defer wg.finish();
 
     comp.updateCObject(c_object, progress_node) catch |err| switch (err) {
         error.AnalysisFail => return,
diff --git a/src/ThreadPool.zig b/src/ThreadPool.zig
index cf9c02fa59..1e91d3f731 100644
--- a/src/ThreadPool.zig
+++ b/src/ThreadPool.zig
@@ -9,8 +9,7 @@ const ThreadPool = @This();
 lock: std.Mutex = .{},
 is_running: bool = true,
 allocator: *std.mem.Allocator,
-spawned: usize = 0,
-threads: []*std.Thread,
+workers: []Worker,
 run_queue: RunQueue = .{},
 idle_queue: IdleQueue = .{},
 
@@ -20,23 +19,69 @@ const Runnable = struct {
     runFn: fn (*Runnable) void,
 };
 
+const Worker = struct {
+    pool: *ThreadPool,
+    thread: *std.Thread,
+    /// The node is for this worker only and must have an already initialized event
+    /// when the thread is spawned.
+    idle_node: IdleQueue.Node,
+
+    fn run(worker: *Worker) void {
+        while (true) {
+            const held = worker.pool.lock.acquire();
+
+            if (worker.pool.run_queue.popFirst()) |run_node| {
+                held.release();
+                (run_node.data.runFn)(&run_node.data);
+                continue;
+            }
+
+            if (worker.pool.is_running) {
+                worker.idle_node.data.reset();
+
+                worker.pool.idle_queue.prepend(&worker.idle_node);
+                held.release();
+
+                worker.idle_node.data.wait();
+                continue;
+            }
+
+            held.release();
+            return;
+        }
+    }
+};
+
 pub fn init(self: *ThreadPool, allocator: *std.mem.Allocator) !void {
     self.* = .{
         .allocator = allocator,
-        .threads = &[_]*std.Thread{},
+        .workers = &[_]Worker{},
     };
     if (std.builtin.single_threaded)
         return;
 
-    errdefer self.deinit();
+    const worker_count = std.math.max(1, std.Thread.cpuCount() catch 1);
+    self.workers = try allocator.alloc(Worker, worker_count);
+    errdefer allocator.free(self.workers);
+
+    var worker_index: usize = 0;
+    errdefer self.destroyWorkers(worker_index);
+    while (worker_index < worker_count) : (worker_index += 1) {
+        const worker = &self.workers[worker_index];
+        worker.pool = self;
 
-    var num_threads = std.math.max(1, std.Thread.cpuCount() catch 1);
-    self.threads = try allocator.alloc(*std.Thread, num_threads);
+        // Each worker requires its ResetEvent to be pre-initialized.
+        try worker.idle_node.data.init();
+        errdefer worker.idle_node.data.deinit();
+
+        worker.thread = try std.Thread.spawn(worker, Worker.run);
+    }
+}
 
-    while (num_threads > 0) : (num_threads -= 1) {
-        const thread = try std.Thread.spawn(self, runWorker);
-        self.threads[self.spawned] = thread;
-        self.spawned += 1;
+fn destroyWorkers(self: *ThreadPool, spawned: usize) void {
+    for (self.workers[0..spawned]) |*worker| {
+        worker.thread.wait();
+        worker.idle_node.data.deinit();
     }
 }
 
@@ -50,9 +95,8 @@ pub fn deinit(self: *ThreadPool) void {
             idle_node.data.set();
     }
 
-    defer self.allocator.free(self.threads);
-    for (self.threads[0..self.spawned]) |thread|
-        thread.wait();
+    self.destroyWorkers(self.workers.len);
+    self.allocator.free(self.workers);
 }
 
 pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
@@ -92,29 +136,3 @@ pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
     if (self.idle_queue.popFirst()) |idle_node|
         idle_node.data.set();
 }
-
-fn runWorker(self: *ThreadPool) void {
-    while (true) {
-        const held = self.lock.acquire();
-
-        if (self.run_queue.popFirst()) |run_node| {
-            held.release();
-            (run_node.data.runFn)(&run_node.data);
-            continue;
-        }
-
-        if (self.is_running) {
-            var idle_node = IdleQueue.Node{ .data = std.ResetEvent.init() };
-
-            self.idle_queue.prepend(&idle_node);
-            held.release();
-
-            idle_node.data.wait();
-            idle_node.data.deinit();
-            continue;
-        }
-
-        held.release();
-        return;
-    }
-}
diff --git a/src/WaitGroup.zig b/src/WaitGroup.zig
index 6a0b12d050..bd6274c10a 100644
--- a/src/WaitGroup.zig
+++ b/src/WaitGroup.zig
@@ -8,7 +8,21 @@ const WaitGroup = @This();
 
 lock: std.Mutex = .{},
 counter: usize = 0,
-event: ?*std.ResetEvent = null,
+event: std.ResetEvent,
+
+pub fn init(self: *WaitGroup) !void {
+    self.* = .{
+        .lock = .{},
+        .counter = 0,
+        .event = undefined,
+    };
+    try self.event.init();
+}
+
+pub fn deinit(self: *WaitGroup) void {
+    self.event.deinit();
+    self.* = undefined;
+}
 
 pub fn start(self: *WaitGroup) void {
     const held = self.lock.acquire();
@@ -17,17 +31,14 @@ pub fn start(self: *WaitGroup) void {
     self.counter += 1;
 }
 
-pub fn stop(self: *WaitGroup) void {
+pub fn finish(self: *WaitGroup) void {
     const held = self.lock.acquire();
     defer held.release();
 
     self.counter -= 1;
 
     if (self.counter == 0) {
-        if (self.event) |event| {
-            self.event = null;
-            event.set();
-        }
+        self.event.set();
     }
 }
 
@@ -40,13 +51,11 @@ pub fn wait(self: *WaitGroup) void {
             return;
         }
 
-        var event = std.ResetEvent.init();
-        defer event.deinit();
-
-        std.debug.assert(self.event == null);
-        self.event = &event;
-
         held.release();
-        event.wait();
+        self.event.wait();
     }
 }
+
+pub fn reset(self: *WaitGroup) void {
+    self.event.reset();
+}
-- 
cgit v1.2.3


From 42b4a48bc96ce22562230cd1a266f93a013c76dd Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 21 Dec 2020 22:18:19 -0700
Subject: WIP start adding support for TSAN

---
 CMakeLists.txt                                     |     1 +
 lib/tsan/interception/interception.h               |   304 +
 lib/tsan/interception/interception_linux.h         |    53 +
 lib/tsan/interception/interception_mac.h           |    27 +
 lib/tsan/interception/interception_win.h           |    83 +
 lib/tsan/sanitizer_common/sancov_flags.h           |    39 +
 lib/tsan/sanitizer_common/sancov_flags.inc         |    20 +
 lib/tsan/sanitizer_common/sanitizer_addrhashmap.h  |   353 +
 lib/tsan/sanitizer_common/sanitizer_allocator.cpp  |   267 +
 lib/tsan/sanitizer_common/sanitizer_allocator.h    |    81 +
 .../sanitizer_common/sanitizer_allocator_bytemap.h |   107 +
 .../sanitizer_common/sanitizer_allocator_checks.h  |    76 +
 .../sanitizer_allocator_combined.h                 |   201 +
 .../sanitizer_allocator_interface.h                |    47 +
 .../sanitizer_allocator_internal.h                 |    55 +
 .../sanitizer_allocator_local_cache.h              |   264 +
 .../sanitizer_allocator_primary32.h                |   380 +
 .../sanitizer_allocator_primary64.h                |   863 ++
 .../sanitizer_common/sanitizer_allocator_report.h  |    39 +
 .../sanitizer_allocator_secondary.h                |   326 +
 .../sanitizer_allocator_size_class_map.h           |   241 +
 .../sanitizer_common/sanitizer_allocator_stats.h   |   106 +
 lib/tsan/sanitizer_common/sanitizer_asm.h          |    68 +
 lib/tsan/sanitizer_common/sanitizer_atomic.h       |    86 +
 lib/tsan/sanitizer_common/sanitizer_atomic_clang.h |   105 +
 .../sanitizer_common/sanitizer_atomic_clang_mips.h |   117 +
 .../sanitizer_atomic_clang_other.h                 |    97 +
 .../sanitizer_common/sanitizer_atomic_clang_x86.h  |   113 +
 lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h  |   256 +
 lib/tsan/sanitizer_common/sanitizer_bitvector.h    |   350 +
 lib/tsan/sanitizer_common/sanitizer_bvgraph.h      |   164 +
 lib/tsan/sanitizer_common/sanitizer_common.cpp     |   348 +
 lib/tsan/sanitizer_common/sanitizer_common.h       |  1002 ++
 .../sanitizer_common_interceptors.inc              | 10175 +++++++++++++++++++
 .../sanitizer_common_interceptors_format.inc       |   562 +
 .../sanitizer_common_interceptors_ioctl.inc        |   609 ++
 ...sanitizer_common_interceptors_netbsd_compat.inc |   128 +
 .../sanitizer_common_interface.inc                 |    41 +
 .../sanitizer_common_interface_posix.inc           |    13 +
 .../sanitizer_common/sanitizer_common_syscalls.inc |  2914 ++++++
 .../sanitizer_coverage_interface.inc               |    33 +
 lib/tsan/sanitizer_common/sanitizer_dbghelp.h      |    41 +
 .../sanitizer_common/sanitizer_deadlock_detector.h |   410 +
 .../sanitizer_deadlock_detector1.cpp               |   194 +
 .../sanitizer_deadlock_detector2.cpp               |   423 +
 .../sanitizer_deadlock_detector_interface.h        |    92 +
 lib/tsan/sanitizer_common/sanitizer_errno.cpp      |    34 +
 lib/tsan/sanitizer_common/sanitizer_errno.h        |    39 +
 lib/tsan/sanitizer_common/sanitizer_errno_codes.h  |    33 +
 lib/tsan/sanitizer_common/sanitizer_file.cpp       |   215 +
 lib/tsan/sanitizer_common/sanitizer_file.h         |   106 +
 .../sanitizer_common/sanitizer_flag_parser.cpp     |   191 +
 lib/tsan/sanitizer_common/sanitizer_flag_parser.h  |   204 +
 lib/tsan/sanitizer_common/sanitizer_flags.cpp      |   129 +
 lib/tsan/sanitizer_common/sanitizer_flags.h        |    67 +
 lib/tsan/sanitizer_common/sanitizer_flags.inc      |   250 +
 lib/tsan/sanitizer_common/sanitizer_freebsd.h      |   137 +
 lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp    |   531 +
 lib/tsan/sanitizer_common/sanitizer_fuchsia.h      |    36 +
 lib/tsan/sanitizer_common/sanitizer_getauxval.h    |    59 +
 .../sanitizer_common/sanitizer_glibc_version.h     |    26 +
 lib/tsan/sanitizer_common/sanitizer_hash.h         |    43 +
 .../sanitizer_interceptors_ioctl_netbsd.inc        |  1535 +++
 .../sanitizer_interface_internal.h                 |   118 +
 .../sanitizer_common/sanitizer_internal_defs.h     |   459 +
 lib/tsan/sanitizer_common/sanitizer_lfstack.h      |    72 +
 lib/tsan/sanitizer_common/sanitizer_libc.cpp       |   280 +
 lib/tsan/sanitizer_common/sanitizer_libc.h         |    85 +
 lib/tsan/sanitizer_common/sanitizer_libignore.cpp  |   129 +
 lib/tsan/sanitizer_common/sanitizer_libignore.h    |   115 +
 lib/tsan/sanitizer_common/sanitizer_linux.cpp      |  2269 +++++
 lib/tsan/sanitizer_common/sanitizer_linux.h        |   162 +
 lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp |   222 +
 lib/tsan/sanitizer_common/sanitizer_list.h         |   166 +
 .../sanitizer_local_address_space_view.h           |    76 +
 lib/tsan/sanitizer_common/sanitizer_mac.cpp        |  1228 +++
 lib/tsan/sanitizer_common/sanitizer_mac.h          |    84 +
 lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc |   420 +
 lib/tsan/sanitizer_common/sanitizer_mutex.h        |   223 +
 lib/tsan/sanitizer_common/sanitizer_netbsd.cpp     |   343 +
 lib/tsan/sanitizer_common/sanitizer_openbsd.cpp    |   115 +
 .../sanitizer_persistent_allocator.cpp             |    18 +
 .../sanitizer_persistent_allocator.h               |    71 +
 .../sanitizer_common/sanitizer_placement_new.h     |    24 +
 lib/tsan/sanitizer_common/sanitizer_platform.h     |   364 +
 .../sanitizer_platform_interceptors.h              |   609 ++
 .../sanitizer_platform_limits_freebsd.cpp          |   531 +
 .../sanitizer_platform_limits_freebsd.h            |   656 ++
 .../sanitizer_platform_limits_linux.cpp            |   108 +
 .../sanitizer_platform_limits_netbsd.cpp           |  2586 +++++
 .../sanitizer_platform_limits_netbsd.h             |  2422 +++++
 .../sanitizer_platform_limits_openbsd.cpp          |   279 +
 .../sanitizer_platform_limits_openbsd.h            |   382 +
 .../sanitizer_platform_limits_posix.cpp            |  1275 +++
 .../sanitizer_platform_limits_posix.h              |  1453 +++
 .../sanitizer_platform_limits_solaris.cpp          |   366 +
 .../sanitizer_platform_limits_solaris.h            |   495 +
 lib/tsan/sanitizer_common/sanitizer_posix.cpp      |   398 +
 lib/tsan/sanitizer_common/sanitizer_posix.h        |   125 +
 lib/tsan/sanitizer_common/sanitizer_printf.cpp     |   358 +
 lib/tsan/sanitizer_common/sanitizer_procmaps.h     |   100 +
 .../sanitizer_common/sanitizer_procmaps_bsd.cpp    |   139 +
 .../sanitizer_common/sanitizer_procmaps_common.cpp |   174 +
 .../sanitizer_procmaps_fuchsia.cpp                 |    80 +
 .../sanitizer_common/sanitizer_procmaps_linux.cpp  |    81 +
 .../sanitizer_common/sanitizer_procmaps_mac.cpp    |   379 +
 .../sanitizer_procmaps_solaris.cpp                 |    67 +
 lib/tsan/sanitizer_common/sanitizer_ptrauth.h      |    21 +
 lib/tsan/sanitizer_common/sanitizer_quarantine.h   |   317 +
 .../sanitizer_common/sanitizer_report_decorator.h  |    48 +
 lib/tsan/sanitizer_common/sanitizer_ring_buffer.h  |   161 +
 lib/tsan/sanitizer_common/sanitizer_rtems.cpp      |   283 +
 lib/tsan/sanitizer_common/sanitizer_rtems.h        |    20 +
 .../sanitizer_signal_interceptors.inc              |    86 +
 lib/tsan/sanitizer_common/sanitizer_solaris.cpp    |   230 +
 lib/tsan/sanitizer_common/sanitizer_stackdepot.h   |    71 +
 .../sanitizer_common/sanitizer_stackdepotbase.h    |   177 +
 lib/tsan/sanitizer_common/sanitizer_stacktrace.h   |   176 +
 .../sanitizer_stacktrace_printer.h                 |    71 +
 lib/tsan/sanitizer_common/sanitizer_stoptheworld.h |    64 +
 .../sanitizer_stoptheworld_fuchsia.cpp             |    42 +
 .../sanitizer_stoptheworld_mac.cpp                 |   182 +
 .../sanitizer_common/sanitizer_suppressions.cpp    |   181 +
 lib/tsan/sanitizer_common/sanitizer_suppressions.h |    56 +
 lib/tsan/sanitizer_common/sanitizer_symbolizer.h   |   223 +
 .../sanitizer_symbolizer_fuchsia.h                 |    42 +
 .../sanitizer_symbolizer_internal.h                |   165 +
 .../sanitizer_symbolizer_libbacktrace.h            |    49 +
 .../sanitizer_common/sanitizer_symbolizer_mac.h    |    48 +
 .../sanitizer_common/sanitizer_symbolizer_rtems.h  |    40 +
 .../sanitizer_common/sanitizer_syscall_generic.inc |    38 +
 .../sanitizer_syscall_linux_aarch64.inc            |   137 +
 .../sanitizer_syscall_linux_arm.inc                |   137 +
 .../sanitizer_syscall_linux_x86_64.inc             |    90 +
 .../sanitizer_common/sanitizer_syscalls_netbsd.inc |  3837 +++++++
 .../sanitizer_common/sanitizer_termination.cpp     |    94 +
 .../sanitizer_common/sanitizer_thread_registry.cpp |   351 +
 .../sanitizer_common/sanitizer_thread_registry.h   |   160 +
 .../sanitizer_common/sanitizer_tls_get_addr.cpp    |   154 +
 lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h |    62 +
 .../sanitizer_common/sanitizer_type_traits.cpp     |    20 +
 lib/tsan/sanitizer_common/sanitizer_type_traits.h  |    62 +
 lib/tsan/sanitizer_common/sanitizer_vector.h       |   124 +
 lib/tsan/sanitizer_common/sanitizer_win.cpp        |  1129 ++
 lib/tsan/sanitizer_common/sanitizer_win.h          |    25 +
 lib/tsan/sanitizer_common/sanitizer_win_defs.h     |   174 +
 .../sanitizer_common/sanitizer_win_dll_thunk.h     |   181 +
 .../sanitizer_win_weak_interception.h              |    32 +
 lib/tsan/tsan_clock.cpp                            |   655 ++
 lib/tsan/tsan_clock.h                              |   283 +
 lib/tsan/tsan_debugging.cpp                        |   262 +
 lib/tsan/tsan_defs.h                               |   195 +
 lib/tsan/tsan_dense_alloc.h                        |   141 +
 lib/tsan/tsan_external.cpp                         |   124 +
 lib/tsan/tsan_fd.cpp                               |   316 +
 lib/tsan/tsan_fd.h                                 |    64 +
 lib/tsan/tsan_flags.cpp                            |   125 +
 lib/tsan/tsan_flags.h                              |    34 +
 lib/tsan/tsan_flags.inc                            |    85 +
 lib/tsan/tsan_ignoreset.cpp                        |    46 +
 lib/tsan/tsan_ignoreset.h                          |    37 +
 lib/tsan/tsan_interceptors.h                       |    76 +
 lib/tsan/tsan_interceptors_posix.cpp               |  2854 ++++++
 lib/tsan/tsan_interface.cpp                        |   160 +
 lib/tsan/tsan_interface.h                          |   422 +
 lib/tsan/tsan_interface_ann.cpp                    |   552 +
 lib/tsan/tsan_interface_ann.h                      |    32 +
 lib/tsan/tsan_interface_atomic.cpp                 |   955 ++
 lib/tsan/tsan_interface_inl.h                      |   132 +
 lib/tsan/tsan_interface_java.cpp                   |   267 +
 lib/tsan/tsan_interface_java.h                     |    99 +
 lib/tsan/tsan_malloc_mac.cpp                       |    71 +
 lib/tsan/tsan_md5.cpp                              |   250 +
 lib/tsan/tsan_mman.cpp                             |   405 +
 lib/tsan/tsan_mman.h                               |    89 +
 lib/tsan/tsan_mutex.cpp                            |   289 +
 lib/tsan/tsan_mutex.h                              |    90 +
 lib/tsan/tsan_mutexset.cpp                         |    88 +
 lib/tsan/tsan_mutexset.h                           |    69 +
 lib/tsan/tsan_platform.h                           |  1028 ++
 lib/tsan/tsan_ppc_regs.h                           |    96 +
 lib/tsan/tsan_preinit.cpp                          |    26 +
 lib/tsan/tsan_report.cpp                           |   486 +
 lib/tsan/tsan_report.h                             |   135 +
 lib/tsan/tsan_rtl.cpp                              |  1127 ++
 lib/tsan/tsan_rtl.h                                |   893 ++
 lib/tsan/tsan_rtl_mutex.cpp                        |   562 +
 lib/tsan/tsan_rtl_proc.cpp                         |    60 +
 lib/tsan/tsan_rtl_report.cpp                       |   754 ++
 lib/tsan/tsan_rtl_thread.cpp                       |   462 +
 lib/tsan/tsan_stack_trace.cpp                      |    63 +
 lib/tsan/tsan_stack_trace.h                        |    42 +
 lib/tsan/tsan_stat.cpp                             |   186 +
 lib/tsan/tsan_stat.h                               |   191 +
 lib/tsan/tsan_suppressions.cpp                     |   161 +
 lib/tsan/tsan_suppressions.h                       |    37 +
 lib/tsan/tsan_symbolize.cpp                        |   122 +
 lib/tsan/tsan_symbolize.h                          |    30 +
 lib/tsan/tsan_sync.cpp                             |   296 +
 lib/tsan/tsan_sync.h                               |   145 +
 lib/tsan/tsan_trace.h                              |    75 +
 lib/tsan/tsan_update_shadow_word_inl.h             |    69 +
 lib/tsan/ubsan/ubsan_flags.h                       |    48 +
 lib/tsan/ubsan/ubsan_flags.inc                     |    28 +
 lib/tsan/ubsan/ubsan_init.h                        |    33 +
 lib/tsan/ubsan/ubsan_platform.h                    |    25 +
 src/Compilation.zig                                |    30 +-
 src/glibc.zig                                      |     1 +
 src/libcxx.zig                                     |     3 +-
 src/libtsan.zig                                    |   219 +
 src/libunwind.zig                                  |     1 +
 src/link.zig                                       |     1 +
 src/link/Elf.zig                                   |     6 +
 src/main.zig                                       |    10 +
 src/musl.zig                                       |     1 +
 src/stage1.zig                                     |     1 +
 src/stage1/all_types.hpp                           |     1 +
 src/stage1/codegen.cpp                             |     4 +
 src/stage1/stage1.cpp                              |     1 +
 src/stage1/stage1.h                                |     1 +
 220 files changed, 72299 insertions(+), 3 deletions(-)
 create mode 100644 lib/tsan/interception/interception.h
 create mode 100644 lib/tsan/interception/interception_linux.h
 create mode 100644 lib/tsan/interception/interception_mac.h
 create mode 100644 lib/tsan/interception/interception_win.h
 create mode 100644 lib/tsan/sanitizer_common/sancov_flags.h
 create mode 100644 lib/tsan/sanitizer_common/sancov_flags.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_addrhashmap.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_checks.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_combined.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_interface.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_internal.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_local_cache.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_report.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_stats.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_asm.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic_clang.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_bitvector.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_bvgraph.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interface.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_dbghelp.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_deadlock_detector1.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_deadlock_detector2.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_deadlock_detector_interface.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_errno.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_errno.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_errno_codes.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_file.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_file.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_flag_parser.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_flags.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_flags.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_flags.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_freebsd.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_fuchsia.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_getauxval.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_glibc_version.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_hash.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_interface_internal.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_internal_defs.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_lfstack.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_libc.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_libc.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_libignore.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_libignore.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_linux.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_linux.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_list.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_mac.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_mac.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_mutex.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_netbsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_openbsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_placement_new.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_posix.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_posix.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_printf.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_fuchsia.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_linux.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_ptrauth.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_quarantine.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_report_decorator.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_ring_buffer.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_rtems.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_rtems.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_solaris.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stackdepot.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stoptheworld.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_suppressions.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_suppressions.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_rtems.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_syscall_linux_arm.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_termination.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_thread_registry.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_type_traits.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_type_traits.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_vector.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_win.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_win.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_win_defs.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h
 create mode 100644 lib/tsan/tsan_clock.cpp
 create mode 100644 lib/tsan/tsan_clock.h
 create mode 100644 lib/tsan/tsan_debugging.cpp
 create mode 100644 lib/tsan/tsan_defs.h
 create mode 100644 lib/tsan/tsan_dense_alloc.h
 create mode 100644 lib/tsan/tsan_external.cpp
 create mode 100644 lib/tsan/tsan_fd.cpp
 create mode 100644 lib/tsan/tsan_fd.h
 create mode 100644 lib/tsan/tsan_flags.cpp
 create mode 100644 lib/tsan/tsan_flags.h
 create mode 100644 lib/tsan/tsan_flags.inc
 create mode 100644 lib/tsan/tsan_ignoreset.cpp
 create mode 100644 lib/tsan/tsan_ignoreset.h
 create mode 100644 lib/tsan/tsan_interceptors.h
 create mode 100644 lib/tsan/tsan_interceptors_posix.cpp
 create mode 100644 lib/tsan/tsan_interface.cpp
 create mode 100644 lib/tsan/tsan_interface.h
 create mode 100644 lib/tsan/tsan_interface_ann.cpp
 create mode 100644 lib/tsan/tsan_interface_ann.h
 create mode 100644 lib/tsan/tsan_interface_atomic.cpp
 create mode 100644 lib/tsan/tsan_interface_inl.h
 create mode 100644 lib/tsan/tsan_interface_java.cpp
 create mode 100644 lib/tsan/tsan_interface_java.h
 create mode 100644 lib/tsan/tsan_malloc_mac.cpp
 create mode 100644 lib/tsan/tsan_md5.cpp
 create mode 100644 lib/tsan/tsan_mman.cpp
 create mode 100644 lib/tsan/tsan_mman.h
 create mode 100644 lib/tsan/tsan_mutex.cpp
 create mode 100644 lib/tsan/tsan_mutex.h
 create mode 100644 lib/tsan/tsan_mutexset.cpp
 create mode 100644 lib/tsan/tsan_mutexset.h
 create mode 100644 lib/tsan/tsan_platform.h
 create mode 100644 lib/tsan/tsan_ppc_regs.h
 create mode 100644 lib/tsan/tsan_preinit.cpp
 create mode 100644 lib/tsan/tsan_report.cpp
 create mode 100644 lib/tsan/tsan_report.h
 create mode 100644 lib/tsan/tsan_rtl.cpp
 create mode 100644 lib/tsan/tsan_rtl.h
 create mode 100644 lib/tsan/tsan_rtl_mutex.cpp
 create mode 100644 lib/tsan/tsan_rtl_proc.cpp
 create mode 100644 lib/tsan/tsan_rtl_report.cpp
 create mode 100644 lib/tsan/tsan_rtl_thread.cpp
 create mode 100644 lib/tsan/tsan_stack_trace.cpp
 create mode 100644 lib/tsan/tsan_stack_trace.h
 create mode 100644 lib/tsan/tsan_stat.cpp
 create mode 100644 lib/tsan/tsan_stat.h
 create mode 100644 lib/tsan/tsan_suppressions.cpp
 create mode 100644 lib/tsan/tsan_suppressions.h
 create mode 100644 lib/tsan/tsan_symbolize.cpp
 create mode 100644 lib/tsan/tsan_symbolize.h
 create mode 100644 lib/tsan/tsan_sync.cpp
 create mode 100644 lib/tsan/tsan_sync.h
 create mode 100644 lib/tsan/tsan_trace.h
 create mode 100644 lib/tsan/tsan_update_shadow_word_inl.h
 create mode 100644 lib/tsan/ubsan/ubsan_flags.h
 create mode 100644 lib/tsan/ubsan/ubsan_flags.inc
 create mode 100644 lib/tsan/ubsan/ubsan_init.h
 create mode 100644 lib/tsan/ubsan/ubsan_platform.h
 create mode 100644 src/libtsan.zig

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b9b9310946..d51f753ac0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -537,6 +537,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/ir.zig"
     "${CMAKE_SOURCE_DIR}/src/libc_installation.zig"
     "${CMAKE_SOURCE_DIR}/src/libcxx.zig"
+    "${CMAKE_SOURCE_DIR}/src/libtsan.zig"
     "${CMAKE_SOURCE_DIR}/src/libunwind.zig"
     "${CMAKE_SOURCE_DIR}/src/link.zig"
     "${CMAKE_SOURCE_DIR}/src/link/C.zig"
diff --git a/lib/tsan/interception/interception.h b/lib/tsan/interception/interception.h
new file mode 100644
index 0000000000..d27a8ccf92
--- /dev/null
+++ b/lib/tsan/interception/interception.h
@@ -0,0 +1,304 @@
+//===-- interception.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Machinery for providing replacements/wrappers for system functions.
+//===----------------------------------------------------------------------===//
+
+#ifndef INTERCEPTION_H
+#define INTERCEPTION_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
+    !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_WINDOWS && \
+    !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS
+# error "Interception doesn't work on this operating system."
+#endif
+
+// These typedefs should be used only in the interceptor definitions to replace
+// the standard system types (e.g. SSIZE_T instead of ssize_t)
+typedef __sanitizer::uptr    SIZE_T;
+typedef __sanitizer::sptr    SSIZE_T;
+typedef __sanitizer::sptr    PTRDIFF_T;
+typedef __sanitizer::s64     INTMAX_T;
+typedef __sanitizer::u64     UINTMAX_T;
+typedef __sanitizer::OFF_T   OFF_T;
+typedef __sanitizer::OFF64_T OFF64_T;
+
+// How to add an interceptor:
+// Suppose you need to wrap/replace system function (generally, from libc):
+//      int foo(const char *bar, double baz);
+// You'll need to:
+//      1) define INTERCEPTOR(int, foo, const char *bar, double baz) { ... } in
+//         your source file. See the notes below for cases when
+//         INTERCEPTOR_WITH_SUFFIX(...) should be used instead.
+//      2) Call "INTERCEPT_FUNCTION(foo)" prior to the first call of "foo".
+//         INTERCEPT_FUNCTION(foo) evaluates to "true" iff the function was
+//         intercepted successfully.
+// You can access original function by calling REAL(foo)(bar, baz).
+// By default, REAL(foo) will be visible only inside your interceptor, and if
+// you want to use it in other parts of RTL, you'll need to:
+//      3a) add DECLARE_REAL(int, foo, const char*, double) to a
+//          header file.
+// However, if the call "INTERCEPT_FUNCTION(foo)" and definition for
+// INTERCEPTOR(..., foo, ...) are in different files, you'll instead need to:
+//      3b) add DECLARE_REAL_AND_INTERCEPTOR(int, foo, const char*, double)
+//          to a header file.
+
+// Notes: 1. Things may not work properly if macro INTERCEPTOR(...) {...} or
+//           DECLARE_REAL(...) are located inside namespaces.
+//        2. On Mac you can also use: "OVERRIDE_FUNCTION(foo, zoo)" to
+//           effectively redirect calls from "foo" to "zoo". In this case
+//           you aren't required to implement
+//           INTERCEPTOR(int, foo, const char *bar, double baz) {...}
+//           but instead you'll have to add
+//           DECLARE_REAL(int, foo, const char *bar, double baz) in your
+//           source file (to define a pointer to overriden function).
+//        3. Some Mac functions have symbol variants discriminated by
+//           additional suffixes, e.g. _$UNIX2003 (see
+//           https://developer.apple.com/library/mac/#releasenotes/Darwin/SymbolVariantsRelNotes/index.html
+//           for more details). To intercept such functions you need to use the
+//           INTERCEPTOR_WITH_SUFFIX(...) macro.
+
+// How it works:
+// To replace system functions on Linux we just need to declare functions
+// with same names in our library and then obtain the real function pointers
+// using dlsym().
+// There is one complication. A user may also intercept some of the functions
+// we intercept. To resolve this we declare our interceptors with __interceptor_
+// prefix, and then make actual interceptors weak aliases to __interceptor_
+// functions.
+//
+// This is not so on Mac OS, where the two-level namespace makes
+// our replacement functions invisible to other libraries. This may be overcomed
+// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared
+// libraries in Chromium were noticed when doing so.
+// Instead we create a dylib containing a __DATA,__interpose section that
+// associates library functions with their wrappers. When this dylib is
+// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all
+// the calls to interposed functions done through stubs to the wrapper
+// functions.
+// As it's decided at compile time which functions are to be intercepted on Mac,
+// INTERCEPT_FUNCTION() is effectively a no-op on this system.
+
+#if SANITIZER_MAC
+#include <sys/cdefs.h>  // For __DARWIN_ALIAS_C().
+
+// Just a pair of pointers.
+struct interpose_substitution {
+  const __sanitizer::uptr replacement;
+  const __sanitizer::uptr original;
+};
+
+// For a function foo() create a global pair of pointers { wrap_foo, foo } in
+// the __DATA,__interpose section.
+// As a result all the calls to foo() will be routed to wrap_foo() at runtime.
+#define INTERPOSER(func_name) __attribute__((used)) \
+const interpose_substitution substitution_##func_name[] \
+    __attribute__((section("__DATA, __interpose"))) = { \
+    { reinterpret_cast<const uptr>(WRAP(func_name)), \
+      reinterpret_cast<const uptr>(func_name) } \
+}
+
+// For a function foo() and a wrapper function bar() create a global pair
+// of pointers { bar, foo } in the __DATA,__interpose section.
+// As a result all the calls to foo() will be routed to bar() at runtime.
+#define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \
+const interpose_substitution substitution_##func_name[] \
+    __attribute__((section("__DATA, __interpose"))) = { \
+    { reinterpret_cast<const uptr>(wrapper_name), \
+      reinterpret_cast<const uptr>(func_name) } \
+}
+
+# define WRAP(x) wrap_##x
+# define WRAPPER_NAME(x) "wrap_"#x
+# define INTERCEPTOR_ATTRIBUTE
+# define DECLARE_WRAPPER(ret_type, func, ...)
+
+#elif SANITIZER_WINDOWS
+# define WRAP(x) __asan_wrap_##x
+# define WRAPPER_NAME(x) "__asan_wrap_"#x
+# define INTERCEPTOR_ATTRIBUTE __declspec(dllexport)
+# define DECLARE_WRAPPER(ret_type, func, ...) \
+    extern "C" ret_type func(__VA_ARGS__);
+# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
+    extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
+#elif SANITIZER_RTEMS
+# define WRAP(x) x
+# define WRAPPER_NAME(x) #x
+# define INTERCEPTOR_ATTRIBUTE
+# define DECLARE_WRAPPER(ret_type, func, ...)
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
+# define WRAP(x) __interceptor_ ## x
+# define WRAPPER_NAME(x) "__interceptor_" #x
+# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+// FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher
+// priority than weak ones so weak aliases won't work for indirect calls
+// in position-independent (-fPIC / -fPIE) mode.
+# define DECLARE_WRAPPER(ret_type, func, ...) \
+     extern "C" ret_type func(__VA_ARGS__) \
+     __attribute__((alias("__interceptor_" #func), visibility("default")));
+#elif !SANITIZER_FUCHSIA
+# define WRAP(x) __interceptor_ ## x
+# define WRAPPER_NAME(x) "__interceptor_" #x
+# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# define DECLARE_WRAPPER(ret_type, func, ...) \
+    extern "C" ret_type func(__VA_ARGS__) \
+    __attribute__((weak, alias("__interceptor_" #func), visibility("default")));
+#endif
+
+#if SANITIZER_FUCHSIA
+// There is no general interception at all on Fuchsia.
+// Sanitizer runtimes just define functions directly to preempt them,
+// and have bespoke ways to access the underlying libc functions.
+# include <zircon/sanitizer.h>
+# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# define REAL(x) __unsanitized_##x
+# define DECLARE_REAL(ret_type, func, ...)
+#elif SANITIZER_RTEMS
+# define REAL(x) __real_ ## x
+# define DECLARE_REAL(ret_type, func, ...) \
+    extern "C" ret_type REAL(func)(__VA_ARGS__);
+#elif !SANITIZER_MAC
+# define PTR_TO_REAL(x) real_##x
+# define REAL(x) __interception::PTR_TO_REAL(x)
+# define FUNC_TYPE(x) x##_type
+
+# define DECLARE_REAL(ret_type, func, ...) \
+    typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
+    namespace __interception { \
+      extern FUNC_TYPE(func) PTR_TO_REAL(func); \
+    }
+# define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src)
+#else  // SANITIZER_MAC
+# define REAL(x) x
+# define DECLARE_REAL(ret_type, func, ...) \
+    extern "C" ret_type func(__VA_ARGS__);
+# define ASSIGN_REAL(x, y)
+#endif  // SANITIZER_MAC
+
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+  DECLARE_REAL(ret_type, func, __VA_ARGS__) \
+  extern "C" ret_type WRAP(func)(__VA_ARGS__);
+// Declare an interceptor and its wrapper defined in a different translation
+// unit (ex. asm).
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)    \
+  extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+  extern "C" ret_type func(__VA_ARGS__);
+#else
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)
+#endif
+
+// Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR
+// macros does its job. In exceptional cases you may need to call REAL(foo)
+// without defining INTERCEPTOR(..., foo, ...). For example, if you override
+// foo with an interceptor for other function.
+#if !SANITIZER_MAC && !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+# define DEFINE_REAL(ret_type, func, ...) \
+    typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
+    namespace __interception { \
+      FUNC_TYPE(func) PTR_TO_REAL(func); \
+    }
+#else
+# define DEFINE_REAL(ret_type, func, ...)
+#endif
+
+#if SANITIZER_FUCHSIA
+
+// We need to define the __interceptor_func name just to get
+// sanitizer_common/scripts/gen_dynamic_list.py to export func.
+// But we don't need to export __interceptor_func to get that.
+#define INTERCEPTOR(ret_type, func, ...)                                \
+  extern "C"[[ gnu::alias(#func), gnu::visibility("hidden") ]] ret_type \
+      __interceptor_##func(__VA_ARGS__);                                \
+  extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__)
+
+#elif !SANITIZER_MAC
+
+#define INTERCEPTOR(ret_type, func, ...) \
+  DEFINE_REAL(ret_type, func, __VA_ARGS__) \
+  DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
+  extern "C" \
+  INTERCEPTOR_ATTRIBUTE \
+  ret_type WRAP(func)(__VA_ARGS__)
+
+// We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now.
+#define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
+  INTERCEPTOR(ret_type, func, __VA_ARGS__)
+
+#else  // SANITIZER_MAC
+
+#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
+  extern "C" ret_type func(__VA_ARGS__) suffix; \
+  extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+  INTERPOSER(func); \
+  extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
+
+#define INTERCEPTOR(ret_type, func, ...) \
+  INTERCEPTOR_ZZZ(/*no symbol variants*/, ret_type, func, __VA_ARGS__)
+
+#define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
+  INTERCEPTOR_ZZZ(__DARWIN_ALIAS_C(func), ret_type, func, __VA_ARGS__)
+
+// Override |overridee| with |overrider|.
+#define OVERRIDE_FUNCTION(overridee, overrider) \
+  INTERPOSER_2(overridee, WRAP(overrider))
+#endif
+
+#if SANITIZER_WINDOWS
+# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
+    typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \
+    namespace __interception { \
+      FUNC_TYPE(func) PTR_TO_REAL(func); \
+    } \
+    extern "C" \
+    INTERCEPTOR_ATTRIBUTE \
+    ret_type __stdcall WRAP(func)(__VA_ARGS__)
+#endif
+
+// ISO C++ forbids casting between pointer-to-function and pointer-to-object,
+// so we use casting via an integral type __interception::uptr,
+// assuming that system is POSIX-compliant. Using other hacks seem
+// challenging, as we don't even pass function type to
+// INTERCEPT_FUNCTION macro, only its name.
+namespace __interception {
+#if defined(_WIN64)
+typedef unsigned long long uptr;
+#else
+typedef unsigned long uptr;
+#endif  // _WIN64
+}  // namespace __interception
+
+#define INCLUDED_FROM_INTERCEPTION_LIB
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+# include "interception_linux.h"
+# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
+# define INTERCEPT_FUNCTION_VER(func, symver) \
+    INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver)
+#elif SANITIZER_MAC
+# include "interception_mac.h"
+# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func)
+# define INTERCEPT_FUNCTION_VER(func, symver) \
+    INTERCEPT_FUNCTION_VER_MAC(func, symver)
+#elif SANITIZER_WINDOWS
+# include "interception_win.h"
+# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_WIN(func)
+# define INTERCEPT_FUNCTION_VER(func, symver) \
+    INTERCEPT_FUNCTION_VER_WIN(func, symver)
+#endif
+
+#undef INCLUDED_FROM_INTERCEPTION_LIB
+
+#endif  // INTERCEPTION_H
diff --git a/lib/tsan/interception/interception_linux.h b/lib/tsan/interception/interception_linux.h
new file mode 100644
index 0000000000..e578da0cf6
--- /dev/null
+++ b/lib/tsan/interception/interception_linux.h
@@ -0,0 +1,53 @@
+//===-- interception_linux.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Linux-specific interception methods.
+//===----------------------------------------------------------------------===//
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
+# error "interception_linux.h should be included from interception library only"
+#endif
+
+#ifndef INTERCEPTION_LINUX_H
+#define INTERCEPTION_LINUX_H
+
+namespace __interception {
+bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func,
+                       uptr wrapper);
+bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
+                       uptr func, uptr wrapper);
+}  // namespace __interception
+
+#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
+  ::__interception::InterceptFunction(            \
+      #func,                                      \
+      (::__interception::uptr *) & REAL(func),    \
+      (::__interception::uptr) & (func),          \
+      (::__interception::uptr) & WRAP(func))
+
+// Android,  Solaris and OpenBSD do not have dlvsym
+#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
+#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
+  ::__interception::InterceptFunction(                        \
+      #func, symver,                                          \
+      (::__interception::uptr *) & REAL(func),                \
+      (::__interception::uptr) & (func),                      \
+      (::__interception::uptr) & WRAP(func))
+#else
+#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
+  INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
+#endif  // !SANITIZER_ANDROID && !SANITIZER_SOLARIS
+
+#endif  // INTERCEPTION_LINUX_H
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+        // SANITIZER_OPENBSD || SANITIZER_SOLARIS
diff --git a/lib/tsan/interception/interception_mac.h b/lib/tsan/interception/interception_mac.h
new file mode 100644
index 0000000000..eddedb8959
--- /dev/null
+++ b/lib/tsan/interception/interception_mac.h
@@ -0,0 +1,27 @@
+//===-- interception_mac.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Mac-specific interception methods.
+//===----------------------------------------------------------------------===//
+
+#if SANITIZER_MAC
+
+#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
+# error "interception_mac.h should be included from interception.h only"
+#endif
+
+#ifndef INTERCEPTION_MAC_H
+#define INTERCEPTION_MAC_H
+
+#define INTERCEPT_FUNCTION_MAC(func)
+#define INTERCEPT_FUNCTION_VER_MAC(func, symver)
+
+#endif  // INTERCEPTION_MAC_H
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/interception/interception_win.h b/lib/tsan/interception/interception_win.h
new file mode 100644
index 0000000000..4590013019
--- /dev/null
+++ b/lib/tsan/interception/interception_win.h
@@ -0,0 +1,83 @@
+//===-- interception_linux.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Windows-specific interception methods.
+//===----------------------------------------------------------------------===//
+
+#if SANITIZER_WINDOWS
+
+#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
+# error "interception_win.h should be included from interception library only"
+#endif
+
+#ifndef INTERCEPTION_WIN_H
+#define INTERCEPTION_WIN_H
+
+namespace __interception {
+// All the functions in the OverrideFunction() family return true on success,
+// false on failure (including "couldn't find the function").
+
+// Overrides a function by its address.
+bool OverrideFunction(uptr old_func, uptr new_func, uptr *orig_old_func = 0);
+
+// Overrides a function in a system DLL or DLL CRT by its exported name.
+bool OverrideFunction(const char *name, uptr new_func, uptr *orig_old_func = 0);
+
+// Windows-only replacement for GetProcAddress. Useful for some sanitizers.
+uptr InternalGetProcAddress(void *module, const char *func_name);
+
+// Overrides a function only when it is called from a specific DLL. For example,
+// this is used to override calls to HeapAlloc/HeapFree from ucrtbase without
+// affecting other third party libraries.
+bool OverrideImportedFunction(const char *module_to_patch,
+                              const char *imported_module,
+                              const char *function_name, uptr new_function,
+                              uptr *orig_old_func);
+
+#if !SANITIZER_WINDOWS64
+// Exposed for unittests
+bool OverrideFunctionWithDetour(
+    uptr old_func, uptr new_func, uptr *orig_old_func);
+#endif
+
+// Exposed for unittests
+bool OverrideFunctionWithRedirectJump(
+    uptr old_func, uptr new_func, uptr *orig_old_func);
+bool OverrideFunctionWithHotPatch(
+    uptr old_func, uptr new_func, uptr *orig_old_func);
+bool OverrideFunctionWithTrampoline(
+    uptr old_func, uptr new_func, uptr *orig_old_func);
+
+// Exposed for unittests
+void TestOnlyReleaseTrampolineRegions();
+
+}  // namespace __interception
+
+#if defined(INTERCEPTION_DYNAMIC_CRT)
+#define INTERCEPT_FUNCTION_WIN(func)                                           \
+  ::__interception::OverrideFunction(#func,                                    \
+                                     (::__interception::uptr)WRAP(func),       \
+                                     (::__interception::uptr *)&REAL(func))
+#else
+#define INTERCEPT_FUNCTION_WIN(func)                                           \
+  ::__interception::OverrideFunction((::__interception::uptr)func,             \
+                                     (::__interception::uptr)WRAP(func),       \
+                                     (::__interception::uptr *)&REAL(func))
+#endif
+
+#define INTERCEPT_FUNCTION_VER_WIN(func, symver) INTERCEPT_FUNCTION_WIN(func)
+
+#define INTERCEPT_FUNCTION_DLLIMPORT(user_dll, provider_dll, func)       \
+  ::__interception::OverrideImportedFunction(                            \
+      user_dll, provider_dll, #func, (::__interception::uptr)WRAP(func), \
+      (::__interception::uptr *)&REAL(func))
+
+#endif  // INTERCEPTION_WIN_H
+#endif  // SANITIZER_WINDOWS
diff --git a/lib/tsan/sanitizer_common/sancov_flags.h b/lib/tsan/sanitizer_common/sancov_flags.h
new file mode 100644
index 0000000000..95d4ee5ca4
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sancov_flags.h
@@ -0,0 +1,39 @@
+//===-- sancov_flags.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Sanitizer Coverage runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANCOV_FLAGS_H
+#define SANCOV_FLAGS_H
+
+#include "sanitizer_flag_parser.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sancov {
+
+struct SancovFlags {
+#define SANCOV_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "sancov_flags.inc"
+#undef SANCOV_FLAG
+
+  void SetDefaults();
+};
+
+extern SancovFlags sancov_flags_dont_use_directly;
+
+inline SancovFlags* sancov_flags() { return &sancov_flags_dont_use_directly; }
+
+void InitializeSancovFlags();
+
+}  // namespace __sancov
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char*
+__sancov_default_options();
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sancov_flags.inc b/lib/tsan/sanitizer_common/sancov_flags.inc
new file mode 100644
index 0000000000..cca33fc359
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sancov_flags.inc
@@ -0,0 +1,20 @@
+//===-- sancov_flags.inc ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Sanitizer Coverage runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANCOV_FLAG
+#error "Defnine SANCOV_FLAG prior to including this file!"
+#endif
+
+SANCOV_FLAG(bool, symbolize, true,
+            "If set, converage information will be symbolized by sancov tool "
+            "after dumping.")
+
+SANCOV_FLAG(bool, help, false, "Print flags help.")
diff --git a/lib/tsan/sanitizer_common/sanitizer_addrhashmap.h b/lib/tsan/sanitizer_common/sanitizer_addrhashmap.h
new file mode 100644
index 0000000000..a033e788cb
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_addrhashmap.h
@@ -0,0 +1,353 @@
+//===-- sanitizer_addrhashmap.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Concurrent uptr->T hashmap.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ADDRHASHMAP_H
+#define SANITIZER_ADDRHASHMAP_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_allocator_internal.h"
+
+namespace __sanitizer {
+
+// Concurrent uptr->T hashmap.
+// T must be a POD type, kSize is preferably a prime but can be any number.
+// Usage example:
+//
+// typedef AddrHashMap<uptr, 11> Map;
+// Map m;
+// {
+//   Map::Handle h(&m, addr);
+//   use h.operator->() to access the data
+//   if h.created() then the element was just created, and the current thread
+//     has exclusive access to it
+//   otherwise the current thread has only read access to the data
+// }
+// {
+//   Map::Handle h(&m, addr, true);
+//   this will remove the data from the map in Handle dtor
+//   the current thread has exclusive access to the data
+//   if !h.exists() then the element never existed
+// }
+template<typename T, uptr kSize>
+class AddrHashMap {
+ private:
+  struct Cell {
+    atomic_uintptr_t addr;
+    T                val;
+  };
+
+  struct AddBucket {
+    uptr cap;
+    uptr size;
+    Cell cells[1];  // variable len
+  };
+
+  static const uptr kBucketSize = 3;
+
+  struct Bucket {
+    RWMutex          mtx;
+    atomic_uintptr_t add;
+    Cell             cells[kBucketSize];
+  };
+
+ public:
+  AddrHashMap();
+
+  class Handle {
+   public:
+    Handle(AddrHashMap<T, kSize> *map, uptr addr);
+    Handle(AddrHashMap<T, kSize> *map, uptr addr, bool remove);
+    Handle(AddrHashMap<T, kSize> *map, uptr addr, bool remove, bool create);
+
+    ~Handle();
+    T *operator->();
+    T &operator*();
+    const T &operator*() const;
+    bool created() const;
+    bool exists() const;
+
+   private:
+    friend AddrHashMap<T, kSize>;
+    AddrHashMap<T, kSize> *map_;
+    Bucket                *bucket_;
+    Cell                  *cell_;
+    uptr                   addr_;
+    uptr                   addidx_;
+    bool                   created_;
+    bool                   remove_;
+    bool                   create_;
+  };
+
+ private:
+  friend class Handle;
+  Bucket *table_;
+
+  void acquire(Handle *h);
+  void release(Handle *h);
+  uptr calcHash(uptr addr);
+};
+
+template<typename T, uptr kSize>
+AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {
+  map_ = map;
+  addr_ = addr;
+  remove_ = false;
+  create_ = true;
+  map_->acquire(this);
+}
+
+template<typename T, uptr kSize>
+AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr,
+    bool remove) {
+  map_ = map;
+  addr_ = addr;
+  remove_ = remove;
+  create_ = true;
+  map_->acquire(this);
+}
+
+template<typename T, uptr kSize>
+AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr,
+    bool remove, bool create) {
+  map_ = map;
+  addr_ = addr;
+  remove_ = remove;
+  create_ = create;
+  map_->acquire(this);
+}
+
+template<typename T, uptr kSize>
+AddrHashMap<T, kSize>::Handle::~Handle() {
+  map_->release(this);
+}
+
+template <typename T, uptr kSize>
+T *AddrHashMap<T, kSize>::Handle::operator->() {
+  return &cell_->val;
+}
+
+template <typename T, uptr kSize>
+const T &AddrHashMap<T, kSize>::Handle::operator*() const {
+  return cell_->val;
+}
+
+template <typename T, uptr kSize>
+T &AddrHashMap<T, kSize>::Handle::operator*() {
+  return cell_->val;
+}
+
+template<typename T, uptr kSize>
+bool AddrHashMap<T, kSize>::Handle::created() const {
+  return created_;
+}
+
+template<typename T, uptr kSize>
+bool AddrHashMap<T, kSize>::Handle::exists() const {
+  return cell_ != nullptr;
+}
+
+template<typename T, uptr kSize>
+AddrHashMap<T, kSize>::AddrHashMap() {
+  table_ = (Bucket*)MmapOrDie(kSize * sizeof(table_[0]), "AddrHashMap");
+}
+
+template<typename T, uptr kSize>
+void AddrHashMap<T, kSize>::acquire(Handle *h) {
+  uptr addr = h->addr_;
+  uptr hash = calcHash(addr);
+  Bucket *b = &table_[hash];
+
+  h->created_ = false;
+  h->addidx_ = -1U;
+  h->bucket_ = b;
+  h->cell_ = nullptr;
+
+  // If we want to remove the element, we need exclusive access to the bucket,
+  // so skip the lock-free phase.
+  if (h->remove_)
+    goto locked;
+
+ retry:
+  // First try to find an existing element w/o read mutex.
+  CHECK(!h->remove_);
+  // Check the embed cells.
+  for (uptr i = 0; i < kBucketSize; i++) {
+    Cell *c = &b->cells[i];
+    uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
+    if (addr1 == addr) {
+      h->cell_ = c;
+      return;
+    }
+  }
+
+  // Check the add cells with read lock.
+  if (atomic_load(&b->add, memory_order_relaxed)) {
+    b->mtx.ReadLock();
+    AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed);
+    for (uptr i = 0; i < add->size; i++) {
+      Cell *c = &add->cells[i];
+      uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+      if (addr1 == addr) {
+        h->addidx_ = i;
+        h->cell_ = c;
+        return;
+      }
+    }
+    b->mtx.ReadUnlock();
+  }
+
+ locked:
+  // Re-check existence under write lock.
+  // Embed cells.
+  b->mtx.Lock();
+  for (uptr i = 0; i < kBucketSize; i++) {
+    Cell *c = &b->cells[i];
+    uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+    if (addr1 == addr) {
+      if (h->remove_) {
+        h->cell_ = c;
+        return;
+      }
+      b->mtx.Unlock();
+      goto retry;
+    }
+  }
+
+  // Add cells.
+  AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed);
+  if (add) {
+    for (uptr i = 0; i < add->size; i++) {
+      Cell *c = &add->cells[i];
+      uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+      if (addr1 == addr) {
+        if (h->remove_) {
+          h->addidx_ = i;
+          h->cell_ = c;
+          return;
+        }
+        b->mtx.Unlock();
+        goto retry;
+      }
+    }
+  }
+
+  // The element does not exist, no need to create it if we want to remove.
+  if (h->remove_ || !h->create_) {
+    b->mtx.Unlock();
+    return;
+  }
+
+  // Now try to create it under the mutex.
+  h->created_ = true;
+  // See if we have a free embed cell.
+  for (uptr i = 0; i < kBucketSize; i++) {
+    Cell *c = &b->cells[i];
+    uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+    if (addr1 == 0) {
+      h->cell_ = c;
+      return;
+    }
+  }
+
+  // Store in the add cells.
+  if (!add) {
+    // Allocate a new add array.
+    const uptr kInitSize = 64;
+    add = (AddBucket*)InternalAlloc(kInitSize);
+    internal_memset(add, 0, kInitSize);
+    add->cap = (kInitSize - sizeof(*add)) / sizeof(add->cells[0]) + 1;
+    add->size = 0;
+    atomic_store(&b->add, (uptr)add, memory_order_relaxed);
+  }
+  if (add->size == add->cap) {
+    // Grow existing add array.
+    uptr oldsize = sizeof(*add) + (add->cap - 1) * sizeof(add->cells[0]);
+    uptr newsize = oldsize * 2;
+    AddBucket *add1 = (AddBucket*)InternalAlloc(newsize);
+    internal_memset(add1, 0, newsize);
+    add1->cap = (newsize - sizeof(*add)) / sizeof(add->cells[0]) + 1;
+    add1->size = add->size;
+    internal_memcpy(add1->cells, add->cells, add->size * sizeof(add->cells[0]));
+    InternalFree(add);
+    atomic_store(&b->add, (uptr)add1, memory_order_relaxed);
+    add = add1;
+  }
+  // Store.
+  uptr i = add->size++;
+  Cell *c = &add->cells[i];
+  CHECK_EQ(atomic_load(&c->addr, memory_order_relaxed), 0);
+  h->addidx_ = i;
+  h->cell_ = c;
+}
+
+template<typename T, uptr kSize>
+void AddrHashMap<T, kSize>::release(Handle *h) {
+  if (!h->cell_)
+    return;
+  Bucket *b = h->bucket_;
+  Cell *c = h->cell_;
+  uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+  if (h->created_) {
+    // Denote completion of insertion.
+    CHECK_EQ(addr1, 0);
+    // After the following store, the element becomes available
+    // for lock-free reads.
+    atomic_store(&c->addr, h->addr_, memory_order_release);
+    b->mtx.Unlock();
+  } else if (h->remove_) {
+    // Denote that the cell is empty now.
+    CHECK_EQ(addr1, h->addr_);
+    atomic_store(&c->addr, 0, memory_order_release);
+    // See if we need to compact the bucket.
+    AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed);
+    if (h->addidx_ == -1U) {
+      // Removed from embed array, move an add element into the freed cell.
+      if (add && add->size != 0) {
+        uptr last = --add->size;
+        Cell *c1 = &add->cells[last];
+        c->val = c1->val;
+        uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed);
+        atomic_store(&c->addr, addr1, memory_order_release);
+        atomic_store(&c1->addr, 0, memory_order_release);
+      }
+    } else {
+      // Removed from add array, compact it.
+      uptr last = --add->size;
+      Cell *c1 = &add->cells[last];
+      if (c != c1) {
+        *c = *c1;
+        atomic_store(&c1->addr, 0, memory_order_relaxed);
+      }
+    }
+    if (add && add->size == 0) {
+      // FIXME(dvyukov): free add?
+    }
+    b->mtx.Unlock();
+  } else {
+    CHECK_EQ(addr1, h->addr_);
+    if (h->addidx_ != -1U)
+      b->mtx.ReadUnlock();
+  }
+}
+
+template<typename T, uptr kSize>
+uptr AddrHashMap<T, kSize>::calcHash(uptr addr) {
+  addr += addr << 10;
+  addr ^= addr >> 6;
+  return addr % kSize;
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ADDRHASHMAP_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator.cpp b/lib/tsan/sanitizer_common/sanitizer_allocator.cpp
new file mode 100644
index 0000000000..ec77b9cbfe
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator.cpp
@@ -0,0 +1,267 @@
+//===-- sanitizer_allocator.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// This allocator is used inside run-times.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator.h"
+
+#include "sanitizer_allocator_checks.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+// Default allocator names.
+const char *PrimaryAllocatorName = "SizeClassAllocator";
+const char *SecondaryAllocatorName = "LargeMmapAllocator";
+
+// ThreadSanitizer for Go uses libc malloc/free.
+#if defined(SANITIZER_USE_MALLOC)
+# if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern "C" void *__libc_malloc(uptr size);
+#  if !SANITIZER_GO
+extern "C" void *__libc_memalign(uptr alignment, uptr size);
+#  endif
+extern "C" void *__libc_realloc(void *ptr, uptr size);
+extern "C" void __libc_free(void *ptr);
+# else
+#  include <stdlib.h>
+#  define __libc_malloc malloc
+#  if !SANITIZER_GO
+static void *__libc_memalign(uptr alignment, uptr size) {
+  void *p;
+  uptr error = posix_memalign(&p, alignment, size);
+  if (error) return nullptr;
+  return p;
+}
+#  endif
+#  define __libc_realloc realloc
+#  define __libc_free free
+# endif
+
+static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
+                              uptr alignment) {
+  (void)cache;
+#if !SANITIZER_GO
+  if (alignment == 0)
+    return __libc_malloc(size);
+  else
+    return __libc_memalign(alignment, size);
+#else
+  // Windows does not provide __libc_memalign/posix_memalign. It provides
+  // __aligned_malloc, but the allocated blocks can't be passed to free,
+  // they need to be passed to __aligned_free. InternalAlloc interface does
+  // not account for such requirement. Alignemnt does not seem to be used
+  // anywhere in runtime, so just call __libc_malloc for now.
+  DCHECK_EQ(alignment, 0);
+  return __libc_malloc(size);
+#endif
+}
+
+static void *RawInternalRealloc(void *ptr, uptr size,
+                                InternalAllocatorCache *cache) {
+  (void)cache;
+  return __libc_realloc(ptr, size);
+}
+
+static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
+  (void)cache;
+  __libc_free(ptr);
+}
+
+InternalAllocator *internal_allocator() {
+  return 0;
+}
+
+#else  // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
+
+static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
+static atomic_uint8_t internal_allocator_initialized;
+static StaticSpinMutex internal_alloc_init_mu;
+
+static InternalAllocatorCache internal_allocator_cache;
+static StaticSpinMutex internal_allocator_cache_mu;
+
+InternalAllocator *internal_allocator() {
+  InternalAllocator *internal_allocator_instance =
+      reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder);
+  if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) {
+    SpinMutexLock l(&internal_alloc_init_mu);
+    if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==
+        0) {
+      internal_allocator_instance->Init(kReleaseToOSIntervalNever);
+      atomic_store(&internal_allocator_initialized, 1, memory_order_release);
+    }
+  }
+  return internal_allocator_instance;
+}
+
+static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
+                              uptr alignment) {
+  if (alignment == 0) alignment = 8;
+  if (cache == 0) {
+    SpinMutexLock l(&internal_allocator_cache_mu);
+    return internal_allocator()->Allocate(&internal_allocator_cache, size,
+                                          alignment);
+  }
+  return internal_allocator()->Allocate(cache, size, alignment);
+}
+
+static void *RawInternalRealloc(void *ptr, uptr size,
+                                InternalAllocatorCache *cache) {
+  uptr alignment = 8;
+  if (cache == 0) {
+    SpinMutexLock l(&internal_allocator_cache_mu);
+    return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
+                                            size, alignment);
+  }
+  return internal_allocator()->Reallocate(cache, ptr, size, alignment);
+}
+
+static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
+  if (!cache) {
+    SpinMutexLock l(&internal_allocator_cache_mu);
+    return internal_allocator()->Deallocate(&internal_allocator_cache, ptr);
+  }
+  internal_allocator()->Deallocate(cache, ptr);
+}
+
+#endif  // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
+
+const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
+
+static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
+  SetAllocatorOutOfMemory();
+  Report("FATAL: %s: internal allocator is out of memory trying to allocate "
+         "0x%zx bytes\n", SanitizerToolName, requested_size);
+  Die();
+}
+
+void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
+  if (size + sizeof(u64) < size)
+    return nullptr;
+  void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
+  if (UNLIKELY(!p))
+    ReportInternalAllocatorOutOfMemory(size + sizeof(u64));
+  ((u64*)p)[0] = kBlockMagic;
+  return (char*)p + sizeof(u64);
+}
+
+void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
+  if (!addr)
+    return InternalAlloc(size, cache);
+  if (size + sizeof(u64) < size)
+    return nullptr;
+  addr = (char*)addr - sizeof(u64);
+  size = size + sizeof(u64);
+  CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
+  void *p = RawInternalRealloc(addr, size, cache);
+  if (UNLIKELY(!p))
+    ReportInternalAllocatorOutOfMemory(size);
+  return (char*)p + sizeof(u64);
+}
+
+void *InternalReallocArray(void *addr, uptr count, uptr size,
+                           InternalAllocatorCache *cache) {
+  if (UNLIKELY(CheckForCallocOverflow(count, size))) {
+    Report(
+        "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
+        "cannot be represented in type size_t\n",
+        SanitizerToolName, count, size);
+    Die();
+  }
+  return InternalRealloc(addr, count * size, cache);
+}
+
+void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
+  if (UNLIKELY(CheckForCallocOverflow(count, size))) {
+    Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
+           "cannot be represented in type size_t\n", SanitizerToolName, count,
+           size);
+    Die();
+  }
+  void *p = InternalAlloc(count * size, cache);
+  if (LIKELY(p))
+    internal_memset(p, 0, count * size);
+  return p;
+}
+
+void InternalFree(void *addr, InternalAllocatorCache *cache) {
+  if (!addr)
+    return;
+  addr = (char*)addr - sizeof(u64);
+  CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
+  ((u64*)addr)[0] = 0;
+  RawInternalFree(addr, cache);
+}
+
+// LowLevelAllocator
+constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
+static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
+static LowLevelAllocateCallback low_level_alloc_callback;
+
+void *LowLevelAllocator::Allocate(uptr size) {
+  // Align allocation size.
+  size = RoundUpTo(size, low_level_alloc_min_alignment);
+  if (allocated_end_ - allocated_current_ < (sptr)size) {
+    uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
+    allocated_current_ =
+        (char*)MmapOrDie(size_to_allocate, __func__);
+    allocated_end_ = allocated_current_ + size_to_allocate;
+    if (low_level_alloc_callback) {
+      low_level_alloc_callback((uptr)allocated_current_,
+                               size_to_allocate);
+    }
+  }
+  CHECK(allocated_end_ - allocated_current_ >= (sptr)size);
+  void *res = allocated_current_;
+  allocated_current_ += size;
+  return res;
+}
+
+void SetLowLevelAllocateMinAlignment(uptr alignment) {
+  CHECK(IsPowerOfTwo(alignment));
+  low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment);
+}
+
+void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
+  low_level_alloc_callback = callback;
+}
+
+// Allocator's OOM and other errors handling support.
+
+static atomic_uint8_t allocator_out_of_memory = {0};
+static atomic_uint8_t allocator_may_return_null = {0};
+
+bool IsAllocatorOutOfMemory() {
+  return atomic_load_relaxed(&allocator_out_of_memory);
+}
+
+void SetAllocatorOutOfMemory() {
+  atomic_store_relaxed(&allocator_out_of_memory, 1);
+}
+
+bool AllocatorMayReturnNull() {
+  return atomic_load(&allocator_may_return_null, memory_order_relaxed);
+}
+
+void SetAllocatorMayReturnNull(bool may_return_null) {
+  atomic_store(&allocator_may_return_null, may_return_null,
+               memory_order_relaxed);
+}
+
+void PrintHintAllocatorCannotReturnNull() {
+  Report("HINT: if you don't care about these errors you may set "
+         "allocator_may_return_null=1\n");
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator.h b/lib/tsan/sanitizer_common/sanitizer_allocator.h
new file mode 100644
index 0000000000..23d589888d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator.h
@@ -0,0 +1,81 @@
+//===-- sanitizer_allocator.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Specialized memory allocator for ThreadSanitizer, MemorySanitizer, etc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_H
+#define SANITIZER_ALLOCATOR_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_lfstack.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_list.h"
+#include "sanitizer_local_address_space_view.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_type_traits.h"
+
+namespace __sanitizer {
+
+// Allows the tools to name their allocations appropriately.
+extern const char *PrimaryAllocatorName;
+extern const char *SecondaryAllocatorName;
+
+// Since flags are immutable and allocator behavior can be changed at runtime
+// (unit tests or ASan on Android are some examples), allocator_may_return_null
+// flag value is cached here and can be altered later.
+bool AllocatorMayReturnNull();
+void SetAllocatorMayReturnNull(bool may_return_null);
+
+// Returns true if allocator detected OOM condition. Can be used to avoid memory
+// hungry operations.
+bool IsAllocatorOutOfMemory();
+// Should be called by a particular allocator when OOM is detected.
+void SetAllocatorOutOfMemory();
+
+void PrintHintAllocatorCannotReturnNull();
+
+// Allocators call these callbacks on mmap/munmap.
+struct NoOpMapUnmapCallback {
+  void OnMap(uptr p, uptr size) const { }
+  void OnUnmap(uptr p, uptr size) const { }
+};
+
+// Callback type for iterating over chunks.
+typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
+
+INLINE u32 Rand(u32 *state) {  // ANSI C linear congruential PRNG.
+  return (*state = *state * 1103515245 + 12345) >> 16;
+}
+
+INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; }  // [0, n)
+
+template<typename T>
+INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) {
+  if (n <= 1) return;
+  u32 state = *rand_state;
+  for (u32 i = n - 1; i > 0; i--)
+    Swap(a[i], a[RandN(&state, i + 1)]);
+  *rand_state = state;
+}
+
+#include "sanitizer_allocator_size_class_map.h"
+#include "sanitizer_allocator_stats.h"
+#include "sanitizer_allocator_primary64.h"
+#include "sanitizer_allocator_bytemap.h"
+#include "sanitizer_allocator_primary32.h"
+#include "sanitizer_allocator_local_cache.h"
+#include "sanitizer_allocator_secondary.h"
+#include "sanitizer_allocator_combined.h"
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ALLOCATOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h b/lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h
new file mode 100644
index 0000000000..0084bb62c8
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h
@@ -0,0 +1,107 @@
+//===-- sanitizer_allocator_bytemap.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// Maps integers in rage [0, kSize) to u8 values.
+template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
+class FlatByteMap {
+ public:
+  using AddressSpaceView = AddressSpaceViewTy;
+  void Init() {
+    internal_memset(map_, 0, sizeof(map_));
+  }
+
+  void set(uptr idx, u8 val) {
+    CHECK_LT(idx, kSize);
+    CHECK_EQ(0U, map_[idx]);
+    map_[idx] = val;
+  }
+  u8 operator[] (uptr idx) {
+    CHECK_LT(idx, kSize);
+    // FIXME: CHECK may be too expensive here.
+    return map_[idx];
+  }
+ private:
+  u8 map_[kSize];
+};
+
+// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values.
+// It is implemented as a two-dimensional array: array of kSize1 pointers
+// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
+// Each value is initially zero and can be set to something else only once.
+// Setting and getting values from multiple threads is safe w/o extra locking.
+template <u64 kSize1, u64 kSize2,
+          typename AddressSpaceViewTy = LocalAddressSpaceView,
+          class MapUnmapCallback = NoOpMapUnmapCallback>
+class TwoLevelByteMap {
+ public:
+  using AddressSpaceView = AddressSpaceViewTy;
+  void Init() {
+    internal_memset(map1_, 0, sizeof(map1_));
+    mu_.Init();
+  }
+
+  void TestOnlyUnmap() {
+    for (uptr i = 0; i < kSize1; i++) {
+      u8 *p = Get(i);
+      if (!p) continue;
+      MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2);
+      UnmapOrDie(p, kSize2);
+    }
+  }
+
+  uptr size() const { return kSize1 * kSize2; }
+  uptr size1() const { return kSize1; }
+  uptr size2() const { return kSize2; }
+
+  void set(uptr idx, u8 val) {
+    CHECK_LT(idx, kSize1 * kSize2);
+    u8 *map2 = GetOrCreate(idx / kSize2);
+    CHECK_EQ(0U, map2[idx % kSize2]);
+    map2[idx % kSize2] = val;
+  }
+
+  u8 operator[] (uptr idx) const {
+    CHECK_LT(idx, kSize1 * kSize2);
+    u8 *map2 = Get(idx / kSize2);
+    if (!map2) return 0;
+    auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]);
+    return *value_ptr;
+  }
+
+ private:
+  u8 *Get(uptr idx) const {
+    CHECK_LT(idx, kSize1);
+    return reinterpret_cast<u8 *>(
+        atomic_load(&map1_[idx], memory_order_acquire));
+  }
+
+  u8 *GetOrCreate(uptr idx) {
+    u8 *res = Get(idx);
+    if (!res) {
+      SpinMutexLock l(&mu_);
+      if (!(res = Get(idx))) {
+        res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap");
+        MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
+        atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
+                     memory_order_release);
+      }
+    }
+    return res;
+  }
+
+  atomic_uintptr_t map1_[kSize1];
+  StaticSpinMutex mu_;
+};
+
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_checks.h b/lib/tsan/sanitizer_common/sanitizer_allocator_checks.h
new file mode 100644
index 0000000000..fc426f0e74
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_checks.h
@@ -0,0 +1,76 @@
+//===-- sanitizer_allocator_checks.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
+// allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_CHECKS_H
+#define SANITIZER_ALLOCATOR_CHECKS_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+
+// The following is defined in a separate compilation unit to avoid pulling in
+// sanitizer_errno.h in this header, which leads to conflicts when other system
+// headers include errno.h. This is usually the result of an unlikely event,
+// and as such we do not care as much about having it inlined.
+void SetErrnoToENOMEM();
+
+// A common errno setting logic shared by almost all sanitizer allocator APIs.
+INLINE void *SetErrnoOnNull(void *ptr) {
+  if (UNLIKELY(!ptr))
+    SetErrnoToENOMEM();
+  return ptr;
+}
+
+// In case of the check failure, the caller of the following Check... functions
+// should "return POLICY::OnBadRequest();" where POLICY is the current allocator
+// failure handling policy.
+
+// Checks aligned_alloc() parameters, verifies that the alignment is a power of
+// two and that the size is a multiple of alignment for POSIX implementation,
+// and a bit relaxed requirement for non-POSIX ones, that the size is a multiple
+// of alignment.
+INLINE bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
+#if SANITIZER_POSIX
+  return alignment != 0 && IsPowerOfTwo(alignment) &&
+         (size & (alignment - 1)) == 0;
+#else
+  return alignment != 0 && size % alignment == 0;
+#endif
+}
+
+// Checks posix_memalign() parameters, verifies that alignment is a power of two
+// and a multiple of sizeof(void *).
+INLINE bool CheckPosixMemalignAlignment(uptr alignment) {
+  return alignment != 0 && IsPowerOfTwo(alignment) &&
+         (alignment % sizeof(void *)) == 0;
+}
+
+// Returns true if calloc(size, n) call overflows on size*n calculation.
+INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
+  if (!size)
+    return false;
+  uptr max = (uptr)-1L;
+  return (max / size) < n;
+}
+
+// Returns true if the size passed to pvalloc overflows when rounded to the next
+// multiple of page_size.
+INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) {
+  return RoundUpTo(size, page_size) < size;
+}
+
+} // namespace __sanitizer
+
+#endif  // SANITIZER_ALLOCATOR_CHECKS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_combined.h b/lib/tsan/sanitizer_common/sanitizer_allocator_combined.h
new file mode 100644
index 0000000000..33f89d6d49
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_combined.h
@@ -0,0 +1,201 @@
+//===-- sanitizer_allocator_combined.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// This class implements a complete memory allocator by using two
+// internal allocators:
+// PrimaryAllocator is efficient, but may not allocate some sizes (alignments).
+//  When allocating 2^x bytes it should return 2^x aligned chunk.
+// PrimaryAllocator is used via a local AllocatorCache.
+// SecondaryAllocator can allocate anything, but is not efficient.
+template <class PrimaryAllocator,
+          class LargeMmapAllocatorPtrArray = DefaultLargeMmapAllocatorPtrArray>
+class CombinedAllocator {
+ public:
+  using AllocatorCache = typename PrimaryAllocator::AllocatorCache;
+  using SecondaryAllocator =
+      LargeMmapAllocator<typename PrimaryAllocator::MapUnmapCallback,
+                         LargeMmapAllocatorPtrArray,
+                         typename PrimaryAllocator::AddressSpaceView>;
+
+  void InitLinkerInitialized(s32 release_to_os_interval_ms) {
+    stats_.InitLinkerInitialized();
+    primary_.Init(release_to_os_interval_ms);
+    secondary_.InitLinkerInitialized();
+  }
+
+  void Init(s32 release_to_os_interval_ms) {
+    stats_.Init();
+    primary_.Init(release_to_os_interval_ms);
+    secondary_.Init();
+  }
+
+  void *Allocate(AllocatorCache *cache, uptr size, uptr alignment) {
+    // Returning 0 on malloc(0) may break a lot of code.
+    if (size == 0)
+      size = 1;
+    if (size + alignment < size) {
+      Report("WARNING: %s: CombinedAllocator allocation overflow: "
+             "0x%zx bytes with 0x%zx alignment requested\n",
+             SanitizerToolName, size, alignment);
+      return nullptr;
+    }
+    uptr original_size = size;
+    // If alignment requirements are to be fulfilled by the frontend allocator
+    // rather than by the primary or secondary, passing an alignment lower than
+    // or equal to 8 will prevent any further rounding up, as well as the later
+    // alignment check.
+    if (alignment > 8)
+      size = RoundUpTo(size, alignment);
+    // The primary allocator should return a 2^x aligned allocation when
+    // requested 2^x bytes, hence using the rounded up 'size' when being
+    // serviced by the primary (this is no longer true when the primary is
+    // using a non-fixed base address). The secondary takes care of the
+    // alignment without such requirement, and allocating 'size' would use
+    // extraneous memory, so we employ 'original_size'.
+    void *res;
+    if (primary_.CanAllocate(size, alignment))
+      res = cache->Allocate(&primary_, primary_.ClassID(size));
+    else
+      res = secondary_.Allocate(&stats_, original_size, alignment);
+    if (alignment > 8)
+      CHECK_EQ(reinterpret_cast<uptr>(res) & (alignment - 1), 0);
+    return res;
+  }
+
+  s32 ReleaseToOSIntervalMs() const {
+    return primary_.ReleaseToOSIntervalMs();
+  }
+
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    primary_.SetReleaseToOSIntervalMs(release_to_os_interval_ms);
+  }
+
+  void ForceReleaseToOS() {
+    primary_.ForceReleaseToOS();
+  }
+
+  void Deallocate(AllocatorCache *cache, void *p) {
+    if (!p) return;
+    if (primary_.PointerIsMine(p))
+      cache->Deallocate(&primary_, primary_.GetSizeClass(p), p);
+    else
+      secondary_.Deallocate(&stats_, p);
+  }
+
+  void *Reallocate(AllocatorCache *cache, void *p, uptr new_size,
+                   uptr alignment) {
+    if (!p)
+      return Allocate(cache, new_size, alignment);
+    if (!new_size) {
+      Deallocate(cache, p);
+      return nullptr;
+    }
+    CHECK(PointerIsMine(p));
+    uptr old_size = GetActuallyAllocatedSize(p);
+    uptr memcpy_size = Min(new_size, old_size);
+    void *new_p = Allocate(cache, new_size, alignment);
+    if (new_p)
+      internal_memcpy(new_p, p, memcpy_size);
+    Deallocate(cache, p);
+    return new_p;
+  }
+
+  bool PointerIsMine(void *p) {
+    if (primary_.PointerIsMine(p))
+      return true;
+    return secondary_.PointerIsMine(p);
+  }
+
+  bool FromPrimary(void *p) {
+    return primary_.PointerIsMine(p);
+  }
+
+  void *GetMetaData(const void *p) {
+    if (primary_.PointerIsMine(p))
+      return primary_.GetMetaData(p);
+    return secondary_.GetMetaData(p);
+  }
+
+  void *GetBlockBegin(const void *p) {
+    if (primary_.PointerIsMine(p))
+      return primary_.GetBlockBegin(p);
+    return secondary_.GetBlockBegin(p);
+  }
+
+  // This function does the same as GetBlockBegin, but is much faster.
+  // Must be called with the allocator locked.
+  void *GetBlockBeginFastLocked(void *p) {
+    if (primary_.PointerIsMine(p))
+      return primary_.GetBlockBegin(p);
+    return secondary_.GetBlockBeginFastLocked(p);
+  }
+
+  uptr GetActuallyAllocatedSize(void *p) {
+    if (primary_.PointerIsMine(p))
+      return primary_.GetActuallyAllocatedSize(p);
+    return secondary_.GetActuallyAllocatedSize(p);
+  }
+
+  uptr TotalMemoryUsed() {
+    return primary_.TotalMemoryUsed() + secondary_.TotalMemoryUsed();
+  }
+
+  void TestOnlyUnmap() { primary_.TestOnlyUnmap(); }
+
+  void InitCache(AllocatorCache *cache) {
+    cache->Init(&stats_);
+  }
+
+  void DestroyCache(AllocatorCache *cache) {
+    cache->Destroy(&primary_, &stats_);
+  }
+
+  void SwallowCache(AllocatorCache *cache) {
+    cache->Drain(&primary_);
+  }
+
+  void GetStats(AllocatorStatCounters s) const {
+    stats_.Get(s);
+  }
+
+  void PrintStats() {
+    primary_.PrintStats();
+    secondary_.PrintStats();
+  }
+
+  // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
+  // introspection API.
+  void ForceLock() {
+    primary_.ForceLock();
+    secondary_.ForceLock();
+  }
+
+  void ForceUnlock() {
+    secondary_.ForceUnlock();
+    primary_.ForceUnlock();
+  }
+
+  // Iterate over all existing chunks.
+  // The allocator must be locked when calling this function.
+  void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+    primary_.ForEachChunk(callback, arg);
+    secondary_.ForEachChunk(callback, arg);
+  }
+
+ private:
+  PrimaryAllocator primary_;
+  SecondaryAllocator secondary_;
+  AllocatorGlobalStats stats_;
+};
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_interface.h b/lib/tsan/sanitizer_common/sanitizer_allocator_interface.h
new file mode 100644
index 0000000000..c1b27563e2
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_interface.h
@@ -0,0 +1,47 @@
+//===-- sanitizer_allocator_interface.h ------------------------- C++ -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Re-declaration of functions from public sanitizer allocator interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_INTERFACE_H
+#define SANITIZER_ALLOCATOR_INTERFACE_H
+
+#include "sanitizer_internal_defs.h"
+
+using __sanitizer::uptr;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __sanitizer_get_estimated_allocated_size(uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_get_ownership(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE uptr
+__sanitizer_get_allocated_size(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes();
+SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size();
+SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes();
+SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_unmapped_bytes();
+
+SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_install_malloc_and_free_hooks(
+    void (*malloc_hook)(const void *, uptr),
+    void (*free_hook)(const void *));
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+    void __sanitizer_malloc_hook(void *ptr, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+    void __sanitizer_free_hook(void *ptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_purge_allocator();
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts);
+}  // extern "C"
+
+#endif  // SANITIZER_ALLOCATOR_INTERFACE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_internal.h b/lib/tsan/sanitizer_common/sanitizer_allocator_internal.h
new file mode 100644
index 0000000000..32849036fd
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_internal.h
@@ -0,0 +1,55 @@
+//===-- sanitizer_allocator_internal.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This allocator is used inside run-times.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_INTERNAL_H
+#define SANITIZER_ALLOCATOR_INTERNAL_H
+
+#include "sanitizer_allocator.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+// FIXME: Check if we may use even more compact size class map for internal
+// purposes.
+typedef CompactSizeClassMap InternalSizeClassMap;
+
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 0;
+  typedef InternalSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = 20;
+  using AddressSpaceView = LocalAddressSpaceView;
+  typedef NoOpMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryInternalAllocator;
+
+typedef CombinedAllocator<PrimaryInternalAllocator,
+                          LargeMmapAllocatorPtrArrayStatic>
+    InternalAllocator;
+typedef InternalAllocator::AllocatorCache InternalAllocatorCache;
+
+void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr,
+                    uptr alignment = 0);
+void *InternalRealloc(void *p, uptr size,
+                      InternalAllocatorCache *cache = nullptr);
+void *InternalReallocArray(void *p, uptr count, uptr size,
+                           InternalAllocatorCache *cache = nullptr);
+void *InternalCalloc(uptr count, uptr size,
+                     InternalAllocatorCache *cache = nullptr);
+void InternalFree(void *p, InternalAllocatorCache *cache = nullptr);
+InternalAllocator *internal_allocator();
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ALLOCATOR_INTERNAL_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_local_cache.h b/lib/tsan/sanitizer_common/sanitizer_allocator_local_cache.h
new file mode 100644
index 0000000000..108dfc231a
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_local_cache.h
@@ -0,0 +1,264 @@
+//===-- sanitizer_allocator_local_cache.h -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// Cache used by SizeClassAllocator64.
+template <class SizeClassAllocator>
+struct SizeClassAllocator64LocalCache {
+  typedef SizeClassAllocator Allocator;
+
+  void Init(AllocatorGlobalStats *s) {
+    stats_.Init();
+    if (s)
+      s->Register(&stats_);
+  }
+
+  void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
+    Drain(allocator);
+    if (s)
+      s->Unregister(&stats_);
+  }
+
+  void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
+    CHECK_NE(class_id, 0UL);
+    CHECK_LT(class_id, kNumClasses);
+    PerClass *c = &per_class_[class_id];
+    if (UNLIKELY(c->count == 0)) {
+      if (UNLIKELY(!Refill(c, allocator, class_id)))
+        return nullptr;
+      DCHECK_GT(c->count, 0);
+    }
+    CompactPtrT chunk = c->chunks[--c->count];
+    stats_.Add(AllocatorStatAllocated, c->class_size);
+    return reinterpret_cast<void *>(allocator->CompactPtrToPointer(
+        allocator->GetRegionBeginBySizeClass(class_id), chunk));
+  }
+
+  void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) {
+    CHECK_NE(class_id, 0UL);
+    CHECK_LT(class_id, kNumClasses);
+    // If the first allocator call on a new thread is a deallocation, then
+    // max_count will be zero, leading to check failure.
+    PerClass *c = &per_class_[class_id];
+    InitCache(c);
+    if (UNLIKELY(c->count == c->max_count))
+      Drain(c, allocator, class_id, c->max_count / 2);
+    CompactPtrT chunk = allocator->PointerToCompactPtr(
+        allocator->GetRegionBeginBySizeClass(class_id),
+        reinterpret_cast<uptr>(p));
+    c->chunks[c->count++] = chunk;
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
+  }
+
+  void Drain(SizeClassAllocator *allocator) {
+    for (uptr i = 1; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
+      while (c->count > 0)
+        Drain(c, allocator, i, c->count);
+    }
+  }
+
+ private:
+  typedef typename Allocator::SizeClassMapT SizeClassMap;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+  typedef typename Allocator::CompactPtrT CompactPtrT;
+
+  struct PerClass {
+    u32 count;
+    u32 max_count;
+    uptr class_size;
+    CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint];
+  };
+  PerClass per_class_[kNumClasses];
+  AllocatorStats stats_;
+
+  void InitCache(PerClass *c) {
+    if (LIKELY(c->max_count))
+      return;
+    for (uptr i = 1; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
+      const uptr size = Allocator::ClassIdToSize(i);
+      c->max_count = 2 * SizeClassMap::MaxCachedHint(size);
+      c->class_size = size;
+    }
+    DCHECK_NE(c->max_count, 0UL);
+  }
+
+  NOINLINE bool Refill(PerClass *c, SizeClassAllocator *allocator,
+                       uptr class_id) {
+    InitCache(c);
+    const uptr num_requested_chunks = c->max_count / 2;
+    if (UNLIKELY(!allocator->GetFromAllocator(&stats_, class_id, c->chunks,
+                                              num_requested_chunks)))
+      return false;
+    c->count = num_requested_chunks;
+    return true;
+  }
+
+  NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id,
+                      uptr count) {
+    CHECK_GE(c->count, count);
+    const uptr first_idx_to_drain = c->count - count;
+    c->count -= count;
+    allocator->ReturnToAllocator(&stats_, class_id,
+                                 &c->chunks[first_idx_to_drain], count);
+  }
+};
+
+// Cache used by SizeClassAllocator32.
+template <class SizeClassAllocator>
+struct SizeClassAllocator32LocalCache {
+  typedef SizeClassAllocator Allocator;
+  typedef typename Allocator::TransferBatch TransferBatch;
+
+  void Init(AllocatorGlobalStats *s) {
+    stats_.Init();
+    if (s)
+      s->Register(&stats_);
+  }
+
+  // Returns a TransferBatch suitable for class_id.
+  TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
+                             TransferBatch *b) {
+    if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+      return (TransferBatch*)Allocate(allocator, batch_class_id);
+    return b;
+  }
+
+  // Destroys TransferBatch b.
+  void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
+                    TransferBatch *b) {
+    if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+      Deallocate(allocator, batch_class_id, b);
+  }
+
+  void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
+    Drain(allocator);
+    if (s)
+      s->Unregister(&stats_);
+  }
+
+  void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
+    CHECK_NE(class_id, 0UL);
+    CHECK_LT(class_id, kNumClasses);
+    PerClass *c = &per_class_[class_id];
+    if (UNLIKELY(c->count == 0)) {
+      if (UNLIKELY(!Refill(c, allocator, class_id)))
+        return nullptr;
+      DCHECK_GT(c->count, 0);
+    }
+    void *res = c->batch[--c->count];
+    PREFETCH(c->batch[c->count - 1]);
+    stats_.Add(AllocatorStatAllocated, c->class_size);
+    return res;
+  }
+
+  void Deallocate(SizeClassAllocator *allocator, uptr class_id, void *p) {
+    CHECK_NE(class_id, 0UL);
+    CHECK_LT(class_id, kNumClasses);
+    // If the first allocator call on a new thread is a deallocation, then
+    // max_count will be zero, leading to check failure.
+    PerClass *c = &per_class_[class_id];
+    InitCache(c);
+    if (UNLIKELY(c->count == c->max_count))
+      Drain(c, allocator, class_id);
+    c->batch[c->count++] = p;
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
+  }
+
+  void Drain(SizeClassAllocator *allocator) {
+    for (uptr i = 1; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
+      while (c->count > 0)
+        Drain(c, allocator, i);
+    }
+  }
+
+ private:
+  typedef typename Allocator::SizeClassMapT SizeClassMap;
+  static const uptr kBatchClassID = SizeClassMap::kBatchClassID;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+  // If kUseSeparateSizeClassForBatch is true, all TransferBatch objects are
+  // allocated from kBatchClassID size class (except for those that are needed
+  // for kBatchClassID itself). The goal is to have TransferBatches in a totally
+  // different region of RAM to improve security.
+  static const bool kUseSeparateSizeClassForBatch =
+      Allocator::kUseSeparateSizeClassForBatch;
+
+  struct PerClass {
+    uptr count;
+    uptr max_count;
+    uptr class_size;
+    uptr batch_class_id;
+    void *batch[2 * TransferBatch::kMaxNumCached];
+  };
+  PerClass per_class_[kNumClasses];
+  AllocatorStats stats_;
+
+  void InitCache(PerClass *c) {
+    if (LIKELY(c->max_count))
+      return;
+    const uptr batch_class_id = SizeClassMap::ClassID(sizeof(TransferBatch));
+    for (uptr i = 1; i < kNumClasses; i++) {
+      PerClass *c = &per_class_[i];
+      const uptr size = Allocator::ClassIdToSize(i);
+      const uptr max_cached = TransferBatch::MaxCached(size);
+      c->max_count = 2 * max_cached;
+      c->class_size = size;
+      // Precompute the class id to use to store batches for the current class
+      // id. 0 means the class size is large enough to store a batch within one
+      // of the chunks. If using a separate size class, it will always be
+      // kBatchClassID, except for kBatchClassID itself.
+      if (kUseSeparateSizeClassForBatch) {
+        c->batch_class_id = (i == kBatchClassID) ? 0 : kBatchClassID;
+      } else {
+        c->batch_class_id = (size <
+          TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ?
+              batch_class_id : 0;
+      }
+    }
+    DCHECK_NE(c->max_count, 0UL);
+  }
+
+  NOINLINE bool Refill(PerClass *c, SizeClassAllocator *allocator,
+                       uptr class_id) {
+    InitCache(c);
+    TransferBatch *b = allocator->AllocateBatch(&stats_, this, class_id);
+    if (UNLIKELY(!b))
+      return false;
+    CHECK_GT(b->Count(), 0);
+    b->CopyToArray(c->batch);
+    c->count = b->Count();
+    DestroyBatch(class_id, allocator, b);
+    return true;
+  }
+
+  NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator,
+                      uptr class_id) {
+    const uptr count = Min(c->max_count / 2, c->count);
+    const uptr first_idx_to_drain = c->count - count;
+    TransferBatch *b = CreateBatch(
+        class_id, allocator, (TransferBatch *)c->batch[first_idx_to_drain]);
+    // Failure to allocate a batch while releasing memory is non recoverable.
+    // TODO(alekseys): Figure out how to do it without allocating a new batch.
+    if (UNLIKELY(!b)) {
+      Report("FATAL: Internal error: %s's allocator failed to allocate a "
+             "transfer batch.\n", SanitizerToolName);
+      Die();
+    }
+    b->SetFromArray(&c->batch[first_idx_to_drain], count);
+    c->count -= count;
+    allocator->DeallocateBatch(&stats_, class_id, b);
+  }
+};
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h b/lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h
new file mode 100644
index 0000000000..3b1838b398
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h
@@ -0,0 +1,380 @@
+//===-- sanitizer_allocator_primary32.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
+
+// SizeClassAllocator32 -- allocator for 32-bit address space.
+// This allocator can theoretically be used on 64-bit arch, but there it is less
+// efficient than SizeClassAllocator64.
+//
+// [kSpaceBeg, kSpaceBeg + kSpaceSize) is the range of addresses which can
+// be returned by MmapOrDie().
+//
+// Region:
+//   a result of a single call to MmapAlignedOrDieOnFatalError(kRegionSize,
+//                                                             kRegionSize).
+// Since the regions are aligned by kRegionSize, there are exactly
+// kNumPossibleRegions possible regions in the address space and so we keep
+// a ByteMap possible_regions to store the size classes of each Region.
+// 0 size class means the region is not used by the allocator.
+//
+// One Region is used to allocate chunks of a single size class.
+// A Region looks like this:
+// UserChunk1 .. UserChunkN <gap> MetaChunkN .. MetaChunk1
+//
+// In order to avoid false sharing the objects of this class should be
+// chache-line aligned.
+
+struct SizeClassAllocator32FlagMasks {  //  Bit masks.
+  enum {
+    kRandomShuffleChunks = 1,
+    kUseSeparateSizeClassForBatch = 2,
+  };
+};
+
+template <class Params>
+class SizeClassAllocator32 {
+ private:
+  static const u64 kTwoLevelByteMapSize1 =
+      (Params::kSpaceSize >> Params::kRegionSizeLog) >> 12;
+  static const u64 kMinFirstMapSizeTwoLevelByteMap = 4;
+
+ public:
+  using AddressSpaceView = typename Params::AddressSpaceView;
+  static const uptr kSpaceBeg = Params::kSpaceBeg;
+  static const u64 kSpaceSize = Params::kSpaceSize;
+  static const uptr kMetadataSize = Params::kMetadataSize;
+  typedef typename Params::SizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = Params::kRegionSizeLog;
+  typedef typename Params::MapUnmapCallback MapUnmapCallback;
+  using ByteMap = typename conditional<
+      (kTwoLevelByteMapSize1 < kMinFirstMapSizeTwoLevelByteMap),
+      FlatByteMap<(Params::kSpaceSize >> Params::kRegionSizeLog),
+                  AddressSpaceView>,
+      TwoLevelByteMap<kTwoLevelByteMapSize1, 1 << 12, AddressSpaceView>>::type;
+
+  COMPILER_CHECK(!SANITIZER_SIGN_EXTENDED_ADDRESSES ||
+                 (kSpaceSize & (kSpaceSize - 1)) == 0);
+
+  static const bool kRandomShuffleChunks = Params::kFlags &
+      SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
+  static const bool kUseSeparateSizeClassForBatch = Params::kFlags &
+      SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
+
+  struct TransferBatch {
+    static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2;
+    void SetFromArray(void *batch[], uptr count) {
+      DCHECK_LE(count, kMaxNumCached);
+      count_ = count;
+      for (uptr i = 0; i < count; i++)
+        batch_[i] = batch[i];
+    }
+    uptr Count() const { return count_; }
+    void Clear() { count_ = 0; }
+    void Add(void *ptr) {
+      batch_[count_++] = ptr;
+      DCHECK_LE(count_, kMaxNumCached);
+    }
+    void CopyToArray(void *to_batch[]) const {
+      for (uptr i = 0, n = Count(); i < n; i++)
+        to_batch[i] = batch_[i];
+    }
+
+    // How much memory do we need for a batch containing n elements.
+    static uptr AllocationSizeRequiredForNElements(uptr n) {
+      return sizeof(uptr) * 2 + sizeof(void *) * n;
+    }
+    static uptr MaxCached(uptr size) {
+      return Min(kMaxNumCached, SizeClassMap::MaxCachedHint(size));
+    }
+
+    TransferBatch *next;
+
+   private:
+    uptr count_;
+    void *batch_[kMaxNumCached];
+  };
+
+  static const uptr kBatchSize = sizeof(TransferBatch);
+  COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0);
+  COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
+
+  static uptr ClassIdToSize(uptr class_id) {
+    return (class_id == SizeClassMap::kBatchClassID) ?
+        kBatchSize : SizeClassMap::Size(class_id);
+  }
+
+  typedef SizeClassAllocator32<Params> ThisT;
+  typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache;
+
+  void Init(s32 release_to_os_interval_ms) {
+    possible_regions.Init();
+    internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
+  }
+
+  s32 ReleaseToOSIntervalMs() const {
+    return kReleaseToOSIntervalNever;
+  }
+
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    // This is empty here. Currently only implemented in 64-bit allocator.
+  }
+
+  void ForceReleaseToOS() {
+    // Currently implemented in 64-bit allocator only.
+  }
+
+  void *MapWithCallback(uptr size) {
+    void *res = MmapOrDie(size, PrimaryAllocatorName);
+    MapUnmapCallback().OnMap((uptr)res, size);
+    return res;
+  }
+
+  void UnmapWithCallback(uptr beg, uptr size) {
+    MapUnmapCallback().OnUnmap(beg, size);
+    UnmapOrDie(reinterpret_cast<void *>(beg), size);
+  }
+
+  static bool CanAllocate(uptr size, uptr alignment) {
+    return size <= SizeClassMap::kMaxSize &&
+      alignment <= SizeClassMap::kMaxSize;
+  }
+
+  void *GetMetaData(const void *p) {
+    CHECK(PointerIsMine(p));
+    uptr mem = reinterpret_cast<uptr>(p);
+    uptr beg = ComputeRegionBeg(mem);
+    uptr size = ClassIdToSize(GetSizeClass(p));
+    u32 offset = mem - beg;
+    uptr n = offset / (u32)size;  // 32-bit division
+    uptr meta = (beg + kRegionSize) - (n + 1) * kMetadataSize;
+    return reinterpret_cast<void*>(meta);
+  }
+
+  NOINLINE TransferBatch *AllocateBatch(AllocatorStats *stat, AllocatorCache *c,
+                                        uptr class_id) {
+    DCHECK_LT(class_id, kNumClasses);
+    SizeClassInfo *sci = GetSizeClassInfo(class_id);
+    SpinMutexLock l(&sci->mutex);
+    if (sci->free_list.empty()) {
+      if (UNLIKELY(!PopulateFreeList(stat, c, sci, class_id)))
+        return nullptr;
+      DCHECK(!sci->free_list.empty());
+    }
+    TransferBatch *b = sci->free_list.front();
+    sci->free_list.pop_front();
+    return b;
+  }
+
+  NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id,
+                                TransferBatch *b) {
+    DCHECK_LT(class_id, kNumClasses);
+    CHECK_GT(b->Count(), 0);
+    SizeClassInfo *sci = GetSizeClassInfo(class_id);
+    SpinMutexLock l(&sci->mutex);
+    sci->free_list.push_front(b);
+  }
+
+  bool PointerIsMine(const void *p) {
+    uptr mem = reinterpret_cast<uptr>(p);
+    if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
+      mem &= (kSpaceSize - 1);
+    if (mem < kSpaceBeg || mem >= kSpaceBeg + kSpaceSize)
+      return false;
+    return GetSizeClass(p) != 0;
+  }
+
+  uptr GetSizeClass(const void *p) {
+    return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];
+  }
+
+  void *GetBlockBegin(const void *p) {
+    CHECK(PointerIsMine(p));
+    uptr mem = reinterpret_cast<uptr>(p);
+    uptr beg = ComputeRegionBeg(mem);
+    uptr size = ClassIdToSize(GetSizeClass(p));
+    u32 offset = mem - beg;
+    u32 n = offset / (u32)size;  // 32-bit division
+    uptr res = beg + (n * (u32)size);
+    return reinterpret_cast<void*>(res);
+  }
+
+  uptr GetActuallyAllocatedSize(void *p) {
+    CHECK(PointerIsMine(p));
+    return ClassIdToSize(GetSizeClass(p));
+  }
+
+  static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); }
+
+  uptr TotalMemoryUsed() {
+    // No need to lock here.
+    uptr res = 0;
+    for (uptr i = 0; i < kNumPossibleRegions; i++)
+      if (possible_regions[i])
+        res += kRegionSize;
+    return res;
+  }
+
+  void TestOnlyUnmap() {
+    for (uptr i = 0; i < kNumPossibleRegions; i++)
+      if (possible_regions[i])
+        UnmapWithCallback((i * kRegionSize), kRegionSize);
+  }
+
+  // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
+  // introspection API.
+  void ForceLock() {
+    for (uptr i = 0; i < kNumClasses; i++) {
+      GetSizeClassInfo(i)->mutex.Lock();
+    }
+  }
+
+  void ForceUnlock() {
+    for (int i = kNumClasses - 1; i >= 0; i--) {
+      GetSizeClassInfo(i)->mutex.Unlock();
+    }
+  }
+
+  // Iterate over all existing chunks.
+  // The allocator must be locked when calling this function.
+  void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+    for (uptr region = 0; region < kNumPossibleRegions; region++)
+      if (possible_regions[region]) {
+        uptr chunk_size = ClassIdToSize(possible_regions[region]);
+        uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
+        uptr region_beg = region * kRegionSize;
+        for (uptr chunk = region_beg;
+             chunk < region_beg + max_chunks_in_region * chunk_size;
+             chunk += chunk_size) {
+          // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk));
+          callback(chunk, arg);
+        }
+      }
+  }
+
+  void PrintStats() {}
+
+  static uptr AdditionalSize() { return 0; }
+
+  typedef SizeClassMap SizeClassMapT;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+
+ private:
+  static const uptr kRegionSize = 1 << kRegionSizeLog;
+  static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize;
+
+  struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {
+    StaticSpinMutex mutex;
+    IntrusiveList<TransferBatch> free_list;
+    u32 rand_state;
+  };
+  COMPILER_CHECK(sizeof(SizeClassInfo) % kCacheLineSize == 0);
+
+  uptr ComputeRegionId(uptr mem) const {
+    if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
+      mem &= (kSpaceSize - 1);
+    const uptr res = mem >> kRegionSizeLog;
+    CHECK_LT(res, kNumPossibleRegions);
+    return res;
+  }
+
+  uptr ComputeRegionBeg(uptr mem) {
+    return mem & ~(kRegionSize - 1);
+  }
+
+  uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
+    DCHECK_LT(class_id, kNumClasses);
+    const uptr res = reinterpret_cast<uptr>(MmapAlignedOrDieOnFatalError(
+        kRegionSize, kRegionSize, PrimaryAllocatorName));
+    if (UNLIKELY(!res))
+      return 0;
+    MapUnmapCallback().OnMap(res, kRegionSize);
+    stat->Add(AllocatorStatMapped, kRegionSize);
+    CHECK(IsAligned(res, kRegionSize));
+    possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
+    return res;
+  }
+
+  SizeClassInfo *GetSizeClassInfo(uptr class_id) {
+    DCHECK_LT(class_id, kNumClasses);
+    return &size_class_info_array[class_id];
+  }
+
+  bool PopulateBatches(AllocatorCache *c, SizeClassInfo *sci, uptr class_id,
+                       TransferBatch **current_batch, uptr max_count,
+                       uptr *pointers_array, uptr count) {
+    // If using a separate class for batches, we do not need to shuffle it.
+    if (kRandomShuffleChunks && (!kUseSeparateSizeClassForBatch ||
+        class_id != SizeClassMap::kBatchClassID))
+      RandomShuffle(pointers_array, count, &sci->rand_state);
+    TransferBatch *b = *current_batch;
+    for (uptr i = 0; i < count; i++) {
+      if (!b) {
+        b = c->CreateBatch(class_id, this, (TransferBatch*)pointers_array[i]);
+        if (UNLIKELY(!b))
+          return false;
+        b->Clear();
+      }
+      b->Add((void*)pointers_array[i]);
+      if (b->Count() == max_count) {
+        sci->free_list.push_back(b);
+        b = nullptr;
+      }
+    }
+    *current_batch = b;
+    return true;
+  }
+
+  bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
+                        SizeClassInfo *sci, uptr class_id) {
+    const uptr region = AllocateRegion(stat, class_id);
+    if (UNLIKELY(!region))
+      return false;
+    if (kRandomShuffleChunks)
+      if (UNLIKELY(sci->rand_state == 0))
+        // The random state is initialized from ASLR (PIE) and time.
+        sci->rand_state = reinterpret_cast<uptr>(sci) ^ NanoTime();
+    const uptr size = ClassIdToSize(class_id);
+    const uptr n_chunks = kRegionSize / (size + kMetadataSize);
+    const uptr max_count = TransferBatch::MaxCached(size);
+    DCHECK_GT(max_count, 0);
+    TransferBatch *b = nullptr;
+    constexpr uptr kShuffleArraySize = 48;
+    uptr shuffle_array[kShuffleArraySize];
+    uptr count = 0;
+    for (uptr i = region; i < region + n_chunks * size; i += size) {
+      shuffle_array[count++] = i;
+      if (count == kShuffleArraySize) {
+        if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
+                                      shuffle_array, count)))
+          return false;
+        count = 0;
+      }
+    }
+    if (count) {
+      if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
+                                    shuffle_array, count)))
+        return false;
+    }
+    if (b) {
+      CHECK_GT(b->Count(), 0);
+      sci->free_list.push_back(b);
+    }
+    return true;
+  }
+
+  ByteMap possible_regions;
+  SizeClassInfo size_class_info_array[kNumClasses];
+};
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h b/lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h
new file mode 100644
index 0000000000..1d9a29c70f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h
@@ -0,0 +1,863 @@
+//===-- sanitizer_allocator_primary64.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+template<class SizeClassAllocator> struct SizeClassAllocator64LocalCache;
+
+// SizeClassAllocator64 -- allocator for 64-bit address space.
+// The template parameter Params is a class containing the actual parameters.
+//
+// Space: a portion of address space of kSpaceSize bytes starting at SpaceBeg.
+// If kSpaceBeg is ~0 then SpaceBeg is chosen dynamically my mmap.
+// Otherwise SpaceBeg=kSpaceBeg (fixed address).
+// kSpaceSize is a power of two.
+// At the beginning the entire space is mprotect-ed, then small parts of it
+// are mapped on demand.
+//
+// Region: a part of Space dedicated to a single size class.
+// There are kNumClasses Regions of equal size.
+//
+// UserChunk: a piece of memory returned to user.
+// MetaChunk: kMetadataSize bytes of metadata associated with a UserChunk.
+
+// FreeArray is an array free-d chunks (stored as 4-byte offsets)
+//
+// A Region looks like this:
+// UserChunk1 ... UserChunkN <gap> MetaChunkN ... MetaChunk1 FreeArray
+
+struct SizeClassAllocator64FlagMasks {  //  Bit masks.
+  enum {
+    kRandomShuffleChunks = 1,
+  };
+};
+
+template <class Params>
+class SizeClassAllocator64 {
+ public:
+  using AddressSpaceView = typename Params::AddressSpaceView;
+  static const uptr kSpaceBeg = Params::kSpaceBeg;
+  static const uptr kSpaceSize = Params::kSpaceSize;
+  static const uptr kMetadataSize = Params::kMetadataSize;
+  typedef typename Params::SizeClassMap SizeClassMap;
+  typedef typename Params::MapUnmapCallback MapUnmapCallback;
+
+  static const bool kRandomShuffleChunks =
+      Params::kFlags & SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
+
+  typedef SizeClassAllocator64<Params> ThisT;
+  typedef SizeClassAllocator64LocalCache<ThisT> AllocatorCache;
+
+  // When we know the size class (the region base) we can represent a pointer
+  // as a 4-byte integer (offset from the region start shifted right by 4).
+  typedef u32 CompactPtrT;
+  static const uptr kCompactPtrScale = 4;
+  CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const {
+    return static_cast<CompactPtrT>((ptr - base) >> kCompactPtrScale);
+  }
+  uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const {
+    return base + (static_cast<uptr>(ptr32) << kCompactPtrScale);
+  }
+
+  void Init(s32 release_to_os_interval_ms) {
+    uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
+    if (kUsingConstantSpaceBeg) {
+      CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
+      CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize,
+                                             PrimaryAllocatorName, kSpaceBeg));
+    } else {
+      // Combined allocator expects that an 2^N allocation is always aligned to
+      // 2^N. For this to work, the start of the space needs to be aligned as
+      // high as the largest size class (which also needs to be a power of 2).
+      NonConstSpaceBeg = address_range.InitAligned(
+          TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
+      CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+    }
+    SetReleaseToOSIntervalMs(release_to_os_interval_ms);
+    MapWithCallbackOrDie(SpaceEnd(), AdditionalSize(),
+                         "SizeClassAllocator: region info");
+    // Check that the RegionInfo array is aligned on the CacheLine size.
+    DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0);
+  }
+
+  s32 ReleaseToOSIntervalMs() const {
+    return atomic_load(&release_to_os_interval_ms_, memory_order_relaxed);
+  }
+
+  void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
+    atomic_store(&release_to_os_interval_ms_, release_to_os_interval_ms,
+                 memory_order_relaxed);
+  }
+
+  void ForceReleaseToOS() {
+    for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
+      BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
+      MaybeReleaseToOS(class_id, true /*force*/);
+    }
+  }
+
+  static bool CanAllocate(uptr size, uptr alignment) {
+    return size <= SizeClassMap::kMaxSize &&
+      alignment <= SizeClassMap::kMaxSize;
+  }
+
+  NOINLINE void ReturnToAllocator(AllocatorStats *stat, uptr class_id,
+                                  const CompactPtrT *chunks, uptr n_chunks) {
+    RegionInfo *region = GetRegionInfo(class_id);
+    uptr region_beg = GetRegionBeginBySizeClass(class_id);
+    CompactPtrT *free_array = GetFreeArray(region_beg);
+
+    BlockingMutexLock l(&region->mutex);
+    uptr old_num_chunks = region->num_freed_chunks;
+    uptr new_num_freed_chunks = old_num_chunks + n_chunks;
+    // Failure to allocate free array space while releasing memory is non
+    // recoverable.
+    if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg,
+                                       new_num_freed_chunks))) {
+      Report("FATAL: Internal error: %s's allocator exhausted the free list "
+             "space for size class %zd (%zd bytes).\n", SanitizerToolName,
+             class_id, ClassIdToSize(class_id));
+      Die();
+    }
+    for (uptr i = 0; i < n_chunks; i++)
+      free_array[old_num_chunks + i] = chunks[i];
+    region->num_freed_chunks = new_num_freed_chunks;
+    region->stats.n_freed += n_chunks;
+
+    MaybeReleaseToOS(class_id, false /*force*/);
+  }
+
+  NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id,
+                                 CompactPtrT *chunks, uptr n_chunks) {
+    RegionInfo *region = GetRegionInfo(class_id);
+    uptr region_beg = GetRegionBeginBySizeClass(class_id);
+    CompactPtrT *free_array = GetFreeArray(region_beg);
+
+    BlockingMutexLock l(&region->mutex);
+    if (UNLIKELY(region->num_freed_chunks < n_chunks)) {
+      if (UNLIKELY(!PopulateFreeArray(stat, class_id, region,
+                                      n_chunks - region->num_freed_chunks)))
+        return false;
+      CHECK_GE(region->num_freed_chunks, n_chunks);
+    }
+    region->num_freed_chunks -= n_chunks;
+    uptr base_idx = region->num_freed_chunks;
+    for (uptr i = 0; i < n_chunks; i++)
+      chunks[i] = free_array[base_idx + i];
+    region->stats.n_allocated += n_chunks;
+    return true;
+  }
+
+  bool PointerIsMine(const void *p) const {
+    uptr P = reinterpret_cast<uptr>(p);
+    if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0)
+      return P / kSpaceSize == kSpaceBeg / kSpaceSize;
+    return P >= SpaceBeg() && P < SpaceEnd();
+  }
+
+  uptr GetRegionBegin(const void *p) {
+    if (kUsingConstantSpaceBeg)
+      return reinterpret_cast<uptr>(p) & ~(kRegionSize - 1);
+    uptr space_beg = SpaceBeg();
+    return ((reinterpret_cast<uptr>(p)  - space_beg) & ~(kRegionSize - 1)) +
+        space_beg;
+  }
+
+  uptr GetRegionBeginBySizeClass(uptr class_id) const {
+    return SpaceBeg() + kRegionSize * class_id;
+  }
+
+  uptr GetSizeClass(const void *p) {
+    if (kUsingConstantSpaceBeg && (kSpaceBeg % kSpaceSize) == 0)
+      return ((reinterpret_cast<uptr>(p)) / kRegionSize) % kNumClassesRounded;
+    return ((reinterpret_cast<uptr>(p) - SpaceBeg()) / kRegionSize) %
+           kNumClassesRounded;
+  }
+
+  void *GetBlockBegin(const void *p) {
+    uptr class_id = GetSizeClass(p);
+    uptr size = ClassIdToSize(class_id);
+    if (!size) return nullptr;
+    uptr chunk_idx = GetChunkIdx((uptr)p, size);
+    uptr reg_beg = GetRegionBegin(p);
+    uptr beg = chunk_idx * size;
+    uptr next_beg = beg + size;
+    if (class_id >= kNumClasses) return nullptr;
+    const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id));
+    if (region->mapped_user >= next_beg)
+      return reinterpret_cast<void*>(reg_beg + beg);
+    return nullptr;
+  }
+
+  uptr GetActuallyAllocatedSize(void *p) {
+    CHECK(PointerIsMine(p));
+    return ClassIdToSize(GetSizeClass(p));
+  }
+
+  static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); }
+
+  void *GetMetaData(const void *p) {
+    uptr class_id = GetSizeClass(p);
+    uptr size = ClassIdToSize(class_id);
+    uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);
+    uptr region_beg = GetRegionBeginBySizeClass(class_id);
+    return reinterpret_cast<void *>(GetMetadataEnd(region_beg) -
+                                    (1 + chunk_idx) * kMetadataSize);
+  }
+
+  uptr TotalMemoryUsed() {
+    uptr res = 0;
+    for (uptr i = 0; i < kNumClasses; i++)
+      res += GetRegionInfo(i)->allocated_user;
+    return res;
+  }
+
+  // Test-only.
+  void TestOnlyUnmap() {
+    UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
+  }
+
+  static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
+                           uptr stats_size) {
+    for (uptr class_id = 0; class_id < stats_size; class_id++)
+      if (stats[class_id] == start)
+        stats[class_id] = rss;
+  }
+
+  void PrintStats(uptr class_id, uptr rss) {
+    RegionInfo *region = GetRegionInfo(class_id);
+    if (region->mapped_user == 0) return;
+    uptr in_use = region->stats.n_allocated - region->stats.n_freed;
+    uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id);
+    Printf(
+        "%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
+        "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
+        "last released: %6zdK region: 0x%zx\n",
+        region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
+        region->mapped_user >> 10, region->stats.n_allocated,
+        region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
+        rss >> 10, region->rtoi.num_releases,
+        region->rtoi.last_released_bytes >> 10,
+        SpaceBeg() + kRegionSize * class_id);
+  }
+
+  void PrintStats() {
+    uptr rss_stats[kNumClasses];
+    for (uptr class_id = 0; class_id < kNumClasses; class_id++)
+      rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
+    GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
+
+    uptr total_mapped = 0;
+    uptr total_rss = 0;
+    uptr n_allocated = 0;
+    uptr n_freed = 0;
+    for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
+      RegionInfo *region = GetRegionInfo(class_id);
+      if (region->mapped_user != 0) {
+        total_mapped += region->mapped_user;
+        total_rss += rss_stats[class_id];
+      }
+      n_allocated += region->stats.n_allocated;
+      n_freed += region->stats.n_freed;
+    }
+
+    Printf("Stats: SizeClassAllocator64: %zdM mapped (%zdM rss) in "
+           "%zd allocations; remains %zd\n", total_mapped >> 20,
+           total_rss >> 20, n_allocated, n_allocated - n_freed);
+    for (uptr class_id = 1; class_id < kNumClasses; class_id++)
+      PrintStats(class_id, rss_stats[class_id]);
+  }
+
+  // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
+  // introspection API.
+  void ForceLock() {
+    for (uptr i = 0; i < kNumClasses; i++) {
+      GetRegionInfo(i)->mutex.Lock();
+    }
+  }
+
+  void ForceUnlock() {
+    for (int i = (int)kNumClasses - 1; i >= 0; i--) {
+      GetRegionInfo(i)->mutex.Unlock();
+    }
+  }
+
+  // Iterate over all existing chunks.
+  // The allocator must be locked when calling this function.
+  void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+    for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
+      RegionInfo *region = GetRegionInfo(class_id);
+      uptr chunk_size = ClassIdToSize(class_id);
+      uptr region_beg = SpaceBeg() + class_id * kRegionSize;
+      uptr region_allocated_user_size =
+          AddressSpaceView::Load(region)->allocated_user;
+      for (uptr chunk = region_beg;
+           chunk < region_beg + region_allocated_user_size;
+           chunk += chunk_size) {
+        // Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk));
+        callback(chunk, arg);
+      }
+    }
+  }
+
+  static uptr ClassIdToSize(uptr class_id) {
+    return SizeClassMap::Size(class_id);
+  }
+
+  static uptr AdditionalSize() {
+    return RoundUpTo(sizeof(RegionInfo) * kNumClassesRounded,
+                     GetPageSizeCached());
+  }
+
+  typedef SizeClassMap SizeClassMapT;
+  static const uptr kNumClasses = SizeClassMap::kNumClasses;
+  static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded;
+
+  // A packed array of counters. Each counter occupies 2^n bits, enough to store
+  // counter's max_value. Ctor will try to allocate the required buffer via
+  // mapper->MapPackedCounterArrayBuffer and the caller is expected to check
+  // whether the initialization was successful by checking IsAllocated() result.
+  // For the performance sake, none of the accessors check the validity of the
+  // arguments, it is assumed that index is always in [0, n) range and the value
+  // is not incremented past max_value.
+  template<class MemoryMapperT>
+  class PackedCounterArray {
+   public:
+    PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper)
+        : n(num_counters), memory_mapper(mapper) {
+      CHECK_GT(num_counters, 0);
+      CHECK_GT(max_value, 0);
+      constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL;
+      // Rounding counter storage size up to the power of two allows for using
+      // bit shifts calculating particular counter's index and offset.
+      uptr counter_size_bits =
+          RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1);
+      CHECK_LE(counter_size_bits, kMaxCounterBits);
+      counter_size_bits_log = Log2(counter_size_bits);
+      counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits);
+
+      uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log;
+      CHECK_GT(packing_ratio, 0);
+      packing_ratio_log = Log2(packing_ratio);
+      bit_offset_mask = packing_ratio - 1;
+
+      buffer_size =
+          (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) *
+          sizeof(*buffer);
+      buffer = reinterpret_cast<u64*>(
+          memory_mapper->MapPackedCounterArrayBuffer(buffer_size));
+    }
+    ~PackedCounterArray() {
+      if (buffer) {
+        memory_mapper->UnmapPackedCounterArrayBuffer(
+            reinterpret_cast<uptr>(buffer), buffer_size);
+      }
+    }
+
+    bool IsAllocated() const {
+      return !!buffer;
+    }
+
+    u64 GetCount() const {
+      return n;
+    }
+
+    uptr Get(uptr i) const {
+      DCHECK_LT(i, n);
+      uptr index = i >> packing_ratio_log;
+      uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+      return (buffer[index] >> bit_offset) & counter_mask;
+    }
+
+    void Inc(uptr i) const {
+      DCHECK_LT(Get(i), counter_mask);
+      uptr index = i >> packing_ratio_log;
+      uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+      buffer[index] += 1ULL << bit_offset;
+    }
+
+    void IncRange(uptr from, uptr to) const {
+      DCHECK_LE(from, to);
+      for (uptr i = from; i <= to; i++)
+        Inc(i);
+    }
+
+   private:
+    const u64 n;
+    u64 counter_size_bits_log;
+    u64 counter_mask;
+    u64 packing_ratio_log;
+    u64 bit_offset_mask;
+
+    MemoryMapperT* const memory_mapper;
+    u64 buffer_size;
+    u64* buffer;
+  };
+
+  template<class MemoryMapperT>
+  class FreePagesRangeTracker {
+   public:
+    explicit FreePagesRangeTracker(MemoryMapperT* mapper)
+        : memory_mapper(mapper),
+          page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)),
+          in_the_range(false), current_page(0), current_range_start_page(0) {}
+
+    void NextPage(bool freed) {
+      if (freed) {
+        if (!in_the_range) {
+          current_range_start_page = current_page;
+          in_the_range = true;
+        }
+      } else {
+        CloseOpenedRange();
+      }
+      current_page++;
+    }
+
+    void Done() {
+      CloseOpenedRange();
+    }
+
+   private:
+    void CloseOpenedRange() {
+      if (in_the_range) {
+        memory_mapper->ReleasePageRangeToOS(
+            current_range_start_page << page_size_scaled_log,
+            current_page << page_size_scaled_log);
+        in_the_range = false;
+      }
+    }
+
+    MemoryMapperT* const memory_mapper;
+    const uptr page_size_scaled_log;
+    bool in_the_range;
+    uptr current_page;
+    uptr current_range_start_page;
+  };
+
+  // Iterates over the free_array to identify memory pages containing freed
+  // chunks only and returns these pages back to OS.
+  // allocated_pages_count is the total number of pages allocated for the
+  // current bucket.
+  template<class MemoryMapperT>
+  static void ReleaseFreeMemoryToOS(CompactPtrT *free_array,
+                                    uptr free_array_count, uptr chunk_size,
+                                    uptr allocated_pages_count,
+                                    MemoryMapperT *memory_mapper) {
+    const uptr page_size = GetPageSizeCached();
+
+    // Figure out the number of chunks per page and whether we can take a fast
+    // path (the number of chunks per page is the same for all pages).
+    uptr full_pages_chunk_count_max;
+    bool same_chunk_count_per_page;
+    if (chunk_size <= page_size && page_size % chunk_size == 0) {
+      // Same number of chunks per page, no cross overs.
+      full_pages_chunk_count_max = page_size / chunk_size;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size <= page_size && page_size % chunk_size != 0 &&
+        chunk_size % (page_size % chunk_size) == 0) {
+      // Some chunks are crossing page boundaries, which means that the page
+      // contains one or two partial chunks, but all pages contain the same
+      // number of chunks.
+      full_pages_chunk_count_max = page_size / chunk_size + 1;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size <= page_size) {
+      // Some chunks are crossing page boundaries, which means that the page
+      // contains one or two partial chunks.
+      full_pages_chunk_count_max = page_size / chunk_size + 2;
+      same_chunk_count_per_page = false;
+    } else if (chunk_size > page_size && chunk_size % page_size == 0) {
+      // One chunk covers multiple pages, no cross overs.
+      full_pages_chunk_count_max = 1;
+      same_chunk_count_per_page = true;
+    } else if (chunk_size > page_size) {
+      // One chunk covers multiple pages, Some chunks are crossing page
+      // boundaries. Some pages contain one chunk, some contain two.
+      full_pages_chunk_count_max = 2;
+      same_chunk_count_per_page = false;
+    } else {
+      UNREACHABLE("All chunk_size/page_size ratios must be handled.");
+    }
+
+    PackedCounterArray<MemoryMapperT> counters(allocated_pages_count,
+                                               full_pages_chunk_count_max,
+                                               memory_mapper);
+    if (!counters.IsAllocated())
+      return;
+
+    const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale;
+    const uptr page_size_scaled = page_size >> kCompactPtrScale;
+    const uptr page_size_scaled_log = Log2(page_size_scaled);
+
+    // Iterate over free chunks and count how many free chunks affect each
+    // allocated page.
+    if (chunk_size <= page_size && page_size % chunk_size == 0) {
+      // Each chunk affects one page only.
+      for (uptr i = 0; i < free_array_count; i++)
+        counters.Inc(free_array[i] >> page_size_scaled_log);
+    } else {
+      // In all other cases chunks might affect more than one page.
+      for (uptr i = 0; i < free_array_count; i++) {
+        counters.IncRange(
+            free_array[i] >> page_size_scaled_log,
+            (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log);
+      }
+    }
+
+    // Iterate over pages detecting ranges of pages with chunk counters equal
+    // to the expected number of chunks for the particular page.
+    FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper);
+    if (same_chunk_count_per_page) {
+      // Fast path, every page has the same number of chunks affecting it.
+      for (uptr i = 0; i < counters.GetCount(); i++)
+        range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max);
+    } else {
+      // Show path, go through the pages keeping count how many chunks affect
+      // each page.
+      const uptr pn =
+          chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1;
+      const uptr pnc = pn * chunk_size_scaled;
+      // The idea is to increment the current page pointer by the first chunk
+      // size, middle portion size (the portion of the page covered by chunks
+      // except the first and the last one) and then the last chunk size, adding
+      // up the number of chunks on the current page and checking on every step
+      // whether the page boundary was crossed.
+      uptr prev_page_boundary = 0;
+      uptr current_boundary = 0;
+      for (uptr i = 0; i < counters.GetCount(); i++) {
+        uptr page_boundary = prev_page_boundary + page_size_scaled;
+        uptr chunks_per_page = pn;
+        if (current_boundary < page_boundary) {
+          if (current_boundary > prev_page_boundary)
+            chunks_per_page++;
+          current_boundary += pnc;
+          if (current_boundary < page_boundary) {
+            chunks_per_page++;
+            current_boundary += chunk_size_scaled;
+          }
+        }
+        prev_page_boundary = page_boundary;
+
+        range_tracker.NextPage(counters.Get(i) == chunks_per_page);
+      }
+    }
+    range_tracker.Done();
+  }
+
+ private:
+  friend class MemoryMapper;
+
+  ReservedAddressRange address_range;
+
+  static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
+  // FreeArray is the array of free-d chunks (stored as 4-byte offsets).
+  // In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize
+  // elements, but in reality this will not happen. For simplicity we
+  // dedicate 1/8 of the region's virtual space to FreeArray.
+  static const uptr kFreeArraySize = kRegionSize / 8;
+
+  static const bool kUsingConstantSpaceBeg = kSpaceBeg != ~(uptr)0;
+  uptr NonConstSpaceBeg;
+  uptr SpaceBeg() const {
+    return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg;
+  }
+  uptr SpaceEnd() const { return  SpaceBeg() + kSpaceSize; }
+  // kRegionSize must be >= 2^32.
+  COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2)));
+  // kRegionSize must be <= 2^36, see CompactPtrT.
+  COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
+  // Call mmap for user memory with at least this size.
+  static const uptr kUserMapSize = 1 << 16;
+  // Call mmap for metadata memory with at least this size.
+  static const uptr kMetaMapSize = 1 << 16;
+  // Call mmap for free array memory with at least this size.
+  static const uptr kFreeArrayMapSize = 1 << 16;
+
+  atomic_sint32_t release_to_os_interval_ms_;
+
+  struct Stats {
+    uptr n_allocated;
+    uptr n_freed;
+  };
+
+  struct ReleaseToOsInfo {
+    uptr n_freed_at_last_release;
+    uptr num_releases;
+    u64 last_release_at_ns;
+    u64 last_released_bytes;
+  };
+
+  struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
+    BlockingMutex mutex;
+    uptr num_freed_chunks;  // Number of elements in the freearray.
+    uptr mapped_free_array;  // Bytes mapped for freearray.
+    uptr allocated_user;  // Bytes allocated for user memory.
+    uptr allocated_meta;  // Bytes allocated for metadata.
+    uptr mapped_user;  // Bytes mapped for user memory.
+    uptr mapped_meta;  // Bytes mapped for metadata.
+    u32 rand_state;  // Seed for random shuffle, used if kRandomShuffleChunks.
+    bool exhausted;  // Whether region is out of space for new chunks.
+    Stats stats;
+    ReleaseToOsInfo rtoi;
+  };
+  COMPILER_CHECK(sizeof(RegionInfo) % kCacheLineSize == 0);
+
+  RegionInfo *GetRegionInfo(uptr class_id) const {
+    DCHECK_LT(class_id, kNumClasses);
+    RegionInfo *regions = reinterpret_cast<RegionInfo *>(SpaceEnd());
+    return &regions[class_id];
+  }
+
+  uptr GetMetadataEnd(uptr region_beg) const {
+    return region_beg + kRegionSize - kFreeArraySize;
+  }
+
+  uptr GetChunkIdx(uptr chunk, uptr size) const {
+    if (!kUsingConstantSpaceBeg)
+      chunk -= SpaceBeg();
+
+    uptr offset = chunk % kRegionSize;
+    // Here we divide by a non-constant. This is costly.
+    // size always fits into 32-bits. If the offset fits too, use 32-bit div.
+    if (offset >> (SANITIZER_WORDSIZE / 2))
+      return offset / size;
+    return (u32)offset / (u32)size;
+  }
+
+  CompactPtrT *GetFreeArray(uptr region_beg) const {
+    return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg));
+  }
+
+  bool MapWithCallback(uptr beg, uptr size, const char *name) {
+    uptr mapped = address_range.Map(beg, size, name);
+    if (UNLIKELY(!mapped))
+      return false;
+    CHECK_EQ(beg, mapped);
+    MapUnmapCallback().OnMap(beg, size);
+    return true;
+  }
+
+  void MapWithCallbackOrDie(uptr beg, uptr size, const char *name) {
+    CHECK_EQ(beg, address_range.MapOrDie(beg, size, name));
+    MapUnmapCallback().OnMap(beg, size);
+  }
+
+  void UnmapWithCallbackOrDie(uptr beg, uptr size) {
+    MapUnmapCallback().OnUnmap(beg, size);
+    address_range.Unmap(beg, size);
+  }
+
+  bool EnsureFreeArraySpace(RegionInfo *region, uptr region_beg,
+                            uptr num_freed_chunks) {
+    uptr needed_space = num_freed_chunks * sizeof(CompactPtrT);
+    if (region->mapped_free_array < needed_space) {
+      uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize);
+      CHECK_LE(new_mapped_free_array, kFreeArraySize);
+      uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) +
+                             region->mapped_free_array;
+      uptr new_map_size = new_mapped_free_array - region->mapped_free_array;
+      if (UNLIKELY(!MapWithCallback(current_map_end, new_map_size,
+                                    "SizeClassAllocator: freearray")))
+        return false;
+      region->mapped_free_array = new_mapped_free_array;
+    }
+    return true;
+  }
+
+  // Check whether this size class is exhausted.
+  bool IsRegionExhausted(RegionInfo *region, uptr class_id,
+                         uptr additional_map_size) {
+    if (LIKELY(region->mapped_user + region->mapped_meta +
+               additional_map_size <= kRegionSize - kFreeArraySize))
+      return false;
+    if (!region->exhausted) {
+      region->exhausted = true;
+      Printf("%s: Out of memory. ", SanitizerToolName);
+      Printf("The process has exhausted %zuMB for size class %zu.\n",
+             kRegionSize >> 20, ClassIdToSize(class_id));
+    }
+    return true;
+  }
+
+  NOINLINE bool PopulateFreeArray(AllocatorStats *stat, uptr class_id,
+                                  RegionInfo *region, uptr requested_count) {
+    // region->mutex is held.
+    const uptr region_beg = GetRegionBeginBySizeClass(class_id);
+    const uptr size = ClassIdToSize(class_id);
+
+    const uptr total_user_bytes =
+        region->allocated_user + requested_count * size;
+    // Map more space for chunks, if necessary.
+    if (LIKELY(total_user_bytes > region->mapped_user)) {
+      if (UNLIKELY(region->mapped_user == 0)) {
+        if (!kUsingConstantSpaceBeg && kRandomShuffleChunks)
+          // The random state is initialized from ASLR.
+          region->rand_state = static_cast<u32>(region_beg >> 12);
+        // Postpone the first release to OS attempt for ReleaseToOSIntervalMs,
+        // preventing just allocated memory from being released sooner than
+        // necessary and also preventing extraneous ReleaseMemoryPagesToOS calls
+        // for short lived processes.
+        // Do it only when the feature is turned on, to avoid a potentially
+        // extraneous syscall.
+        if (ReleaseToOSIntervalMs() >= 0)
+          region->rtoi.last_release_at_ns = MonotonicNanoTime();
+      }
+      // Do the mmap for the user memory.
+      const uptr user_map_size =
+          RoundUpTo(total_user_bytes - region->mapped_user, kUserMapSize);
+      if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size)))
+        return false;
+      if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user,
+                                    user_map_size,
+                                    "SizeClassAllocator: region data")))
+        return false;
+      stat->Add(AllocatorStatMapped, user_map_size);
+      region->mapped_user += user_map_size;
+    }
+    const uptr new_chunks_count =
+        (region->mapped_user - region->allocated_user) / size;
+
+    if (kMetadataSize) {
+      // Calculate the required space for metadata.
+      const uptr total_meta_bytes =
+          region->allocated_meta + new_chunks_count * kMetadataSize;
+      const uptr meta_map_size = (total_meta_bytes > region->mapped_meta) ?
+          RoundUpTo(total_meta_bytes - region->mapped_meta, kMetaMapSize) : 0;
+      // Map more space for metadata, if necessary.
+      if (meta_map_size) {
+        if (UNLIKELY(IsRegionExhausted(region, class_id, meta_map_size)))
+          return false;
+        if (UNLIKELY(!MapWithCallback(
+            GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size,
+            meta_map_size, "SizeClassAllocator: region metadata")))
+          return false;
+        region->mapped_meta += meta_map_size;
+      }
+    }
+
+    // If necessary, allocate more space for the free array and populate it with
+    // newly allocated chunks.
+    const uptr total_freed_chunks = region->num_freed_chunks + new_chunks_count;
+    if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, total_freed_chunks)))
+      return false;
+    CompactPtrT *free_array = GetFreeArray(region_beg);
+    for (uptr i = 0, chunk = region->allocated_user; i < new_chunks_count;
+         i++, chunk += size)
+      free_array[total_freed_chunks - 1 - i] = PointerToCompactPtr(0, chunk);
+    if (kRandomShuffleChunks)
+      RandomShuffle(&free_array[region->num_freed_chunks], new_chunks_count,
+                    &region->rand_state);
+
+    // All necessary memory is mapped and now it is safe to advance all
+    // 'allocated_*' counters.
+    region->num_freed_chunks += new_chunks_count;
+    region->allocated_user += new_chunks_count * size;
+    CHECK_LE(region->allocated_user, region->mapped_user);
+    region->allocated_meta += new_chunks_count * kMetadataSize;
+    CHECK_LE(region->allocated_meta, region->mapped_meta);
+    region->exhausted = false;
+
+    // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent
+    // MaybeReleaseToOS from releasing just allocated pages or protect these
+    // not yet used chunks some other way.
+
+    return true;
+  }
+
+  class MemoryMapper {
+   public:
+    MemoryMapper(const ThisT& base_allocator, uptr class_id)
+        : allocator(base_allocator),
+          region_base(base_allocator.GetRegionBeginBySizeClass(class_id)),
+          released_ranges_count(0),
+          released_bytes(0) {
+    }
+
+    uptr GetReleasedRangesCount() const {
+      return released_ranges_count;
+    }
+
+    uptr GetReleasedBytes() const {
+      return released_bytes;
+    }
+
+    uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
+      // TODO(alekseyshl): The idea to explore is to check if we have enough
+      // space between num_freed_chunks*sizeof(CompactPtrT) and
+      // mapped_free_array to fit buffer_size bytes and use that space instead
+      // of mapping a temporary one.
+      return reinterpret_cast<uptr>(
+          MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters"));
+    }
+
+    void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {
+      UnmapOrDie(reinterpret_cast<void *>(buffer), buffer_size);
+    }
+
+    // Releases [from, to) range of pages back to OS.
+    void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) {
+      const uptr from_page = allocator.CompactPtrToPointer(region_base, from);
+      const uptr to_page = allocator.CompactPtrToPointer(region_base, to);
+      ReleaseMemoryPagesToOS(from_page, to_page);
+      released_ranges_count++;
+      released_bytes += to_page - from_page;
+    }
+
+   private:
+    const ThisT& allocator;
+    const uptr region_base;
+    uptr released_ranges_count;
+    uptr released_bytes;
+  };
+
+  // Attempts to release RAM occupied by freed chunks back to OS. The region is
+  // expected to be locked.
+  void MaybeReleaseToOS(uptr class_id, bool force) {
+    RegionInfo *region = GetRegionInfo(class_id);
+    const uptr chunk_size = ClassIdToSize(class_id);
+    const uptr page_size = GetPageSizeCached();
+
+    uptr n = region->num_freed_chunks;
+    if (n * chunk_size < page_size)
+      return;  // No chance to release anything.
+    if ((region->stats.n_freed -
+         region->rtoi.n_freed_at_last_release) * chunk_size < page_size) {
+      return;  // Nothing new to release.
+    }
+
+    if (!force) {
+      s32 interval_ms = ReleaseToOSIntervalMs();
+      if (interval_ms < 0)
+        return;
+
+      if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL >
+          MonotonicNanoTime()) {
+        return;  // Memory was returned recently.
+      }
+    }
+
+    MemoryMapper memory_mapper(*this, class_id);
+
+    ReleaseFreeMemoryToOS<MemoryMapper>(
+        GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size,
+        RoundUpTo(region->allocated_user, page_size) / page_size,
+        &memory_mapper);
+
+    if (memory_mapper.GetReleasedRangesCount() > 0) {
+      region->rtoi.n_freed_at_last_release = region->stats.n_freed;
+      region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
+      region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
+    }
+    region->rtoi.last_release_at_ns = MonotonicNanoTime();
+  }
+};
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_report.h b/lib/tsan/sanitizer_common/sanitizer_allocator_report.h
new file mode 100644
index 0000000000..0653c365c1
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_report.h
@@ -0,0 +1,39 @@
+//===-- sanitizer_allocator_report.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Shared allocator error reporting for ThreadSanitizer, MemorySanitizer, etc.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_REPORT_H
+#define SANITIZER_ALLOCATOR_REPORT_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+void NORETURN ReportCallocOverflow(uptr count, uptr size,
+                                   const StackTrace *stack);
+void NORETURN ReportReallocArrayOverflow(uptr count, uptr size,
+                                         const StackTrace *stack);
+void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack);
+void NORETURN ReportInvalidAllocationAlignment(uptr alignment,
+                                               const StackTrace *stack);
+void NORETURN ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
+                                                 const StackTrace *stack);
+void NORETURN ReportInvalidPosixMemalignAlignment(uptr alignment,
+                                                  const StackTrace *stack);
+void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
+                                         const StackTrace *stack);
+void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ALLOCATOR_REPORT_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h b/lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h
new file mode 100644
index 0000000000..1d128f55de
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h
@@ -0,0 +1,326 @@
+//===-- sanitizer_allocator_secondary.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// Fixed array to store LargeMmapAllocator chunks list, limited to 32K total
+// allocated chunks. To be used in memory constrained or not memory hungry cases
+// (currently, 32 bits and internal allocator).
+class LargeMmapAllocatorPtrArrayStatic {
+ public:
+  INLINE void *Init() { return &p_[0]; }
+  INLINE void EnsureSpace(uptr n) { CHECK_LT(n, kMaxNumChunks); }
+ private:
+  static const int kMaxNumChunks = 1 << 15;
+  uptr p_[kMaxNumChunks];
+};
+
+// Much less restricted LargeMmapAllocator chunks list (comparing to
+// PtrArrayStatic). Backed by mmaped memory region and can hold up to 1M chunks.
+// ReservedAddressRange was used instead of just MAP_NORESERVE to achieve the
+// same functionality in Fuchsia case, which does not support MAP_NORESERVE.
+class LargeMmapAllocatorPtrArrayDynamic {
+ public:
+  INLINE void *Init() {
+    uptr p = address_range_.Init(kMaxNumChunks * sizeof(uptr),
+                                 SecondaryAllocatorName);
+    CHECK(p);
+    return reinterpret_cast<void*>(p);
+  }
+
+  INLINE void EnsureSpace(uptr n) {
+    CHECK_LT(n, kMaxNumChunks);
+    DCHECK(n <= n_reserved_);
+    if (UNLIKELY(n == n_reserved_)) {
+      address_range_.MapOrDie(
+          reinterpret_cast<uptr>(address_range_.base()) +
+              n_reserved_ * sizeof(uptr),
+          kChunksBlockCount * sizeof(uptr));
+      n_reserved_ += kChunksBlockCount;
+    }
+  }
+
+ private:
+  static const int kMaxNumChunks = 1 << 20;
+  static const int kChunksBlockCount = 1 << 14;
+  ReservedAddressRange address_range_;
+  uptr n_reserved_;
+};
+
+#if SANITIZER_WORDSIZE == 32
+typedef LargeMmapAllocatorPtrArrayStatic DefaultLargeMmapAllocatorPtrArray;
+#else
+typedef LargeMmapAllocatorPtrArrayDynamic DefaultLargeMmapAllocatorPtrArray;
+#endif
+
+// This class can (de)allocate only large chunks of memory using mmap/unmap.
+// The main purpose of this allocator is to cover large and rare allocation
+// sizes not covered by more efficient allocators (e.g. SizeClassAllocator64).
+template <class MapUnmapCallback = NoOpMapUnmapCallback,
+          class PtrArrayT = DefaultLargeMmapAllocatorPtrArray,
+          class AddressSpaceViewTy = LocalAddressSpaceView>
+class LargeMmapAllocator {
+ public:
+  using AddressSpaceView = AddressSpaceViewTy;
+  void InitLinkerInitialized() {
+    page_size_ = GetPageSizeCached();
+    chunks_ = reinterpret_cast<Header**>(ptr_array_.Init());
+  }
+
+  void Init() {
+    internal_memset(this, 0, sizeof(*this));
+    InitLinkerInitialized();
+  }
+
+  void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) {
+    CHECK(IsPowerOfTwo(alignment));
+    uptr map_size = RoundUpMapSize(size);
+    if (alignment > page_size_)
+      map_size += alignment;
+    // Overflow.
+    if (map_size < size) {
+      Report("WARNING: %s: LargeMmapAllocator allocation overflow: "
+             "0x%zx bytes with 0x%zx alignment requested\n",
+             SanitizerToolName, map_size, alignment);
+      return nullptr;
+    }
+    uptr map_beg = reinterpret_cast<uptr>(
+        MmapOrDieOnFatalError(map_size, SecondaryAllocatorName));
+    if (!map_beg)
+      return nullptr;
+    CHECK(IsAligned(map_beg, page_size_));
+    MapUnmapCallback().OnMap(map_beg, map_size);
+    uptr map_end = map_beg + map_size;
+    uptr res = map_beg + page_size_;
+    if (res & (alignment - 1))  // Align.
+      res += alignment - (res & (alignment - 1));
+    CHECK(IsAligned(res, alignment));
+    CHECK(IsAligned(res, page_size_));
+    CHECK_GE(res + size, map_beg);
+    CHECK_LE(res + size, map_end);
+    Header *h = GetHeader(res);
+    h->size = size;
+    h->map_beg = map_beg;
+    h->map_size = map_size;
+    uptr size_log = MostSignificantSetBitIndex(map_size);
+    CHECK_LT(size_log, ARRAY_SIZE(stats.by_size_log));
+    {
+      SpinMutexLock l(&mutex_);
+      ptr_array_.EnsureSpace(n_chunks_);
+      uptr idx = n_chunks_++;
+      h->chunk_idx = idx;
+      chunks_[idx] = h;
+      chunks_sorted_ = false;
+      stats.n_allocs++;
+      stats.currently_allocated += map_size;
+      stats.max_allocated = Max(stats.max_allocated, stats.currently_allocated);
+      stats.by_size_log[size_log]++;
+      stat->Add(AllocatorStatAllocated, map_size);
+      stat->Add(AllocatorStatMapped, map_size);
+    }
+    return reinterpret_cast<void*>(res);
+  }
+
+  void Deallocate(AllocatorStats *stat, void *p) {
+    Header *h = GetHeader(p);
+    {
+      SpinMutexLock l(&mutex_);
+      uptr idx = h->chunk_idx;
+      CHECK_EQ(chunks_[idx], h);
+      CHECK_LT(idx, n_chunks_);
+      chunks_[idx] = chunks_[--n_chunks_];
+      chunks_[idx]->chunk_idx = idx;
+      chunks_sorted_ = false;
+      stats.n_frees++;
+      stats.currently_allocated -= h->map_size;
+      stat->Sub(AllocatorStatAllocated, h->map_size);
+      stat->Sub(AllocatorStatMapped, h->map_size);
+    }
+    MapUnmapCallback().OnUnmap(h->map_beg, h->map_size);
+    UnmapOrDie(reinterpret_cast<void*>(h->map_beg), h->map_size);
+  }
+
+  uptr TotalMemoryUsed() {
+    SpinMutexLock l(&mutex_);
+    uptr res = 0;
+    for (uptr i = 0; i < n_chunks_; i++) {
+      Header *h = chunks_[i];
+      CHECK_EQ(h->chunk_idx, i);
+      res += RoundUpMapSize(h->size);
+    }
+    return res;
+  }
+
+  bool PointerIsMine(const void *p) {
+    return GetBlockBegin(p) != nullptr;
+  }
+
+  uptr GetActuallyAllocatedSize(void *p) {
+    return RoundUpTo(GetHeader(p)->size, page_size_);
+  }
+
+  // At least page_size_/2 metadata bytes is available.
+  void *GetMetaData(const void *p) {
+    // Too slow: CHECK_EQ(p, GetBlockBegin(p));
+    if (!IsAligned(reinterpret_cast<uptr>(p), page_size_)) {
+      Printf("%s: bad pointer %p\n", SanitizerToolName, p);
+      CHECK(IsAligned(reinterpret_cast<uptr>(p), page_size_));
+    }
+    return GetHeader(p) + 1;
+  }
+
+  void *GetBlockBegin(const void *ptr) {
+    uptr p = reinterpret_cast<uptr>(ptr);
+    SpinMutexLock l(&mutex_);
+    uptr nearest_chunk = 0;
+    Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_);
+    // Cache-friendly linear search.
+    for (uptr i = 0; i < n_chunks_; i++) {
+      uptr ch = reinterpret_cast<uptr>(chunks[i]);
+      if (p < ch) continue;  // p is at left to this chunk, skip it.
+      if (p - ch < p - nearest_chunk)
+        nearest_chunk = ch;
+    }
+    if (!nearest_chunk)
+      return nullptr;
+    const Header *h =
+        AddressSpaceView::Load(reinterpret_cast<Header *>(nearest_chunk));
+    Header *h_ptr = reinterpret_cast<Header *>(nearest_chunk);
+    CHECK_GE(nearest_chunk, h->map_beg);
+    CHECK_LT(nearest_chunk, h->map_beg + h->map_size);
+    CHECK_LE(nearest_chunk, p);
+    if (h->map_beg + h->map_size <= p)
+      return nullptr;
+    return GetUser(h_ptr);
+  }
+
+  void EnsureSortedChunks() {
+    if (chunks_sorted_) return;
+    Header **chunks = AddressSpaceView::LoadWritable(chunks_, n_chunks_);
+    Sort(reinterpret_cast<uptr *>(chunks), n_chunks_);
+    for (uptr i = 0; i < n_chunks_; i++)
+      AddressSpaceView::LoadWritable(chunks[i])->chunk_idx = i;
+    chunks_sorted_ = true;
+  }
+
+  // This function does the same as GetBlockBegin, but is much faster.
+  // Must be called with the allocator locked.
+  void *GetBlockBeginFastLocked(void *ptr) {
+    mutex_.CheckLocked();
+    uptr p = reinterpret_cast<uptr>(ptr);
+    uptr n = n_chunks_;
+    if (!n) return nullptr;
+    EnsureSortedChunks();
+    Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_);
+    auto min_mmap_ = reinterpret_cast<uptr>(chunks[0]);
+    auto max_mmap_ = reinterpret_cast<uptr>(chunks[n - 1]) +
+                     AddressSpaceView::Load(chunks[n - 1])->map_size;
+    if (p < min_mmap_ || p >= max_mmap_)
+      return nullptr;
+    uptr beg = 0, end = n - 1;
+    // This loop is a log(n) lower_bound. It does not check for the exact match
+    // to avoid expensive cache-thrashing loads.
+    while (end - beg >= 2) {
+      uptr mid = (beg + end) / 2;  // Invariant: mid >= beg + 1
+      if (p < reinterpret_cast<uptr>(chunks[mid]))
+        end = mid - 1;  // We are not interested in chunks[mid].
+      else
+        beg = mid;  // chunks[mid] may still be what we want.
+    }
+
+    if (beg < end) {
+      CHECK_EQ(beg + 1, end);
+      // There are 2 chunks left, choose one.
+      if (p >= reinterpret_cast<uptr>(chunks[end]))
+        beg = end;
+    }
+
+    const Header *h = AddressSpaceView::Load(chunks[beg]);
+    Header *h_ptr = chunks[beg];
+    if (h->map_beg + h->map_size <= p || p < h->map_beg)
+      return nullptr;
+    return GetUser(h_ptr);
+  }
+
+  void PrintStats() {
+    Printf("Stats: LargeMmapAllocator: allocated %zd times, "
+           "remains %zd (%zd K) max %zd M; by size logs: ",
+           stats.n_allocs, stats.n_allocs - stats.n_frees,
+           stats.currently_allocated >> 10, stats.max_allocated >> 20);
+    for (uptr i = 0; i < ARRAY_SIZE(stats.by_size_log); i++) {
+      uptr c = stats.by_size_log[i];
+      if (!c) continue;
+      Printf("%zd:%zd; ", i, c);
+    }
+    Printf("\n");
+  }
+
+  // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
+  // introspection API.
+  void ForceLock() {
+    mutex_.Lock();
+  }
+
+  void ForceUnlock() {
+    mutex_.Unlock();
+  }
+
+  // Iterate over all existing chunks.
+  // The allocator must be locked when calling this function.
+  void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+    EnsureSortedChunks();  // Avoid doing the sort while iterating.
+    const Header *const *chunks = AddressSpaceView::Load(chunks_, n_chunks_);
+    for (uptr i = 0; i < n_chunks_; i++) {
+      const Header *t = chunks[i];
+      callback(reinterpret_cast<uptr>(GetUser(t)), arg);
+      // Consistency check: verify that the array did not change.
+      CHECK_EQ(chunks[i], t);
+      CHECK_EQ(AddressSpaceView::Load(chunks[i])->chunk_idx, i);
+    }
+  }
+
+ private:
+  struct Header {
+    uptr map_beg;
+    uptr map_size;
+    uptr size;
+    uptr chunk_idx;
+  };
+
+  Header *GetHeader(uptr p) {
+    CHECK(IsAligned(p, page_size_));
+    return reinterpret_cast<Header*>(p - page_size_);
+  }
+  Header *GetHeader(const void *p) {
+    return GetHeader(reinterpret_cast<uptr>(p));
+  }
+
+  void *GetUser(const Header *h) {
+    CHECK(IsAligned((uptr)h, page_size_));
+    return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);
+  }
+
+  uptr RoundUpMapSize(uptr size) {
+    return RoundUpTo(size, page_size_) + page_size_;
+  }
+
+  uptr page_size_;
+  Header **chunks_;
+  PtrArrayT ptr_array_;
+  uptr n_chunks_;
+  bool chunks_sorted_;
+  struct Stats {
+    uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
+  } stats;
+  StaticSpinMutex mutex_;
+};
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h b/lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h
new file mode 100644
index 0000000000..12d8c89230
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h
@@ -0,0 +1,241 @@
+//===-- sanitizer_allocator_size_class_map.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// SizeClassMap maps allocation sizes into size classes and back.
+// Class 0 always corresponds to size 0.
+// The other sizes are controlled by the template parameters:
+//   kMinSizeLog: defines the class 1    as 2^kMinSizeLog.
+//   kMaxSizeLog: defines the last class as 2^kMaxSizeLog.
+//   kMidSizeLog: the classes starting from 1 increase with step
+//                2^kMinSizeLog until 2^kMidSizeLog.
+//   kNumBits: the number of non-zero bits in sizes after 2^kMidSizeLog.
+//             E.g. with kNumBits==3 all size classes after 2^kMidSizeLog
+//             look like 0b1xx0..0, where x is either 0 or 1.
+//
+// Example: kNumBits=3, kMidSizeLog=4, kMidSizeLog=8, kMaxSizeLog=17:
+//
+// Classes 1 - 16 correspond to sizes 16 to 256 (size = class_id * 16).
+// Next 4 classes: 256 + i * 64  (i = 1 to 4).
+// Next 4 classes: 512 + i * 128 (i = 1 to 4).
+// ...
+// Next 4 classes: 2^k + i * 2^(k-2) (i = 1 to 4).
+// Last class corresponds to kMaxSize = 1 << kMaxSizeLog.
+//
+// This structure of the size class map gives us:
+//   - Efficient table-free class-to-size and size-to-class functions.
+//   - Difference between two consequent size classes is between 14% and 25%
+//
+// This class also gives a hint to a thread-caching allocator about the amount
+// of chunks that need to be cached per-thread:
+//  - kMaxNumCachedHint is a hint for maximal number of chunks per size class.
+//    The actual number is computed in TransferBatch.
+//  - (1 << kMaxBytesCachedLog) is the maximal number of bytes per size class.
+//
+// Part of output of SizeClassMap::Print():
+// c00 => s: 0 diff: +0 00% l 0 cached: 0 0; id 0
+// c01 => s: 16 diff: +16 00% l 4 cached: 256 4096; id 1
+// c02 => s: 32 diff: +16 100% l 5 cached: 256 8192; id 2
+// c03 => s: 48 diff: +16 50% l 5 cached: 256 12288; id 3
+// c04 => s: 64 diff: +16 33% l 6 cached: 256 16384; id 4
+// c05 => s: 80 diff: +16 25% l 6 cached: 256 20480; id 5
+// c06 => s: 96 diff: +16 20% l 6 cached: 256 24576; id 6
+// c07 => s: 112 diff: +16 16% l 6 cached: 256 28672; id 7
+//
+// c08 => s: 128 diff: +16 14% l 7 cached: 256 32768; id 8
+// c09 => s: 144 diff: +16 12% l 7 cached: 256 36864; id 9
+// c10 => s: 160 diff: +16 11% l 7 cached: 256 40960; id 10
+// c11 => s: 176 diff: +16 10% l 7 cached: 256 45056; id 11
+// c12 => s: 192 diff: +16 09% l 7 cached: 256 49152; id 12
+// c13 => s: 208 diff: +16 08% l 7 cached: 256 53248; id 13
+// c14 => s: 224 diff: +16 07% l 7 cached: 256 57344; id 14
+// c15 => s: 240 diff: +16 07% l 7 cached: 256 61440; id 15
+//
+// c16 => s: 256 diff: +16 06% l 8 cached: 256 65536; id 16
+// c17 => s: 320 diff: +64 25% l 8 cached: 204 65280; id 17
+// c18 => s: 384 diff: +64 20% l 8 cached: 170 65280; id 18
+// c19 => s: 448 diff: +64 16% l 8 cached: 146 65408; id 19
+//
+// c20 => s: 512 diff: +64 14% l 9 cached: 128 65536; id 20
+// c21 => s: 640 diff: +128 25% l 9 cached: 102 65280; id 21
+// c22 => s: 768 diff: +128 20% l 9 cached: 85 65280; id 22
+// c23 => s: 896 diff: +128 16% l 9 cached: 73 65408; id 23
+//
+// c24 => s: 1024 diff: +128 14% l 10 cached: 64 65536; id 24
+// c25 => s: 1280 diff: +256 25% l 10 cached: 51 65280; id 25
+// c26 => s: 1536 diff: +256 20% l 10 cached: 42 64512; id 26
+// c27 => s: 1792 diff: +256 16% l 10 cached: 36 64512; id 27
+//
+// ...
+//
+// c48 => s: 65536 diff: +8192 14% l 16 cached: 1 65536; id 48
+// c49 => s: 81920 diff: +16384 25% l 16 cached: 1 81920; id 49
+// c50 => s: 98304 diff: +16384 20% l 16 cached: 1 98304; id 50
+// c51 => s: 114688 diff: +16384 16% l 16 cached: 1 114688; id 51
+//
+// c52 => s: 131072 diff: +16384 14% l 17 cached: 1 131072; id 52
+//
+//
+// Another example (kNumBits=2):
+// c00 => s: 0 diff: +0 00% l 0 cached: 0 0; id 0
+// c01 => s: 32 diff: +32 00% l 5 cached: 64 2048; id 1
+// c02 => s: 64 diff: +32 100% l 6 cached: 64 4096; id 2
+// c03 => s: 96 diff: +32 50% l 6 cached: 64 6144; id 3
+// c04 => s: 128 diff: +32 33% l 7 cached: 64 8192; id 4
+// c05 => s: 160 diff: +32 25% l 7 cached: 64 10240; id 5
+// c06 => s: 192 diff: +32 20% l 7 cached: 64 12288; id 6
+// c07 => s: 224 diff: +32 16% l 7 cached: 64 14336; id 7
+// c08 => s: 256 diff: +32 14% l 8 cached: 64 16384; id 8
+// c09 => s: 384 diff: +128 50% l 8 cached: 42 16128; id 9
+// c10 => s: 512 diff: +128 33% l 9 cached: 32 16384; id 10
+// c11 => s: 768 diff: +256 50% l 9 cached: 21 16128; id 11
+// c12 => s: 1024 diff: +256 33% l 10 cached: 16 16384; id 12
+// c13 => s: 1536 diff: +512 50% l 10 cached: 10 15360; id 13
+// c14 => s: 2048 diff: +512 33% l 11 cached: 8 16384; id 14
+// c15 => s: 3072 diff: +1024 50% l 11 cached: 5 15360; id 15
+// c16 => s: 4096 diff: +1024 33% l 12 cached: 4 16384; id 16
+// c17 => s: 6144 diff: +2048 50% l 12 cached: 2 12288; id 17
+// c18 => s: 8192 diff: +2048 33% l 13 cached: 2 16384; id 18
+// c19 => s: 12288 diff: +4096 50% l 13 cached: 1 12288; id 19
+// c20 => s: 16384 diff: +4096 33% l 14 cached: 1 16384; id 20
+// c21 => s: 24576 diff: +8192 50% l 14 cached: 1 24576; id 21
+// c22 => s: 32768 diff: +8192 33% l 15 cached: 1 32768; id 22
+// c23 => s: 49152 diff: +16384 50% l 15 cached: 1 49152; id 23
+// c24 => s: 65536 diff: +16384 33% l 16 cached: 1 65536; id 24
+// c25 => s: 98304 diff: +32768 50% l 16 cached: 1 98304; id 25
+// c26 => s: 131072 diff: +32768 33% l 17 cached: 1 131072; id 26
+
+template <uptr kNumBits, uptr kMinSizeLog, uptr kMidSizeLog, uptr kMaxSizeLog,
+          uptr kMaxNumCachedHintT, uptr kMaxBytesCachedLog>
+class SizeClassMap {
+  static const uptr kMinSize = 1 << kMinSizeLog;
+  static const uptr kMidSize = 1 << kMidSizeLog;
+  static const uptr kMidClass = kMidSize / kMinSize;
+  static const uptr S = kNumBits - 1;
+  static const uptr M = (1 << S) - 1;
+
+ public:
+  // kMaxNumCachedHintT is a power of two. It serves as a hint
+  // for the size of TransferBatch, the actual size could be a bit smaller.
+  static const uptr kMaxNumCachedHint = kMaxNumCachedHintT;
+  COMPILER_CHECK((kMaxNumCachedHint & (kMaxNumCachedHint - 1)) == 0);
+
+  static const uptr kMaxSize = 1UL << kMaxSizeLog;
+  static const uptr kNumClasses =
+      kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1 + 1;
+  static const uptr kLargestClassID = kNumClasses - 2;
+  static const uptr kBatchClassID = kNumClasses - 1;
+  COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256);
+  static const uptr kNumClassesRounded =
+      kNumClasses <= 32  ? 32 :
+      kNumClasses <= 64  ? 64 :
+      kNumClasses <= 128 ? 128 : 256;
+
+  static uptr Size(uptr class_id) {
+    // Estimate the result for kBatchClassID because this class does not know
+    // the exact size of TransferBatch. It's OK since we are using the actual
+    // sizeof(TransferBatch) where it matters.
+    if (UNLIKELY(class_id == kBatchClassID))
+      return kMaxNumCachedHint * sizeof(uptr);
+    if (class_id <= kMidClass)
+      return kMinSize * class_id;
+    class_id -= kMidClass;
+    uptr t = kMidSize << (class_id >> S);
+    return t + (t >> S) * (class_id & M);
+  }
+
+  static uptr ClassID(uptr size) {
+    if (UNLIKELY(size > kMaxSize))
+      return 0;
+    if (size <= kMidSize)
+      return (size + kMinSize - 1) >> kMinSizeLog;
+    const uptr l = MostSignificantSetBitIndex(size);
+    const uptr hbits = (size >> (l - S)) & M;
+    const uptr lbits = size & ((1U << (l - S)) - 1);
+    const uptr l1 = l - kMidSizeLog;
+    return kMidClass + (l1 << S) + hbits + (lbits > 0);
+  }
+
+  static uptr MaxCachedHint(uptr size) {
+    DCHECK_LE(size, kMaxSize);
+    if (UNLIKELY(size == 0))
+      return 0;
+    uptr n;
+    // Force a 32-bit division if the template parameters allow for it.
+    if (kMaxBytesCachedLog > 31 || kMaxSizeLog > 31)
+      n = (1UL << kMaxBytesCachedLog) / size;
+    else
+      n = (1U << kMaxBytesCachedLog) / static_cast<u32>(size);
+    return Max<uptr>(1U, Min(kMaxNumCachedHint, n));
+  }
+
+  static void Print() {
+    uptr prev_s = 0;
+    uptr total_cached = 0;
+    for (uptr i = 0; i < kNumClasses; i++) {
+      uptr s = Size(i);
+      if (s >= kMidSize / 2 && (s & (s - 1)) == 0)
+        Printf("\n");
+      uptr d = s - prev_s;
+      uptr p = prev_s ? (d * 100 / prev_s) : 0;
+      uptr l = s ? MostSignificantSetBitIndex(s) : 0;
+      uptr cached = MaxCachedHint(s) * s;
+      if (i == kBatchClassID)
+        d = p = l = 0;
+      Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "
+             "cached: %zd %zd; id %zd\n",
+             i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
+      total_cached += cached;
+      prev_s = s;
+    }
+    Printf("Total cached: %zd\n", total_cached);
+  }
+
+  static void Validate() {
+    for (uptr c = 1; c < kNumClasses; c++) {
+      // Printf("Validate: c%zd\n", c);
+      uptr s = Size(c);
+      CHECK_NE(s, 0U);
+      if (c == kBatchClassID)
+        continue;
+      CHECK_EQ(ClassID(s), c);
+      if (c < kLargestClassID)
+        CHECK_EQ(ClassID(s + 1), c + 1);
+      CHECK_EQ(ClassID(s - 1), c);
+      CHECK_GT(Size(c), Size(c - 1));
+    }
+    CHECK_EQ(ClassID(kMaxSize + 1), 0);
+
+    for (uptr s = 1; s <= kMaxSize; s++) {
+      uptr c = ClassID(s);
+      // Printf("s%zd => c%zd\n", s, c);
+      CHECK_LT(c, kNumClasses);
+      CHECK_GE(Size(c), s);
+      if (c > 0)
+        CHECK_LT(Size(c - 1), s);
+    }
+  }
+};
+
+typedef SizeClassMap<3, 4, 8, 17, 128, 16> DefaultSizeClassMap;
+typedef SizeClassMap<3, 4, 8, 17, 64, 14> CompactSizeClassMap;
+typedef SizeClassMap<2, 5, 9, 16, 64, 14> VeryCompactSizeClassMap;
+
+// The following SizeClassMap only holds a way small number of cached entries,
+// allowing for denser per-class arrays, smaller memory footprint and usually
+// better performances in threaded environments.
+typedef SizeClassMap<3, 4, 8, 17, 8, 10> DenseSizeClassMap;
+// Similar to VeryCompact map above, this one has a small number of different
+// size classes, and also reduced thread-local caches.
+typedef SizeClassMap<2, 5, 9, 16, 8, 10> VeryDenseSizeClassMap;
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_stats.h b/lib/tsan/sanitizer_common/sanitizer_allocator_stats.h
new file mode 100644
index 0000000000..6f14e3863c
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_stats.h
@@ -0,0 +1,106 @@
+//===-- sanitizer_allocator_stats.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_ALLOCATOR_H
+#error This file must be included inside sanitizer_allocator.h
+#endif
+
+// Memory allocator statistics
+enum AllocatorStat {
+  AllocatorStatAllocated,
+  AllocatorStatMapped,
+  AllocatorStatCount
+};
+
+typedef uptr AllocatorStatCounters[AllocatorStatCount];
+
+// Per-thread stats, live in per-thread cache.
+class AllocatorStats {
+ public:
+  void Init() {
+    internal_memset(this, 0, sizeof(*this));
+  }
+  void InitLinkerInitialized() {}
+
+  void Add(AllocatorStat i, uptr v) {
+    v += atomic_load(&stats_[i], memory_order_relaxed);
+    atomic_store(&stats_[i], v, memory_order_relaxed);
+  }
+
+  void Sub(AllocatorStat i, uptr v) {
+    v = atomic_load(&stats_[i], memory_order_relaxed) - v;
+    atomic_store(&stats_[i], v, memory_order_relaxed);
+  }
+
+  void Set(AllocatorStat i, uptr v) {
+    atomic_store(&stats_[i], v, memory_order_relaxed);
+  }
+
+  uptr Get(AllocatorStat i) const {
+    return atomic_load(&stats_[i], memory_order_relaxed);
+  }
+
+ private:
+  friend class AllocatorGlobalStats;
+  AllocatorStats *next_;
+  AllocatorStats *prev_;
+  atomic_uintptr_t stats_[AllocatorStatCount];
+};
+
+// Global stats, used for aggregation and querying.
+class AllocatorGlobalStats : public AllocatorStats {
+ public:
+  void InitLinkerInitialized() {
+    next_ = this;
+    prev_ = this;
+  }
+  void Init() {
+    internal_memset(this, 0, sizeof(*this));
+    InitLinkerInitialized();
+  }
+
+  void Register(AllocatorStats *s) {
+    SpinMutexLock l(&mu_);
+    s->next_ = next_;
+    s->prev_ = this;
+    next_->prev_ = s;
+    next_ = s;
+  }
+
+  void Unregister(AllocatorStats *s) {
+    SpinMutexLock l(&mu_);
+    s->prev_->next_ = s->next_;
+    s->next_->prev_ = s->prev_;
+    for (int i = 0; i < AllocatorStatCount; i++)
+      Add(AllocatorStat(i), s->Get(AllocatorStat(i)));
+  }
+
+  void Get(AllocatorStatCounters s) const {
+    internal_memset(s, 0, AllocatorStatCount * sizeof(uptr));
+    SpinMutexLock l(&mu_);
+    const AllocatorStats *stats = this;
+    for (;;) {
+      for (int i = 0; i < AllocatorStatCount; i++)
+        s[i] += stats->Get(AllocatorStat(i));
+      stats = stats->next_;
+      if (stats == this)
+        break;
+    }
+    // All stats must be non-negative.
+    for (int i = 0; i < AllocatorStatCount; i++)
+      s[i] = ((sptr)s[i]) >= 0 ? s[i] : 0;
+  }
+
+ private:
+  mutable StaticSpinMutex mu_;
+};
+
+
diff --git a/lib/tsan/sanitizer_common/sanitizer_asm.h b/lib/tsan/sanitizer_common/sanitizer_asm.h
new file mode 100644
index 0000000000..803af3285e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_asm.h
@@ -0,0 +1,68 @@
+//===-- sanitizer_asm.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various support for assemebler.
+//
+//===----------------------------------------------------------------------===//
+
+// Some toolchains do not support .cfi asm directives, so we have to hide
+// them inside macros.
+#if defined(__clang__) ||                                                      \
+    (defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM))
+  // GCC defined __GCC_HAVE_DWARF2_CFI_ASM if it supports CFI.
+  // Clang seems to support CFI by default (or not?).
+  // We need two versions of macros: for inline asm and standalone asm files.
+# define CFI_INL_ADJUST_CFA_OFFSET(n) ".cfi_adjust_cfa_offset " #n ";"
+
+# define CFI_STARTPROC .cfi_startproc
+# define CFI_ENDPROC .cfi_endproc
+# define CFI_ADJUST_CFA_OFFSET(n) .cfi_adjust_cfa_offset n
+# define CFI_DEF_CFA_OFFSET(n) .cfi_def_cfa_offset n
+# define CFI_REL_OFFSET(reg, n) .cfi_rel_offset reg, n
+# define CFI_OFFSET(reg, n) .cfi_offset reg, n
+# define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg
+# define CFI_DEF_CFA(reg, n) .cfi_def_cfa reg, n
+# define CFI_RESTORE(reg) .cfi_restore reg
+
+#else  // No CFI
+# define CFI_INL_ADJUST_CFA_OFFSET(n)
+# define CFI_STARTPROC
+# define CFI_ENDPROC
+# define CFI_ADJUST_CFA_OFFSET(n)
+# define CFI_DEF_CFA_OFFSET(n)
+# define CFI_REL_OFFSET(reg, n)
+# define CFI_OFFSET(reg, n)
+# define CFI_DEF_CFA_REGISTER(reg)
+# define CFI_DEF_CFA(reg, n)
+# define CFI_RESTORE(reg)
+#endif
+
+#if !defined(__APPLE__)
+# define ASM_HIDDEN(symbol) .hidden symbol
+# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
+# define ASM_SIZE(symbol) .size symbol, .-symbol
+# define ASM_SYMBOL(symbol) symbol
+# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
+# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
+#else
+# define ASM_HIDDEN(symbol)
+# define ASM_TYPE_FUNCTION(symbol)
+# define ASM_SIZE(symbol)
+# define ASM_SYMBOL(symbol) _##symbol
+# define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol
+# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
+#endif
+
+#if defined(__ELF__) && (defined(__GNU__) || defined(__FreeBSD__) || \
+                         defined(__Fuchsia__) || defined(__linux__))
+// clang-format off
+#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits  // NOLINT
+// clang-format on
+#else
+#define NO_EXEC_STACK_DIRECTIVE
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic.h b/lib/tsan/sanitizer_common/sanitizer_atomic.h
new file mode 100644
index 0000000000..a798a0cf25
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic.h
@@ -0,0 +1,86 @@
+//===-- sanitizer_atomic.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_H
+#define SANITIZER_ATOMIC_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+enum memory_order {
+  memory_order_relaxed = 1 << 0,
+  memory_order_consume = 1 << 1,
+  memory_order_acquire = 1 << 2,
+  memory_order_release = 1 << 3,
+  memory_order_acq_rel = 1 << 4,
+  memory_order_seq_cst = 1 << 5
+};
+
+struct atomic_uint8_t {
+  typedef u8 Type;
+  volatile Type val_dont_use;
+};
+
+struct atomic_uint16_t {
+  typedef u16 Type;
+  volatile Type val_dont_use;
+};
+
+struct atomic_sint32_t {
+  typedef s32 Type;
+  volatile Type val_dont_use;
+};
+
+struct atomic_uint32_t {
+  typedef u32 Type;
+  volatile Type val_dont_use;
+};
+
+struct atomic_uint64_t {
+  typedef u64 Type;
+  // On 32-bit platforms u64 is not necessary aligned on 8 bytes.
+  volatile ALIGNED(8) Type val_dont_use;
+};
+
+struct atomic_uintptr_t {
+  typedef uptr Type;
+  volatile Type val_dont_use;
+};
+
+}  // namespace __sanitizer
+
+#if defined(__clang__) || defined(__GNUC__)
+# include "sanitizer_atomic_clang.h"
+#elif defined(_MSC_VER)
+# include "sanitizer_atomic_msvc.h"
+#else
+# error "Unsupported compiler"
+#endif
+
+namespace __sanitizer {
+
+// Clutter-reducing helpers.
+
+template<typename T>
+INLINE typename T::Type atomic_load_relaxed(const volatile T *a) {
+  return atomic_load(a, memory_order_relaxed);
+}
+
+template<typename T>
+INLINE void atomic_store_relaxed(volatile T *a, typename T::Type v) {
+  atomic_store(a, v, memory_order_relaxed);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ATOMIC_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic_clang.h b/lib/tsan/sanitizer_common/sanitizer_atomic_clang.h
new file mode 100644
index 0000000000..c40461ebc3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic_clang.h
@@ -0,0 +1,105 @@
+//===-- sanitizer_atomic_clang.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+// Not intended for direct inclusion. Include sanitizer_atomic.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_CLANG_H
+#define SANITIZER_ATOMIC_CLANG_H
+
+#if defined(__i386__) || defined(__x86_64__)
+# include "sanitizer_atomic_clang_x86.h"
+#else
+# include "sanitizer_atomic_clang_other.h"
+#endif
+
+namespace __sanitizer {
+
+// We would like to just use compiler builtin atomic operations
+// for loads and stores, but they are mostly broken in clang:
+// - they lead to vastly inefficient code generation
+// (http://llvm.org/bugs/show_bug.cgi?id=17281)
+// - 64-bit atomic operations are not implemented on x86_32
+// (http://llvm.org/bugs/show_bug.cgi?id=15034)
+// - they are not implemented on ARM
+// error: undefined reference to '__atomic_load_4'
+
+// See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+// for mappings of the memory model to different processors.
+
+INLINE void atomic_signal_fence(memory_order) {
+  __asm__ __volatile__("" ::: "memory");
+}
+
+INLINE void atomic_thread_fence(memory_order) {
+  __sync_synchronize();
+}
+
+template<typename T>
+INLINE typename T::Type atomic_fetch_add(volatile T *a,
+    typename T::Type v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return __sync_fetch_and_add(&a->val_dont_use, v);
+}
+
+template<typename T>
+INLINE typename T::Type atomic_fetch_sub(volatile T *a,
+    typename T::Type v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return __sync_fetch_and_add(&a->val_dont_use, -v);
+}
+
+template<typename T>
+INLINE typename T::Type atomic_exchange(volatile T *a,
+    typename T::Type v, memory_order mo) {
+  DCHECK(!((uptr)a % sizeof(*a)));
+  if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
+    __sync_synchronize();
+  v = __sync_lock_test_and_set(&a->val_dont_use, v);
+  if (mo == memory_order_seq_cst)
+    __sync_synchronize();
+  return v;
+}
+
+template <typename T>
+INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
+                                           typename T::Type xchg,
+                                           memory_order mo) {
+  typedef typename T::Type Type;
+  Type cmpv = *cmp;
+  Type prev;
+  prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
+  if (prev == cmpv) return true;
+  *cmp = prev;
+  return false;
+}
+
+template<typename T>
+INLINE bool atomic_compare_exchange_weak(volatile T *a,
+                                         typename T::Type *cmp,
+                                         typename T::Type xchg,
+                                         memory_order mo) {
+  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
+}
+
+}  // namespace __sanitizer
+
+// This include provides explicit template instantiations for atomic_uint64_t
+// on MIPS32, which does not directly support 8 byte atomics. It has to
+// proceed the template definitions above.
+#if defined(_MIPS_SIM) && defined(_ABIO32)
+  #include "sanitizer_atomic_clang_mips.h"
+#endif
+
+#undef ATOMIC_ORDER
+
+#endif  // SANITIZER_ATOMIC_CLANG_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h
new file mode 100644
index 0000000000..d369aeb993
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h
@@ -0,0 +1,117 @@
+//===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+// Not intended for direct inclusion. Include sanitizer_atomic.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H
+#define SANITIZER_ATOMIC_CLANG_MIPS_H
+
+namespace __sanitizer {
+
+// MIPS32 does not support atomics > 4 bytes. To address this lack of
+// functionality, the sanitizer library provides helper methods which use an
+// internal spin lock mechanism to emulate atomic oprations when the size is
+// 8 bytes.
+static void __spin_lock(volatile int *lock) {
+  while (__sync_lock_test_and_set(lock, 1))
+    while (*lock) {
+    }
+}
+
+static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); }
+
+// Make sure the lock is on its own cache line to prevent false sharing.
+// Put it inside a struct that is aligned and padded to the typical MIPS
+// cacheline which is 32 bytes.
+static struct {
+  int lock;
+  char pad[32 - sizeof(int)];
+} __attribute__((aligned(32))) lock = {0, {0}};
+
+template <>
+INLINE atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
+                                              atomic_uint64_t::Type val,
+                                              memory_order mo) {
+  DCHECK(mo &
+         (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+  DCHECK(!((uptr)ptr % sizeof(*ptr)));
+
+  atomic_uint64_t::Type ret;
+
+  __spin_lock(&lock.lock);
+  ret = *(const_cast<atomic_uint64_t::Type volatile *>(&ptr->val_dont_use));
+  ptr->val_dont_use = ret + val;
+  __spin_unlock(&lock.lock);
+
+  return ret;
+}
+
+template <>
+INLINE atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
+                                              atomic_uint64_t::Type val,
+                                              memory_order mo) {
+  return atomic_fetch_add(ptr, -val, mo);
+}
+
+template <>
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
+                                           atomic_uint64_t::Type *cmp,
+                                           atomic_uint64_t::Type xchg,
+                                           memory_order mo) {
+  DCHECK(mo &
+         (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+  DCHECK(!((uptr)ptr % sizeof(*ptr)));
+
+  typedef atomic_uint64_t::Type Type;
+  Type cmpv = *cmp;
+  Type prev;
+  bool ret = false;
+
+  __spin_lock(&lock.lock);
+  prev = *(const_cast<Type volatile *>(&ptr->val_dont_use));
+  if (prev == cmpv) {
+    ret = true;
+    ptr->val_dont_use = xchg;
+  }
+  __spin_unlock(&lock.lock);
+
+  return ret;
+}
+
+template <>
+INLINE atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
+                                         memory_order mo) {
+  DCHECK(mo &
+         (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+  DCHECK(!((uptr)ptr % sizeof(*ptr)));
+
+  atomic_uint64_t::Type zero = 0;
+  volatile atomic_uint64_t *Newptr =
+      const_cast<volatile atomic_uint64_t *>(ptr);
+  return atomic_fetch_add(Newptr, zero, mo);
+}
+
+template <>
+INLINE void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
+                         memory_order mo) {
+  DCHECK(mo &
+         (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+  DCHECK(!((uptr)ptr % sizeof(*ptr)));
+
+  __spin_lock(&lock.lock);
+  ptr->val_dont_use = v;
+  __spin_unlock(&lock.lock);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ATOMIC_CLANG_MIPS_H
+
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h
new file mode 100644
index 0000000000..b8685a8542
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_other.h
@@ -0,0 +1,97 @@
+//===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+// Not intended for direct inclusion. Include sanitizer_atomic.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H
+#define SANITIZER_ATOMIC_CLANG_OTHER_H
+
+namespace __sanitizer {
+
+
+INLINE void proc_yield(int cnt) {
+  __asm__ __volatile__("" ::: "memory");
+}
+
+template<typename T>
+INLINE typename T::Type atomic_load(
+    const volatile T *a, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_consume
+      | memory_order_acquire | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+  typename T::Type v;
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
+    if (mo == memory_order_relaxed) {
+      v = a->val_dont_use;
+    } else if (mo == memory_order_consume) {
+      // Assume that processor respects data dependencies
+      // (and that compiler won't break them).
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      __asm__ __volatile__("" ::: "memory");
+    } else if (mo == memory_order_acquire) {
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      __sync_synchronize();
+    } else {  // seq_cst
+      // E.g. on POWER we need a hw fence even before the store.
+      __sync_synchronize();
+      v = a->val_dont_use;
+      __sync_synchronize();
+    }
+  } else {
+    // 64-bit load on 32-bit platform.
+    // Gross, but simple and reliable.
+    // Assume that it is not in read-only memory.
+    v = __sync_fetch_and_add(
+        const_cast<typename T::Type volatile *>(&a->val_dont_use), 0);
+  }
+  return v;
+}
+
+template<typename T>
+INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_release
+      | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
+    if (mo == memory_order_relaxed) {
+      a->val_dont_use = v;
+    } else if (mo == memory_order_release) {
+      __sync_synchronize();
+      a->val_dont_use = v;
+      __asm__ __volatile__("" ::: "memory");
+    } else {  // seq_cst
+      __sync_synchronize();
+      a->val_dont_use = v;
+      __sync_synchronize();
+    }
+  } else {
+    // 64-bit store on 32-bit platform.
+    // Gross, but simple and reliable.
+    typename T::Type cmp = a->val_dont_use;
+    typename T::Type cur;
+    for (;;) {
+      cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v);
+      if (cur == cmp || cur == v)
+        break;
+      cmp = cur;
+    }
+  }
+}
+
+}  // namespace __sanitizer
+
+#endif  // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h
new file mode 100644
index 0000000000..f2ce553baa
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic_clang_x86.h
@@ -0,0 +1,113 @@
+//===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+// Not intended for direct inclusion. Include sanitizer_atomic.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_CLANG_X86_H
+#define SANITIZER_ATOMIC_CLANG_X86_H
+
+namespace __sanitizer {
+
+INLINE void proc_yield(int cnt) {
+  __asm__ __volatile__("" ::: "memory");
+  for (int i = 0; i < cnt; i++)
+    __asm__ __volatile__("pause");
+  __asm__ __volatile__("" ::: "memory");
+}
+
+template<typename T>
+INLINE typename T::Type atomic_load(
+    const volatile T *a, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_consume
+      | memory_order_acquire | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+  typename T::Type v;
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
+    if (mo == memory_order_relaxed) {
+      v = a->val_dont_use;
+    } else if (mo == memory_order_consume) {
+      // Assume that processor respects data dependencies
+      // (and that compiler won't break them).
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      __asm__ __volatile__("" ::: "memory");
+    } else if (mo == memory_order_acquire) {
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      // On x86 loads are implicitly acquire.
+      __asm__ __volatile__("" ::: "memory");
+    } else {  // seq_cst
+      // On x86 plain MOV is enough for seq_cst store.
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      __asm__ __volatile__("" ::: "memory");
+    }
+  } else {
+    // 64-bit load on 32-bit platform.
+    __asm__ __volatile__(
+        "movq %1, %%mm0;"  // Use mmx reg for 64-bit atomic moves
+        "movq %%mm0, %0;"  // (ptr could be read-only)
+        "emms;"            // Empty mmx state/Reset FP regs
+        : "=m" (v)
+        : "m" (a->val_dont_use)
+        : // mark the mmx registers as clobbered
+#ifdef __MMX__
+          "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
+#endif  // #ifdef __MMX__
+          "memory");
+  }
+  return v;
+}
+
+template<typename T>
+INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_release
+      | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
+    if (mo == memory_order_relaxed) {
+      a->val_dont_use = v;
+    } else if (mo == memory_order_release) {
+      // On x86 stores are implicitly release.
+      __asm__ __volatile__("" ::: "memory");
+      a->val_dont_use = v;
+      __asm__ __volatile__("" ::: "memory");
+    } else {  // seq_cst
+      // On x86 stores are implicitly release.
+      __asm__ __volatile__("" ::: "memory");
+      a->val_dont_use = v;
+      __sync_synchronize();
+    }
+  } else {
+    // 64-bit store on 32-bit platform.
+    __asm__ __volatile__(
+        "movq %1, %%mm0;"  // Use mmx reg for 64-bit atomic moves
+        "movq %%mm0, %0;"
+        "emms;"            // Empty mmx state/Reset FP regs
+        : "=m" (a->val_dont_use)
+        : "m" (v)
+        : // mark the mmx registers as clobbered
+#ifdef __MMX__
+          "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
+#endif  // #ifdef __MMX__
+          "memory");
+    if (mo == memory_order_seq_cst)
+      __sync_synchronize();
+  }
+}
+
+}  // namespace __sanitizer
+
+#endif  // #ifndef SANITIZER_ATOMIC_CLANG_X86_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h b/lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h
new file mode 100644
index 0000000000..6a7c5465dc
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_atomic_msvc.h
@@ -0,0 +1,256 @@
+//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+// Not intended for direct inclusion. Include sanitizer_atomic.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ATOMIC_MSVC_H
+#define SANITIZER_ATOMIC_MSVC_H
+
+extern "C" void _ReadWriteBarrier();
+#pragma intrinsic(_ReadWriteBarrier)
+extern "C" void _mm_mfence();
+#pragma intrinsic(_mm_mfence)
+extern "C" void _mm_pause();
+#pragma intrinsic(_mm_pause)
+extern "C" char _InterlockedExchange8(char volatile *Addend, char Value);
+#pragma intrinsic(_InterlockedExchange8)
+extern "C" short _InterlockedExchange16(short volatile *Addend, short Value);
+#pragma intrinsic(_InterlockedExchange16)
+extern "C" long _InterlockedExchange(long volatile *Addend, long Value);
+#pragma intrinsic(_InterlockedExchange)
+extern "C" long _InterlockedExchangeAdd(long volatile *Addend, long Value);
+#pragma intrinsic(_InterlockedExchangeAdd)
+extern "C" char _InterlockedCompareExchange8(char volatile *Destination,
+                                             char Exchange, char Comparand);
+#pragma intrinsic(_InterlockedCompareExchange8)
+extern "C" short _InterlockedCompareExchange16(short volatile *Destination,
+                                               short Exchange, short Comparand);
+#pragma intrinsic(_InterlockedCompareExchange16)
+extern "C" long long _InterlockedCompareExchange64(
+    long long volatile *Destination, long long Exchange, long long Comparand);
+#pragma intrinsic(_InterlockedCompareExchange64)
+extern "C" void *_InterlockedCompareExchangePointer(
+    void *volatile *Destination,
+    void *Exchange, void *Comparand);
+#pragma intrinsic(_InterlockedCompareExchangePointer)
+extern "C" long __cdecl _InterlockedCompareExchange(long volatile *Destination,
+                                                    long Exchange,
+                                                    long Comparand);
+#pragma intrinsic(_InterlockedCompareExchange)
+
+#ifdef _WIN64
+extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend,
+                                               long long Value);
+#pragma intrinsic(_InterlockedExchangeAdd64)
+#endif
+
+namespace __sanitizer {
+
+INLINE void atomic_signal_fence(memory_order) {
+  _ReadWriteBarrier();
+}
+
+INLINE void atomic_thread_fence(memory_order) {
+  _mm_mfence();
+}
+
+INLINE void proc_yield(int cnt) {
+  for (int i = 0; i < cnt; i++)
+    _mm_pause();
+}
+
+template<typename T>
+INLINE typename T::Type atomic_load(
+    const volatile T *a, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_consume
+      | memory_order_acquire | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+  typename T::Type v;
+  // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
+  if (mo == memory_order_relaxed) {
+    v = a->val_dont_use;
+  } else {
+    atomic_signal_fence(memory_order_seq_cst);
+    v = a->val_dont_use;
+    atomic_signal_fence(memory_order_seq_cst);
+  }
+  return v;
+}
+
+template<typename T>
+INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+  DCHECK(mo & (memory_order_relaxed | memory_order_release
+      | memory_order_seq_cst));
+  DCHECK(!((uptr)a % sizeof(*a)));
+  // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
+  if (mo == memory_order_relaxed) {
+    a->val_dont_use = v;
+  } else {
+    atomic_signal_fence(memory_order_seq_cst);
+    a->val_dont_use = v;
+    atomic_signal_fence(memory_order_seq_cst);
+  }
+  if (mo == memory_order_seq_cst)
+    atomic_thread_fence(memory_order_seq_cst);
+}
+
+INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
+    u32 v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
+                                      (long)v);
+}
+
+INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
+    uptr v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+#ifdef _WIN64
+  return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
+                                         (long long)v);
+#else
+  return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
+                                       (long)v);
+#endif
+}
+
+INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
+    u32 v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
+                                      -(long)v);
+}
+
+INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
+    uptr v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+#ifdef _WIN64
+  return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
+                                         -(long long)v);
+#else
+  return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
+                                       -(long)v);
+#endif
+}
+
+INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
+    u8 v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
+}
+
+INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
+    u16 v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
+}
+
+INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
+    u32 v, memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
+}
+
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
+                                           u8 *cmp,
+                                           u8 xchgv,
+                                           memory_order mo) {
+  (void)mo;
+  DCHECK(!((uptr)a % sizeof(*a)));
+  u8 cmpv = *cmp;
+#ifdef _WIN64
+  u8 prev = (u8)_InterlockedCompareExchange8(
+      (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
+#else
+  u8 prev;
+  __asm {
+    mov al, cmpv
+    mov ecx, a
+    mov dl, xchgv
+    lock cmpxchg [ecx], dl
+    mov prev, al
+  }
+#endif
+  if (prev == cmpv)
+    return true;
+  *cmp = prev;
+  return false;
+}
+
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
+                                           uptr *cmp,
+                                           uptr xchg,
+                                           memory_order mo) {
+  uptr cmpv = *cmp;
+  uptr prev = (uptr)_InterlockedCompareExchangePointer(
+      (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
+  if (prev == cmpv)
+    return true;
+  *cmp = prev;
+  return false;
+}
+
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
+                                           u16 *cmp,
+                                           u16 xchg,
+                                           memory_order mo) {
+  u16 cmpv = *cmp;
+  u16 prev = (u16)_InterlockedCompareExchange16(
+      (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
+  if (prev == cmpv)
+    return true;
+  *cmp = prev;
+  return false;
+}
+
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
+                                           u32 *cmp,
+                                           u32 xchg,
+                                           memory_order mo) {
+  u32 cmpv = *cmp;
+  u32 prev = (u32)_InterlockedCompareExchange(
+      (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
+  if (prev == cmpv)
+    return true;
+  *cmp = prev;
+  return false;
+}
+
+INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
+                                           u64 *cmp,
+                                           u64 xchg,
+                                           memory_order mo) {
+  u64 cmpv = *cmp;
+  u64 prev = (u64)_InterlockedCompareExchange64(
+      (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
+  if (prev == cmpv)
+    return true;
+  *cmp = prev;
+  return false;
+}
+
+template<typename T>
+INLINE bool atomic_compare_exchange_weak(volatile T *a,
+                                         typename T::Type *cmp,
+                                         typename T::Type xchg,
+                                         memory_order mo) {
+  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ATOMIC_CLANG_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_bitvector.h b/lib/tsan/sanitizer_common/sanitizer_bitvector.h
new file mode 100644
index 0000000000..07a59ab11c
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_bitvector.h
@@ -0,0 +1,350 @@
+//===-- sanitizer_bitvector.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Specializer BitVector implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_BITVECTOR_H
+#define SANITIZER_BITVECTOR_H
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+// Fixed size bit vector based on a single basic integer.
+template <class basic_int_t = uptr>
+class BasicBitVector {
+ public:
+  enum SizeEnum : uptr { kSize = sizeof(basic_int_t) * 8 };
+
+  uptr size() const { return kSize; }
+  // No CTOR.
+  void clear() { bits_ = 0; }
+  void setAll() { bits_ = ~(basic_int_t)0; }
+  bool empty() const { return bits_ == 0; }
+
+  // Returns true if the bit has changed from 0 to 1.
+  bool setBit(uptr idx) {
+    basic_int_t old = bits_;
+    bits_ |= mask(idx);
+    return bits_ != old;
+  }
+
+  // Returns true if the bit has changed from 1 to 0.
+  bool clearBit(uptr idx) {
+    basic_int_t old = bits_;
+    bits_ &= ~mask(idx);
+    return bits_ != old;
+  }
+
+  bool getBit(uptr idx) const { return (bits_ & mask(idx)) != 0; }
+
+  uptr getAndClearFirstOne() {
+    CHECK(!empty());
+    uptr idx = LeastSignificantSetBitIndex(bits_);
+    clearBit(idx);
+    return idx;
+  }
+
+  // Do "this |= v" and return whether new bits have been added.
+  bool setUnion(const BasicBitVector &v) {
+    basic_int_t old = bits_;
+    bits_ |= v.bits_;
+    return bits_ != old;
+  }
+
+  // Do "this &= v" and return whether any bits have been removed.
+  bool setIntersection(const BasicBitVector &v) {
+    basic_int_t old = bits_;
+    bits_ &= v.bits_;
+    return bits_ != old;
+  }
+
+  // Do "this &= ~v" and return whether any bits have been removed.
+  bool setDifference(const BasicBitVector &v) {
+    basic_int_t old = bits_;
+    bits_ &= ~v.bits_;
+    return bits_ != old;
+  }
+
+  void copyFrom(const BasicBitVector &v) { bits_ = v.bits_; }
+
+  // Returns true if 'this' intersects with 'v'.
+  bool intersectsWith(const BasicBitVector &v) const {
+    return (bits_ & v.bits_) != 0;
+  }
+
+  // for (BasicBitVector<>::Iterator it(bv); it.hasNext();) {
+  //   uptr idx = it.next();
+  //   use(idx);
+  // }
+  class Iterator {
+   public:
+    Iterator() { }
+    explicit Iterator(const BasicBitVector &bv) : bv_(bv) {}
+    bool hasNext() const { return !bv_.empty(); }
+    uptr next() { return bv_.getAndClearFirstOne(); }
+    void clear() { bv_.clear(); }
+   private:
+    BasicBitVector bv_;
+  };
+
+ private:
+  basic_int_t mask(uptr idx) const {
+    CHECK_LT(idx, size());
+    return (basic_int_t)1UL << idx;
+  }
+  basic_int_t bits_;
+};
+
+// Fixed size bit vector of (kLevel1Size*BV::kSize**2) bits.
+// The implementation is optimized for better performance on
+// sparse bit vectors, i.e. the those with few set bits.
+template <uptr kLevel1Size = 1, class BV = BasicBitVector<> >
+class TwoLevelBitVector {
+  // This is essentially a 2-level bit vector.
+  // Set bit in the first level BV indicates that there are set bits
+  // in the corresponding BV of the second level.
+  // This structure allows O(kLevel1Size) time for clear() and empty(),
+  // as well fast handling of sparse BVs.
+ public:
+  enum SizeEnum : uptr { kSize = BV::kSize * BV::kSize * kLevel1Size };
+  // No CTOR.
+
+  uptr size() const { return kSize; }
+
+  void clear() {
+    for (uptr i = 0; i < kLevel1Size; i++)
+      l1_[i].clear();
+  }
+
+  void setAll() {
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      l1_[i0].setAll();
+      for (uptr i1 = 0; i1 < BV::kSize; i1++)
+        l2_[i0][i1].setAll();
+    }
+  }
+
+  bool empty() const {
+    for (uptr i = 0; i < kLevel1Size; i++)
+      if (!l1_[i].empty())
+        return false;
+    return true;
+  }
+
+  // Returns true if the bit has changed from 0 to 1.
+  bool setBit(uptr idx) {
+    check(idx);
+    uptr i0 = idx0(idx);
+    uptr i1 = idx1(idx);
+    uptr i2 = idx2(idx);
+    if (!l1_[i0].getBit(i1)) {
+      l1_[i0].setBit(i1);
+      l2_[i0][i1].clear();
+    }
+    bool res = l2_[i0][i1].setBit(i2);
+    // Printf("%s: %zd => %zd %zd %zd; %d\n", __func__,
+    // idx, i0, i1, i2, res);
+    return res;
+  }
+
+  bool clearBit(uptr idx) {
+    check(idx);
+    uptr i0 = idx0(idx);
+    uptr i1 = idx1(idx);
+    uptr i2 = idx2(idx);
+    bool res = false;
+    if (l1_[i0].getBit(i1)) {
+      res = l2_[i0][i1].clearBit(i2);
+      if (l2_[i0][i1].empty())
+        l1_[i0].clearBit(i1);
+    }
+    return res;
+  }
+
+  bool getBit(uptr idx) const {
+    check(idx);
+    uptr i0 = idx0(idx);
+    uptr i1 = idx1(idx);
+    uptr i2 = idx2(idx);
+    // Printf("%s: %zd => %zd %zd %zd\n", __func__, idx, i0, i1, i2);
+    return l1_[i0].getBit(i1) && l2_[i0][i1].getBit(i2);
+  }
+
+  uptr getAndClearFirstOne() {
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      if (l1_[i0].empty()) continue;
+      uptr i1 = l1_[i0].getAndClearFirstOne();
+      uptr i2 = l2_[i0][i1].getAndClearFirstOne();
+      if (!l2_[i0][i1].empty())
+        l1_[i0].setBit(i1);
+      uptr res = i0 * BV::kSize * BV::kSize + i1 * BV::kSize + i2;
+      // Printf("getAndClearFirstOne: %zd %zd %zd => %zd\n", i0, i1, i2, res);
+      return res;
+    }
+    CHECK(0);
+    return 0;
+  }
+
+  // Do "this |= v" and return whether new bits have been added.
+  bool setUnion(const TwoLevelBitVector &v) {
+    bool res = false;
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      BV t = v.l1_[i0];
+      while (!t.empty()) {
+        uptr i1 = t.getAndClearFirstOne();
+        if (l1_[i0].setBit(i1))
+          l2_[i0][i1].clear();
+        if (l2_[i0][i1].setUnion(v.l2_[i0][i1]))
+          res = true;
+      }
+    }
+    return res;
+  }
+
+  // Do "this &= v" and return whether any bits have been removed.
+  bool setIntersection(const TwoLevelBitVector &v) {
+    bool res = false;
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      if (l1_[i0].setIntersection(v.l1_[i0]))
+        res = true;
+      if (!l1_[i0].empty()) {
+        BV t = l1_[i0];
+        while (!t.empty()) {
+          uptr i1 = t.getAndClearFirstOne();
+          if (l2_[i0][i1].setIntersection(v.l2_[i0][i1]))
+            res = true;
+          if (l2_[i0][i1].empty())
+            l1_[i0].clearBit(i1);
+        }
+      }
+    }
+    return res;
+  }
+
+  // Do "this &= ~v" and return whether any bits have been removed.
+  bool setDifference(const TwoLevelBitVector &v) {
+    bool res = false;
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      BV t = l1_[i0];
+      t.setIntersection(v.l1_[i0]);
+      while (!t.empty()) {
+        uptr i1 = t.getAndClearFirstOne();
+        if (l2_[i0][i1].setDifference(v.l2_[i0][i1]))
+          res = true;
+        if (l2_[i0][i1].empty())
+          l1_[i0].clearBit(i1);
+      }
+    }
+    return res;
+  }
+
+  void copyFrom(const TwoLevelBitVector &v) {
+    clear();
+    setUnion(v);
+  }
+
+  // Returns true if 'this' intersects with 'v'.
+  bool intersectsWith(const TwoLevelBitVector &v) const {
+    for (uptr i0 = 0; i0 < kLevel1Size; i0++) {
+      BV t = l1_[i0];
+      t.setIntersection(v.l1_[i0]);
+      while (!t.empty()) {
+        uptr i1 = t.getAndClearFirstOne();
+        if (!v.l1_[i0].getBit(i1)) continue;
+        if (l2_[i0][i1].intersectsWith(v.l2_[i0][i1]))
+          return true;
+      }
+    }
+    return false;
+  }
+
+  // for (TwoLevelBitVector<>::Iterator it(bv); it.hasNext();) {
+  //   uptr idx = it.next();
+  //   use(idx);
+  // }
+  class Iterator {
+   public:
+    Iterator() { }
+    explicit Iterator(const TwoLevelBitVector &bv) : bv_(bv), i0_(0), i1_(0) {
+      it1_.clear();
+      it2_.clear();
+    }
+
+    bool hasNext() const {
+      if (it1_.hasNext()) return true;
+      for (uptr i = i0_; i < kLevel1Size; i++)
+        if (!bv_.l1_[i].empty()) return true;
+      return false;
+    }
+
+    uptr next() {
+      // Printf("++++: %zd %zd; %d %d; size %zd\n", i0_, i1_, it1_.hasNext(),
+      //       it2_.hasNext(), kSize);
+      if (!it1_.hasNext() && !it2_.hasNext()) {
+        for (; i0_ < kLevel1Size; i0_++) {
+          if (bv_.l1_[i0_].empty()) continue;
+          it1_ = typename BV::Iterator(bv_.l1_[i0_]);
+          // Printf("+i0: %zd %zd; %d %d; size %zd\n", i0_, i1_, it1_.hasNext(),
+          //   it2_.hasNext(), kSize);
+          break;
+        }
+      }
+      if (!it2_.hasNext()) {
+        CHECK(it1_.hasNext());
+        i1_ = it1_.next();
+        it2_ = typename BV::Iterator(bv_.l2_[i0_][i1_]);
+        // Printf("++i1: %zd %zd; %d %d; size %zd\n", i0_, i1_, it1_.hasNext(),
+        //       it2_.hasNext(), kSize);
+      }
+      CHECK(it2_.hasNext());
+      uptr i2 = it2_.next();
+      uptr res = i0_ * BV::kSize * BV::kSize + i1_ * BV::kSize + i2;
+      // Printf("+ret: %zd %zd; %d %d; size %zd; res: %zd\n", i0_, i1_,
+      //       it1_.hasNext(), it2_.hasNext(), kSize, res);
+      if (!it1_.hasNext() && !it2_.hasNext())
+        i0_++;
+      return res;
+    }
+
+   private:
+    const TwoLevelBitVector &bv_;
+    uptr i0_, i1_;
+    typename BV::Iterator it1_, it2_;
+  };
+
+ private:
+  void check(uptr idx) const { CHECK_LE(idx, size()); }
+
+  uptr idx0(uptr idx) const {
+    uptr res = idx / (BV::kSize * BV::kSize);
+    CHECK_LE(res, kLevel1Size);
+    return res;
+  }
+
+  uptr idx1(uptr idx) const {
+    uptr res = (idx / BV::kSize) % BV::kSize;
+    CHECK_LE(res, BV::kSize);
+    return res;
+  }
+
+  uptr idx2(uptr idx) const {
+    uptr res = idx % BV::kSize;
+    CHECK_LE(res, BV::kSize);
+    return res;
+  }
+
+  BV l1_[kLevel1Size];
+  BV l2_[kLevel1Size][BV::kSize];
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_BITVECTOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_bvgraph.h b/lib/tsan/sanitizer_common/sanitizer_bvgraph.h
new file mode 100644
index 0000000000..e7249055be
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_bvgraph.h
@@ -0,0 +1,164 @@
+//===-- sanitizer_bvgraph.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime.
+// BVGraph -- a directed graph.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_BVGRAPH_H
+#define SANITIZER_BVGRAPH_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_bitvector.h"
+
+namespace __sanitizer {
+
+// Directed graph of fixed size implemented as an array of bit vectors.
+// Not thread-safe, all accesses should be protected by an external lock.
+template<class BV>
+class BVGraph {
+ public:
+  enum SizeEnum : uptr { kSize = BV::kSize };
+  uptr size() const { return kSize; }
+  // No CTOR.
+  void clear() {
+    for (uptr i = 0; i < size(); i++)
+      v[i].clear();
+  }
+
+  bool empty() const {
+    for (uptr i = 0; i < size(); i++)
+      if (!v[i].empty())
+        return false;
+    return true;
+  }
+
+  // Returns true if a new edge was added.
+  bool addEdge(uptr from, uptr to) {
+    check(from, to);
+    return v[from].setBit(to);
+  }
+
+  // Returns true if at least one new edge was added.
+  uptr addEdges(const BV &from, uptr to, uptr added_edges[],
+                uptr max_added_edges) {
+    uptr res = 0;
+    t1.copyFrom(from);
+    while (!t1.empty()) {
+      uptr node = t1.getAndClearFirstOne();
+      if (v[node].setBit(to))
+        if (res < max_added_edges)
+          added_edges[res++] = node;
+    }
+    return res;
+  }
+
+  // *EXPERIMENTAL*
+  // Returns true if an edge from=>to exist.
+  // This function does not use any global state except for 'this' itself,
+  // and thus can be called from different threads w/o locking.
+  // This would be racy.
+  // FIXME: investigate how much we can prove about this race being "benign".
+  bool hasEdge(uptr from, uptr to) { return v[from].getBit(to); }
+
+  // Returns true if the edge from=>to was removed.
+  bool removeEdge(uptr from, uptr to) {
+    return v[from].clearBit(to);
+  }
+
+  // Returns true if at least one edge *=>to was removed.
+  bool removeEdgesTo(const BV &to) {
+    bool res = 0;
+    for (uptr from = 0; from < size(); from++) {
+      if (v[from].setDifference(to))
+        res = true;
+    }
+    return res;
+  }
+
+  // Returns true if at least one edge from=>* was removed.
+  bool removeEdgesFrom(const BV &from) {
+    bool res = false;
+    t1.copyFrom(from);
+    while (!t1.empty()) {
+      uptr idx = t1.getAndClearFirstOne();
+      if (!v[idx].empty()) {
+        v[idx].clear();
+        res = true;
+      }
+    }
+    return res;
+  }
+
+  void removeEdgesFrom(uptr from) {
+    return v[from].clear();
+  }
+
+  bool hasEdge(uptr from, uptr to) const {
+    check(from, to);
+    return v[from].getBit(to);
+  }
+
+  // Returns true if there is a path from the node 'from'
+  // to any of the nodes in 'targets'.
+  bool isReachable(uptr from, const BV &targets) {
+    BV &to_visit = t1,
+       &visited = t2;
+    to_visit.copyFrom(v[from]);
+    visited.clear();
+    visited.setBit(from);
+    while (!to_visit.empty()) {
+      uptr idx = to_visit.getAndClearFirstOne();
+      if (visited.setBit(idx))
+        to_visit.setUnion(v[idx]);
+    }
+    return targets.intersectsWith(visited);
+  }
+
+  // Finds a path from 'from' to one of the nodes in 'target',
+  // stores up to 'path_size' items of the path into 'path',
+  // returns the path length, or 0 if there is no path of size 'path_size'.
+  uptr findPath(uptr from, const BV &targets, uptr *path, uptr path_size) {
+    if (path_size == 0)
+      return 0;
+    path[0] = from;
+    if (targets.getBit(from))
+      return 1;
+    // The function is recursive, so we don't want to create BV on stack.
+    // Instead of a getAndClearFirstOne loop we use the slower iterator.
+    for (typename BV::Iterator it(v[from]); it.hasNext(); ) {
+      uptr idx = it.next();
+      if (uptr res = findPath(idx, targets, path + 1, path_size - 1))
+        return res + 1;
+    }
+    return 0;
+  }
+
+  // Same as findPath, but finds a shortest path.
+  uptr findShortestPath(uptr from, const BV &targets, uptr *path,
+                        uptr path_size) {
+    for (uptr p = 1; p <= path_size; p++)
+      if (findPath(from, targets, path, p) == p)
+        return p;
+    return 0;
+  }
+
+ private:
+  void check(uptr idx1, uptr idx2) const {
+    CHECK_LT(idx1, size());
+    CHECK_LT(idx2, size());
+  }
+  BV v[kSize];
+  // Keep temporary vectors here since we can not create large objects on stack.
+  BV t1, t2;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_BVGRAPH_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_common.cpp b/lib/tsan/sanitizer_common/sanitizer_common.cpp
new file mode 100644
index 0000000000..87efda5bd3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common.cpp
@@ -0,0 +1,348 @@
+//===-- sanitizer_common.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_allocator_interface.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+
+namespace __sanitizer {
+
+const char *SanitizerToolName = "SanitizerTool";
+
+atomic_uint32_t current_verbosity;
+uptr PageSizeCached;
+u32 NumberOfCPUsCached;
+
+// PID of the tracer task in StopTheWorld. It shares the address space with the
+// main process, but has a different PID and thus requires special handling.
+uptr stoptheworld_tracer_pid = 0;
+// Cached pid of parent process - if the parent process dies, we want to keep
+// writing to the same log file.
+uptr stoptheworld_tracer_ppid = 0;
+
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      const char *mmap_type, error_t err,
+                                      bool raw_report) {
+  static int recursion_count;
+  if (SANITIZER_RTEMS || raw_report || recursion_count) {
+    // If we are on RTEMS or raw report is requested or we went into recursion,
+    // just die.  The Report() and CHECK calls below may call mmap recursively
+    // and fail.
+    RawWrite("ERROR: Failed to mmap\n");
+    Die();
+  }
+  recursion_count++;
+  Report("ERROR: %s failed to "
+         "%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
+         SanitizerToolName, mmap_type, size, size, mem_type, err);
+#if !SANITIZER_GO
+  DumpProcessMap();
+#endif
+  UNREACHABLE("unable to mmap");
+}
+
+typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
+typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
+
+const char *StripPathPrefix(const char *filepath,
+                            const char *strip_path_prefix) {
+  if (!filepath) return nullptr;
+  if (!strip_path_prefix) return filepath;
+  const char *res = filepath;
+  if (const char *pos = internal_strstr(filepath, strip_path_prefix))
+    res = pos + internal_strlen(strip_path_prefix);
+  if (res[0] == '.' && res[1] == '/')
+    res += 2;
+  return res;
+}
+
+const char *StripModuleName(const char *module) {
+  if (!module)
+    return nullptr;
+  if (SANITIZER_WINDOWS) {
+    // On Windows, both slash and backslash are possible.
+    // Pick the one that goes last.
+    if (const char *bslash_pos = internal_strrchr(module, '\\'))
+      return StripModuleName(bslash_pos + 1);
+  }
+  if (const char *slash_pos = internal_strrchr(module, '/')) {
+    return slash_pos + 1;
+  }
+  return module;
+}
+
+void ReportErrorSummary(const char *error_message, const char *alt_tool_name) {
+  if (!common_flags()->print_summary)
+    return;
+  InternalScopedString buff(kMaxSummaryLength);
+  buff.append("SUMMARY: %s: %s",
+              alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
+  __sanitizer_report_error_summary(buff.data());
+}
+
+// Removes the ANSI escape sequences from the input string (in-place).
+void RemoveANSIEscapeSequencesFromString(char *str) {
+  if (!str)
+    return;
+
+  // We are going to remove the escape sequences in place.
+  char *s = str;
+  char *z = str;
+  while (*s != '\0') {
+    CHECK_GE(s, z);
+    // Skip over ANSI escape sequences with pointer 's'.
+    if (*s == '\033' && *(s + 1) == '[') {
+      s = internal_strchrnul(s, 'm');
+      if (*s == '\0') {
+        break;
+      }
+      s++;
+      continue;
+    }
+    // 's' now points at a character we want to keep. Copy over the buffer
+    // content if the escape sequence has been perviously skipped andadvance
+    // both pointers.
+    if (s != z)
+      *z = *s;
+
+    // If we have not seen an escape sequence, just advance both pointers.
+    z++;
+    s++;
+  }
+
+  // Null terminate the string.
+  *z = '\0';
+}
+
+void LoadedModule::set(const char *module_name, uptr base_address) {
+  clear();
+  full_name_ = internal_strdup(module_name);
+  base_address_ = base_address;
+}
+
+void LoadedModule::set(const char *module_name, uptr base_address,
+                       ModuleArch arch, u8 uuid[kModuleUUIDSize],
+                       bool instrumented) {
+  set(module_name, base_address);
+  arch_ = arch;
+  internal_memcpy(uuid_, uuid, sizeof(uuid_));
+  instrumented_ = instrumented;
+}
+
+void LoadedModule::clear() {
+  InternalFree(full_name_);
+  base_address_ = 0;
+  max_executable_address_ = 0;
+  full_name_ = nullptr;
+  arch_ = kModuleArchUnknown;
+  internal_memset(uuid_, 0, kModuleUUIDSize);
+  instrumented_ = false;
+  while (!ranges_.empty()) {
+    AddressRange *r = ranges_.front();
+    ranges_.pop_front();
+    InternalFree(r);
+  }
+}
+
+void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable,
+                                   bool writable, const char *name) {
+  void *mem = InternalAlloc(sizeof(AddressRange));
+  AddressRange *r =
+      new(mem) AddressRange(beg, end, executable, writable, name);
+  ranges_.push_back(r);
+  if (executable && end > max_executable_address_)
+    max_executable_address_ = end;
+}
+
+bool LoadedModule::containsAddress(uptr address) const {
+  for (const AddressRange &r : ranges()) {
+    if (r.beg <= address && address < r.end)
+      return true;
+  }
+  return false;
+}
+
+static atomic_uintptr_t g_total_mmaped;
+
+void IncreaseTotalMmap(uptr size) {
+  if (!common_flags()->mmap_limit_mb) return;
+  uptr total_mmaped =
+      atomic_fetch_add(&g_total_mmaped, size, memory_order_relaxed) + size;
+  // Since for now mmap_limit_mb is not a user-facing flag, just kill
+  // a program. Use RAW_CHECK to avoid extra mmaps in reporting.
+  RAW_CHECK((total_mmaped >> 20) < common_flags()->mmap_limit_mb);
+}
+
+void DecreaseTotalMmap(uptr size) {
+  if (!common_flags()->mmap_limit_mb) return;
+  atomic_fetch_sub(&g_total_mmaped, size, memory_order_relaxed);
+}
+
+bool TemplateMatch(const char *templ, const char *str) {
+  if ((!str) || str[0] == 0)
+    return false;
+  bool start = false;
+  if (templ && templ[0] == '^') {
+    start = true;
+    templ++;
+  }
+  bool asterisk = false;
+  while (templ && templ[0]) {
+    if (templ[0] == '*') {
+      templ++;
+      start = false;
+      asterisk = true;
+      continue;
+    }
+    if (templ[0] == '$')
+      return str[0] == 0 || asterisk;
+    if (str[0] == 0)
+      return false;
+    char *tpos = (char*)internal_strchr(templ, '*');
+    char *tpos1 = (char*)internal_strchr(templ, '$');
+    if ((!tpos) || (tpos1 && tpos1 < tpos))
+      tpos = tpos1;
+    if (tpos)
+      tpos[0] = 0;
+    const char *str0 = str;
+    const char *spos = internal_strstr(str, templ);
+    str = spos + internal_strlen(templ);
+    templ = tpos;
+    if (tpos)
+      tpos[0] = tpos == tpos1 ? '$' : '*';
+    if (!spos)
+      return false;
+    if (start && spos != str0)
+      return false;
+    start = false;
+    asterisk = false;
+  }
+  return true;
+}
+
+static char binary_name_cache_str[kMaxPathLength];
+static char process_name_cache_str[kMaxPathLength];
+
+const char *GetProcessName() {
+  return process_name_cache_str;
+}
+
+static uptr ReadProcessName(/*out*/ char *buf, uptr buf_len) {
+  ReadLongProcessName(buf, buf_len);
+  char *s = const_cast<char *>(StripModuleName(buf));
+  uptr len = internal_strlen(s);
+  if (s != buf) {
+    internal_memmove(buf, s, len);
+    buf[len] = '\0';
+  }
+  return len;
+}
+
+void UpdateProcessName() {
+  ReadProcessName(process_name_cache_str, sizeof(process_name_cache_str));
+}
+
+// Call once to make sure that binary_name_cache_str is initialized
+void CacheBinaryName() {
+  if (binary_name_cache_str[0] != '\0')
+    return;
+  ReadBinaryName(binary_name_cache_str, sizeof(binary_name_cache_str));
+  ReadProcessName(process_name_cache_str, sizeof(process_name_cache_str));
+}
+
+uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) {
+  CacheBinaryName();
+  uptr name_len = internal_strlen(binary_name_cache_str);
+  name_len = (name_len < buf_len - 1) ? name_len : buf_len - 1;
+  if (buf_len == 0)
+    return 0;
+  internal_memcpy(buf, binary_name_cache_str, name_len);
+  buf[name_len] = '\0';
+  return name_len;
+}
+
+#if !SANITIZER_GO
+void PrintCmdline() {
+  char **argv = GetArgv();
+  if (!argv) return;
+  Printf("\nCommand: ");
+  for (uptr i = 0; argv[i]; ++i)
+    Printf("%s ", argv[i]);
+  Printf("\n\n");
+}
+#endif
+
+// Malloc hooks.
+static const int kMaxMallocFreeHooks = 5;
+struct MallocFreeHook {
+  void (*malloc_hook)(const void *, uptr);
+  void (*free_hook)(const void *);
+};
+
+static MallocFreeHook MFHooks[kMaxMallocFreeHooks];
+
+void RunMallocHooks(const void *ptr, uptr size) {
+  for (int i = 0; i < kMaxMallocFreeHooks; i++) {
+    auto hook = MFHooks[i].malloc_hook;
+    if (!hook) return;
+    hook(ptr, size);
+  }
+}
+
+void RunFreeHooks(const void *ptr) {
+  for (int i = 0; i < kMaxMallocFreeHooks; i++) {
+    auto hook = MFHooks[i].free_hook;
+    if (!hook) return;
+    hook(ptr);
+  }
+}
+
+static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr),
+                                  void (*free_hook)(const void *)) {
+  if (!malloc_hook || !free_hook) return 0;
+  for (int i = 0; i < kMaxMallocFreeHooks; i++) {
+    if (MFHooks[i].malloc_hook == nullptr) {
+      MFHooks[i].malloc_hook = malloc_hook;
+      MFHooks[i].free_hook = free_hook;
+      return i + 1;
+    }
+  }
+  return 0;
+}
+
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+extern "C" {
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_report_error_summary,
+                             const char *error_summary) {
+  Printf("%s\n", error_summary);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_acquire_crash_state() {
+  static atomic_uint8_t in_crash_state = {};
+  return !atomic_exchange(&in_crash_state, 1, memory_order_relaxed);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *,
+                                                                  uptr),
+                                              void (*free_hook)(const void *)) {
+  return InstallMallocFreeHooks(malloc_hook, free_hook);
+}
+} // extern "C"
diff --git a/lib/tsan/sanitizer_common/sanitizer_common.h b/lib/tsan/sanitizer_common/sanitizer_common.h
new file mode 100644
index 0000000000..07b307a602
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common.h
@@ -0,0 +1,1002 @@
+//===-- sanitizer_common.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between run-time libraries of sanitizers.
+//
+// It declares common functions and classes that are used in both runtimes.
+// Implementation of some functions are provided in sanitizer_common, while
+// others must be defined by run-time library itself.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_COMMON_H
+#define SANITIZER_COMMON_H
+
+#include "sanitizer_flags.h"
+#include "sanitizer_interface_internal.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_list.h"
+#include "sanitizer_mutex.h"
+
+#if defined(_MSC_VER) && !defined(__clang__)
+extern "C" void _ReadWriteBarrier();
+#pragma intrinsic(_ReadWriteBarrier)
+#endif
+
+namespace __sanitizer {
+
+struct AddressInfo;
+struct BufferedStackTrace;
+struct SignalContext;
+struct StackTrace;
+
+// Constants.
+const uptr kWordSize = SANITIZER_WORDSIZE / 8;
+const uptr kWordSizeInBits = 8 * kWordSize;
+
+const uptr kCacheLineSize = SANITIZER_CACHE_LINE_SIZE;
+
+const uptr kMaxPathLength = 4096;
+
+const uptr kMaxThreadStackSize = 1 << 30;  // 1Gb
+
+static const uptr kErrorMessageBufferSize = 1 << 16;
+
+// Denotes fake PC values that come from JIT/JAVA/etc.
+// For such PC values __tsan_symbolize_external_ex() will be called.
+const u64 kExternalPCBit = 1ULL << 60;
+
+extern const char *SanitizerToolName;  // Can be changed by the tool.
+
+extern atomic_uint32_t current_verbosity;
+INLINE void SetVerbosity(int verbosity) {
+  atomic_store(&current_verbosity, verbosity, memory_order_relaxed);
+}
+INLINE int Verbosity() {
+  return atomic_load(&current_verbosity, memory_order_relaxed);
+}
+
+#if SANITIZER_ANDROID
+INLINE uptr GetPageSize() {
+// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
+  return 4096;
+}
+INLINE uptr GetPageSizeCached() {
+  return 4096;
+}
+#else
+uptr GetPageSize();
+extern uptr PageSizeCached;
+INLINE uptr GetPageSizeCached() {
+  if (!PageSizeCached)
+    PageSizeCached = GetPageSize();
+  return PageSizeCached;
+}
+#endif
+uptr GetMmapGranularity();
+uptr GetMaxVirtualAddress();
+uptr GetMaxUserVirtualAddress();
+// Threads
+tid_t GetTid();
+int TgKill(pid_t pid, tid_t tid, int sig);
+uptr GetThreadSelf();
+void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
+                                uptr *stack_bottom);
+void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+                          uptr *tls_addr, uptr *tls_size);
+
+// Memory management
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
+INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
+  return MmapOrDie(size, mem_type, /*raw_report*/ true);
+}
+void UnmapOrDie(void *addr, uptr size);
+// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
+// case returns nullptr.
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type);
+bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr)
+     WARN_UNUSED_RESULT;
+bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size,
+                             const char *name = nullptr) WARN_UNUSED_RESULT;
+void *MmapNoReserveOrDie(uptr size, const char *mem_type);
+void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name = nullptr);
+// Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in
+// that case returns nullptr.
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size,
+                                 const char *name = nullptr);
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr);
+void *MmapNoAccess(uptr size);
+// Map aligned chunk of address space; size and alignment are powers of two.
+// Dies on all but out of memory errors, in the latter case returns nullptr.
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type);
+// Disallow access to a memory range.  Use MmapFixedNoAccess to allocate an
+// unaccessible memory.
+bool MprotectNoAccess(uptr addr, uptr size);
+bool MprotectReadOnly(uptr addr, uptr size);
+
+void MprotectMallocZones(void *addr, int prot);
+
+// Find an available address space.
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found, uptr *max_occupied_addr);
+
+// Used to check if we can map shadow memory to a fixed location.
+bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
+// Releases memory pages entirely within the [beg, end] address range. Noop if
+// the provided range does not contain at least one entire page.
+void ReleaseMemoryPagesToOS(uptr beg, uptr end);
+void IncreaseTotalMmap(uptr size);
+void DecreaseTotalMmap(uptr size);
+uptr GetRSS();
+void SetShadowRegionHugePageMode(uptr addr, uptr length);
+bool DontDumpShadowMemory(uptr addr, uptr length);
+// Check if the built VMA size matches the runtime one.
+void CheckVMASize();
+void RunMallocHooks(const void *ptr, uptr size);
+void RunFreeHooks(const void *ptr);
+
+class ReservedAddressRange {
+ public:
+  uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0);
+  uptr InitAligned(uptr size, uptr align, const char *name = nullptr);
+  uptr Map(uptr fixed_addr, uptr size, const char *name = nullptr);
+  uptr MapOrDie(uptr fixed_addr, uptr size, const char *name = nullptr);
+  void Unmap(uptr addr, uptr size);
+  void *base() const { return base_; }
+  uptr size() const { return size_; }
+
+ private:
+  void* base_;
+  uptr size_;
+  const char* name_;
+  uptr os_handle_;
+};
+
+typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
+                               /*out*/uptr *stats, uptr stats_size);
+
+// Parse the contents of /proc/self/smaps and generate a memory profile.
+// |cb| is a tool-specific callback that fills the |stats| array containing
+// |stats_size| elements.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+
+// Simple low-level (mmap-based) allocator for internal use. Doesn't have
+// constructor, so all instances of LowLevelAllocator should be
+// linker initialized.
+class LowLevelAllocator {
+ public:
+  // Requires an external lock.
+  void *Allocate(uptr size);
+ private:
+  char *allocated_end_;
+  char *allocated_current_;
+};
+// Set the min alignment of LowLevelAllocator to at least alignment.
+void SetLowLevelAllocateMinAlignment(uptr alignment);
+typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
+// Allows to register tool-specific callbacks for LowLevelAllocator.
+// Passing NULL removes the callback.
+void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
+
+// IO
+void CatastrophicErrorWrite(const char *buffer, uptr length);
+void RawWrite(const char *buffer);
+bool ColorizeReports();
+void RemoveANSIEscapeSequencesFromString(char *buffer);
+void Printf(const char *format, ...);
+void Report(const char *format, ...);
+void SetPrintfAndReportCallback(void (*callback)(const char *));
+#define VReport(level, ...)                                              \
+  do {                                                                   \
+    if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \
+  } while (0)
+#define VPrintf(level, ...)                                              \
+  do {                                                                   \
+    if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
+  } while (0)
+
+// Lock sanitizer error reporting and protects against nested errors.
+class ScopedErrorReportLock {
+ public:
+  ScopedErrorReportLock();
+  ~ScopedErrorReportLock();
+
+  static void CheckLocked();
+};
+
+extern uptr stoptheworld_tracer_pid;
+extern uptr stoptheworld_tracer_ppid;
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size);
+
+// Error report formatting.
+const char *StripPathPrefix(const char *filepath,
+                            const char *strip_file_prefix);
+// Strip the directories from the module name.
+const char *StripModuleName(const char *module);
+
+// OS
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len);
+uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len);
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len);
+const char *GetProcessName();
+void UpdateProcessName();
+void CacheBinaryName();
+void DisableCoreDumperIfNecessary();
+void DumpProcessMap();
+void PrintModuleMap();
+const char *GetEnv(const char *name);
+bool SetEnv(const char *name, const char *value);
+
+u32 GetUid();
+void ReExec();
+void CheckASLR();
+void CheckMPROTECT();
+char **GetArgv();
+char **GetEnviron();
+void PrintCmdline();
+bool StackSizeIsUnlimited();
+void SetStackSizeLimitInBytes(uptr limit);
+bool AddressSpaceIsUnlimited();
+void SetAddressSpaceUnlimited();
+void AdjustStackSize(void *attr);
+void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
+void SetSandboxingCallback(void (*f)());
+
+void InitializeCoverage(bool enabled, const char *coverage_dir);
+
+void InitTlsSize();
+uptr GetTlsSize();
+
+// Other
+void SleepForSeconds(int seconds);
+void SleepForMillis(int millis);
+u64 NanoTime();
+u64 MonotonicNanoTime();
+int Atexit(void (*function)(void));
+bool TemplateMatch(const char *templ, const char *str);
+
+// Exit
+void NORETURN Abort();
+void NORETURN Die();
+void NORETURN
+CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2);
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      const char *mmap_type, error_t err,
+                                      bool raw_report = false);
+
+// Specific tools may override behavior of "Die" and "CheckFailed" functions
+// to do tool-specific job.
+typedef void (*DieCallbackType)(void);
+
+// It's possible to add several callbacks that would be run when "Die" is
+// called. The callbacks will be run in the opposite order. The tools are
+// strongly recommended to setup all callbacks during initialization, when there
+// is only a single thread.
+bool AddDieCallback(DieCallbackType callback);
+bool RemoveDieCallback(DieCallbackType callback);
+
+void SetUserDieCallback(DieCallbackType callback);
+
+typedef void (*CheckFailedCallbackType)(const char *, int, const char *,
+                                       u64, u64);
+void SetCheckFailedCallback(CheckFailedCallbackType callback);
+
+// Callback will be called if soft_rss_limit_mb is given and the limit is
+// exceeded (exceeded==true) or if rss went down below the limit
+// (exceeded==false).
+// The callback should be registered once at the tool init time.
+void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
+
+// Functions related to signal handling.
+typedef void (*SignalHandlerType)(int, void *, void *);
+HandleSignalMode GetHandleSignalMode(int signum);
+void InstallDeadlySignalHandlers(SignalHandlerType handler);
+
+// Signal reporting.
+// Each sanitizer uses slightly different implementation of stack unwinding.
+typedef void (*UnwindSignalStackCallbackType)(const SignalContext &sig,
+                                              const void *callback_context,
+                                              BufferedStackTrace *stack);
+// Print deadly signal report and die.
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context);
+
+// Part of HandleDeadlySignal, exposed for asan.
+void StartReportDeadlySignal();
+// Part of HandleDeadlySignal, exposed for asan.
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context);
+
+// Alternative signal stack (POSIX-only).
+void SetAlternateSignalStack();
+void UnsetAlternateSignalStack();
+
+// We don't want a summary too long.
+const int kMaxSummaryLength = 1024;
+// Construct a one-line string:
+//   SUMMARY: SanitizerToolName: error_message
+// and pass it to __sanitizer_report_error_summary.
+// If alt_tool_name is provided, it's used in place of SanitizerToolName.
+void ReportErrorSummary(const char *error_message,
+                        const char *alt_tool_name = nullptr);
+// Same as above, but construct error_message as:
+//   error_type file:line[:column][ function]
+void ReportErrorSummary(const char *error_type, const AddressInfo &info,
+                        const char *alt_tool_name = nullptr);
+// Same as above, but obtains AddressInfo by symbolizing top stack trace frame.
+void ReportErrorSummary(const char *error_type, const StackTrace *trace,
+                        const char *alt_tool_name = nullptr);
+
+void ReportMmapWriteExec(int prot);
+
+// Math
+#if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
+extern "C" {
+unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
+unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
+#if defined(_WIN64)
+unsigned char _BitScanForward64(unsigned long *index, unsigned __int64 mask);
+unsigned char _BitScanReverse64(unsigned long *index, unsigned __int64 mask);
+#endif
+}
+#endif
+
+INLINE uptr MostSignificantSetBitIndex(uptr x) {
+  CHECK_NE(x, 0U);
+  unsigned long up;
+#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
+# ifdef _WIN64
+  up = SANITIZER_WORDSIZE - 1 - __builtin_clzll(x);
+# else
+  up = SANITIZER_WORDSIZE - 1 - __builtin_clzl(x);
+# endif
+#elif defined(_WIN64)
+  _BitScanReverse64(&up, x);
+#else
+  _BitScanReverse(&up, x);
+#endif
+  return up;
+}
+
+INLINE uptr LeastSignificantSetBitIndex(uptr x) {
+  CHECK_NE(x, 0U);
+  unsigned long up;
+#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
+# ifdef _WIN64
+  up = __builtin_ctzll(x);
+# else
+  up = __builtin_ctzl(x);
+# endif
+#elif defined(_WIN64)
+  _BitScanForward64(&up, x);
+#else
+  _BitScanForward(&up, x);
+#endif
+  return up;
+}
+
+INLINE bool IsPowerOfTwo(uptr x) {
+  return (x & (x - 1)) == 0;
+}
+
+INLINE uptr RoundUpToPowerOfTwo(uptr size) {
+  CHECK(size);
+  if (IsPowerOfTwo(size)) return size;
+
+  uptr up = MostSignificantSetBitIndex(size);
+  CHECK_LT(size, (1ULL << (up + 1)));
+  CHECK_GT(size, (1ULL << up));
+  return 1ULL << (up + 1);
+}
+
+INLINE uptr RoundUpTo(uptr size, uptr boundary) {
+  RAW_CHECK(IsPowerOfTwo(boundary));
+  return (size + boundary - 1) & ~(boundary - 1);
+}
+
+INLINE uptr RoundDownTo(uptr x, uptr boundary) {
+  return x & ~(boundary - 1);
+}
+
+INLINE bool IsAligned(uptr a, uptr alignment) {
+  return (a & (alignment - 1)) == 0;
+}
+
+INLINE uptr Log2(uptr x) {
+  CHECK(IsPowerOfTwo(x));
+  return LeastSignificantSetBitIndex(x);
+}
+
+// Don't use std::min, std::max or std::swap, to minimize dependency
+// on libstdc++.
+template<class T> T Min(T a, T b) { return a < b ? a : b; }
+template<class T> T Max(T a, T b) { return a > b ? a : b; }
+template<class T> void Swap(T& a, T& b) {
+  T tmp = a;
+  a = b;
+  b = tmp;
+}
+
+// Char handling
+INLINE bool IsSpace(int c) {
+  return (c == ' ') || (c == '\n') || (c == '\t') ||
+         (c == '\f') || (c == '\r') || (c == '\v');
+}
+INLINE bool IsDigit(int c) {
+  return (c >= '0') && (c <= '9');
+}
+INLINE int ToLower(int c) {
+  return (c >= 'A' && c <= 'Z') ? (c + 'a' - 'A') : c;
+}
+
+// A low-level vector based on mmap. May incur a significant memory overhead for
+// small vectors.
+// WARNING: The current implementation supports only POD types.
+template<typename T>
+class InternalMmapVectorNoCtor {
+ public:
+  void Initialize(uptr initial_capacity) {
+    capacity_bytes_ = 0;
+    size_ = 0;
+    data_ = 0;
+    reserve(initial_capacity);
+  }
+  void Destroy() { UnmapOrDie(data_, capacity_bytes_); }
+  T &operator[](uptr i) {
+    CHECK_LT(i, size_);
+    return data_[i];
+  }
+  const T &operator[](uptr i) const {
+    CHECK_LT(i, size_);
+    return data_[i];
+  }
+  void push_back(const T &element) {
+    CHECK_LE(size_, capacity());
+    if (size_ == capacity()) {
+      uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1);
+      Realloc(new_capacity);
+    }
+    internal_memcpy(&data_[size_++], &element, sizeof(T));
+  }
+  T &back() {
+    CHECK_GT(size_, 0);
+    return data_[size_ - 1];
+  }
+  void pop_back() {
+    CHECK_GT(size_, 0);
+    size_--;
+  }
+  uptr size() const {
+    return size_;
+  }
+  const T *data() const {
+    return data_;
+  }
+  T *data() {
+    return data_;
+  }
+  uptr capacity() const { return capacity_bytes_ / sizeof(T); }
+  void reserve(uptr new_size) {
+    // Never downsize internal buffer.
+    if (new_size > capacity())
+      Realloc(new_size);
+  }
+  void resize(uptr new_size) {
+    if (new_size > size_) {
+      reserve(new_size);
+      internal_memset(&data_[size_], 0, sizeof(T) * (new_size - size_));
+    }
+    size_ = new_size;
+  }
+
+  void clear() { size_ = 0; }
+  bool empty() const { return size() == 0; }
+
+  const T *begin() const {
+    return data();
+  }
+  T *begin() {
+    return data();
+  }
+  const T *end() const {
+    return data() + size();
+  }
+  T *end() {
+    return data() + size();
+  }
+
+  void swap(InternalMmapVectorNoCtor &other) {
+    Swap(data_, other.data_);
+    Swap(capacity_bytes_, other.capacity_bytes_);
+    Swap(size_, other.size_);
+  }
+
+ private:
+  void Realloc(uptr new_capacity) {
+    CHECK_GT(new_capacity, 0);
+    CHECK_LE(size_, new_capacity);
+    uptr new_capacity_bytes =
+        RoundUpTo(new_capacity * sizeof(T), GetPageSizeCached());
+    T *new_data = (T *)MmapOrDie(new_capacity_bytes, "InternalMmapVector");
+    internal_memcpy(new_data, data_, size_ * sizeof(T));
+    UnmapOrDie(data_, capacity_bytes_);
+    data_ = new_data;
+    capacity_bytes_ = new_capacity_bytes;
+  }
+
+  T *data_;
+  uptr capacity_bytes_;
+  uptr size_;
+};
+
+template <typename T>
+bool operator==(const InternalMmapVectorNoCtor<T> &lhs,
+                const InternalMmapVectorNoCtor<T> &rhs) {
+  if (lhs.size() != rhs.size()) return false;
+  return internal_memcmp(lhs.data(), rhs.data(), lhs.size() * sizeof(T)) == 0;
+}
+
+template <typename T>
+bool operator!=(const InternalMmapVectorNoCtor<T> &lhs,
+                const InternalMmapVectorNoCtor<T> &rhs) {
+  return !(lhs == rhs);
+}
+
+template<typename T>
+class InternalMmapVector : public InternalMmapVectorNoCtor<T> {
+ public:
+  InternalMmapVector() { InternalMmapVectorNoCtor<T>::Initialize(0); }
+  explicit InternalMmapVector(uptr cnt) {
+    InternalMmapVectorNoCtor<T>::Initialize(cnt);
+    this->resize(cnt);
+  }
+  ~InternalMmapVector() { InternalMmapVectorNoCtor<T>::Destroy(); }
+  // Disallow copies and moves.
+  InternalMmapVector(const InternalMmapVector &) = delete;
+  InternalMmapVector &operator=(const InternalMmapVector &) = delete;
+  InternalMmapVector(InternalMmapVector &&) = delete;
+  InternalMmapVector &operator=(InternalMmapVector &&) = delete;
+};
+
+class InternalScopedString : public InternalMmapVector<char> {
+ public:
+  explicit InternalScopedString(uptr max_length)
+      : InternalMmapVector<char>(max_length), length_(0) {
+    (*this)[0] = '\0';
+  }
+  uptr length() { return length_; }
+  void clear() {
+    (*this)[0] = '\0';
+    length_ = 0;
+  }
+  void append(const char *format, ...);
+
+ private:
+  uptr length_;
+};
+
+template <class T>
+struct CompareLess {
+  bool operator()(const T &a, const T &b) const { return a < b; }
+};
+
+// HeapSort for arrays and InternalMmapVector.
+template <class T, class Compare = CompareLess<T>>
+void Sort(T *v, uptr size, Compare comp = {}) {
+  if (size < 2)
+    return;
+  // Stage 1: insert elements to the heap.
+  for (uptr i = 1; i < size; i++) {
+    uptr j, p;
+    for (j = i; j > 0; j = p) {
+      p = (j - 1) / 2;
+      if (comp(v[p], v[j]))
+        Swap(v[j], v[p]);
+      else
+        break;
+    }
+  }
+  // Stage 2: swap largest element with the last one,
+  // and sink the new top.
+  for (uptr i = size - 1; i > 0; i--) {
+    Swap(v[0], v[i]);
+    uptr j, max_ind;
+    for (j = 0; j < i; j = max_ind) {
+      uptr left = 2 * j + 1;
+      uptr right = 2 * j + 2;
+      max_ind = j;
+      if (left < i && comp(v[max_ind], v[left]))
+        max_ind = left;
+      if (right < i && comp(v[max_ind], v[right]))
+        max_ind = right;
+      if (max_ind != j)
+        Swap(v[j], v[max_ind]);
+      else
+        break;
+    }
+  }
+}
+
+// Works like std::lower_bound: finds the first element that is not less
+// than the val.
+template <class Container, class Value, class Compare>
+uptr InternalLowerBound(const Container &v, uptr first, uptr last,
+                        const Value &val, Compare comp) {
+  while (last > first) {
+    uptr mid = (first + last) / 2;
+    if (comp(v[mid], val))
+      first = mid + 1;
+    else
+      last = mid;
+  }
+  return first;
+}
+
+enum ModuleArch {
+  kModuleArchUnknown,
+  kModuleArchI386,
+  kModuleArchX86_64,
+  kModuleArchX86_64H,
+  kModuleArchARMV6,
+  kModuleArchARMV7,
+  kModuleArchARMV7S,
+  kModuleArchARMV7K,
+  kModuleArchARM64
+};
+
+// Opens the file 'file_name" and reads up to 'max_len' bytes.
+// The resulting buffer is mmaped and stored in '*buff'.
+// Returns true if file was successfully opened and read.
+bool ReadFileToVector(const char *file_name,
+                      InternalMmapVectorNoCtor<char> *buff,
+                      uptr max_len = 1 << 26, error_t *errno_p = nullptr);
+
+// Opens the file 'file_name" and reads up to 'max_len' bytes.
+// This function is less I/O efficient than ReadFileToVector as it may reread
+// file multiple times to avoid mmap during read attempts. It's used to read
+// procmap, so short reads with mmap in between can produce inconsistent result.
+// The resulting buffer is mmaped and stored in '*buff'.
+// The size of the mmaped region is stored in '*buff_size'.
+// The total number of read bytes is stored in '*read_len'.
+// Returns true if file was successfully opened and read.
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+                      uptr *read_len, uptr max_len = 1 << 26,
+                      error_t *errno_p = nullptr);
+
+// When adding a new architecture, don't forget to also update
+// script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cpp.
+inline const char *ModuleArchToString(ModuleArch arch) {
+  switch (arch) {
+    case kModuleArchUnknown:
+      return "";
+    case kModuleArchI386:
+      return "i386";
+    case kModuleArchX86_64:
+      return "x86_64";
+    case kModuleArchX86_64H:
+      return "x86_64h";
+    case kModuleArchARMV6:
+      return "armv6";
+    case kModuleArchARMV7:
+      return "armv7";
+    case kModuleArchARMV7S:
+      return "armv7s";
+    case kModuleArchARMV7K:
+      return "armv7k";
+    case kModuleArchARM64:
+      return "arm64";
+  }
+  CHECK(0 && "Invalid module arch");
+  return "";
+}
+
+const uptr kModuleUUIDSize = 16;
+const uptr kMaxSegName = 16;
+
+// Represents a binary loaded into virtual memory (e.g. this can be an
+// executable or a shared object).
+class LoadedModule {
+ public:
+  LoadedModule()
+      : full_name_(nullptr),
+        base_address_(0),
+        max_executable_address_(0),
+        arch_(kModuleArchUnknown),
+        instrumented_(false) {
+    internal_memset(uuid_, 0, kModuleUUIDSize);
+    ranges_.clear();
+  }
+  void set(const char *module_name, uptr base_address);
+  void set(const char *module_name, uptr base_address, ModuleArch arch,
+           u8 uuid[kModuleUUIDSize], bool instrumented);
+  void clear();
+  void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
+                       const char *name = nullptr);
+  bool containsAddress(uptr address) const;
+
+  const char *full_name() const { return full_name_; }
+  uptr base_address() const { return base_address_; }
+  uptr max_executable_address() const { return max_executable_address_; }
+  ModuleArch arch() const { return arch_; }
+  const u8 *uuid() const { return uuid_; }
+  bool instrumented() const { return instrumented_; }
+
+  struct AddressRange {
+    AddressRange *next;
+    uptr beg;
+    uptr end;
+    bool executable;
+    bool writable;
+    char name[kMaxSegName];
+
+    AddressRange(uptr beg, uptr end, bool executable, bool writable,
+                 const char *name)
+        : next(nullptr),
+          beg(beg),
+          end(end),
+          executable(executable),
+          writable(writable) {
+      internal_strncpy(this->name, (name ? name : ""), ARRAY_SIZE(this->name));
+    }
+  };
+
+  const IntrusiveList<AddressRange> &ranges() const { return ranges_; }
+
+ private:
+  char *full_name_;  // Owned.
+  uptr base_address_;
+  uptr max_executable_address_;
+  ModuleArch arch_;
+  u8 uuid_[kModuleUUIDSize];
+  bool instrumented_;
+  IntrusiveList<AddressRange> ranges_;
+};
+
+// List of LoadedModules. OS-dependent implementation is responsible for
+// filling this information.
+class ListOfModules {
+ public:
+  ListOfModules() : initialized(false) {}
+  ~ListOfModules() { clear(); }
+  void init();
+  void fallbackInit();  // Uses fallback init if available, otherwise clears
+  const LoadedModule *begin() const { return modules_.begin(); }
+  LoadedModule *begin() { return modules_.begin(); }
+  const LoadedModule *end() const { return modules_.end(); }
+  LoadedModule *end() { return modules_.end(); }
+  uptr size() const { return modules_.size(); }
+  const LoadedModule &operator[](uptr i) const {
+    CHECK_LT(i, modules_.size());
+    return modules_[i];
+  }
+
+ private:
+  void clear() {
+    for (auto &module : modules_) module.clear();
+    modules_.clear();
+  }
+  void clearOrInit() {
+    initialized ? clear() : modules_.Initialize(kInitialCapacity);
+    initialized = true;
+  }
+
+  InternalMmapVectorNoCtor<LoadedModule> modules_;
+  // We rarely have more than 16K loaded modules.
+  static const uptr kInitialCapacity = 1 << 14;
+  bool initialized;
+};
+
+// Callback type for iterating over a set of memory ranges.
+typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg);
+
+enum AndroidApiLevel {
+  ANDROID_NOT_ANDROID = 0,
+  ANDROID_KITKAT = 19,
+  ANDROID_LOLLIPOP_MR1 = 22,
+  ANDROID_POST_LOLLIPOP = 23
+};
+
+void WriteToSyslog(const char *buffer);
+
+#if defined(SANITIZER_WINDOWS) && defined(_MSC_VER) && !defined(__clang__)
+#define SANITIZER_WIN_TRACE 1
+#else
+#define SANITIZER_WIN_TRACE 0
+#endif
+
+#if SANITIZER_MAC || SANITIZER_WIN_TRACE
+void LogFullErrorReport(const char *buffer);
+#else
+INLINE void LogFullErrorReport(const char *buffer) {}
+#endif
+
+#if SANITIZER_LINUX || SANITIZER_MAC
+void WriteOneLineToSyslog(const char *s);
+void LogMessageOnPrintf(const char *str);
+#else
+INLINE void WriteOneLineToSyslog(const char *s) {}
+INLINE void LogMessageOnPrintf(const char *str) {}
+#endif
+
+#if SANITIZER_LINUX || SANITIZER_WIN_TRACE
+// Initialize Android logging. Any writes before this are silently lost.
+void AndroidLogInit();
+void SetAbortMessage(const char *);
+#else
+INLINE void AndroidLogInit() {}
+// FIXME: MacOS implementation could use CRSetCrashLogMessage.
+INLINE void SetAbortMessage(const char *) {}
+#endif
+
+#if SANITIZER_ANDROID
+void SanitizerInitializeUnwinder();
+AndroidApiLevel AndroidGetApiLevel();
+#else
+INLINE void AndroidLogWrite(const char *buffer_unused) {}
+INLINE void SanitizerInitializeUnwinder() {}
+INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
+#endif
+
+INLINE uptr GetPthreadDestructorIterations() {
+#if SANITIZER_ANDROID
+  return (AndroidGetApiLevel() == ANDROID_LOLLIPOP_MR1) ? 8 : 4;
+#elif SANITIZER_POSIX
+  return 4;
+#else
+// Unused on Windows.
+  return 0;
+#endif
+}
+
+void *internal_start_thread(void *(*func)(void*), void *arg);
+void internal_join_thread(void *th);
+void MaybeStartBackgroudThread();
+
+// Make the compiler think that something is going on there.
+// Use this inside a loop that looks like memset/memcpy/etc to prevent the
+// compiler from recognising it and turning it into an actual call to
+// memset/memcpy/etc.
+static inline void SanitizerBreakOptimization(void *arg) {
+#if defined(_MSC_VER) && !defined(__clang__)
+  _ReadWriteBarrier();
+#else
+  __asm__ __volatile__("" : : "r" (arg) : "memory");
+#endif
+}
+
+struct SignalContext {
+  void *siginfo;
+  void *context;
+  uptr addr;
+  uptr pc;
+  uptr sp;
+  uptr bp;
+  bool is_memory_access;
+  enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
+
+  // In some cases the kernel cannot provide the true faulting address; `addr`
+  // will be zero then.  This field allows to distinguish between these cases
+  // and dereferences of null.
+  bool is_true_faulting_addr;
+
+  // VS2013 doesn't implement unrestricted unions, so we need a trivial default
+  // constructor
+  SignalContext() = default;
+
+  // Creates signal context in a platform-specific manner.
+  // SignalContext is going to keep pointers to siginfo and context without
+  // owning them.
+  SignalContext(void *siginfo, void *context)
+      : siginfo(siginfo),
+        context(context),
+        addr(GetAddress()),
+        is_memory_access(IsMemoryAccess()),
+        write_flag(GetWriteFlag()),
+        is_true_faulting_addr(IsTrueFaultingAddress()) {
+    InitPcSpBp();
+  }
+
+  static void DumpAllRegisters(void *context);
+
+  // Type of signal e.g. SIGSEGV or EXCEPTION_ACCESS_VIOLATION.
+  int GetType() const;
+
+  // String description of the signal.
+  const char *Describe() const;
+
+  // Returns true if signal is stack overflow.
+  bool IsStackOverflow() const;
+
+ private:
+  // Platform specific initialization.
+  void InitPcSpBp();
+  uptr GetAddress() const;
+  WriteFlag GetWriteFlag() const;
+  bool IsMemoryAccess() const;
+  bool IsTrueFaultingAddress() const;
+};
+
+void InitializePlatformEarly();
+void MaybeReexec();
+
+template <typename Fn>
+class RunOnDestruction {
+ public:
+  explicit RunOnDestruction(Fn fn) : fn_(fn) {}
+  ~RunOnDestruction() { fn_(); }
+
+ private:
+  Fn fn_;
+};
+
+// A simple scope guard. Usage:
+// auto cleanup = at_scope_exit([]{ do_cleanup; });
+template <typename Fn>
+RunOnDestruction<Fn> at_scope_exit(Fn fn) {
+  return RunOnDestruction<Fn>(fn);
+}
+
+// Linux on 64-bit s390 had a nasty bug that crashes the whole machine
+// if a process uses virtual memory over 4TB (as many sanitizers like
+// to do).  This function will abort the process if running on a kernel
+// that looks vulnerable.
+#if SANITIZER_LINUX && SANITIZER_S390_64
+void AvoidCVE_2016_2143();
+#else
+INLINE void AvoidCVE_2016_2143() {}
+#endif
+
+struct StackDepotStats {
+  uptr n_uniq_ids;
+  uptr allocated;
+};
+
+// The default value for allocator_release_to_os_interval_ms common flag to
+// indicate that sanitizer allocator should not attempt to release memory to OS.
+const s32 kReleaseToOSIntervalNever = -1;
+
+void CheckNoDeepBind(const char *filename, int flag);
+
+// Returns the requested amount of random data (up to 256 bytes) that can then
+// be used to seed a PRNG. Defaults to blocking like the underlying syscall.
+bool GetRandom(void *buffer, uptr length, bool blocking = true);
+
+// Returns the number of logical processors on the system.
+u32 GetNumberOfCPUs();
+extern u32 NumberOfCPUsCached;
+INLINE u32 GetNumberOfCPUsCached() {
+  if (!NumberOfCPUsCached)
+    NumberOfCPUsCached = GetNumberOfCPUs();
+  return NumberOfCPUsCached;
+}
+
+template <typename T>
+class ArrayRef {
+ public:
+  ArrayRef() {}
+  ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {}
+
+  T *begin() { return begin_; }
+  T *end() { return end_; }
+
+ private:
+  T *begin_ = nullptr;
+  T *end_ = nullptr;
+};
+
+}  // namespace __sanitizer
+
+inline void *operator new(__sanitizer::operator_new_size_type size,
+                          __sanitizer::LowLevelAllocator &alloc) {  // NOLINT
+  return alloc.Allocate(size);
+}
+
+#endif  // SANITIZER_COMMON_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc
new file mode 100644
index 0000000000..d7e0bba762
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc
@@ -0,0 +1,10175 @@
+//===-- sanitizer_common_interceptors.inc -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common function interceptors for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// This file should be included into the tool's interceptor file,
+// which has to define its own macros:
+//   COMMON_INTERCEPTOR_ENTER
+//   COMMON_INTERCEPTOR_ENTER_NOIGNORE
+//   COMMON_INTERCEPTOR_READ_RANGE
+//   COMMON_INTERCEPTOR_WRITE_RANGE
+//   COMMON_INTERCEPTOR_INITIALIZE_RANGE
+//   COMMON_INTERCEPTOR_DIR_ACQUIRE
+//   COMMON_INTERCEPTOR_FD_ACQUIRE
+//   COMMON_INTERCEPTOR_FD_RELEASE
+//   COMMON_INTERCEPTOR_FD_ACCESS
+//   COMMON_INTERCEPTOR_SET_THREAD_NAME
+//   COMMON_INTERCEPTOR_ON_DLOPEN
+//   COMMON_INTERCEPTOR_ON_EXIT
+//   COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
+//   COMMON_INTERCEPTOR_MUTEX_POST_LOCK
+//   COMMON_INTERCEPTOR_MUTEX_UNLOCK
+//   COMMON_INTERCEPTOR_MUTEX_REPAIR
+//   COMMON_INTERCEPTOR_SET_PTHREAD_NAME
+//   COMMON_INTERCEPTOR_HANDLE_RECVMSG
+//   COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
+//   COMMON_INTERCEPTOR_MEMSET_IMPL
+//   COMMON_INTERCEPTOR_MEMMOVE_IMPL
+//   COMMON_INTERCEPTOR_MEMCPY_IMPL
+//   COMMON_INTERCEPTOR_MMAP_IMPL
+//   COMMON_INTERCEPTOR_COPY_STRING
+//   COMMON_INTERCEPTOR_STRNDUP_IMPL
+//   COMMON_INTERCEPTOR_STRERROR
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_addrhashmap.h"
+#include "sanitizer_errno.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_platform_interceptors.h"
+#include "sanitizer_symbolizer.h"
+#include "sanitizer_tls_get_addr.h"
+
+#include <stdarg.h>
+
+#if SANITIZER_INTERCEPTOR_HOOKS
+#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__);
+#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \
+  SANITIZER_INTERFACE_WEAK_DEF(void, f, __VA_ARGS__) {}
+#else
+#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...)
+#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...)
+
+#endif  // SANITIZER_INTERCEPTOR_HOOKS
+
+#if SANITIZER_WINDOWS && !defined(va_copy)
+#define va_copy(dst, src) ((dst) = (src))
+#endif // _WIN32
+
+#if SANITIZER_FREEBSD
+#define pthread_setname_np pthread_set_name_np
+#define inet_aton __inet_aton
+#define inet_pton __inet_pton
+#define iconv __bsd_iconv
+#endif
+
+#if SANITIZER_NETBSD
+#define clock_getres __clock_getres50
+#define clock_gettime __clock_gettime50
+#define clock_settime __clock_settime50
+#define ctime __ctime50
+#define ctime_r __ctime_r50
+#define devname __devname50
+#define fgetpos __fgetpos50
+#define fsetpos __fsetpos50
+#define fstatvfs __fstatvfs90
+#define fstatvfs1 __fstatvfs190
+#define fts_children __fts_children60
+#define fts_close __fts_close60
+#define fts_open __fts_open60
+#define fts_read __fts_read60
+#define fts_set __fts_set60
+#define getitimer __getitimer50
+#define getmntinfo __getmntinfo90
+#define getpwent __getpwent50
+#define getpwnam __getpwnam50
+#define getpwnam_r __getpwnam_r50
+#define getpwuid __getpwuid50
+#define getpwuid_r __getpwuid_r50
+#define getutent __getutent50
+#define getutxent __getutxent50
+#define getutxid __getutxid50
+#define getutxline __getutxline50
+#define getvfsstat __getvfsstat90
+#define pututxline __pututxline50
+#define glob __glob30
+#define gmtime __gmtime50
+#define gmtime_r __gmtime_r50
+#define localtime __locatime50
+#define localtime_r __localtime_r50
+#define mktime __mktime50
+#define lstat __lstat50
+#define opendir __opendir30
+#define readdir __readdir30
+#define readdir_r __readdir_r30
+#define scandir __scandir30
+#define setitimer __setitimer50
+#define setlocale __setlocale50
+#define shmctl __shmctl50
+#define sigaltstack __sigaltstack14
+#define sigemptyset __sigemptyset14
+#define sigfillset __sigfillset14
+#define sigpending __sigpending14
+#define sigprocmask __sigprocmask14
+#define sigtimedwait __sigtimedwait50
+#define stat __stat50
+#define statvfs __statvfs90
+#define statvfs1 __statvfs190
+#define time __time50
+#define times __times13
+#define unvis __unvis50
+#define wait3 __wait350
+#define wait4 __wait450
+extern const unsigned short *_ctype_tab_;
+extern const short *_toupper_tab_;
+extern const short *_tolower_tab_;
+#endif
+
+// Platform-specific options.
+#if SANITIZER_MAC
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#elif SANITIZER_WINDOWS64
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#else
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true
+#endif  // SANITIZER_MAC
+
+#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
+#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_UNPOISON_PARAM
+#define COMMON_INTERCEPTOR_UNPOISON_PARAM(count) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_FD_ACCESS
+#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
+#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_POST_LOCK
+#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_UNLOCK
+#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_REPAIR
+#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MUTEX_INVALID
+#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG
+#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg))
+#endif
+
+#ifndef COMMON_INTERCEPTOR_FILE_OPEN
+#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_FILE_CLOSE
+#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_LIBRARY_LOADED
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_LIBRARY_UNLOADED
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_ENTER_NOIGNORE
+#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, ...) \
+  COMMON_INTERCEPTOR_ENTER(ctx, __VA_ARGS__)
+#endif
+
+#ifndef COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (0)
+#endif
+
+#define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n)                   \
+    COMMON_INTERCEPTOR_READ_RANGE((ctx), (s),                       \
+      common_flags()->strict_string_checks ? (REAL(strlen)(s)) + 1 : (n) )
+
+#ifndef COMMON_INTERCEPTOR_ON_DLOPEN
+#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
+  CheckNoDeepBind(filename, flag);
+#endif
+
+#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) *begin = *end = 0;
+#endif
+
+#ifndef COMMON_INTERCEPTOR_ACQUIRE
+#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_RELEASE
+#define COMMON_INTERCEPTOR_RELEASE(ctx, u) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_USER_CALLBACK_START
+#define COMMON_INTERCEPTOR_USER_CALLBACK_START() {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_USER_CALLBACK_END
+#define COMMON_INTERCEPTOR_USER_CALLBACK_END() {}
+#endif
+
+#ifdef SANITIZER_NLDBL_VERSION
+#define COMMON_INTERCEPT_FUNCTION_LDBL(fn)                          \
+    COMMON_INTERCEPT_FUNCTION_VER(fn, SANITIZER_NLDBL_VERSION)
+#else
+#define COMMON_INTERCEPT_FUNCTION_LDBL(fn)                          \
+    COMMON_INTERCEPT_FUNCTION(fn)
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
+  {                                                       \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)        \
+      return internal_memset(dst, v, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size);  \
+    if (common_flags()->intercept_intrin)                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);     \
+    return REAL(memset)(dst, v, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
+  {                                                          \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)           \
+      return internal_memmove(dst, src, size);               \
+    COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                  \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);        \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);         \
+    }                                                        \
+    return REAL(memmove)(dst, src, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
+  {                                                         \
+    if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) {        \
+      return internal_memmove(dst, src, size);              \
+    }                                                       \
+    COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size);  \
+    if (common_flags()->intercept_intrin) {                 \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size);       \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size);        \
+    }                                                       \
+    return REAL(memcpy)(dst, src, size);                    \
+  }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MMAP_IMPL
+#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
+                                     off)                                  \
+  { return REAL(mmap)(addr, sz, prot, flags, fd, off); }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_COPY_STRING
+#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
+#endif
+
+#ifndef COMMON_INTERCEPTOR_STRNDUP_IMPL
+#define COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size)                         \
+  COMMON_INTERCEPTOR_ENTER(ctx, strndup, s, size);                            \
+  uptr copy_length = internal_strnlen(s, size);                               \
+  char *new_mem = (char *)WRAP(malloc)(copy_length + 1);                      \
+  if (common_flags()->intercept_strndup) {                                    \
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1));       \
+  }                                                                           \
+  COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length);               \
+  internal_memcpy(new_mem, s, copy_length);                                   \
+  new_mem[copy_length] = '\0';                                                \
+  return new_mem;
+#endif
+
+#ifndef COMMON_INTERCEPTOR_STRERROR
+#define COMMON_INTERCEPTOR_STRERROR() {}
+#endif
+
+struct FileMetadata {
+  // For open_memstream().
+  char **addr;
+  SIZE_T *size;
+};
+
+struct CommonInterceptorMetadata {
+  enum {
+    CIMT_INVALID = 0,
+    CIMT_FILE
+  } type;
+  union {
+    FileMetadata file;
+  };
+};
+
+#if SI_POSIX
+typedef AddrHashMap<CommonInterceptorMetadata, 31051> MetadataHashMap;
+
+static MetadataHashMap *interceptor_metadata_map;
+
+UNUSED static void SetInterceptorMetadata(__sanitizer_FILE *addr,
+                                          const FileMetadata &file) {
+  MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr);
+  CHECK(h.created());
+  h->type = CommonInterceptorMetadata::CIMT_FILE;
+  h->file = file;
+}
+
+UNUSED static const FileMetadata *GetInterceptorMetadata(
+    __sanitizer_FILE *addr) {
+  MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr,
+                            /* remove */ false,
+                            /* create */ false);
+  if (addr && h.exists()) {
+    CHECK(!h.created());
+    CHECK(h->type == CommonInterceptorMetadata::CIMT_FILE);
+    return &h->file;
+  } else {
+    return 0;
+  }
+}
+
+UNUSED static void DeleteInterceptorMetadata(void *addr) {
+  MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, true);
+  CHECK(h.exists());
+}
+#endif  // SI_POSIX
+
+#if SANITIZER_INTERCEPT_STRLEN
+INTERCEPTOR(SIZE_T, strlen, const char *s) {
+  // Sometimes strlen is called prior to InitializeCommonInterceptors,
+  // in which case the REAL(strlen) typically used in
+  // COMMON_INTERCEPTOR_ENTER will fail.  We use internal_strlen here
+  // to handle that.
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_strlen(s);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strlen, s);
+  SIZE_T result = REAL(strlen)(s);
+  if (common_flags()->intercept_strlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, result + 1);
+  return result;
+}
+#define INIT_STRLEN COMMON_INTERCEPT_FUNCTION(strlen)
+#else
+#define INIT_STRLEN
+#endif
+
+#if SANITIZER_INTERCEPT_STRNLEN
+INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strnlen, s, maxlen);
+  SIZE_T length = REAL(strnlen)(s, maxlen);
+  if (common_flags()->intercept_strlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, Min(length + 1, maxlen));
+  return length;
+}
+#define INIT_STRNLEN COMMON_INTERCEPT_FUNCTION(strnlen)
+#else
+#define INIT_STRNLEN
+#endif
+
+#if SANITIZER_INTERCEPT_STRNDUP
+INTERCEPTOR(char*, strndup, const char *s, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
+}
+#define INIT_STRNDUP COMMON_INTERCEPT_FUNCTION(strndup)
+#else
+#define INIT_STRNDUP
+#endif // SANITIZER_INTERCEPT_STRNDUP
+
+#if SANITIZER_INTERCEPT___STRNDUP
+INTERCEPTOR(char*, __strndup, const char *s, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
+}
+#define INIT___STRNDUP COMMON_INTERCEPT_FUNCTION(__strndup)
+#else
+#define INIT___STRNDUP
+#endif // SANITIZER_INTERCEPT___STRNDUP
+
+#if SANITIZER_INTERCEPT_TEXTDOMAIN
+INTERCEPTOR(char*, textdomain, const char *domainname) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, textdomain, domainname);
+  if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
+  char *domain = REAL(textdomain)(domainname);
+  if (domain) {
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, REAL(strlen)(domain) + 1);
+  }
+  return domain;
+}
+#define INIT_TEXTDOMAIN COMMON_INTERCEPT_FUNCTION(textdomain)
+#else
+#define INIT_TEXTDOMAIN
+#endif
+
+#if SANITIZER_INTERCEPT_STRCMP
+static inline int CharCmpX(unsigned char c1, unsigned char c2) {
+  return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1;
+}
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc,
+                              const char *s1, const char *s2, int result)
+
+INTERCEPTOR(int, strcmp, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strcmp, s1, s2);
+  unsigned char c1, c2;
+  uptr i;
+  for (i = 0;; i++) {
+    c1 = (unsigned char)s1[i];
+    c2 = (unsigned char)s2[i];
+    if (c1 != c2 || c1 == '\0') break;
+  }
+  COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+  int result = CharCmpX(c1, c2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1,
+                             s2, result);
+  return result;
+}
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc,
+                              const char *s1, const char *s2, uptr n,
+                              int result)
+
+INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_strncmp(s1, s2, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size);
+  unsigned char c1 = 0, c2 = 0;
+  uptr i;
+  for (i = 0; i < size; i++) {
+    c1 = (unsigned char)s1[i];
+    c2 = (unsigned char)s2[i];
+    if (c1 != c2 || c1 == '\0') break;
+  }
+  uptr i1 = i;
+  uptr i2 = i;
+  if (common_flags()->strict_string_checks) {
+    for (; i1 < size && s1[i1]; i1++) {}
+    for (; i2 < size && s2[i2]; i2++) {}
+  }
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s1), Min(i1 + 1, size));
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s2), Min(i2 + 1, size));
+  int result = CharCmpX(c1, c2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, GET_CALLER_PC(), s1,
+                             s2, size, result);
+  return result;
+}
+
+#define INIT_STRCMP COMMON_INTERCEPT_FUNCTION(strcmp)
+#define INIT_STRNCMP COMMON_INTERCEPT_FUNCTION(strncmp)
+#else
+#define INIT_STRCMP
+#define INIT_STRNCMP
+#endif
+
+#if SANITIZER_INTERCEPT_STRCASECMP
+static inline int CharCaseCmp(unsigned char c1, unsigned char c2) {
+  int c1_low = ToLower(c1);
+  int c2_low = ToLower(c2);
+  return c1_low - c2_low;
+}
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasecmp, uptr called_pc,
+                              const char *s1, const char *s2, int result)
+
+INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strcasecmp, s1, s2);
+  unsigned char c1 = 0, c2 = 0;
+  uptr i;
+  for (i = 0;; i++) {
+    c1 = (unsigned char)s1[i];
+    c2 = (unsigned char)s2[i];
+    if (CharCaseCmp(c1, c2) != 0 || c1 == '\0') break;
+  }
+  COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+  int result = CharCaseCmp(c1, c2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasecmp, GET_CALLER_PC(),
+                             s1, s2, result);
+  return result;
+}
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc,
+                              const char *s1, const char *s2, uptr size,
+                              int result)
+
+INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, size);
+  unsigned char c1 = 0, c2 = 0;
+  uptr i;
+  for (i = 0; i < size; i++) {
+    c1 = (unsigned char)s1[i];
+    c2 = (unsigned char)s2[i];
+    if (CharCaseCmp(c1, c2) != 0 || c1 == '\0') break;
+  }
+  uptr i1 = i;
+  uptr i2 = i;
+  if (common_flags()->strict_string_checks) {
+    for (; i1 < size && s1[i1]; i1++) {}
+    for (; i2 < size && s2[i2]; i2++) {}
+  }
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s1), Min(i1 + 1, size));
+  COMMON_INTERCEPTOR_READ_RANGE((ctx), (s2), Min(i2 + 1, size));
+  int result = CharCaseCmp(c1, c2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, GET_CALLER_PC(),
+                             s1, s2, size, result);
+  return result;
+}
+
+#define INIT_STRCASECMP COMMON_INTERCEPT_FUNCTION(strcasecmp)
+#define INIT_STRNCASECMP COMMON_INTERCEPT_FUNCTION(strncasecmp)
+#else
+#define INIT_STRCASECMP
+#define INIT_STRNCASECMP
+#endif
+
+#if SANITIZER_INTERCEPT_STRSTR || SANITIZER_INTERCEPT_STRCASESTR
+static inline void StrstrCheck(void *ctx, char *r, const char *s1,
+                               const char *s2) {
+    uptr len1 = REAL(strlen)(s1);
+    uptr len2 = REAL(strlen)(s2);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_STRSTR
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strstr, uptr called_pc,
+                              const char *s1, const char *s2, char *result)
+
+INTERCEPTOR(char*, strstr, const char *s1, const char *s2) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_strstr(s1, s2);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strstr, s1, s2);
+  char *r = REAL(strstr)(s1, s2);
+  if (common_flags()->intercept_strstr)
+    StrstrCheck(ctx, r, s1, s2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strstr, GET_CALLER_PC(), s1,
+                             s2, r);
+  return r;
+}
+
+#define INIT_STRSTR COMMON_INTERCEPT_FUNCTION(strstr);
+#else
+#define INIT_STRSTR
+#endif
+
+#if SANITIZER_INTERCEPT_STRCASESTR
+
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasestr, uptr called_pc,
+                              const char *s1, const char *s2, char *result)
+
+INTERCEPTOR(char*, strcasestr, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strcasestr, s1, s2);
+  char *r = REAL(strcasestr)(s1, s2);
+  if (common_flags()->intercept_strstr)
+    StrstrCheck(ctx, r, s1, s2);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcasestr, GET_CALLER_PC(),
+                             s1, s2, r);
+  return r;
+}
+
+#define INIT_STRCASESTR COMMON_INTERCEPT_FUNCTION(strcasestr);
+#else
+#define INIT_STRCASESTR
+#endif
+
+#if SANITIZER_INTERCEPT_STRTOK
+
+INTERCEPTOR(char*, strtok, char *str, const char *delimiters) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtok, str, delimiters);
+  if (!common_flags()->intercept_strtok) {
+    return REAL(strtok)(str, delimiters);
+  }
+  if (common_flags()->strict_string_checks) {
+    // If strict_string_checks is enabled, we check the whole first argument
+    // string on the first call (strtok saves this string in a static buffer
+    // for subsequent calls). We do not need to check strtok's result.
+    // As the delimiters can change, we check them every call.
+    if (str != nullptr) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+    }
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,
+                                  REAL(strlen)(delimiters) + 1);
+    return REAL(strtok)(str, delimiters);
+  } else {
+    // However, when strict_string_checks is disabled we cannot check the
+    // whole string on the first call. Instead, we check the result string
+    // which is guaranteed to be a NULL-terminated substring of the first
+    // argument. We also conservatively check one character of str and the
+    // delimiters.
+    if (str != nullptr) {
+      COMMON_INTERCEPTOR_READ_STRING(ctx, str, 1);
+    }
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);
+    char *result = REAL(strtok)(str, delimiters);
+    if (result != nullptr) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, result, REAL(strlen)(result) + 1);
+    } else if (str != nullptr) {
+      // No delimiter were found, it's safe to assume that the entire str was
+      // scanned.
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+    }
+    return result;
+  }
+}
+
+#define INIT_STRTOK COMMON_INTERCEPT_FUNCTION(strtok)
+#else
+#define INIT_STRTOK
+#endif
+
+#if SANITIZER_INTERCEPT_MEMMEM
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memmem, uptr called_pc,
+                              const void *s1, SIZE_T len1, const void *s2,
+                              SIZE_T len2, void *result)
+
+INTERCEPTOR(void*, memmem, const void *s1, SIZE_T len1, const void *s2,
+            SIZE_T len2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, memmem, s1, len1, s2, len2);
+  void *r = REAL(memmem)(s1, len1, s2, len2);
+  if (common_flags()->intercept_memmem) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, len1);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2);
+  }
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memmem, GET_CALLER_PC(),
+                             s1, len1, s2, len2, r);
+  return r;
+}
+
+#define INIT_MEMMEM COMMON_INTERCEPT_FUNCTION(memmem);
+#else
+#define INIT_MEMMEM
+#endif  // SANITIZER_INTERCEPT_MEMMEM
+
+#if SANITIZER_INTERCEPT_STRCHR
+INTERCEPTOR(char*, strchr, const char *s, int c) {
+  void *ctx;
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_strchr(s, c);
+  COMMON_INTERCEPTOR_ENTER(ctx, strchr, s, c);
+  char *result = REAL(strchr)(s, c);
+  if (common_flags()->intercept_strchr) {
+    // Keep strlen as macro argument, as macro may ignore it.
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s,
+      (result ? result - s : REAL(strlen)(s)) + 1);
+  }
+  return result;
+}
+#define INIT_STRCHR COMMON_INTERCEPT_FUNCTION(strchr)
+#else
+#define INIT_STRCHR
+#endif
+
+#if SANITIZER_INTERCEPT_STRCHRNUL
+INTERCEPTOR(char*, strchrnul, const char *s, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strchrnul, s, c);
+  char *result = REAL(strchrnul)(s, c);
+  uptr len = result - s + 1;
+  if (common_flags()->intercept_strchr)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s, len);
+  return result;
+}
+#define INIT_STRCHRNUL COMMON_INTERCEPT_FUNCTION(strchrnul)
+#else
+#define INIT_STRCHRNUL
+#endif
+
+#if SANITIZER_INTERCEPT_STRRCHR
+INTERCEPTOR(char*, strrchr, const char *s, int c) {
+  void *ctx;
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_strrchr(s, c);
+  COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);
+  if (common_flags()->intercept_strchr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  return REAL(strrchr)(s, c);
+}
+#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)
+#else
+#define INIT_STRRCHR
+#endif
+
+#if SANITIZER_INTERCEPT_STRSPN
+INTERCEPTOR(SIZE_T, strspn, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strspn, s1, s2);
+  SIZE_T r = REAL(strspn)(s1, s2);
+  if (common_flags()->intercept_strspn) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
+  }
+  return r;
+}
+
+INTERCEPTOR(SIZE_T, strcspn, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strcspn, s1, s2);
+  SIZE_T r = REAL(strcspn)(s1, s2);
+  if (common_flags()->intercept_strspn) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
+  }
+  return r;
+}
+
+#define INIT_STRSPN \
+  COMMON_INTERCEPT_FUNCTION(strspn); \
+  COMMON_INTERCEPT_FUNCTION(strcspn);
+#else
+#define INIT_STRSPN
+#endif
+
+#if SANITIZER_INTERCEPT_STRPBRK
+INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strpbrk, s1, s2);
+  char *r = REAL(strpbrk)(s1, s2);
+  if (common_flags()->intercept_strpbrk) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s1,
+        r ? r - s1 + 1 : REAL(strlen)(s1) + 1);
+  }
+  return r;
+}
+
+#define INIT_STRPBRK COMMON_INTERCEPT_FUNCTION(strpbrk);
+#else
+#define INIT_STRPBRK
+#endif
+
+#if SANITIZER_INTERCEPT_MEMSET
+INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
+}
+
+#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
+#else
+#define INIT_MEMSET
+#endif
+
+#if SANITIZER_INTERCEPT_MEMMOVE
+INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+}
+
+#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
+#else
+#define INIT_MEMMOVE
+#endif
+
+#if SANITIZER_INTERCEPT_MEMCPY
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+  // On OS X, calling internal_memcpy here will cause memory corruptions,
+  // because memcpy and memmove are actually aliases of the same
+  // implementation.  We need to use internal_memmove here.
+  // N.B.: If we switch this to internal_ we'll have to use internal_memmove
+  // due to memcpy being an alias of memmove on OS X.
+  void *ctx;
+  if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
+    COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+  } else {
+    COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+  }
+}
+
+#define INIT_MEMCPY                                  \
+  do {                                               \
+    if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
+      COMMON_INTERCEPT_FUNCTION(memcpy);             \
+    } else {                                         \
+      ASSIGN_REAL(memcpy, memmove);                  \
+    }                                                \
+    CHECK(REAL(memcpy));                             \
+  } while (false)
+
+#else
+#define INIT_MEMCPY
+#endif
+
+#if SANITIZER_INTERCEPT_MEMCMP
+DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
+                              const void *s1, const void *s2, uptr n,
+                              int result)
+
+// Common code for `memcmp` and `bcmp`.
+int MemcmpInterceptorCommon(void *ctx,
+                            int (*real_fn)(const void *, const void *, uptr),
+                            const void *a1, const void *a2, uptr size) {
+  if (common_flags()->intercept_memcmp) {
+    if (common_flags()->strict_memcmp) {
+      // Check the entire regions even if the first bytes of the buffers are
+      // different.
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, a1, size);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, a2, size);
+      // Fallthrough to REAL(memcmp) below.
+    } else {
+      unsigned char c1 = 0, c2 = 0;
+      const unsigned char *s1 = (const unsigned char*)a1;
+      const unsigned char *s2 = (const unsigned char*)a2;
+      uptr i;
+      for (i = 0; i < size; i++) {
+        c1 = s1[i];
+        c2 = s2[i];
+        if (c1 != c2) break;
+      }
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, size));
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, size));
+      int r = CharCmpX(c1, c2);
+      CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(),
+                                 a1, a2, size, r);
+      return r;
+    }
+  }
+  int result = real_fn(a1, a2, size);
+  CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1,
+                             a2, size, result);
+  return result;
+}
+
+INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_memcmp(a1, a2, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size);
+  return MemcmpInterceptorCommon(ctx, REAL(memcmp), a1, a2, size);
+}
+
+#define INIT_MEMCMP COMMON_INTERCEPT_FUNCTION(memcmp)
+#else
+#define INIT_MEMCMP
+#endif
+
+#if SANITIZER_INTERCEPT_BCMP
+INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_memcmp(a1, a2, size);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, bcmp, a1, a2, size);
+  return MemcmpInterceptorCommon(ctx, REAL(bcmp), a1, a2, size);
+}
+
+#define INIT_BCMP COMMON_INTERCEPT_FUNCTION(bcmp)
+#else
+#define INIT_BCMP
+#endif
+
+#if SANITIZER_INTERCEPT_MEMCHR
+INTERCEPTOR(void*, memchr, const void *s, int c, SIZE_T n) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_memchr(s, c, n);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, memchr, s, c, n);
+#if SANITIZER_WINDOWS
+  void *res;
+  if (REAL(memchr)) {
+    res = REAL(memchr)(s, c, n);
+  } else {
+    res = internal_memchr(s, c, n);
+  }
+#else
+  void *res = REAL(memchr)(s, c, n);
+#endif
+  uptr len = res ? (char *)res - (const char *)s + 1 : n;
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, len);
+  return res;
+}
+
+#define INIT_MEMCHR COMMON_INTERCEPT_FUNCTION(memchr)
+#else
+#define INIT_MEMCHR
+#endif
+
+#if SANITIZER_INTERCEPT_MEMRCHR
+INTERCEPTOR(void*, memrchr, const void *s, int c, SIZE_T n) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, memrchr, s, c, n);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, n);
+  return REAL(memrchr)(s, c, n);
+}
+
+#define INIT_MEMRCHR COMMON_INTERCEPT_FUNCTION(memrchr)
+#else
+#define INIT_MEMRCHR
+#endif
+
+#if SANITIZER_INTERCEPT_FREXP
+INTERCEPTOR(double, frexp, double x, int *exp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, frexp, x, exp);
+  // Assuming frexp() always writes to |exp|.
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+  double res = REAL(frexp)(x, exp);
+  return res;
+}
+
+#define INIT_FREXP COMMON_INTERCEPT_FUNCTION(frexp);
+#else
+#define INIT_FREXP
+#endif  // SANITIZER_INTERCEPT_FREXP
+
+#if SANITIZER_INTERCEPT_FREXPF_FREXPL
+INTERCEPTOR(float, frexpf, float x, int *exp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  float res = REAL(frexpf)(x, exp);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+  return res;
+}
+
+INTERCEPTOR(long double, frexpl, long double x, int *exp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  long double res = REAL(frexpl)(x, exp);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+  return res;
+}
+
+#define INIT_FREXPF_FREXPL           \
+  COMMON_INTERCEPT_FUNCTION(frexpf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(frexpl)
+#else
+#define INIT_FREXPF_FREXPL
+#endif  // SANITIZER_INTERCEPT_FREXPF_FREXPL
+
+#if SI_POSIX
+static void write_iovec(void *ctx, struct __sanitizer_iovec *iovec,
+                        SIZE_T iovlen, SIZE_T maxlen) {
+  for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
+    SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec[i].iov_base, sz);
+    maxlen -= sz;
+  }
+}
+
+static void read_iovec(void *ctx, struct __sanitizer_iovec *iovec,
+                       SIZE_T iovlen, SIZE_T maxlen) {
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec) * iovlen);
+  for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
+    SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec[i].iov_base, sz);
+    maxlen -= sz;
+  }
+}
+#endif
+
+#if SANITIZER_INTERCEPT_READ
+INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, read, fd, ptr, count);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(read)(fd, ptr, count);
+  if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_READ COMMON_INTERCEPT_FUNCTION(read)
+#else
+#define INIT_READ
+#endif
+
+#if SANITIZER_INTERCEPT_FREAD
+INTERCEPTOR(SIZE_T, fread, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fread, ptr, size, nmemb, file);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(fread)(ptr, size, nmemb, file);
+  if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res * size);
+  return res;
+}
+#define INIT_FREAD COMMON_INTERCEPT_FUNCTION(fread)
+#else
+#define INIT_FREAD
+#endif
+
+#if SANITIZER_INTERCEPT_PREAD
+INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pread, fd, ptr, count, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(pread)(fd, ptr, count, offset);
+  if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_PREAD COMMON_INTERCEPT_FUNCTION(pread)
+#else
+#define INIT_PREAD
+#endif
+
+#if SANITIZER_INTERCEPT_PREAD64
+INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pread64, fd, ptr, count, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(pread64)(fd, ptr, count, offset);
+  if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_PREAD64 COMMON_INTERCEPT_FUNCTION(pread64)
+#else
+#define INIT_PREAD64
+#endif
+
+#if SANITIZER_INTERCEPT_READV
+INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov,
+                        int iovcnt) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readv, fd, iov, iovcnt);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SSIZE_T res = REAL(readv)(fd, iov, iovcnt);
+  if (res > 0) write_iovec(ctx, iov, iovcnt, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_READV COMMON_INTERCEPT_FUNCTION(readv)
+#else
+#define INIT_READV
+#endif
+
+#if SANITIZER_INTERCEPT_PREADV
+INTERCEPTOR(SSIZE_T, preadv, int fd, __sanitizer_iovec *iov, int iovcnt,
+            OFF_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, preadv, fd, iov, iovcnt, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SSIZE_T res = REAL(preadv)(fd, iov, iovcnt, offset);
+  if (res > 0) write_iovec(ctx, iov, iovcnt, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_PREADV COMMON_INTERCEPT_FUNCTION(preadv)
+#else
+#define INIT_PREADV
+#endif
+
+#if SANITIZER_INTERCEPT_PREADV64
+INTERCEPTOR(SSIZE_T, preadv64, int fd, __sanitizer_iovec *iov, int iovcnt,
+            OFF64_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, preadv64, fd, iov, iovcnt, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SSIZE_T res = REAL(preadv64)(fd, iov, iovcnt, offset);
+  if (res > 0) write_iovec(ctx, iov, iovcnt, res);
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#define INIT_PREADV64 COMMON_INTERCEPT_FUNCTION(preadv64)
+#else
+#define INIT_PREADV64
+#endif
+
+#if SANITIZER_INTERCEPT_WRITE
+INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, write, fd, ptr, count);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(write)(fd, ptr, count);
+  // FIXME: this check should be _before_ the call to REAL(write), not after
+  if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
+  return res;
+}
+#define INIT_WRITE COMMON_INTERCEPT_FUNCTION(write)
+#else
+#define INIT_WRITE
+#endif
+
+#if SANITIZER_INTERCEPT_FWRITE
+INTERCEPTOR(SIZE_T, fwrite, const void *p, uptr size, uptr nmemb, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file);
+  SIZE_T res = REAL(fwrite)(p, size, nmemb, file);
+  if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, p, res * size);
+  return res;
+}
+#define INIT_FWRITE COMMON_INTERCEPT_FUNCTION(fwrite)
+#else
+#define INIT_FWRITE
+#endif
+
+#if SANITIZER_INTERCEPT_PWRITE
+INTERCEPTOR(SSIZE_T, pwrite, int fd, void *ptr, SIZE_T count, OFF_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pwrite, fd, ptr, count, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(pwrite)(fd, ptr, count, offset);
+  if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
+  return res;
+}
+#define INIT_PWRITE COMMON_INTERCEPT_FUNCTION(pwrite)
+#else
+#define INIT_PWRITE
+#endif
+
+#if SANITIZER_INTERCEPT_PWRITE64
+INTERCEPTOR(SSIZE_T, pwrite64, int fd, void *ptr, OFF64_T count,
+            OFF64_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pwrite64, fd, ptr, count, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(pwrite64)(fd, ptr, count, offset);
+  if (res > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, res);
+  return res;
+}
+#define INIT_PWRITE64 COMMON_INTERCEPT_FUNCTION(pwrite64)
+#else
+#define INIT_PWRITE64
+#endif
+
+#if SANITIZER_INTERCEPT_WRITEV
+INTERCEPTOR_WITH_SUFFIX(SSIZE_T, writev, int fd, __sanitizer_iovec *iov,
+                        int iovcnt) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, writev, fd, iov, iovcnt);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(writev)(fd, iov, iovcnt);
+  if (res > 0) read_iovec(ctx, iov, iovcnt, res);
+  return res;
+}
+#define INIT_WRITEV COMMON_INTERCEPT_FUNCTION(writev)
+#else
+#define INIT_WRITEV
+#endif
+
+#if SANITIZER_INTERCEPT_PWRITEV
+INTERCEPTOR(SSIZE_T, pwritev, int fd, __sanitizer_iovec *iov, int iovcnt,
+            OFF_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pwritev, fd, iov, iovcnt, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(pwritev)(fd, iov, iovcnt, offset);
+  if (res > 0) read_iovec(ctx, iov, iovcnt, res);
+  return res;
+}
+#define INIT_PWRITEV COMMON_INTERCEPT_FUNCTION(pwritev)
+#else
+#define INIT_PWRITEV
+#endif
+
+#if SANITIZER_INTERCEPT_PWRITEV64
+INTERCEPTOR(SSIZE_T, pwritev64, int fd, __sanitizer_iovec *iov, int iovcnt,
+            OFF64_T offset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pwritev64, fd, iov, iovcnt, offset);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  if (fd >= 0) COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  SSIZE_T res = REAL(pwritev64)(fd, iov, iovcnt, offset);
+  if (res > 0) read_iovec(ctx, iov, iovcnt, res);
+  return res;
+}
+#define INIT_PWRITEV64 COMMON_INTERCEPT_FUNCTION(pwritev64)
+#else
+#define INIT_PWRITEV64
+#endif
+
+#if SANITIZER_INTERCEPT_FGETS
+INTERCEPTOR(char *, fgets, char *s, SIZE_T size, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgets, s, size, file);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(fgets)(s, size, file);
+  if (res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  return res;
+}
+#define INIT_FGETS COMMON_INTERCEPT_FUNCTION(fgets)
+#else
+#define INIT_FGETS
+#endif
+
+#if SANITIZER_INTERCEPT_FPUTS
+INTERCEPTOR_WITH_SUFFIX(int, fputs, char *s, void *file) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file);
+  if (!SANITIZER_MAC || s) {  // `fputs(NULL, file)` is supported on Darwin.
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  }
+  return REAL(fputs)(s, file);
+}
+#define INIT_FPUTS COMMON_INTERCEPT_FUNCTION(fputs)
+#else
+#define INIT_FPUTS
+#endif
+
+#if SANITIZER_INTERCEPT_PUTS
+INTERCEPTOR(int, puts, char *s) {
+  // libc file streams can call user-supplied functions, see fopencookie.
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, puts, s);
+  if (!SANITIZER_MAC || s) {  // `puts(NULL)` is supported on Darwin.
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  }
+  return REAL(puts)(s);
+}
+#define INIT_PUTS COMMON_INTERCEPT_FUNCTION(puts)
+#else
+#define INIT_PUTS
+#endif
+
+#if SANITIZER_INTERCEPT_PRCTL
+INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
+            unsigned long arg4, unsigned long arg5) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
+  static const int PR_SET_NAME = 15;
+  int res = REAL(prctl(option, arg2, arg3, arg4, arg5));
+  if (option == PR_SET_NAME) {
+    char buff[16];
+    internal_strncpy(buff, (char *)arg2, 15);
+    buff[15] = 0;
+    COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
+  }
+  return res;
+}
+#define INIT_PRCTL COMMON_INTERCEPT_FUNCTION(prctl)
+#else
+#define INIT_PRCTL
+#endif  // SANITIZER_INTERCEPT_PRCTL
+
+#if SANITIZER_INTERCEPT_TIME
+INTERCEPTOR(unsigned long, time, unsigned long *t) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, time, t);
+  unsigned long local_t;
+  unsigned long res = REAL(time)(&local_t);
+  if (t && res != (unsigned long)-1) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, t, sizeof(*t));
+    *t = local_t;
+  }
+  return res;
+}
+#define INIT_TIME COMMON_INTERCEPT_FUNCTION(time);
+#else
+#define INIT_TIME
+#endif  // SANITIZER_INTERCEPT_TIME
+
+#if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS
+static void unpoison_tm(void *ctx, __sanitizer_tm *tm) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm));
+#if !SANITIZER_SOLARIS
+  if (tm->tm_zone) {
+    // Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
+    // can point to shared memory and tsan would report a data race.
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,
+                                        REAL(strlen(tm->tm_zone)) + 1);
+  }
+#endif
+}
+INTERCEPTOR(__sanitizer_tm *, localtime, unsigned long *timep) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, localtime, timep);
+  __sanitizer_tm *res = REAL(localtime)(timep);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    unpoison_tm(ctx, res);
+  }
+  return res;
+}
+INTERCEPTOR(__sanitizer_tm *, localtime_r, unsigned long *timep, void *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, localtime_r, timep, result);
+  __sanitizer_tm *res = REAL(localtime_r)(timep, result);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    unpoison_tm(ctx, res);
+  }
+  return res;
+}
+INTERCEPTOR(__sanitizer_tm *, gmtime, unsigned long *timep) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gmtime, timep);
+  __sanitizer_tm *res = REAL(gmtime)(timep);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    unpoison_tm(ctx, res);
+  }
+  return res;
+}
+INTERCEPTOR(__sanitizer_tm *, gmtime_r, unsigned long *timep, void *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gmtime_r, timep, result);
+  __sanitizer_tm *res = REAL(gmtime_r)(timep, result);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    unpoison_tm(ctx, res);
+  }
+  return res;
+}
+INTERCEPTOR(char *, ctime, unsigned long *timep) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ctime, timep);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(ctime)(timep);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+INTERCEPTOR(char *, ctime_r, unsigned long *timep, char *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ctime_r, timep, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(ctime_r)(timep, result);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+INTERCEPTOR(char *, asctime, __sanitizer_tm *tm) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, asctime, tm);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(asctime)(tm);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+INTERCEPTOR(char *, asctime_r, __sanitizer_tm *tm, char *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, asctime_r, tm, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(asctime_r)(tm, result);
+  if (res) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+INTERCEPTOR(long, mktime, __sanitizer_tm *tm) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mktime, tm);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_sec, sizeof(tm->tm_sec));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_min, sizeof(tm->tm_min));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_hour, sizeof(tm->tm_hour));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_mday, sizeof(tm->tm_mday));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_mon, sizeof(tm->tm_mon));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_year, sizeof(tm->tm_year));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &tm->tm_isdst, sizeof(tm->tm_isdst));
+  long res = REAL(mktime)(tm);
+  if (res != -1) unpoison_tm(ctx, tm);
+  return res;
+}
+#define INIT_LOCALTIME_AND_FRIENDS        \
+  COMMON_INTERCEPT_FUNCTION(localtime);   \
+  COMMON_INTERCEPT_FUNCTION(localtime_r); \
+  COMMON_INTERCEPT_FUNCTION(gmtime);      \
+  COMMON_INTERCEPT_FUNCTION(gmtime_r);    \
+  COMMON_INTERCEPT_FUNCTION(ctime);       \
+  COMMON_INTERCEPT_FUNCTION(ctime_r);     \
+  COMMON_INTERCEPT_FUNCTION(asctime);     \
+  COMMON_INTERCEPT_FUNCTION(asctime_r);   \
+  COMMON_INTERCEPT_FUNCTION(mktime);
+#else
+#define INIT_LOCALTIME_AND_FRIENDS
+#endif  // SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS
+
+#if SANITIZER_INTERCEPT_STRPTIME
+INTERCEPTOR(char *, strptime, char *s, char *format, __sanitizer_tm *tm) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strptime, s, format, tm);
+  if (format)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(strptime)(s, format, tm);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, s, res ? res - s : 0);
+  if (res && tm) {
+    // Do not call unpoison_tm here, because strptime does not, in fact,
+    // initialize the entire struct tm. For example, tm_zone pointer is left
+    // uninitialized.
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm));
+  }
+  return res;
+}
+#define INIT_STRPTIME COMMON_INTERCEPT_FUNCTION(strptime);
+#else
+#define INIT_STRPTIME
+#endif
+
+#if SANITIZER_INTERCEPT_SCANF || SANITIZER_INTERCEPT_PRINTF
+#include "sanitizer_common_interceptors_format.inc"
+
+#define FORMAT_INTERCEPTOR_IMPL(name, vname, ...)                              \
+  {                                                                            \
+    void *ctx;                                                                 \
+    va_list ap;                                                                \
+    va_start(ap, format);                                                      \
+    COMMON_INTERCEPTOR_ENTER(ctx, vname, __VA_ARGS__, ap);                     \
+    int res = WRAP(vname)(__VA_ARGS__, ap);                                    \
+    va_end(ap);                                                                \
+    return res;                                                                \
+  }
+
+#endif
+
+#if SANITIZER_INTERCEPT_SCANF
+
+#define VSCANF_INTERCEPTOR_IMPL(vname, allowGnuMalloc, ...)                    \
+  {                                                                            \
+    void *ctx;                                                                 \
+    COMMON_INTERCEPTOR_ENTER(ctx, vname, __VA_ARGS__);                         \
+    va_list aq;                                                                \
+    va_copy(aq, ap);                                                           \
+    int res = REAL(vname)(__VA_ARGS__);                                        \
+    if (res > 0)                                                               \
+      scanf_common(ctx, res, allowGnuMalloc, format, aq);                      \
+    va_end(aq);                                                                \
+    return res;                                                                \
+  }
+
+INTERCEPTOR(int, vscanf, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(vscanf, true, format, ap)
+
+INTERCEPTOR(int, vsscanf, const char *str, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(vsscanf, true, str, format, ap)
+
+INTERCEPTOR(int, vfscanf, void *stream, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(vfscanf, true, stream, format, ap)
+
+#if SANITIZER_INTERCEPT_ISOC99_SCANF
+INTERCEPTOR(int, __isoc99_vscanf, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc99_vscanf, false, format, ap)
+
+INTERCEPTOR(int, __isoc99_vsscanf, const char *str, const char *format,
+            va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap)
+
+INTERCEPTOR(int, __isoc99_vfscanf, void *stream, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap)
+#endif  // SANITIZER_INTERCEPT_ISOC99_SCANF
+
+INTERCEPTOR(int, scanf, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(scanf, vscanf, format)
+
+INTERCEPTOR(int, fscanf, void *stream, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(fscanf, vfscanf, stream, format)
+
+INTERCEPTOR(int, sscanf, const char *str, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(sscanf, vsscanf, str, format)
+
+#if SANITIZER_INTERCEPT_ISOC99_SCANF
+INTERCEPTOR(int, __isoc99_scanf, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_scanf, __isoc99_vscanf, format)
+
+INTERCEPTOR(int, __isoc99_fscanf, void *stream, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format)
+
+INTERCEPTOR(int, __isoc99_sscanf, const char *str, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
+#endif
+
+#endif
+
+#if SANITIZER_INTERCEPT_SCANF
+#define INIT_SCANF                    \
+  COMMON_INTERCEPT_FUNCTION_LDBL(scanf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(sscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(fscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vscanf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsscanf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vfscanf);
+#else
+#define INIT_SCANF
+#endif
+
+#if SANITIZER_INTERCEPT_ISOC99_SCANF
+#define INIT_ISOC99_SCANF                      \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_scanf);   \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_sscanf);  \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf);  \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf);  \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf);
+#else
+#define INIT_ISOC99_SCANF
+#endif
+
+#if SANITIZER_INTERCEPT_PRINTF
+
+#define VPRINTF_INTERCEPTOR_ENTER(vname, ...)                                  \
+  void *ctx;                                                                   \
+  COMMON_INTERCEPTOR_ENTER(ctx, vname, __VA_ARGS__);                           \
+  va_list aq;                                                                  \
+  va_copy(aq, ap);
+
+#define VPRINTF_INTERCEPTOR_RETURN()                                           \
+  va_end(aq);
+
+#define VPRINTF_INTERCEPTOR_IMPL(vname, ...)                                   \
+  {                                                                            \
+    VPRINTF_INTERCEPTOR_ENTER(vname, __VA_ARGS__);                             \
+    if (common_flags()->check_printf)                                          \
+      printf_common(ctx, format, aq);                                          \
+    int res = REAL(vname)(__VA_ARGS__);                                        \
+    VPRINTF_INTERCEPTOR_RETURN();                                              \
+    return res;                                                                \
+  }
+
+// FIXME: under ASan the REAL() call below may write to freed memory and
+// corrupt its metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#define VSPRINTF_INTERCEPTOR_IMPL(vname, str, ...)                             \
+  {                                                                            \
+    VPRINTF_INTERCEPTOR_ENTER(vname, str, __VA_ARGS__)                         \
+    if (common_flags()->check_printf) {                                        \
+      printf_common(ctx, format, aq);                                          \
+    }                                                                          \
+    int res = REAL(vname)(str, __VA_ARGS__);                                   \
+    if (res >= 0) {                                                            \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, str, res + 1);                       \
+    }                                                                          \
+    VPRINTF_INTERCEPTOR_RETURN();                                              \
+    return res;                                                                \
+  }
+
+// FIXME: under ASan the REAL() call below may write to freed memory and
+// corrupt its metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#define VSNPRINTF_INTERCEPTOR_IMPL(vname, str, size, ...)                      \
+  {                                                                            \
+    VPRINTF_INTERCEPTOR_ENTER(vname, str, size, __VA_ARGS__)                   \
+    if (common_flags()->check_printf) {                                        \
+      printf_common(ctx, format, aq);                                          \
+    }                                                                          \
+    int res = REAL(vname)(str, size, __VA_ARGS__);                             \
+    if (res >= 0) {                                                            \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, str, Min(size, (SIZE_T)(res + 1)));  \
+    }                                                                          \
+    VPRINTF_INTERCEPTOR_RETURN();                                              \
+    return res;                                                                \
+  }
+
+// FIXME: under ASan the REAL() call below may write to freed memory and
+// corrupt its metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#define VASPRINTF_INTERCEPTOR_IMPL(vname, strp, ...)                           \
+  {                                                                            \
+    VPRINTF_INTERCEPTOR_ENTER(vname, strp, __VA_ARGS__)                        \
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, strp, sizeof(char *));                 \
+    if (common_flags()->check_printf) {                                        \
+      printf_common(ctx, format, aq);                                          \
+    }                                                                          \
+    int res = REAL(vname)(strp, __VA_ARGS__);                                  \
+    if (res >= 0) {                                                            \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *strp, res + 1);                     \
+    }                                                                          \
+    VPRINTF_INTERCEPTOR_RETURN();                                              \
+    return res;                                                                \
+  }
+
+INTERCEPTOR(int, vprintf, const char *format, va_list ap)
+VPRINTF_INTERCEPTOR_IMPL(vprintf, format, ap)
+
+INTERCEPTOR(int, vfprintf, __sanitizer_FILE *stream, const char *format,
+            va_list ap)
+VPRINTF_INTERCEPTOR_IMPL(vfprintf, stream, format, ap)
+
+INTERCEPTOR(int, vsnprintf, char *str, SIZE_T size, const char *format,
+            va_list ap)
+VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap)
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __vsnprintf_chk, char *str, SIZE_T size, int flag,
+            SIZE_T size_to, const char *format, va_list ap)
+VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap)
+#endif
+
+#if SANITIZER_INTERCEPT_PRINTF_L
+INTERCEPTOR(int, vsnprintf_l, char *str, SIZE_T size, void *loc,
+            const char *format, va_list ap)
+VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf_l, str, size, loc, format, ap)
+
+INTERCEPTOR(int, snprintf_l, char *str, SIZE_T size, void *loc,
+            const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(snprintf_l, vsnprintf_l, str, size, loc, format)
+#endif  // SANITIZER_INTERCEPT_PRINTF_L
+
+INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap)
+VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap)
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __vsprintf_chk, char *str, int flag, SIZE_T size_to,
+            const char *format, va_list ap)
+VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap)
+#endif
+
+INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap)
+VASPRINTF_INTERCEPTOR_IMPL(vasprintf, strp, format, ap)
+
+#if SANITIZER_INTERCEPT_ISOC99_PRINTF
+INTERCEPTOR(int, __isoc99_vprintf, const char *format, va_list ap)
+VPRINTF_INTERCEPTOR_IMPL(__isoc99_vprintf, format, ap)
+
+INTERCEPTOR(int, __isoc99_vfprintf, __sanitizer_FILE *stream,
+            const char *format, va_list ap)
+VPRINTF_INTERCEPTOR_IMPL(__isoc99_vfprintf, stream, format, ap)
+
+INTERCEPTOR(int, __isoc99_vsnprintf, char *str, SIZE_T size, const char *format,
+            va_list ap)
+VSNPRINTF_INTERCEPTOR_IMPL(__isoc99_vsnprintf, str, size, format, ap)
+
+INTERCEPTOR(int, __isoc99_vsprintf, char *str, const char *format,
+            va_list ap)
+VSPRINTF_INTERCEPTOR_IMPL(__isoc99_vsprintf, str, format,
+                          ap)
+
+#endif  // SANITIZER_INTERCEPT_ISOC99_PRINTF
+
+INTERCEPTOR(int, printf, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(printf, vprintf, format)
+
+INTERCEPTOR(int, fprintf, __sanitizer_FILE *stream, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(fprintf, vfprintf, stream, format)
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __fprintf_chk, __sanitizer_FILE *stream, SIZE_T size,
+            const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__fprintf_chk, vfprintf, stream, format)
+#endif
+
+INTERCEPTOR(int, sprintf, char *str, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(sprintf, vsprintf, str, format)
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __sprintf_chk, char *str, int flag, SIZE_T size_to,
+            const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__sprintf_chk, vsprintf, str, format)
+#endif
+
+INTERCEPTOR(int, snprintf, char *str, SIZE_T size, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(snprintf, vsnprintf, str, size, format)
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __snprintf_chk, char *str, SIZE_T size, int flag,
+            SIZE_T size_to, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__snprintf_chk, vsnprintf, str, size, format)
+#endif
+
+INTERCEPTOR(int, asprintf, char **strp, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(asprintf, vasprintf, strp, format)
+
+#if SANITIZER_INTERCEPT_ISOC99_PRINTF
+INTERCEPTOR(int, __isoc99_printf, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_printf, __isoc99_vprintf, format)
+
+INTERCEPTOR(int, __isoc99_fprintf, __sanitizer_FILE *stream, const char *format,
+            ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_fprintf, __isoc99_vfprintf, stream, format)
+
+INTERCEPTOR(int, __isoc99_sprintf, char *str, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_sprintf, __isoc99_vsprintf, str, format)
+
+INTERCEPTOR(int, __isoc99_snprintf, char *str, SIZE_T size,
+            const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size,
+                        format)
+
+#endif  // SANITIZER_INTERCEPT_ISOC99_PRINTF
+
+#endif  // SANITIZER_INTERCEPT_PRINTF
+
+#if SANITIZER_INTERCEPT_PRINTF
+#define INIT_PRINTF                     \
+  COMMON_INTERCEPT_FUNCTION_LDBL(printf);    \
+  COMMON_INTERCEPT_FUNCTION_LDBL(sprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(snprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(asprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(fprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vprintf);   \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsprintf);  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vsnprintf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vasprintf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(vfprintf);
+#else
+#define INIT_PRINTF
+#endif
+
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+#define INIT___PRINTF_CHK                     \
+  COMMON_INTERCEPT_FUNCTION(__sprintf_chk);   \
+  COMMON_INTERCEPT_FUNCTION(__snprintf_chk);  \
+  COMMON_INTERCEPT_FUNCTION(__vsprintf_chk);  \
+  COMMON_INTERCEPT_FUNCTION(__vsnprintf_chk); \
+  COMMON_INTERCEPT_FUNCTION(__fprintf_chk);
+#else
+#define INIT___PRINTF_CHK
+#endif
+
+#if SANITIZER_INTERCEPT_PRINTF_L
+#define INIT_PRINTF_L                     \
+  COMMON_INTERCEPT_FUNCTION(snprintf_l);  \
+  COMMON_INTERCEPT_FUNCTION(vsnprintf_l);
+#else
+#define INIT_PRINTF_L
+#endif
+
+#if SANITIZER_INTERCEPT_ISOC99_PRINTF
+#define INIT_ISOC99_PRINTF                       \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_printf);    \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_sprintf);   \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_snprintf);  \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_fprintf);   \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vprintf);   \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vsprintf);  \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vsnprintf); \
+  COMMON_INTERCEPT_FUNCTION(__isoc99_vfprintf);
+#else
+#define INIT_ISOC99_PRINTF
+#endif
+
+#if SANITIZER_INTERCEPT_IOCTL
+#include "sanitizer_common_interceptors_ioctl.inc"
+#include "sanitizer_interceptors_ioctl_netbsd.inc"
+INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) {
+  // We need a frame pointer, because we call into ioctl_common_[pre|post] which
+  // can trigger a report and we need to be able to unwind through this
+  // function.  On Mac in debug mode we might not have a frame pointer, because
+  // ioctl_common_[pre|post] doesn't get inlined here.
+  ENABLE_FRAME_POINTER;
+
+  void *ctx;
+  va_list ap;
+  va_start(ap, request);
+  void *arg = va_arg(ap, void *);
+  va_end(ap);
+  COMMON_INTERCEPTOR_ENTER(ctx, ioctl, d, request, arg);
+
+  CHECK(ioctl_initialized);
+
+  // Note: TSan does not use common flags, and they are zero-initialized.
+  // This effectively disables ioctl handling in TSan.
+  if (!common_flags()->handle_ioctl) return REAL(ioctl)(d, request, arg);
+
+  // Although request is unsigned long, the rest of the interceptor uses it
+  // as just "unsigned" to save space, because we know that all values fit in
+  // "unsigned" - they are compile-time constants.
+
+  const ioctl_desc *desc = ioctl_lookup(request);
+  ioctl_desc decoded_desc;
+  if (!desc) {
+    VPrintf(2, "Decoding unknown ioctl 0x%x\n", request);
+    if (!ioctl_decode(request, &decoded_desc))
+      Printf("WARNING: failed decoding unknown ioctl 0x%x\n", request);
+    else
+      desc = &decoded_desc;
+  }
+
+  if (desc) ioctl_common_pre(ctx, desc, d, request, arg);
+  int res = REAL(ioctl)(d, request, arg);
+  // FIXME: some ioctls have different return values for success and failure.
+  if (desc && res != -1) ioctl_common_post(ctx, desc, res, d, request, arg);
+  return res;
+}
+#define INIT_IOCTL \
+  ioctl_init();    \
+  COMMON_INTERCEPT_FUNCTION(ioctl);
+#else
+#define INIT_IOCTL
+#endif
+
+#if SANITIZER_POSIX
+UNUSED static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) {
+  if (pwd) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd));
+    if (pwd->pw_name)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name,
+                                     REAL(strlen)(pwd->pw_name) + 1);
+    if (pwd->pw_passwd)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd,
+                                     REAL(strlen)(pwd->pw_passwd) + 1);
+#if !SANITIZER_ANDROID
+    if (pwd->pw_gecos)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
+                                     REAL(strlen)(pwd->pw_gecos) + 1);
+#endif
+#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+    if (pwd->pw_class)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
+                                     REAL(strlen)(pwd->pw_class) + 1);
+#endif
+    if (pwd->pw_dir)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir,
+                                     REAL(strlen)(pwd->pw_dir) + 1);
+    if (pwd->pw_shell)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell,
+                                     REAL(strlen)(pwd->pw_shell) + 1);
+  }
+}
+
+UNUSED static void unpoison_group(void *ctx, __sanitizer_group *grp) {
+  if (grp) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp));
+    if (grp->gr_name)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name,
+                                     REAL(strlen)(grp->gr_name) + 1);
+    if (grp->gr_passwd)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd,
+                                     REAL(strlen)(grp->gr_passwd) + 1);
+    char **p = grp->gr_mem;
+    for (; *p; ++p) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+    }
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem,
+                                   (p - grp->gr_mem + 1) * sizeof(*p));
+  }
+}
+#endif  // SANITIZER_POSIX
+
+#if SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS
+INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  __sanitizer_passwd *res = REAL(getpwnam)(name);
+  unpoison_passwd(ctx, res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwuid, uid);
+  __sanitizer_passwd *res = REAL(getpwuid)(uid);
+  unpoison_passwd(ctx, res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  __sanitizer_group *res = REAL(getgrnam)(name);
+  unpoison_group(ctx, res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_group *, getgrgid, u32 gid) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrgid, gid);
+  __sanitizer_group *res = REAL(getgrgid)(gid);
+  unpoison_group(ctx, res);
+  return res;
+}
+#define INIT_GETPWNAM_AND_FRIENDS      \
+  COMMON_INTERCEPT_FUNCTION(getpwnam); \
+  COMMON_INTERCEPT_FUNCTION(getpwuid); \
+  COMMON_INTERCEPT_FUNCTION(getgrnam); \
+  COMMON_INTERCEPT_FUNCTION(getgrgid);
+#else
+#define INIT_GETPWNAM_AND_FRIENDS
+#endif
+
+#if SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS
+INTERCEPTOR(int, getpwnam_r, const char *name, __sanitizer_passwd *pwd,
+            char *buf, SIZE_T buflen, __sanitizer_passwd **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwnam_r, name, pwd, buf, buflen, result);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getpwnam_r)(name, pwd, buf, buflen, result);
+  if (!res && result)
+    unpoison_passwd(ctx, *result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+INTERCEPTOR(int, getpwuid_r, u32 uid, __sanitizer_passwd *pwd, char *buf,
+            SIZE_T buflen, __sanitizer_passwd **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwuid_r, uid, pwd, buf, buflen, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getpwuid_r)(uid, pwd, buf, buflen, result);
+  if (!res && result)
+    unpoison_passwd(ctx, *result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+INTERCEPTOR(int, getgrnam_r, const char *name, __sanitizer_group *grp,
+            char *buf, SIZE_T buflen, __sanitizer_group **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrnam_r, name, grp, buf, buflen, result);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getgrnam_r)(name, grp, buf, buflen, result);
+  if (!res && result)
+    unpoison_group(ctx, *result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+INTERCEPTOR(int, getgrgid_r, u32 gid, __sanitizer_group *grp, char *buf,
+            SIZE_T buflen, __sanitizer_group **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrgid_r, gid, grp, buf, buflen, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getgrgid_r)(gid, grp, buf, buflen, result);
+  if (!res && result)
+    unpoison_group(ctx, *result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+#define INIT_GETPWNAM_R_AND_FRIENDS      \
+  COMMON_INTERCEPT_FUNCTION(getpwnam_r); \
+  COMMON_INTERCEPT_FUNCTION(getpwuid_r); \
+  COMMON_INTERCEPT_FUNCTION(getgrnam_r); \
+  COMMON_INTERCEPT_FUNCTION(getgrgid_r);
+#else
+#define INIT_GETPWNAM_R_AND_FRIENDS
+#endif
+
+#if SANITIZER_INTERCEPT_GETPWENT
+INTERCEPTOR(__sanitizer_passwd *, getpwent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwent, dummy);
+  __sanitizer_passwd *res = REAL(getpwent)(dummy);
+  unpoison_passwd(ctx, res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_group *, getgrent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrent, dummy);
+  __sanitizer_group *res = REAL(getgrent)(dummy);
+  unpoison_group(ctx, res);
+  return res;
+}
+#define INIT_GETPWENT                  \
+  COMMON_INTERCEPT_FUNCTION(getpwent); \
+  COMMON_INTERCEPT_FUNCTION(getgrent);
+#else
+#define INIT_GETPWENT
+#endif
+
+#if SANITIZER_INTERCEPT_FGETPWENT
+INTERCEPTOR(__sanitizer_passwd *, fgetpwent, void *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent, fp);
+  __sanitizer_passwd *res = REAL(fgetpwent)(fp);
+  unpoison_passwd(ctx, res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_group *, fgetgrent, void *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent, fp);
+  __sanitizer_group *res = REAL(fgetgrent)(fp);
+  unpoison_group(ctx, res);
+  return res;
+}
+#define INIT_FGETPWENT                  \
+  COMMON_INTERCEPT_FUNCTION(fgetpwent); \
+  COMMON_INTERCEPT_FUNCTION(fgetgrent);
+#else
+#define INIT_FGETPWENT
+#endif
+
+#if SANITIZER_INTERCEPT_GETPWENT_R
+INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf,
+            SIZE_T buflen, __sanitizer_passwd **pwbufp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpwent_r, pwbuf, buf, buflen, pwbufp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getpwent_r)(pwbuf, buf, buflen, pwbufp);
+  if (!res && pwbufp)
+    unpoison_passwd(ctx, *pwbufp);
+  if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp));
+  return res;
+}
+INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen,
+            __sanitizer_group **pwbufp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrent_r, pwbuf, buf, buflen, pwbufp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp);
+  if (!res && pwbufp)
+    unpoison_group(ctx, *pwbufp);
+  if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp));
+  return res;
+}
+#define INIT_GETPWENT_R                   \
+  COMMON_INTERCEPT_FUNCTION(getpwent_r);  \
+  COMMON_INTERCEPT_FUNCTION(getgrent_r);
+#else
+#define INIT_GETPWENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_FGETPWENT_R
+INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf,
+            SIZE_T buflen, __sanitizer_passwd **pwbufp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp);
+  if (!res && pwbufp)
+    unpoison_passwd(ctx, *pwbufp);
+  if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp));
+  return res;
+}
+#define INIT_FGETPWENT_R                  \
+  COMMON_INTERCEPT_FUNCTION(fgetpwent_r);
+#else
+#define INIT_FGETPWENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_FGETGRENT_R
+INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf,
+            SIZE_T buflen, __sanitizer_group **pwbufp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent_r, fp, pwbuf, buf, buflen, pwbufp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fgetgrent_r)(fp, pwbuf, buf, buflen, pwbufp);
+  if (!res && pwbufp)
+    unpoison_group(ctx, *pwbufp);
+  if (pwbufp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwbufp, sizeof(*pwbufp));
+  return res;
+}
+#define INIT_FGETGRENT_R                  \
+  COMMON_INTERCEPT_FUNCTION(fgetgrent_r);
+#else
+#define INIT_FGETGRENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_SETPWENT
+// The only thing these interceptors do is disable any nested interceptors.
+// These functions may open nss modules and call uninstrumented functions from
+// them, and we don't want things like strlen() to trigger.
+INTERCEPTOR(void, setpwent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setpwent, dummy);
+  REAL(setpwent)(dummy);
+}
+INTERCEPTOR(void, endpwent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, endpwent, dummy);
+  REAL(endpwent)(dummy);
+}
+INTERCEPTOR(void, setgrent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setgrent, dummy);
+  REAL(setgrent)(dummy);
+}
+INTERCEPTOR(void, endgrent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, endgrent, dummy);
+  REAL(endgrent)(dummy);
+}
+#define INIT_SETPWENT                  \
+  COMMON_INTERCEPT_FUNCTION(setpwent); \
+  COMMON_INTERCEPT_FUNCTION(endpwent); \
+  COMMON_INTERCEPT_FUNCTION(setgrent); \
+  COMMON_INTERCEPT_FUNCTION(endgrent);
+#else
+#define INIT_SETPWENT
+#endif
+
+#if SANITIZER_INTERCEPT_CLOCK_GETTIME
+INTERCEPTOR(int, clock_getres, u32 clk_id, void *tp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, clock_getres, clk_id, tp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(clock_getres)(clk_id, tp);
+  if (!res && tp) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz);
+  }
+  return res;
+}
+INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, clock_gettime, clk_id, tp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(clock_gettime)(clk_id, tp);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz);
+  }
+  return res;
+}
+namespace __sanitizer {
+extern "C" {
+int real_clock_gettime(u32 clk_id, void *tp) {
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_clock_gettime(clk_id, tp);
+  return REAL(clock_gettime)(clk_id, tp);
+}
+}  // extern "C"
+}  // namespace __sanitizer
+INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, tp, struct_timespec_sz);
+  return REAL(clock_settime)(clk_id, tp);
+}
+#define INIT_CLOCK_GETTIME                  \
+  COMMON_INTERCEPT_FUNCTION(clock_getres);  \
+  COMMON_INTERCEPT_FUNCTION(clock_gettime); \
+  COMMON_INTERCEPT_FUNCTION(clock_settime);
+#else
+#define INIT_CLOCK_GETTIME
+#endif
+
+#if SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID
+INTERCEPTOR(int, clock_getcpuclockid, pid_t pid,
+            __sanitizer_clockid_t *clockid) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, clock_getcpuclockid, pid, clockid);
+  int res = REAL(clock_getcpuclockid)(pid, clockid);
+  if (!res && clockid) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, clockid, sizeof *clockid);
+  }
+  return res;
+}
+
+#define INIT_CLOCK_GETCPUCLOCKID                  \
+  COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid);
+#else
+#define INIT_CLOCK_GETCPUCLOCKID
+#endif
+
+#if SANITIZER_INTERCEPT_GETITIMER
+INTERCEPTOR(int, getitimer, int which, void *curr_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getitimer, which, curr_value);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getitimer)(which, curr_value);
+  if (!res && curr_value) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, struct_itimerval_sz);
+  }
+  return res;
+}
+INTERCEPTOR(int, setitimer, int which, const void *new_value, void *old_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setitimer, which, new_value, old_value);
+  if (new_value) {
+    // itimerval can contain padding that may be legitimately uninitialized
+    const struct __sanitizer_itimerval *nv =
+        (const struct __sanitizer_itimerval *)new_value;
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &nv->it_interval.tv_sec,
+                                  sizeof(__sanitizer_time_t));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &nv->it_interval.tv_usec,
+                                  sizeof(__sanitizer_suseconds_t));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &nv->it_value.tv_sec,
+                                  sizeof(__sanitizer_time_t));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &nv->it_value.tv_usec,
+                                  sizeof(__sanitizer_suseconds_t));
+  }
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(setitimer)(which, new_value, old_value);
+  if (!res && old_value) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, struct_itimerval_sz);
+  }
+  return res;
+}
+#define INIT_GETITIMER                  \
+  COMMON_INTERCEPT_FUNCTION(getitimer); \
+  COMMON_INTERCEPT_FUNCTION(setitimer);
+#else
+#define INIT_GETITIMER
+#endif
+
+#if SANITIZER_INTERCEPT_GLOB
+static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pglob, sizeof(*pglob));
+  // +1 for NULL pointer at the end.
+  if (pglob->gl_pathv)
+    COMMON_INTERCEPTOR_WRITE_RANGE(
+        ctx, pglob->gl_pathv, (pglob->gl_pathc + 1) * sizeof(*pglob->gl_pathv));
+  for (SIZE_T i = 0; i < pglob->gl_pathc; ++i) {
+    char *p = pglob->gl_pathv[i];
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
+  }
+}
+
+#if SANITIZER_SOLARIS
+INTERCEPTOR(int, glob, const char *pattern, int flags,
+            int (*errfunc)(const char *epath, int eerrno),
+            __sanitizer_glob_t *pglob) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0);
+  int res = REAL(glob)(pattern, flags, errfunc, pglob);
+  if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
+  return res;
+}
+#else
+static THREADLOCAL __sanitizer_glob_t *pglob_copy;
+
+static void wrapped_gl_closedir(void *dir) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  pglob_copy->gl_closedir(dir);
+}
+
+static void *wrapped_gl_readdir(void *dir) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  return pglob_copy->gl_readdir(dir);
+}
+
+static void *wrapped_gl_opendir(const char *s) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+  return pglob_copy->gl_opendir(s);
+}
+
+static int wrapped_gl_lstat(const char *s, void *st) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+  return pglob_copy->gl_lstat(s, st);
+}
+
+static int wrapped_gl_stat(const char *s, void *st) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
+  return pglob_copy->gl_stat(s, st);
+}
+
+static const __sanitizer_glob_t kGlobCopy = {
+      0,                  0,                   0,
+      0,                  wrapped_gl_closedir, wrapped_gl_readdir,
+      wrapped_gl_opendir, wrapped_gl_lstat,    wrapped_gl_stat};
+
+INTERCEPTOR(int, glob, const char *pattern, int flags,
+            int (*errfunc)(const char *epath, int eerrno),
+            __sanitizer_glob_t *pglob) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0);
+  __sanitizer_glob_t glob_copy;
+  internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy));
+  if (flags & glob_altdirfunc) {
+    Swap(pglob->gl_closedir, glob_copy.gl_closedir);
+    Swap(pglob->gl_readdir, glob_copy.gl_readdir);
+    Swap(pglob->gl_opendir, glob_copy.gl_opendir);
+    Swap(pglob->gl_lstat, glob_copy.gl_lstat);
+    Swap(pglob->gl_stat, glob_copy.gl_stat);
+    pglob_copy = &glob_copy;
+  }
+  int res = REAL(glob)(pattern, flags, errfunc, pglob);
+  if (flags & glob_altdirfunc) {
+    Swap(pglob->gl_closedir, glob_copy.gl_closedir);
+    Swap(pglob->gl_readdir, glob_copy.gl_readdir);
+    Swap(pglob->gl_opendir, glob_copy.gl_opendir);
+    Swap(pglob->gl_lstat, glob_copy.gl_lstat);
+    Swap(pglob->gl_stat, glob_copy.gl_stat);
+  }
+  pglob_copy = 0;
+  if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
+  return res;
+}
+#endif  // SANITIZER_SOLARIS
+#define INIT_GLOB                  \
+  COMMON_INTERCEPT_FUNCTION(glob);
+#else  // SANITIZER_INTERCEPT_GLOB
+#define INIT_GLOB
+#endif  // SANITIZER_INTERCEPT_GLOB
+
+#if SANITIZER_INTERCEPT_GLOB64
+INTERCEPTOR(int, glob64, const char *pattern, int flags,
+            int (*errfunc)(const char *epath, int eerrno),
+            __sanitizer_glob_t *pglob) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, glob64, pattern, flags, errfunc, pglob);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0);
+  __sanitizer_glob_t glob_copy;
+  internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy));
+  if (flags & glob_altdirfunc) {
+    Swap(pglob->gl_closedir, glob_copy.gl_closedir);
+    Swap(pglob->gl_readdir, glob_copy.gl_readdir);
+    Swap(pglob->gl_opendir, glob_copy.gl_opendir);
+    Swap(pglob->gl_lstat, glob_copy.gl_lstat);
+    Swap(pglob->gl_stat, glob_copy.gl_stat);
+    pglob_copy = &glob_copy;
+  }
+  int res = REAL(glob64)(pattern, flags, errfunc, pglob);
+  if (flags & glob_altdirfunc) {
+    Swap(pglob->gl_closedir, glob_copy.gl_closedir);
+    Swap(pglob->gl_readdir, glob_copy.gl_readdir);
+    Swap(pglob->gl_opendir, glob_copy.gl_opendir);
+    Swap(pglob->gl_lstat, glob_copy.gl_lstat);
+    Swap(pglob->gl_stat, glob_copy.gl_stat);
+  }
+  pglob_copy = 0;
+  if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
+  return res;
+}
+#define INIT_GLOB64                \
+  COMMON_INTERCEPT_FUNCTION(glob64);
+#else  // SANITIZER_INTERCEPT_GLOB64
+#define INIT_GLOB64
+#endif  // SANITIZER_INTERCEPT_GLOB64
+
+#if SANITIZER_INTERCEPT_WAIT
+// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
+// suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for
+// details.
+INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wait, status);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(wait)(status);
+  if (res != -1 && status)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
+  return res;
+}
+// On FreeBSD id_t is always 64-bit wide.
+#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
+INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, long long id, void *infop,
+                        int options) {
+#else
+INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop,
+                        int options) {
+#endif
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, waitid, idtype, id, infop, options);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(waitid)(idtype, id, infop, options);
+  if (res != -1 && infop)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz);
+  return res;
+}
+INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, waitpid, pid, status, options);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(waitpid)(pid, status, options);
+  if (res != -1 && status)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
+  return res;
+}
+INTERCEPTOR(int, wait3, int *status, int options, void *rusage) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wait3, status, options, rusage);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(wait3)(status, options, rusage);
+  if (res != -1) {
+    if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
+    if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
+  }
+  return res;
+}
+#if SANITIZER_ANDROID
+INTERCEPTOR(int, __wait4, int pid, int *status, int options, void *rusage) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __wait4, pid, status, options, rusage);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(__wait4)(pid, status, options, rusage);
+  if (res != -1) {
+    if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
+    if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
+  }
+  return res;
+}
+#define INIT_WAIT4 COMMON_INTERCEPT_FUNCTION(__wait4);
+#else
+INTERCEPTOR(int, wait4, int pid, int *status, int options, void *rusage) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wait4, pid, status, options, rusage);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(wait4)(pid, status, options, rusage);
+  if (res != -1) {
+    if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status));
+    if (rusage) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rusage, struct_rusage_sz);
+  }
+  return res;
+}
+#define INIT_WAIT4 COMMON_INTERCEPT_FUNCTION(wait4);
+#endif  // SANITIZER_ANDROID
+#define INIT_WAIT                     \
+  COMMON_INTERCEPT_FUNCTION(wait);    \
+  COMMON_INTERCEPT_FUNCTION(waitid);  \
+  COMMON_INTERCEPT_FUNCTION(waitpid); \
+  COMMON_INTERCEPT_FUNCTION(wait3);
+#else
+#define INIT_WAIT
+#define INIT_WAIT4
+#endif
+
+#if SANITIZER_INTERCEPT_INET
+INTERCEPTOR(char *, inet_ntop, int af, const void *src, char *dst, u32 size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, inet_ntop, af, src, dst, size);
+  uptr sz = __sanitizer_in_addr_sz(af);
+  if (sz) COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sz);
+  // FIXME: figure out read size based on the address family.
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(inet_ntop)(af, src, dst, size);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+INTERCEPTOR(int, inet_pton, int af, const char *src, void *dst) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, inet_pton, af, src, dst);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, src, 0);
+  // FIXME: figure out read size based on the address family.
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(inet_pton)(af, src, dst);
+  if (res == 1) {
+    uptr sz = __sanitizer_in_addr_sz(af);
+    if (sz) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sz);
+  }
+  return res;
+}
+#define INIT_INET                       \
+  COMMON_INTERCEPT_FUNCTION(inet_ntop); \
+  COMMON_INTERCEPT_FUNCTION(inet_pton);
+#else
+#define INIT_INET
+#endif
+
+#if SANITIZER_INTERCEPT_INET
+INTERCEPTOR(int, inet_aton, const char *cp, void *dst) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, inet_aton, cp, dst);
+  if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(inet_aton)(cp, dst);
+  if (res != 0) {
+    uptr sz = __sanitizer_in_addr_sz(af_inet);
+    if (sz) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sz);
+  }
+  return res;
+}
+#define INIT_INET_ATON COMMON_INTERCEPT_FUNCTION(inet_aton);
+#else
+#define INIT_INET_ATON
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM
+INTERCEPTOR(int, pthread_getschedparam, uptr thread, int *policy, int *param) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_getschedparam, thread, policy, param);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(pthread_getschedparam)(thread, policy, param);
+  if (res == 0) {
+    if (policy) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, policy, sizeof(*policy));
+    if (param) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, param, sizeof(*param));
+  }
+  return res;
+}
+#define INIT_PTHREAD_GETSCHEDPARAM \
+  COMMON_INTERCEPT_FUNCTION(pthread_getschedparam);
+#else
+#define INIT_PTHREAD_GETSCHEDPARAM
+#endif
+
+#if SANITIZER_INTERCEPT_GETADDRINFO
+INTERCEPTOR(int, getaddrinfo, char *node, char *service,
+            struct __sanitizer_addrinfo *hints,
+            struct __sanitizer_addrinfo **out) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getaddrinfo, node, service, hints, out);
+  if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, REAL(strlen)(node) + 1);
+  if (service)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, service, REAL(strlen)(service) + 1);
+  if (hints)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getaddrinfo)(node, service, hints, out);
+  if (res == 0 && out) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, out, sizeof(*out));
+    struct __sanitizer_addrinfo *p = *out;
+    while (p) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+      if (p->ai_addr)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_addr, p->ai_addrlen);
+      if (p->ai_canonname)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_canonname,
+                                       REAL(strlen)(p->ai_canonname) + 1);
+      p = p->ai_next;
+    }
+  }
+  return res;
+}
+#define INIT_GETADDRINFO COMMON_INTERCEPT_FUNCTION(getaddrinfo);
+#else
+#define INIT_GETADDRINFO
+#endif
+
+#if SANITIZER_INTERCEPT_GETNAMEINFO
+INTERCEPTOR(int, getnameinfo, void *sockaddr, unsigned salen, char *host,
+            unsigned hostlen, char *serv, unsigned servlen, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getnameinfo, sockaddr, salen, host, hostlen,
+                           serv, servlen, flags);
+  // FIXME: consider adding READ_RANGE(sockaddr, salen)
+  // There is padding in in_addr that may make this too noisy
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res =
+      REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags);
+  if (res == 0) {
+    if (host && hostlen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, REAL(strlen)(host) + 1);
+    if (serv && servlen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, REAL(strlen)(serv) + 1);
+  }
+  return res;
+}
+#define INIT_GETNAMEINFO COMMON_INTERCEPT_FUNCTION(getnameinfo);
+#else
+#define INIT_GETNAMEINFO
+#endif
+
+#if SANITIZER_INTERCEPT_GETSOCKNAME
+INTERCEPTOR(int, getsockname, int sock_fd, void *addr, int *addrlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+  int addrlen_in = *addrlen;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getsockname)(sock_fd, addr, addrlen);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen));
+  }
+  return res;
+}
+#define INIT_GETSOCKNAME COMMON_INTERCEPT_FUNCTION(getsockname);
+#else
+#define INIT_GETSOCKNAME
+#endif
+
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME || SANITIZER_INTERCEPT_GETHOSTBYNAME_R
+static void write_hostent(void *ctx, struct __sanitizer_hostent *h) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h, sizeof(__sanitizer_hostent));
+  if (h->h_name)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, REAL(strlen)(h->h_name) + 1);
+  char **p = h->h_aliases;
+  while (*p) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+    ++p;
+  }
+  COMMON_INTERCEPTOR_WRITE_RANGE(
+      ctx, h->h_aliases, (p - h->h_aliases + 1) * sizeof(*h->h_aliases));
+  p = h->h_addr_list;
+  while (*p) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, h->h_length);
+    ++p;
+  }
+  COMMON_INTERCEPTOR_WRITE_RANGE(
+      ctx, h->h_addr_list, (p - h->h_addr_list + 1) * sizeof(*h->h_addr_list));
+}
+#endif
+
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME
+INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname, char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname, name);
+  struct __sanitizer_hostent *res = REAL(gethostbyname)(name);
+  if (res) write_hostent(ctx, res);
+  return res;
+}
+
+INTERCEPTOR(struct __sanitizer_hostent *, gethostbyaddr, void *addr, int len,
+            int type) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyaddr, addr, len, type);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, len);
+  struct __sanitizer_hostent *res = REAL(gethostbyaddr)(addr, len, type);
+  if (res) write_hostent(ctx, res);
+  return res;
+}
+
+INTERCEPTOR(struct __sanitizer_hostent *, gethostent, int fake) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostent, fake);
+  struct __sanitizer_hostent *res = REAL(gethostent)(fake);
+  if (res) write_hostent(ctx, res);
+  return res;
+}
+#define INIT_GETHOSTBYNAME                  \
+  COMMON_INTERCEPT_FUNCTION(gethostent);    \
+  COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \
+  COMMON_INTERCEPT_FUNCTION(gethostbyname);
+#else
+#define INIT_GETHOSTBYNAME
+#endif  // SANITIZER_INTERCEPT_GETHOSTBYNAME
+
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME2
+INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname2, name, af);
+  struct __sanitizer_hostent *res = REAL(gethostbyname2)(name, af);
+  if (res) write_hostent(ctx, res);
+  return res;
+}
+#define INIT_GETHOSTBYNAME2 COMMON_INTERCEPT_FUNCTION(gethostbyname2);
+#else
+#define INIT_GETHOSTBYNAME2
+#endif  // SANITIZER_INTERCEPT_GETHOSTBYNAME2
+
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME_R
+INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret,
+            char *buf, SIZE_T buflen, __sanitizer_hostent **result,
+            int *h_errnop) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname_r, name, ret, buf, buflen, result,
+                           h_errnop);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(gethostbyname_r)(name, ret, buf, buflen, result, h_errnop);
+  if (result) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (res == 0 && *result) write_hostent(ctx, *result);
+  }
+  if (h_errnop)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h_errnop, sizeof(*h_errnop));
+  return res;
+}
+#define INIT_GETHOSTBYNAME_R COMMON_INTERCEPT_FUNCTION(gethostbyname_r);
+#else
+#define INIT_GETHOSTBYNAME_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETHOSTENT_R
+INTERCEPTOR(int, gethostent_r, struct __sanitizer_hostent *ret, char *buf,
+            SIZE_T buflen, __sanitizer_hostent **result, int *h_errnop) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostent_r, ret, buf, buflen, result,
+                           h_errnop);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(gethostent_r)(ret, buf, buflen, result, h_errnop);
+  if (result) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (res == 0 && *result) write_hostent(ctx, *result);
+  }
+  if (h_errnop)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h_errnop, sizeof(*h_errnop));
+  return res;
+}
+#define INIT_GETHOSTENT_R                  \
+  COMMON_INTERCEPT_FUNCTION(gethostent_r);
+#else
+#define INIT_GETHOSTENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETHOSTBYADDR_R
+INTERCEPTOR(int, gethostbyaddr_r, void *addr, int len, int type,
+            struct __sanitizer_hostent *ret, char *buf, SIZE_T buflen,
+            __sanitizer_hostent **result, int *h_errnop) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyaddr_r, addr, len, type, ret, buf,
+                           buflen, result, h_errnop);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, len);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(gethostbyaddr_r)(addr, len, type, ret, buf, buflen, result,
+                                  h_errnop);
+  if (result) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (res == 0 && *result) write_hostent(ctx, *result);
+  }
+  if (h_errnop)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h_errnop, sizeof(*h_errnop));
+  return res;
+}
+#define INIT_GETHOSTBYADDR_R                  \
+  COMMON_INTERCEPT_FUNCTION(gethostbyaddr_r);
+#else
+#define INIT_GETHOSTBYADDR_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME2_R
+INTERCEPTOR(int, gethostbyname2_r, char *name, int af,
+            struct __sanitizer_hostent *ret, char *buf, SIZE_T buflen,
+            __sanitizer_hostent **result, int *h_errnop) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname2_r, name, af, ret, buf, buflen,
+                           result, h_errnop);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res =
+      REAL(gethostbyname2_r)(name, af, ret, buf, buflen, result, h_errnop);
+  if (result) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (res == 0 && *result) write_hostent(ctx, *result);
+  }
+  if (h_errnop)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h_errnop, sizeof(*h_errnop));
+  return res;
+}
+#define INIT_GETHOSTBYNAME2_R                  \
+  COMMON_INTERCEPT_FUNCTION(gethostbyname2_r);
+#else
+#define INIT_GETHOSTBYNAME2_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETSOCKOPT
+INTERCEPTOR(int, getsockopt, int sockfd, int level, int optname, void *optval,
+            int *optlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getsockopt, sockfd, level, optname, optval,
+                           optlen);
+  if (optlen) COMMON_INTERCEPTOR_READ_RANGE(ctx, optlen, sizeof(*optlen));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getsockopt)(sockfd, level, optname, optval, optlen);
+  if (res == 0)
+    if (optval && optlen) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, optval, *optlen);
+  return res;
+}
+#define INIT_GETSOCKOPT COMMON_INTERCEPT_FUNCTION(getsockopt);
+#else
+#define INIT_GETSOCKOPT
+#endif
+
+#if SANITIZER_INTERCEPT_ACCEPT
+INTERCEPTOR(int, accept, int fd, void *addr, unsigned *addrlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, accept, fd, addr, addrlen);
+  unsigned addrlen0 = 0;
+  if (addrlen) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+    addrlen0 = *addrlen;
+  }
+  int fd2 = REAL(accept)(fd, addr, addrlen);
+  if (fd2 >= 0) {
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
+    if (addr && addrlen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(*addrlen, addrlen0));
+  }
+  return fd2;
+}
+#define INIT_ACCEPT COMMON_INTERCEPT_FUNCTION(accept);
+#else
+#define INIT_ACCEPT
+#endif
+
+#if SANITIZER_INTERCEPT_ACCEPT4
+INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, accept4, fd, addr, addrlen, f);
+  unsigned addrlen0 = 0;
+  if (addrlen) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+    addrlen0 = *addrlen;
+  }
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int fd2 = REAL(accept4)(fd, addr, addrlen, f);
+  if (fd2 >= 0) {
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
+    if (addr && addrlen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(*addrlen, addrlen0));
+  }
+  return fd2;
+}
+#define INIT_ACCEPT4 COMMON_INTERCEPT_FUNCTION(accept4);
+#else
+#define INIT_ACCEPT4
+#endif
+
+#if SANITIZER_INTERCEPT_PACCEPT
+INTERCEPTOR(int, paccept, int fd, void *addr, unsigned *addrlen,
+            __sanitizer_sigset_t *set, int f) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, paccept, fd, addr, addrlen, set, f);
+  unsigned addrlen0 = 0;
+  if (addrlen) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+    addrlen0 = *addrlen;
+  }
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  int fd2 = REAL(paccept)(fd, addr, addrlen, set, f);
+  if (fd2 >= 0) {
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2);
+    if (addr && addrlen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(*addrlen, addrlen0));
+  }
+  return fd2;
+}
+#define INIT_PACCEPT COMMON_INTERCEPT_FUNCTION(paccept);
+#else
+#define INIT_PACCEPT
+#endif
+
+#if SANITIZER_INTERCEPT_MODF
+INTERCEPTOR(double, modf, double x, double *iptr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, modf, x, iptr);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  double res = REAL(modf)(x, iptr);
+  if (iptr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr));
+  }
+  return res;
+}
+INTERCEPTOR(float, modff, float x, float *iptr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, modff, x, iptr);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  float res = REAL(modff)(x, iptr);
+  if (iptr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr));
+  }
+  return res;
+}
+INTERCEPTOR(long double, modfl, long double x, long double *iptr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, modfl, x, iptr);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  long double res = REAL(modfl)(x, iptr);
+  if (iptr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr));
+  }
+  return res;
+}
+#define INIT_MODF                   \
+  COMMON_INTERCEPT_FUNCTION(modf);  \
+  COMMON_INTERCEPT_FUNCTION(modff); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(modfl);
+#else
+#define INIT_MODF
+#endif
+
+#if SANITIZER_INTERCEPT_RECVMSG || SANITIZER_INTERCEPT_RECVMMSG
+static void write_msghdr(void *ctx, struct __sanitizer_msghdr *msg,
+                         SSIZE_T maxlen) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msg, sizeof(*msg));
+  if (msg->msg_name && msg->msg_namelen)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msg->msg_name, msg->msg_namelen);
+  if (msg->msg_iov && msg->msg_iovlen)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msg->msg_iov,
+                                   sizeof(*msg->msg_iov) * msg->msg_iovlen);
+  write_iovec(ctx, msg->msg_iov, msg->msg_iovlen, maxlen);
+  if (msg->msg_control && msg->msg_controllen)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msg->msg_control, msg->msg_controllen);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_RECVMSG
+INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg,
+            int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, recvmsg, fd, msg, flags);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(recvmsg)(fd, msg, flags);
+  if (res >= 0) {
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+    if (msg) {
+      write_msghdr(ctx, msg, res);
+      COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg);
+    }
+  }
+  return res;
+}
+#define INIT_RECVMSG COMMON_INTERCEPT_FUNCTION(recvmsg);
+#else
+#define INIT_RECVMSG
+#endif
+
+#if SANITIZER_INTERCEPT_RECVMMSG
+INTERCEPTOR(int, recvmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
+            unsigned int vlen, int flags, void *timeout) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, recvmmsg, fd, msgvec, vlen, flags, timeout);
+  if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
+  int res = REAL(recvmmsg)(fd, msgvec, vlen, flags, timeout);
+  if (res >= 0) {
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+    for (int i = 0; i < res; ++i) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &msgvec[i].msg_len,
+                                     sizeof(msgvec[i].msg_len));
+      write_msghdr(ctx, &msgvec[i].msg_hdr, msgvec[i].msg_len);
+      COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, &msgvec[i].msg_hdr);
+    }
+  }
+  return res;
+}
+#define INIT_RECVMMSG COMMON_INTERCEPT_FUNCTION(recvmmsg);
+#else
+#define INIT_RECVMMSG
+#endif
+
+#if SANITIZER_INTERCEPT_SENDMSG || SANITIZER_INTERCEPT_SENDMMSG
+static void read_msghdr_control(void *ctx, void *control, uptr controllen) {
+  const unsigned kCmsgDataOffset =
+      RoundUpTo(sizeof(__sanitizer_cmsghdr), sizeof(uptr));
+
+  char *p = (char *)control;
+  char *const control_end = p + controllen;
+  while (true) {
+    if (p + sizeof(__sanitizer_cmsghdr) > control_end) break;
+    __sanitizer_cmsghdr *cmsg = (__sanitizer_cmsghdr *)p;
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_len, sizeof(cmsg->cmsg_len));
+
+    if (p + RoundUpTo(cmsg->cmsg_len, sizeof(uptr)) > control_end) break;
+
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_level,
+                                  sizeof(cmsg->cmsg_level));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &cmsg->cmsg_type,
+                                  sizeof(cmsg->cmsg_type));
+
+    if (cmsg->cmsg_len > kCmsgDataOffset) {
+      char *data = p + kCmsgDataOffset;
+      unsigned data_len = cmsg->cmsg_len - kCmsgDataOffset;
+      if (data_len > 0) COMMON_INTERCEPTOR_READ_RANGE(ctx, data, data_len);
+    }
+
+    p += RoundUpTo(cmsg->cmsg_len, sizeof(uptr));
+  }
+}
+
+static void read_msghdr(void *ctx, struct __sanitizer_msghdr *msg,
+                        SSIZE_T maxlen) {
+#define R(f) \
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &msg->msg_##f, sizeof(msg->msg_##f))
+  R(name);
+  R(namelen);
+  R(iov);
+  R(iovlen);
+  R(control);
+  R(controllen);
+  R(flags);
+#undef R
+  if (msg->msg_name && msg->msg_namelen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, msg->msg_name, msg->msg_namelen);
+  if (msg->msg_iov && msg->msg_iovlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, msg->msg_iov,
+                                  sizeof(*msg->msg_iov) * msg->msg_iovlen);
+  read_iovec(ctx, msg->msg_iov, msg->msg_iovlen, maxlen);
+  if (msg->msg_control && msg->msg_controllen)
+    read_msghdr_control(ctx, msg->msg_control, msg->msg_controllen);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_SENDMSG
+INTERCEPTOR(SSIZE_T, sendmsg, int fd, struct __sanitizer_msghdr *msg,
+            int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sendmsg, fd, msg, flags);
+  if (fd >= 0) {
+    COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+    COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  }
+  SSIZE_T res = REAL(sendmsg)(fd, msg, flags);
+  if (common_flags()->intercept_send && res >= 0 && msg)
+    read_msghdr(ctx, msg, res);
+  return res;
+}
+#define INIT_SENDMSG COMMON_INTERCEPT_FUNCTION(sendmsg);
+#else
+#define INIT_SENDMSG
+#endif
+
+#if SANITIZER_INTERCEPT_SENDMMSG
+INTERCEPTOR(int, sendmmsg, int fd, struct __sanitizer_mmsghdr *msgvec,
+            unsigned vlen, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sendmmsg, fd, msgvec, vlen, flags);
+  if (fd >= 0) {
+    COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+    COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  }
+  int res = REAL(sendmmsg)(fd, msgvec, vlen, flags);
+  if (res >= 0 && msgvec) {
+    for (int i = 0; i < res; ++i) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &msgvec[i].msg_len,
+                                     sizeof(msgvec[i].msg_len));
+      if (common_flags()->intercept_send)
+        read_msghdr(ctx, &msgvec[i].msg_hdr, msgvec[i].msg_len);
+    }
+  }
+  return res;
+}
+#define INIT_SENDMMSG COMMON_INTERCEPT_FUNCTION(sendmmsg);
+#else
+#define INIT_SENDMMSG
+#endif
+
+#if SANITIZER_INTERCEPT_SYSMSG
+INTERCEPTOR(int, msgsnd, int msqid, const void *msgp, SIZE_T msgsz,
+            int msgflg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, msgsnd, msqid, msgp, msgsz, msgflg);
+  if (msgp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, msgp, sizeof(long) + msgsz);
+  int res = REAL(msgsnd)(msqid, msgp, msgsz, msgflg);
+  return res;
+}
+
+INTERCEPTOR(SSIZE_T, msgrcv, int msqid, void *msgp, SIZE_T msgsz,
+            long msgtyp, int msgflg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, msgrcv, msqid, msgp, msgsz, msgtyp, msgflg);
+  SSIZE_T len = REAL(msgrcv)(msqid, msgp, msgsz, msgtyp, msgflg);
+  if (len != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, msgp, sizeof(long) + len);
+  return len;
+}
+
+#define INIT_SYSMSG                  \
+  COMMON_INTERCEPT_FUNCTION(msgsnd); \
+  COMMON_INTERCEPT_FUNCTION(msgrcv);
+#else
+#define INIT_SYSMSG
+#endif
+
+#if SANITIZER_INTERCEPT_GETPEERNAME
+INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);
+  unsigned addr_sz;
+  if (addrlen) addr_sz = *addrlen;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getpeername)(sockfd, addr, addrlen);
+  if (!res && addr && addrlen)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
+  return res;
+}
+#define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);
+#else
+#define INIT_GETPEERNAME
+#endif
+
+#if SANITIZER_INTERCEPT_SYSINFO
+INTERCEPTOR(int, sysinfo, void *info) {
+  void *ctx;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  COMMON_INTERCEPTOR_ENTER(ctx, sysinfo, info);
+  int res = REAL(sysinfo)(info);
+  if (!res && info)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, struct_sysinfo_sz);
+  return res;
+}
+#define INIT_SYSINFO COMMON_INTERCEPT_FUNCTION(sysinfo);
+#else
+#define INIT_SYSINFO
+#endif
+
+#if SANITIZER_INTERCEPT_READDIR
+INTERCEPTOR(__sanitizer_dirent *, opendir, const char *path) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, opendir, path);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  __sanitizer_dirent *res = REAL(opendir)(path);
+  if (res)
+    COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path);
+  return res;
+}
+
+INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readdir, dirp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_dirent *res = REAL(readdir)(dirp);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
+  return res;
+}
+
+INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry,
+            __sanitizer_dirent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readdir_r, dirp, entry, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(readdir_r)(dirp, entry, result);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (*result)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen);
+  }
+  return res;
+}
+
+#define INIT_READDIR                  \
+  COMMON_INTERCEPT_FUNCTION(opendir); \
+  COMMON_INTERCEPT_FUNCTION(readdir); \
+  COMMON_INTERCEPT_FUNCTION(readdir_r);
+#else
+#define INIT_READDIR
+#endif
+
+#if SANITIZER_INTERCEPT_READDIR64
+INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readdir64, dirp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_dirent64 *res = REAL(readdir64)(dirp);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
+  return res;
+}
+
+INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry,
+            __sanitizer_dirent64 **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readdir64_r, dirp, entry, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(readdir64_r)(dirp, entry, result);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+    if (*result)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen);
+  }
+  return res;
+}
+#define INIT_READDIR64                  \
+  COMMON_INTERCEPT_FUNCTION(readdir64); \
+  COMMON_INTERCEPT_FUNCTION(readdir64_r);
+#else
+#define INIT_READDIR64
+#endif
+
+#if SANITIZER_INTERCEPT_PTRACE
+INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ptrace, request, pid, addr, data);
+  __sanitizer_iovec local_iovec;
+
+  if (data) {
+    if (request == ptrace_setregs) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_regs_struct_sz);
+    } else if (request == ptrace_setfpregs) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpregs_struct_sz);
+    } else if (request == ptrace_setfpxregs) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
+    } else if (request == ptrace_setvfpregs) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
+    } else if (request == ptrace_setsiginfo) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, siginfo_t_sz);
+
+    // Some kernel might zero the iovec::iov_base in case of invalid
+    // write access.  In this case copy the invalid address for further
+    // inspection.
+    } else if (request == ptrace_setregset || request == ptrace_getregset) {
+      __sanitizer_iovec *iovec = (__sanitizer_iovec*)data;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec));
+      local_iovec = *iovec;
+      if (request == ptrace_setregset)
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec->iov_base, iovec->iov_len);
+    }
+  }
+
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  uptr res = REAL(ptrace)(request, pid, addr, data);
+
+  if (!res && data) {
+    // Note that PEEK* requests assign different meaning to the return value.
+    // This function does not handle them (nor does it need to).
+    if (request == ptrace_getregs) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_regs_struct_sz);
+    } else if (request == ptrace_getfpregs) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpregs_struct_sz);
+    } else if (request == ptrace_getfpxregs) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
+    } else if (request == ptrace_getvfpregs) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
+    } else if (request == ptrace_getsiginfo) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, siginfo_t_sz);
+    } else if (request == ptrace_geteventmsg) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(unsigned long));
+    } else if (request == ptrace_getregset) {
+      __sanitizer_iovec *iovec = (__sanitizer_iovec*)data;
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec, sizeof(*iovec));
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, local_iovec.iov_base,
+                                     local_iovec.iov_len);
+    }
+  }
+  return res;
+}
+
+#define INIT_PTRACE COMMON_INTERCEPT_FUNCTION(ptrace);
+#else
+#define INIT_PTRACE
+#endif
+
+#if SANITIZER_INTERCEPT_SETLOCALE
+static void unpoison_ctype_arrays(void *ctx) {
+#if SANITIZER_NETBSD
+  // These arrays contain 256 regular elements in unsigned char range + 1 EOF
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, _ctype_tab_, 257 * sizeof(short));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, _toupper_tab_, 257 * sizeof(short));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, _tolower_tab_, 257 * sizeof(short));
+#endif
+}
+
+INTERCEPTOR(char *, setlocale, int category, char *locale) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setlocale, category, locale);
+  if (locale)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, REAL(strlen)(locale) + 1);
+  char *res = REAL(setlocale)(category, locale);
+  if (res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+    unpoison_ctype_arrays(ctx);
+  }
+  return res;
+}
+
+#define INIT_SETLOCALE COMMON_INTERCEPT_FUNCTION(setlocale);
+#else
+#define INIT_SETLOCALE
+#endif
+
+#if SANITIZER_INTERCEPT_GETCWD
+INTERCEPTOR(char *, getcwd, char *buf, SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getcwd, buf, size);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(getcwd)(buf, size);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_GETCWD COMMON_INTERCEPT_FUNCTION(getcwd);
+#else
+#define INIT_GETCWD
+#endif
+
+#if SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME
+INTERCEPTOR(char *, get_current_dir_name, int fake) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, get_current_dir_name, fake);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(get_current_dir_name)(fake);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+
+#define INIT_GET_CURRENT_DIR_NAME \
+  COMMON_INTERCEPT_FUNCTION(get_current_dir_name);
+#else
+#define INIT_GET_CURRENT_DIR_NAME
+#endif
+
+UNUSED static inline void FixRealStrtolEndptr(const char *nptr, char **endptr) {
+  CHECK(endptr);
+  if (nptr == *endptr) {
+    // No digits were found at strtol call, we need to find out the last
+    // symbol accessed by strtoll on our own.
+    // We get this symbol by skipping leading blanks and optional +/- sign.
+    while (IsSpace(*nptr)) nptr++;
+    if (*nptr == '+' || *nptr == '-') nptr++;
+    *endptr = const_cast<char *>(nptr);
+  }
+  CHECK(*endptr >= nptr);
+}
+
+UNUSED static inline void StrtolFixAndCheck(void *ctx, const char *nptr,
+                             char **endptr, char *real_endptr, int base) {
+  if (endptr) {
+    *endptr = real_endptr;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, endptr, sizeof(*endptr));
+  }
+  // If base has unsupported value, strtol can exit with EINVAL
+  // without reading any characters. So do additional checks only
+  // if base is valid.
+  bool is_valid_base = (base == 0) || (2 <= base && base <= 36);
+  if (is_valid_base) {
+    FixRealStrtolEndptr(nptr, &real_endptr);
+  }
+  COMMON_INTERCEPTOR_READ_STRING(ctx, nptr, is_valid_base ?
+                                 (real_endptr - nptr) + 1 : 0);
+}
+
+
+#if SANITIZER_INTERCEPT_STRTOIMAX
+INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *real_endptr;
+  INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base);
+  StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+  return res;
+}
+
+INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *real_endptr;
+  UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base);
+  StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+  return res;
+}
+
+#define INIT_STRTOIMAX                  \
+  COMMON_INTERCEPT_FUNCTION(strtoimax); \
+  COMMON_INTERCEPT_FUNCTION(strtoumax);
+#else
+#define INIT_STRTOIMAX
+#endif
+
+#if SANITIZER_INTERCEPT_MBSTOWCS
+INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mbstowcs, dest, src, len);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(mbstowcs)(dest, src, len);
+  if (res != (SIZE_T) - 1 && dest) {
+    SIZE_T write_cnt = res + (res < len);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt * sizeof(wchar_t));
+  }
+  return res;
+}
+
+INTERCEPTOR(SIZE_T, mbsrtowcs, wchar_t *dest, const char **src, SIZE_T len,
+            void *ps) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mbsrtowcs, dest, src, len, ps);
+  if (src) COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+  if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(mbsrtowcs)(dest, src, len, ps);
+  if (res != (SIZE_T)(-1) && dest && src) {
+    // This function, and several others, may or may not write the terminating
+    // \0 character. They write it iff they clear *src.
+    SIZE_T write_cnt = res + !*src;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt * sizeof(wchar_t));
+  }
+  return res;
+}
+
+#define INIT_MBSTOWCS                  \
+  COMMON_INTERCEPT_FUNCTION(mbstowcs); \
+  COMMON_INTERCEPT_FUNCTION(mbsrtowcs);
+#else
+#define INIT_MBSTOWCS
+#endif
+
+#if SANITIZER_INTERCEPT_MBSNRTOWCS
+INTERCEPTOR(SIZE_T, mbsnrtowcs, wchar_t *dest, const char **src, SIZE_T nms,
+            SIZE_T len, void *ps) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mbsnrtowcs, dest, src, nms, len, ps);
+  if (src) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+    if (nms) COMMON_INTERCEPTOR_READ_RANGE(ctx, *src, nms);
+  }
+  if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(mbsnrtowcs)(dest, src, nms, len, ps);
+  if (res != (SIZE_T)(-1) && dest && src) {
+    SIZE_T write_cnt = res + !*src;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt * sizeof(wchar_t));
+  }
+  return res;
+}
+
+#define INIT_MBSNRTOWCS COMMON_INTERCEPT_FUNCTION(mbsnrtowcs);
+#else
+#define INIT_MBSNRTOWCS
+#endif
+
+#if SANITIZER_INTERCEPT_WCSTOMBS
+INTERCEPTOR(SIZE_T, wcstombs, char *dest, const wchar_t *src, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcstombs, dest, src, len);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(wcstombs)(dest, src, len);
+  if (res != (SIZE_T) - 1 && dest) {
+    SIZE_T write_cnt = res + (res < len);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt);
+  }
+  return res;
+}
+
+INTERCEPTOR(SIZE_T, wcsrtombs, char *dest, const wchar_t **src, SIZE_T len,
+            void *ps) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsrtombs, dest, src, len, ps);
+  if (src) COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+  if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(wcsrtombs)(dest, src, len, ps);
+  if (res != (SIZE_T) - 1 && dest && src) {
+    SIZE_T write_cnt = res + !*src;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt);
+  }
+  return res;
+}
+
+#define INIT_WCSTOMBS                  \
+  COMMON_INTERCEPT_FUNCTION(wcstombs); \
+  COMMON_INTERCEPT_FUNCTION(wcsrtombs);
+#else
+#define INIT_WCSTOMBS
+#endif
+
+#if SANITIZER_INTERCEPT_WCSNRTOMBS
+INTERCEPTOR(SIZE_T, wcsnrtombs, char *dest, const wchar_t **src, SIZE_T nms,
+            SIZE_T len, void *ps) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsnrtombs, dest, src, nms, len, ps);
+  if (src) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+    if (nms) COMMON_INTERCEPTOR_READ_RANGE(ctx, *src, nms);
+  }
+  if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(wcsnrtombs)(dest, src, nms, len, ps);
+  if (res != ((SIZE_T)-1) && dest && src) {
+    SIZE_T write_cnt = res + !*src;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt);
+  }
+  return res;
+}
+
+#define INIT_WCSNRTOMBS COMMON_INTERCEPT_FUNCTION(wcsnrtombs);
+#else
+#define INIT_WCSNRTOMBS
+#endif
+
+
+#if SANITIZER_INTERCEPT_WCRTOMB
+INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcrtomb, dest, src, ps);
+  if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz);
+
+  if (!dest)
+    return REAL(wcrtomb)(dest, src, ps);
+
+  char local_dest[32];
+  SIZE_T res = REAL(wcrtomb)(local_dest, src, ps);
+  if (res != ((SIZE_T)-1)) {
+    CHECK_LE(res, sizeof(local_dest));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res);
+    REAL(memcpy)(dest, local_dest, res);
+  }
+  return res;
+}
+
+#define INIT_WCRTOMB COMMON_INTERCEPT_FUNCTION(wcrtomb);
+#else
+#define INIT_WCRTOMB
+#endif
+
+#if SANITIZER_INTERCEPT_WCTOMB
+INTERCEPTOR(int, wctomb, char *dest, wchar_t src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wctomb, dest, src);
+  if (!dest)
+    return REAL(wctomb)(dest, src);
+
+  char local_dest[32];
+  int res = REAL(wctomb)(local_dest, src);
+  if (res != -1) {
+    CHECK_LE(res, sizeof(local_dest));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, res);
+    REAL(memcpy)(dest, local_dest, res);
+  }
+  return res;
+}
+
+#define INIT_WCTOMB COMMON_INTERCEPT_FUNCTION(wctomb);
+#else
+#define INIT_WCTOMB
+#endif
+
+#if SANITIZER_INTERCEPT_TCGETATTR
+INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, tcgetattr, fd, termios_p);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(tcgetattr)(fd, termios_p);
+  if (!res && termios_p)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, termios_p, struct_termios_sz);
+  return res;
+}
+
+#define INIT_TCGETATTR COMMON_INTERCEPT_FUNCTION(tcgetattr);
+#else
+#define INIT_TCGETATTR
+#endif
+
+#if SANITIZER_INTERCEPT_REALPATH
+INTERCEPTOR(char *, realpath, const char *path, char *resolved_path) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, realpath, path, resolved_path);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+
+  // Workaround a bug in glibc where dlsym(RTLD_NEXT, ...) returns the oldest
+  // version of a versioned symbol. For realpath(), this gives us something
+  // (called __old_realpath) that does not handle NULL in the second argument.
+  // Handle it as part of the interceptor.
+  char *allocated_path = nullptr;
+  if (!resolved_path)
+    allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1);
+
+  char *res = REAL(realpath)(path, resolved_path);
+  if (allocated_path && !res) WRAP(free)(allocated_path);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
+#else
+#define INIT_REALPATH
+#endif
+
+#if SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME
+INTERCEPTOR(char *, canonicalize_file_name, const char *path) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, canonicalize_file_name, path);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  char *res = REAL(canonicalize_file_name)(path);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_CANONICALIZE_FILE_NAME \
+  COMMON_INTERCEPT_FUNCTION(canonicalize_file_name);
+#else
+#define INIT_CANONICALIZE_FILE_NAME
+#endif
+
+#if SANITIZER_INTERCEPT_CONFSTR
+INTERCEPTOR(SIZE_T, confstr, int name, char *buf, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, confstr, name, buf, len);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(confstr)(name, buf, len);
+  if (buf && res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res < len ? res : len);
+  return res;
+}
+#define INIT_CONFSTR COMMON_INTERCEPT_FUNCTION(confstr);
+#else
+#define INIT_CONFSTR
+#endif
+
+#if SANITIZER_INTERCEPT_SCHED_GETAFFINITY
+INTERCEPTOR(int, sched_getaffinity, int pid, SIZE_T cpusetsize, void *mask) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sched_getaffinity, pid, cpusetsize, mask);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sched_getaffinity)(pid, cpusetsize, mask);
+  if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize);
+  return res;
+}
+#define INIT_SCHED_GETAFFINITY COMMON_INTERCEPT_FUNCTION(sched_getaffinity);
+#else
+#define INIT_SCHED_GETAFFINITY
+#endif
+
+#if SANITIZER_INTERCEPT_SCHED_GETPARAM
+INTERCEPTOR(int, sched_getparam, int pid, void *param) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sched_getparam, pid, param);
+  int res = REAL(sched_getparam)(pid, param);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, param, struct_sched_param_sz);
+  return res;
+}
+#define INIT_SCHED_GETPARAM COMMON_INTERCEPT_FUNCTION(sched_getparam);
+#else
+#define INIT_SCHED_GETPARAM
+#endif
+
+#if SANITIZER_INTERCEPT_STRERROR
+INTERCEPTOR(char *, strerror, int errnum) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);
+  COMMON_INTERCEPTOR_STRERROR();
+  char *res = REAL(strerror)(errnum);
+  if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_STRERROR COMMON_INTERCEPT_FUNCTION(strerror);
+#else
+#define INIT_STRERROR
+#endif
+
+#if SANITIZER_INTERCEPT_STRERROR_R
+// There are 2 versions of strerror_r:
+//  * POSIX version returns 0 on success, negative error code on failure,
+//    writes message to buf.
+//  * GNU version returns message pointer, which points to either buf or some
+//    static storage.
+#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
+    SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD ||                 \
+    SANITIZER_FREEBSD || SANITIZER_OPENBSD
+// POSIX version. Spec is not clear on whether buf is NULL-terminated.
+// At least on OSX, buf contents are valid even when the call fails.
+INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(strerror_r)(errnum, buf, buflen);
+
+  SIZE_T sz = internal_strnlen(buf, buflen);
+  if (sz < buflen) ++sz;
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sz);
+  return res;
+}
+#else
+// GNU version.
+INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(strerror_r)(errnum, buf, buflen);
+  if (res == buf)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  else
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  return res;
+}
+#endif //(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE ||
+       //SANITIZER_MAC
+#define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);
+#else
+#define INIT_STRERROR_R
+#endif
+
+#if SANITIZER_INTERCEPT_XPG_STRERROR_R
+INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __xpg_strerror_r, errnum, buf, buflen);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(__xpg_strerror_r)(errnum, buf, buflen);
+  // This version always returns a null-terminated string.
+  if (buf && buflen)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+  return res;
+}
+#define INIT_XPG_STRERROR_R COMMON_INTERCEPT_FUNCTION(__xpg_strerror_r);
+#else
+#define INIT_XPG_STRERROR_R
+#endif
+
+#if SANITIZER_INTERCEPT_SCANDIR
+typedef int (*scandir_filter_f)(const struct __sanitizer_dirent *);
+typedef int (*scandir_compar_f)(const struct __sanitizer_dirent **,
+                                const struct __sanitizer_dirent **);
+
+static THREADLOCAL scandir_filter_f scandir_filter;
+static THREADLOCAL scandir_compar_f scandir_compar;
+
+static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
+  return scandir_filter(dir);
+}
+
+static int wrapped_scandir_compar(const struct __sanitizer_dirent **a,
+                                  const struct __sanitizer_dirent **b) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen);
+  return scandir_compar(a, b);
+}
+
+INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist,
+            scandir_filter_f filter, scandir_compar_f compar) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, scandir, dirp, namelist, filter, compar);
+  if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
+  scandir_filter = filter;
+  scandir_compar = compar;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(scandir)(dirp, namelist,
+                          filter ? wrapped_scandir_filter : nullptr,
+                          compar ? wrapped_scandir_compar : nullptr);
+  scandir_filter = nullptr;
+  scandir_compar = nullptr;
+  if (namelist && res > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(*namelist));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res);
+    for (int i = 0; i < res; ++i)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
+                                     (*namelist)[i]->d_reclen);
+  }
+  return res;
+}
+#define INIT_SCANDIR COMMON_INTERCEPT_FUNCTION(scandir);
+#else
+#define INIT_SCANDIR
+#endif
+
+#if SANITIZER_INTERCEPT_SCANDIR64
+typedef int (*scandir64_filter_f)(const struct __sanitizer_dirent64 *);
+typedef int (*scandir64_compar_f)(const struct __sanitizer_dirent64 **,
+                                  const struct __sanitizer_dirent64 **);
+
+static THREADLOCAL scandir64_filter_f scandir64_filter;
+static THREADLOCAL scandir64_compar_f scandir64_compar;
+
+static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
+  return scandir64_filter(dir);
+}
+
+static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a,
+                                    const struct __sanitizer_dirent64 **b) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen);
+  return scandir64_compar(a, b);
+}
+
+INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist,
+            scandir64_filter_f filter, scandir64_compar_f compar) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, scandir64, dirp, namelist, filter, compar);
+  if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
+  scandir64_filter = filter;
+  scandir64_compar = compar;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res =
+      REAL(scandir64)(dirp, namelist,
+                      filter ? wrapped_scandir64_filter : nullptr,
+                      compar ? wrapped_scandir64_compar : nullptr);
+  scandir64_filter = nullptr;
+  scandir64_compar = nullptr;
+  if (namelist && res > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(*namelist));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res);
+    for (int i = 0; i < res; ++i)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
+                                     (*namelist)[i]->d_reclen);
+  }
+  return res;
+}
+#define INIT_SCANDIR64 COMMON_INTERCEPT_FUNCTION(scandir64);
+#else
+#define INIT_SCANDIR64
+#endif
+
+#if SANITIZER_INTERCEPT_GETGROUPS
+INTERCEPTOR(int, getgroups, int size, u32 *lst) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgroups, size, lst);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getgroups)(size, lst);
+  if (res >= 0 && lst && size > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lst, res * sizeof(*lst));
+  return res;
+}
+#define INIT_GETGROUPS COMMON_INTERCEPT_FUNCTION(getgroups);
+#else
+#define INIT_GETGROUPS
+#endif
+
+#if SANITIZER_INTERCEPT_POLL
+static void read_pollfd(void *ctx, __sanitizer_pollfd *fds,
+                        __sanitizer_nfds_t nfds) {
+  for (unsigned i = 0; i < nfds; ++i) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &fds[i].fd, sizeof(fds[i].fd));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, &fds[i].events, sizeof(fds[i].events));
+  }
+}
+
+static void write_pollfd(void *ctx, __sanitizer_pollfd *fds,
+                         __sanitizer_nfds_t nfds) {
+  for (unsigned i = 0; i < nfds; ++i)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &fds[i].revents,
+                                   sizeof(fds[i].revents));
+}
+
+INTERCEPTOR(int, poll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
+            int timeout) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, poll, fds, nfds, timeout);
+  if (fds && nfds) read_pollfd(ctx, fds, nfds);
+  int res = COMMON_INTERCEPTOR_BLOCK_REAL(poll)(fds, nfds, timeout);
+  if (fds && nfds) write_pollfd(ctx, fds, nfds);
+  return res;
+}
+#define INIT_POLL COMMON_INTERCEPT_FUNCTION(poll);
+#else
+#define INIT_POLL
+#endif
+
+#if SANITIZER_INTERCEPT_PPOLL
+INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
+            void *timeout_ts, __sanitizer_sigset_t *sigmask) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ppoll, fds, nfds, timeout_ts, sigmask);
+  if (fds && nfds) read_pollfd(ctx, fds, nfds);
+  if (timeout_ts)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout_ts, struct_timespec_sz);
+  if (sigmask) COMMON_INTERCEPTOR_READ_RANGE(ctx, sigmask, sizeof(*sigmask));
+  int res =
+      COMMON_INTERCEPTOR_BLOCK_REAL(ppoll)(fds, nfds, timeout_ts, sigmask);
+  if (fds && nfds) write_pollfd(ctx, fds, nfds);
+  return res;
+}
+#define INIT_PPOLL COMMON_INTERCEPT_FUNCTION(ppoll);
+#else
+#define INIT_PPOLL
+#endif
+
+#if SANITIZER_INTERCEPT_WORDEXP
+INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wordexp, s, p, flags);
+  if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(wordexp)(s, p, flags);
+  if (!res && p) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+    if (p->we_wordc)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
+                                     sizeof(*p->we_wordv) * p->we_wordc);
+    for (uptr i = 0; i < p->we_wordc; ++i) {
+      char *w = p->we_wordv[i];
+      if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, REAL(strlen)(w) + 1);
+    }
+  }
+  return res;
+}
+#define INIT_WORDEXP COMMON_INTERCEPT_FUNCTION(wordexp);
+#else
+#define INIT_WORDEXP
+#endif
+
+#if SANITIZER_INTERCEPT_SIGWAIT
+INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigwait, set, sig);
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigwait)(set, sig);
+  if (!res && sig) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sig, sizeof(*sig));
+  return res;
+}
+#define INIT_SIGWAIT COMMON_INTERCEPT_FUNCTION(sigwait);
+#else
+#define INIT_SIGWAIT
+#endif
+
+#if SANITIZER_INTERCEPT_SIGWAITINFO
+INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigwaitinfo, set, info);
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigwaitinfo)(set, info);
+  if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz);
+  return res;
+}
+#define INIT_SIGWAITINFO COMMON_INTERCEPT_FUNCTION(sigwaitinfo);
+#else
+#define INIT_SIGWAITINFO
+#endif
+
+#if SANITIZER_INTERCEPT_SIGTIMEDWAIT
+INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info,
+            void *timeout) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigtimedwait, set, info, timeout);
+  if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigtimedwait)(set, info, timeout);
+  if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz);
+  return res;
+}
+#define INIT_SIGTIMEDWAIT COMMON_INTERCEPT_FUNCTION(sigtimedwait);
+#else
+#define INIT_SIGTIMEDWAIT
+#endif
+
+#if SANITIZER_INTERCEPT_SIGSETOPS
+INTERCEPTOR(int, sigemptyset, __sanitizer_sigset_t *set) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigemptyset, set);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigemptyset)(set);
+  if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set));
+  return res;
+}
+
+INTERCEPTOR(int, sigfillset, __sanitizer_sigset_t *set) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigfillset, set);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigfillset)(set);
+  if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set));
+  return res;
+}
+#define INIT_SIGSETOPS                    \
+  COMMON_INTERCEPT_FUNCTION(sigemptyset); \
+  COMMON_INTERCEPT_FUNCTION(sigfillset);
+#else
+#define INIT_SIGSETOPS
+#endif
+
+#if SANITIZER_INTERCEPT_SIGPENDING
+INTERCEPTOR(int, sigpending, __sanitizer_sigset_t *set) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigpending, set);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigpending)(set);
+  if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set));
+  return res;
+}
+#define INIT_SIGPENDING COMMON_INTERCEPT_FUNCTION(sigpending);
+#else
+#define INIT_SIGPENDING
+#endif
+
+#if SANITIZER_INTERCEPT_SIGPROCMASK
+INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set,
+            __sanitizer_sigset_t *oldset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset);
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(sigprocmask)(how, set, oldset);
+  if (!res && oldset)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset));
+  return res;
+}
+#define INIT_SIGPROCMASK COMMON_INTERCEPT_FUNCTION(sigprocmask);
+#else
+#define INIT_SIGPROCMASK
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_SIGMASK
+INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set,
+            __sanitizer_sigset_t *oldset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_sigmask, how, set, oldset);
+  if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(pthread_sigmask)(how, set, oldset);
+  if (!res && oldset)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset));
+  return res;
+}
+#define INIT_PTHREAD_SIGMASK COMMON_INTERCEPT_FUNCTION(pthread_sigmask);
+#else
+#define INIT_PTHREAD_SIGMASK
+#endif
+
+#if SANITIZER_INTERCEPT_BACKTRACE
+INTERCEPTOR(int, backtrace, void **buffer, int size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(backtrace)(buffer, size);
+  if (res && buffer)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer));
+  return res;
+}
+
+INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size);
+  if (buffer && size)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char **res = REAL(backtrace_symbols)(buffer, size);
+  if (res && size) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));
+    for (int i = 0; i < size; ++i)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], REAL(strlen(res[i])) + 1);
+  }
+  return res;
+}
+#define INIT_BACKTRACE                  \
+  COMMON_INTERCEPT_FUNCTION(backtrace); \
+  COMMON_INTERCEPT_FUNCTION(backtrace_symbols);
+#else
+#define INIT_BACKTRACE
+#endif
+
+#if SANITIZER_INTERCEPT__EXIT
+INTERCEPTOR(void, _exit, int status) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, _exit, status);
+  COMMON_INTERCEPTOR_USER_CALLBACK_START();
+  int status1 = COMMON_INTERCEPTOR_ON_EXIT(ctx);
+  COMMON_INTERCEPTOR_USER_CALLBACK_END();
+  if (status == 0) status = status1;
+  REAL(_exit)(status);
+}
+#define INIT__EXIT COMMON_INTERCEPT_FUNCTION(_exit);
+#else
+#define INIT__EXIT
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEX
+INTERCEPTOR(int, pthread_mutex_lock, void *m) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_lock, m);
+  COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
+  int res = REAL(pthread_mutex_lock)(m);
+  if (res == errno_EOWNERDEAD)
+    COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
+  if (res == 0 || res == errno_EOWNERDEAD)
+    COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
+  if (res == errno_EINVAL)
+    COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
+  return res;
+}
+
+INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_unlock, m);
+  COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
+  int res = REAL(pthread_mutex_unlock)(m);
+  if (res == errno_EINVAL)
+    COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
+  return res;
+}
+
+#define INIT_PTHREAD_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(pthread_mutex_lock)
+#define INIT_PTHREAD_MUTEX_UNLOCK \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutex_unlock)
+#else
+#define INIT_PTHREAD_MUTEX_LOCK
+#define INIT_PTHREAD_MUTEX_UNLOCK
+#endif
+
+#if SANITIZER_INTERCEPT___PTHREAD_MUTEX
+INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_lock, m);
+  COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
+  int res = REAL(__pthread_mutex_lock)(m);
+  if (res == errno_EOWNERDEAD)
+    COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
+  if (res == 0 || res == errno_EOWNERDEAD)
+    COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
+  if (res == errno_EINVAL)
+    COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
+  return res;
+}
+
+INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_unlock, m);
+  COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
+  int res = REAL(__pthread_mutex_unlock)(m);
+  if (res == errno_EINVAL)
+    COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
+  return res;
+}
+
+#define INIT___PTHREAD_MUTEX_LOCK \
+  COMMON_INTERCEPT_FUNCTION(__pthread_mutex_lock)
+#define INIT___PTHREAD_MUTEX_UNLOCK \
+  COMMON_INTERCEPT_FUNCTION(__pthread_mutex_unlock)
+#else
+#define INIT___PTHREAD_MUTEX_LOCK
+#define INIT___PTHREAD_MUTEX_UNLOCK
+#endif
+
+#if SANITIZER_INTERCEPT___LIBC_MUTEX
+INTERCEPTOR(int, __libc_mutex_lock, void *m)
+ALIAS(WRAPPER_NAME(pthread_mutex_lock));
+
+INTERCEPTOR(int, __libc_mutex_unlock, void *m)
+ALIAS(WRAPPER_NAME(pthread_mutex_unlock));
+
+INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)
+ALIAS(WRAPPER_NAME(pthread_setcancelstate));
+
+#define INIT___LIBC_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock)
+#define INIT___LIBC_MUTEX_UNLOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock)
+#define INIT___LIBC_THR_SETCANCELSTATE \
+  COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)
+#else
+#define INIT___LIBC_MUTEX_LOCK
+#define INIT___LIBC_MUTEX_UNLOCK
+#define INIT___LIBC_THR_SETCANCELSTATE
+#endif
+
+#if SANITIZER_INTERCEPT_GETMNTENT || SANITIZER_INTERCEPT_GETMNTENT_R
+static void write_mntent(void *ctx, __sanitizer_mntent *mnt) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));
+  if (mnt->mnt_fsname)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_fsname,
+                                   REAL(strlen)(mnt->mnt_fsname) + 1);
+  if (mnt->mnt_dir)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_dir,
+                                   REAL(strlen)(mnt->mnt_dir) + 1);
+  if (mnt->mnt_type)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_type,
+                                   REAL(strlen)(mnt->mnt_type) + 1);
+  if (mnt->mnt_opts)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_opts,
+                                   REAL(strlen)(mnt->mnt_opts) + 1);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_GETMNTENT
+INTERCEPTOR(__sanitizer_mntent *, getmntent, void *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getmntent, fp);
+  __sanitizer_mntent *res = REAL(getmntent)(fp);
+  if (res) write_mntent(ctx, res);
+  return res;
+}
+#define INIT_GETMNTENT COMMON_INTERCEPT_FUNCTION(getmntent);
+#else
+#define INIT_GETMNTENT
+#endif
+
+#if SANITIZER_INTERCEPT_GETMNTENT_R
+INTERCEPTOR(__sanitizer_mntent *, getmntent_r, void *fp,
+            __sanitizer_mntent *mntbuf, char *buf, int buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getmntent_r, fp, mntbuf, buf, buflen);
+  __sanitizer_mntent *res = REAL(getmntent_r)(fp, mntbuf, buf, buflen);
+  if (res) write_mntent(ctx, res);
+  return res;
+}
+#define INIT_GETMNTENT_R COMMON_INTERCEPT_FUNCTION(getmntent_r);
+#else
+#define INIT_GETMNTENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_STATFS
+INTERCEPTOR(int, statfs, char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statfs, path, buf);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(statfs)(path, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz);
+  return res;
+}
+INTERCEPTOR(int, fstatfs, int fd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatfs, fd, buf);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fstatfs)(fd, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz);
+  return res;
+}
+#define INIT_STATFS                  \
+  COMMON_INTERCEPT_FUNCTION(statfs); \
+  COMMON_INTERCEPT_FUNCTION(fstatfs);
+#else
+#define INIT_STATFS
+#endif
+
+#if SANITIZER_INTERCEPT_STATFS64
+INTERCEPTOR(int, statfs64, char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statfs64, path, buf);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(statfs64)(path, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz);
+  return res;
+}
+INTERCEPTOR(int, fstatfs64, int fd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatfs64, fd, buf);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fstatfs64)(fd, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz);
+  return res;
+}
+#define INIT_STATFS64                  \
+  COMMON_INTERCEPT_FUNCTION(statfs64); \
+  COMMON_INTERCEPT_FUNCTION(fstatfs64);
+#else
+#define INIT_STATFS64
+#endif
+
+#if SANITIZER_INTERCEPT_STATVFS
+INTERCEPTOR(int, statvfs, char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(statvfs)(path, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
+  return res;
+}
+INTERCEPTOR(int, fstatvfs, int fd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fstatvfs)(fd, buf);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
+    if (fd >= 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return res;
+}
+#define INIT_STATVFS                  \
+  COMMON_INTERCEPT_FUNCTION(statvfs); \
+  COMMON_INTERCEPT_FUNCTION(fstatvfs);
+#else
+#define INIT_STATVFS
+#endif
+
+#if SANITIZER_INTERCEPT_STATVFS64
+INTERCEPTOR(int, statvfs64, char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statvfs64, path, buf);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(statvfs64)(path, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz);
+  return res;
+}
+INTERCEPTOR(int, fstatvfs64, int fd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs64, fd, buf);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fstatvfs64)(fd, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz);
+  return res;
+}
+#define INIT_STATVFS64                  \
+  COMMON_INTERCEPT_FUNCTION(statvfs64); \
+  COMMON_INTERCEPT_FUNCTION(fstatvfs64);
+#else
+#define INIT_STATVFS64
+#endif
+
+#if SANITIZER_INTERCEPT_INITGROUPS
+INTERCEPTOR(int, initgroups, char *user, u32 group) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, initgroups, user, group);
+  if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, REAL(strlen)(user) + 1);
+  int res = REAL(initgroups)(user, group);
+  return res;
+}
+#define INIT_INITGROUPS COMMON_INTERCEPT_FUNCTION(initgroups);
+#else
+#define INIT_INITGROUPS
+#endif
+
+#if SANITIZER_INTERCEPT_ETHER_NTOA_ATON
+INTERCEPTOR(char *, ether_ntoa, __sanitizer_ether_addr *addr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa, addr);
+  if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
+  char *res = REAL(ether_ntoa)(addr);
+  if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  return res;
+}
+INTERCEPTOR(__sanitizer_ether_addr *, ether_aton, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_aton, buf);
+  if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+  __sanitizer_ether_addr *res = REAL(ether_aton)(buf);
+  if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, sizeof(*res));
+  return res;
+}
+#define INIT_ETHER_NTOA_ATON             \
+  COMMON_INTERCEPT_FUNCTION(ether_ntoa); \
+  COMMON_INTERCEPT_FUNCTION(ether_aton);
+#else
+#define INIT_ETHER_NTOA_ATON
+#endif
+
+#if SANITIZER_INTERCEPT_ETHER_HOST
+INTERCEPTOR(int, ether_ntohost, char *hostname, __sanitizer_ether_addr *addr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_ntohost, hostname, addr);
+  if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(ether_ntohost)(hostname, addr);
+  if (!res && hostname)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+  return res;
+}
+INTERCEPTOR(int, ether_hostton, char *hostname, __sanitizer_ether_addr *addr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_hostton, hostname, addr);
+  if (hostname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(ether_hostton)(hostname, addr);
+  if (!res && addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
+  return res;
+}
+INTERCEPTOR(int, ether_line, char *line, __sanitizer_ether_addr *addr,
+            char *hostname) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_line, line, addr, hostname);
+  if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(ether_line)(line, addr, hostname);
+  if (!res) {
+    if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
+    if (hostname)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
+  }
+  return res;
+}
+#define INIT_ETHER_HOST                     \
+  COMMON_INTERCEPT_FUNCTION(ether_ntohost); \
+  COMMON_INTERCEPT_FUNCTION(ether_hostton); \
+  COMMON_INTERCEPT_FUNCTION(ether_line);
+#else
+#define INIT_ETHER_HOST
+#endif
+
+#if SANITIZER_INTERCEPT_ETHER_R
+INTERCEPTOR(char *, ether_ntoa_r, __sanitizer_ether_addr *addr, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa_r, addr, buf);
+  if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(ether_ntoa_r)(addr, buf);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+INTERCEPTOR(__sanitizer_ether_addr *, ether_aton_r, char *buf,
+            __sanitizer_ether_addr *addr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ether_aton_r, buf, addr);
+  if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_ether_addr *res = REAL(ether_aton_r)(buf, addr);
+  if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(*res));
+  return res;
+}
+#define INIT_ETHER_R                       \
+  COMMON_INTERCEPT_FUNCTION(ether_ntoa_r); \
+  COMMON_INTERCEPT_FUNCTION(ether_aton_r);
+#else
+#define INIT_ETHER_R
+#endif
+
+#if SANITIZER_INTERCEPT_SHMCTL
+INTERCEPTOR(int, shmctl, int shmid, int cmd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, shmctl, shmid, cmd, buf);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(shmctl)(shmid, cmd, buf);
+  if (res >= 0) {
+    unsigned sz = 0;
+    if (cmd == shmctl_ipc_stat || cmd == shmctl_shm_stat)
+      sz = sizeof(__sanitizer_shmid_ds);
+    else if (cmd == shmctl_ipc_info)
+      sz = struct_shminfo_sz;
+    else if (cmd == shmctl_shm_info)
+      sz = struct_shm_info_sz;
+    if (sz) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sz);
+  }
+  return res;
+}
+#define INIT_SHMCTL COMMON_INTERCEPT_FUNCTION(shmctl);
+#else
+#define INIT_SHMCTL
+#endif
+
+#if SANITIZER_INTERCEPT_RANDOM_R
+INTERCEPTOR(int, random_r, void *buf, u32 *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, random_r, buf, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(random_r)(buf, result);
+  if (!res && result)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+#define INIT_RANDOM_R COMMON_INTERCEPT_FUNCTION(random_r);
+#else
+#define INIT_RANDOM_R
+#endif
+
+// FIXME: under ASan the REAL() call below may write to freed memory and corrupt
+// its metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#if SANITIZER_INTERCEPT_PTHREAD_ATTR_GET ||              \
+    SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED ||        \
+    SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSSCHED || \
+    SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GET ||         \
+    SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GET ||        \
+    SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GET ||          \
+    SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GET
+#define INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(fn, sz)            \
+  INTERCEPTOR(int, fn, void *attr, void *r) {                  \
+    void *ctx;                                                 \
+    COMMON_INTERCEPTOR_ENTER(ctx, fn, attr, r);                \
+    int res = REAL(fn)(attr, r);                               \
+    if (!res && r) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, r, sz); \
+    return res;                                                \
+  }
+#define INTERCEPTOR_PTHREAD_ATTR_GET(what, sz) \
+  INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(pthread_attr_get##what, sz)
+#define INTERCEPTOR_PTHREAD_MUTEXATTR_GET(what, sz) \
+  INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(pthread_mutexattr_get##what, sz)
+#define INTERCEPTOR_PTHREAD_RWLOCKATTR_GET(what, sz) \
+  INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(pthread_rwlockattr_get##what, sz)
+#define INTERCEPTOR_PTHREAD_CONDATTR_GET(what, sz) \
+  INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(pthread_condattr_get##what, sz)
+#define INTERCEPTOR_PTHREAD_BARRIERATTR_GET(what, sz) \
+  INTERCEPTOR_PTHREAD_OBJECT_ATTR_GET(pthread_barrierattr_get##what, sz)
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_ATTR_GET
+INTERCEPTOR_PTHREAD_ATTR_GET(detachstate, sizeof(int))
+INTERCEPTOR_PTHREAD_ATTR_GET(guardsize, sizeof(SIZE_T))
+INTERCEPTOR_PTHREAD_ATTR_GET(scope, sizeof(int))
+INTERCEPTOR_PTHREAD_ATTR_GET(stacksize, sizeof(SIZE_T))
+INTERCEPTOR(int, pthread_attr_getstack, void *attr, void **addr, SIZE_T *size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_attr_getstack, attr, addr, size);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(pthread_attr_getstack)(attr, addr, size);
+  if (!res) {
+    if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
+    if (size) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, size, sizeof(*size));
+  }
+  return res;
+}
+
+// We may need to call the real pthread_attr_getstack from the run-time
+// in sanitizer_common, but we don't want to include the interception headers
+// there. So, just define this function here.
+namespace __sanitizer {
+extern "C" {
+int real_pthread_attr_getstack(void *attr, void **addr, SIZE_T *size) {
+  return REAL(pthread_attr_getstack)(attr, addr, size);
+}
+}  // extern "C"
+}  // namespace __sanitizer
+
+#define INIT_PTHREAD_ATTR_GET                             \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getdetachstate); \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getguardsize);   \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getscope);       \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getstacksize);   \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getstack);
+#else
+#define INIT_PTHREAD_ATTR_GET
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED
+INTERCEPTOR_PTHREAD_ATTR_GET(schedparam, struct_sched_param_sz)
+INTERCEPTOR_PTHREAD_ATTR_GET(schedpolicy, sizeof(int))
+
+#define INIT_PTHREAD_ATTR_GET_SCHED                      \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getschedparam); \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getschedpolicy);
+#else
+#define INIT_PTHREAD_ATTR_GET_SCHED
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED
+INTERCEPTOR_PTHREAD_ATTR_GET(inheritsched, sizeof(int))
+
+#define INIT_PTHREAD_ATTR_GETINHERITSCHED \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getinheritsched);
+#else
+#define INIT_PTHREAD_ATTR_GETINHERITSCHED
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP
+INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize,
+            void *cpuset) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_attr_getaffinity_np, attr, cpusetsize,
+                           cpuset);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(pthread_attr_getaffinity_np)(attr, cpusetsize, cpuset);
+  if (!res && cpusetsize && cpuset)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize);
+  return res;
+}
+
+#define INIT_PTHREAD_ATTR_GETAFFINITY_NP \
+  COMMON_INTERCEPT_FUNCTION(pthread_attr_getaffinity_np);
+#else
+#define INIT_PTHREAD_ATTR_GETAFFINITY_NP
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETPSHARED \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_getpshared);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETPSHARED
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(type, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETTYPE \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_gettype);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETTYPE
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(protocol, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETPROTOCOL \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_getprotocol);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETPROTOCOL
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(prioceiling, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETPRIOCEILING \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_getprioceiling);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETPRIOCEILING
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(robust, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETROBUST \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_getrobust);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETROBUST
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP
+INTERCEPTOR_PTHREAD_MUTEXATTR_GET(robust_np, sizeof(int))
+#define INIT_PTHREAD_MUTEXATTR_GETROBUST_NP \
+  COMMON_INTERCEPT_FUNCTION(pthread_mutexattr_getrobust_np);
+#else
+#define INIT_PTHREAD_MUTEXATTR_GETROBUST_NP
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED
+INTERCEPTOR_PTHREAD_RWLOCKATTR_GET(pshared, sizeof(int))
+#define INIT_PTHREAD_RWLOCKATTR_GETPSHARED \
+  COMMON_INTERCEPT_FUNCTION(pthread_rwlockattr_getpshared);
+#else
+#define INIT_PTHREAD_RWLOCKATTR_GETPSHARED
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP
+INTERCEPTOR_PTHREAD_RWLOCKATTR_GET(kind_np, sizeof(int))
+#define INIT_PTHREAD_RWLOCKATTR_GETKIND_NP \
+  COMMON_INTERCEPT_FUNCTION(pthread_rwlockattr_getkind_np);
+#else
+#define INIT_PTHREAD_RWLOCKATTR_GETKIND_NP
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED
+INTERCEPTOR_PTHREAD_CONDATTR_GET(pshared, sizeof(int))
+#define INIT_PTHREAD_CONDATTR_GETPSHARED \
+  COMMON_INTERCEPT_FUNCTION(pthread_condattr_getpshared);
+#else
+#define INIT_PTHREAD_CONDATTR_GETPSHARED
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK
+INTERCEPTOR_PTHREAD_CONDATTR_GET(clock, sizeof(int))
+#define INIT_PTHREAD_CONDATTR_GETCLOCK \
+  COMMON_INTERCEPT_FUNCTION(pthread_condattr_getclock);
+#else
+#define INIT_PTHREAD_CONDATTR_GETCLOCK
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED
+INTERCEPTOR_PTHREAD_BARRIERATTR_GET(pshared, sizeof(int)) // !mac !android
+#define INIT_PTHREAD_BARRIERATTR_GETPSHARED \
+  COMMON_INTERCEPT_FUNCTION(pthread_barrierattr_getpshared);
+#else
+#define INIT_PTHREAD_BARRIERATTR_GETPSHARED
+#endif
+
+#if SANITIZER_INTERCEPT_TMPNAM
+INTERCEPTOR(char *, tmpnam, char *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, tmpnam, s);
+  char *res = REAL(tmpnam)(s);
+  if (res) {
+    if (s)
+      // FIXME: under ASan the call below may write to freed memory and corrupt
+      // its metadata. See
+      // https://github.com/google/sanitizers/issues/321.
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+    else
+      COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+#define INIT_TMPNAM COMMON_INTERCEPT_FUNCTION(tmpnam);
+#else
+#define INIT_TMPNAM
+#endif
+
+#if SANITIZER_INTERCEPT_TMPNAM_R
+INTERCEPTOR(char *, tmpnam_r, char *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, tmpnam_r, s);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(tmpnam_r)(s);
+  if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
+  return res;
+}
+#define INIT_TMPNAM_R COMMON_INTERCEPT_FUNCTION(tmpnam_r);
+#else
+#define INIT_TMPNAM_R
+#endif
+
+#if SANITIZER_INTERCEPT_TTYNAME
+INTERCEPTOR(char *, ttyname, int fd) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd);
+  char *res = REAL(ttyname)(fd);
+  if (res != nullptr)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname);
+#else
+#define INIT_TTYNAME
+#endif
+
+#if SANITIZER_INTERCEPT_TTYNAME_R
+INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize);
+  int res = REAL(ttyname_r)(fd, name, namesize);
+  if (res == 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  return res;
+}
+#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r);
+#else
+#define INIT_TTYNAME_R
+#endif
+
+#if SANITIZER_INTERCEPT_TEMPNAM
+INTERCEPTOR(char *, tempnam, char *dir, char *pfx) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, tempnam, dir, pfx);
+  if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, REAL(strlen)(dir) + 1);
+  if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, REAL(strlen)(pfx) + 1);
+  char *res = REAL(tempnam)(dir, pfx);
+  if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  return res;
+}
+#define INIT_TEMPNAM COMMON_INTERCEPT_FUNCTION(tempnam);
+#else
+#define INIT_TEMPNAM
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && !SANITIZER_NETBSD
+INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0);
+  COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name);
+  return REAL(pthread_setname_np)(thread, name);
+}
+#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np);
+#elif SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && SANITIZER_NETBSD
+INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name, void *arg) {
+  void *ctx;
+  char newname[32]; // PTHREAD_MAX_NAMELEN_NP=32
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name, arg);
+  COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0);
+  internal_snprintf(newname, sizeof(newname), name, arg);
+  COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, newname);
+  return REAL(pthread_setname_np)(thread, name, arg);
+}
+#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np);
+#else
+#define INIT_PTHREAD_SETNAME_NP
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP
+INTERCEPTOR(int, pthread_getname_np, uptr thread, char *name, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_getname_np, thread, name, len);
+  int res = REAL(pthread_getname_np)(thread, name, len);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strnlen(name, len) + 1);
+  return res;
+}
+#define INIT_PTHREAD_GETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_getname_np);
+#else
+#define INIT_PTHREAD_GETNAME_NP
+#endif
+
+#if SANITIZER_INTERCEPT_SINCOS
+INTERCEPTOR(void, sincos, double x, double *sin, double *cos) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sincos, x, sin, cos);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  REAL(sincos)(x, sin, cos);
+  if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin));
+  if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos));
+}
+INTERCEPTOR(void, sincosf, float x, float *sin, float *cos) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sincosf, x, sin, cos);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  REAL(sincosf)(x, sin, cos);
+  if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin));
+  if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos));
+}
+INTERCEPTOR(void, sincosl, long double x, long double *sin, long double *cos) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sincosl, x, sin, cos);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  REAL(sincosl)(x, sin, cos);
+  if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin));
+  if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos));
+}
+#define INIT_SINCOS                   \
+  COMMON_INTERCEPT_FUNCTION(sincos);  \
+  COMMON_INTERCEPT_FUNCTION(sincosf); \
+  COMMON_INTERCEPT_FUNCTION_LDBL(sincosl);
+#else
+#define INIT_SINCOS
+#endif
+
+#if SANITIZER_INTERCEPT_REMQUO
+INTERCEPTOR(double, remquo, double x, double y, int *quo) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, remquo, x, y, quo);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  double res = REAL(remquo)(x, y, quo);
+  if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo));
+  return res;
+}
+INTERCEPTOR(float, remquof, float x, float y, int *quo) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, remquof, x, y, quo);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  float res = REAL(remquof)(x, y, quo);
+  if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo));
+  return res;
+}
+#define INIT_REMQUO                   \
+  COMMON_INTERCEPT_FUNCTION(remquo);  \
+  COMMON_INTERCEPT_FUNCTION(remquof);
+#else
+#define INIT_REMQUO
+#endif
+
+#if SANITIZER_INTERCEPT_REMQUOL
+INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, remquol, x, y, quo);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  long double res = REAL(remquol)(x, y, quo);
+  if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo));
+  return res;
+}
+#define INIT_REMQUOL                  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(remquol);
+#else
+#define INIT_REMQUOL
+#endif
+
+#if SANITIZER_INTERCEPT_LGAMMA
+extern int signgam;
+INTERCEPTOR(double, lgamma, double x) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgamma, x);
+  double res = REAL(lgamma)(x);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam));
+  return res;
+}
+INTERCEPTOR(float, lgammaf, float x) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgammaf, x);
+  float res = REAL(lgammaf)(x);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam));
+  return res;
+}
+#define INIT_LGAMMA                   \
+  COMMON_INTERCEPT_FUNCTION(lgamma);  \
+  COMMON_INTERCEPT_FUNCTION(lgammaf);
+#else
+#define INIT_LGAMMA
+#endif
+
+#if SANITIZER_INTERCEPT_LGAMMAL
+INTERCEPTOR(long double, lgammal, long double x) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgammal, x);
+  long double res = REAL(lgammal)(x);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, &signgam, sizeof(signgam));
+  return res;
+}
+#define INIT_LGAMMAL                  \
+  COMMON_INTERCEPT_FUNCTION_LDBL(lgammal);
+#else
+#define INIT_LGAMMAL
+#endif
+
+#if SANITIZER_INTERCEPT_LGAMMA_R
+INTERCEPTOR(double, lgamma_r, double x, int *signp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgamma_r, x, signp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  double res = REAL(lgamma_r)(x, signp);
+  if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp));
+  return res;
+}
+INTERCEPTOR(float, lgammaf_r, float x, int *signp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgammaf_r, x, signp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  float res = REAL(lgammaf_r)(x, signp);
+  if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp));
+  return res;
+}
+#define INIT_LGAMMA_R                   \
+  COMMON_INTERCEPT_FUNCTION(lgamma_r);  \
+  COMMON_INTERCEPT_FUNCTION(lgammaf_r);
+#else
+#define INIT_LGAMMA_R
+#endif
+
+#if SANITIZER_INTERCEPT_LGAMMAL_R
+INTERCEPTOR(long double, lgammal_r, long double x, int *signp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgammal_r, x, signp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  long double res = REAL(lgammal_r)(x, signp);
+  if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp));
+  return res;
+}
+#define INIT_LGAMMAL_R COMMON_INTERCEPT_FUNCTION_LDBL(lgammal_r);
+#else
+#define INIT_LGAMMAL_R
+#endif
+
+#if SANITIZER_INTERCEPT_DRAND48_R
+INTERCEPTOR(int, drand48_r, void *buffer, double *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, drand48_r, buffer, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(drand48_r)(buffer, result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+INTERCEPTOR(int, lrand48_r, void *buffer, long *result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lrand48_r, buffer, result);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(lrand48_r)(buffer, result);
+  if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
+  return res;
+}
+#define INIT_DRAND48_R                  \
+  COMMON_INTERCEPT_FUNCTION(drand48_r); \
+  COMMON_INTERCEPT_FUNCTION(lrand48_r);
+#else
+#define INIT_DRAND48_R
+#endif
+
+#if SANITIZER_INTERCEPT_RAND_R
+INTERCEPTOR(int, rand_r, unsigned *seedp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, rand_r, seedp);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, seedp, sizeof(*seedp));
+  return REAL(rand_r)(seedp);
+}
+#define INIT_RAND_R COMMON_INTERCEPT_FUNCTION(rand_r);
+#else
+#define INIT_RAND_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETLINE
+INTERCEPTOR(SSIZE_T, getline, char **lineptr, SIZE_T *n, void *stream) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getline, lineptr, n, stream);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(getline)(lineptr, n, stream);
+  if (res > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineptr, sizeof(*lineptr));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *lineptr, res + 1);
+  }
+  return res;
+}
+
+// FIXME: under ASan the call below may write to freed memory and corrupt its
+// metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#define GETDELIM_INTERCEPTOR_IMPL(vname)                                       \
+  {                                                                            \
+    void *ctx;                                                                 \
+    COMMON_INTERCEPTOR_ENTER(ctx, vname, lineptr, n, delim, stream);           \
+    SSIZE_T res = REAL(vname)(lineptr, n, delim, stream);                      \
+    if (res > 0) {                                                             \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineptr, sizeof(*lineptr));          \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));                      \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *lineptr, res + 1);                  \
+    }                                                                          \
+    return res;                                                                \
+  }
+
+INTERCEPTOR(SSIZE_T, __getdelim, char **lineptr, SIZE_T *n, int delim,
+            void *stream)
+GETDELIM_INTERCEPTOR_IMPL(__getdelim)
+
+// There's no __getdelim() on FreeBSD so we supply the getdelim() interceptor
+// with its own body.
+INTERCEPTOR(SSIZE_T, getdelim, char **lineptr, SIZE_T *n, int delim,
+            void *stream)
+GETDELIM_INTERCEPTOR_IMPL(getdelim)
+
+#define INIT_GETLINE                     \
+  COMMON_INTERCEPT_FUNCTION(getline);    \
+  COMMON_INTERCEPT_FUNCTION(__getdelim); \
+  COMMON_INTERCEPT_FUNCTION(getdelim);
+#else
+#define INIT_GETLINE
+#endif
+
+#if SANITIZER_INTERCEPT_ICONV
+INTERCEPTOR(SIZE_T, iconv, void *cd, char **inbuf, SIZE_T *inbytesleft,
+            char **outbuf, SIZE_T *outbytesleft) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, iconv, cd, inbuf, inbytesleft, outbuf,
+                           outbytesleft);
+  if (inbytesleft)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, inbytesleft, sizeof(*inbytesleft));
+  if (inbuf && inbytesleft)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, *inbuf, *inbytesleft);
+  if (outbytesleft)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, outbytesleft, sizeof(*outbytesleft));
+  void *outbuf_orig = outbuf ? *outbuf : nullptr;
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SIZE_T res = REAL(iconv)(cd, inbuf, inbytesleft, outbuf, outbytesleft);
+  if (outbuf && *outbuf > outbuf_orig) {
+    SIZE_T sz = (char *)*outbuf - (char *)outbuf_orig;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, outbuf_orig, sz);
+  }
+  return res;
+}
+#define INIT_ICONV COMMON_INTERCEPT_FUNCTION(iconv);
+#else
+#define INIT_ICONV
+#endif
+
+#if SANITIZER_INTERCEPT_TIMES
+INTERCEPTOR(__sanitizer_clock_t, times, void *tms) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, times, tms);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_clock_t res = REAL(times)(tms);
+  if (res != (__sanitizer_clock_t)-1 && tms)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tms, struct_tms_sz);
+  return res;
+}
+#define INIT_TIMES COMMON_INTERCEPT_FUNCTION(times);
+#else
+#define INIT_TIMES
+#endif
+
+#if SANITIZER_INTERCEPT_TLS_GET_ADDR
+#if !SANITIZER_S390
+#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr)
+// If you see any crashes around this functions, there are 2 known issues with
+// it: 1. __tls_get_addr can be called with mis-aligned stack due to:
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
+// 2. It can be called recursively if sanitizer code uses __tls_get_addr
+// to access thread local variables (it should not happen normally,
+// because sanitizers use initial-exec tls model).
+INTERCEPTOR(void *, __tls_get_addr, void *arg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr, arg);
+  void *res = REAL(__tls_get_addr)(arg);
+  uptr tls_begin, tls_end;
+  COMMON_INTERCEPTOR_GET_TLS_RANGE(&tls_begin, &tls_end);
+  DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, tls_begin, tls_end);
+  if (dtv) {
+    // New DTLS block has been allocated.
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE((void *)dtv->beg, dtv->size);
+  }
+  return res;
+}
+#if SANITIZER_PPC
+// On PowerPC, we also need to intercept __tls_get_addr_opt, which has
+// mostly the same semantics as __tls_get_addr, but its presence enables
+// some optimizations in linker (which are safe to ignore here).
+extern "C" __attribute__((alias("__interceptor___tls_get_addr"),
+                          visibility("default")))
+void *__tls_get_addr_opt(void *arg);
+#endif
+#else // SANITIZER_S390
+// On s390, we have to intercept two functions here:
+// - __tls_get_addr_internal, which is a glibc-internal function that is like
+//   the usual __tls_get_addr, but returns a TP-relative offset instead of
+//   a proper pointer.  It is used by dlsym for TLS symbols.
+// - __tls_get_offset, which is like the above, but also takes a GOT-relative
+//   descriptor offset as an argument instead of a pointer.  GOT address
+//   is passed in r12, so it's necessary to write it in assembly.  This is
+//   the function used by the compiler.
+extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg));
+#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_offset)
+DEFINE_REAL(uptr, __tls_get_offset, void *arg)
+extern "C" uptr __tls_get_offset(void *arg);
+extern "C" uptr __interceptor___tls_get_offset(void *arg);
+INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr_internal, arg);
+  uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset));
+  uptr tp = reinterpret_cast<uptr>(__builtin_thread_pointer());
+  void *ptr = reinterpret_cast<void *>(res + tp);
+  uptr tls_begin, tls_end;
+  COMMON_INTERCEPTOR_GET_TLS_RANGE(&tls_begin, &tls_end);
+  DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, ptr, tls_begin, tls_end);
+  if (dtv) {
+    // New DTLS block has been allocated.
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE((void *)dtv->beg, dtv->size);
+  }
+  return res;
+}
+// We need a hidden symbol aliasing the above, so that we can jump
+// directly to it from the assembly below.
+extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),
+                          visibility("hidden")))
+uptr __tls_get_addr_hidden(void *arg);
+// Now carefully intercept __tls_get_offset.
+asm(
+  ".text\n"
+// The __intercept_ version has to exist, so that gen_dynamic_list.py
+// exports our symbol.
+  ".weak __tls_get_offset\n"
+  ".type __tls_get_offset, @function\n"
+  "__tls_get_offset:\n"
+  ".global __interceptor___tls_get_offset\n"
+  ".type __interceptor___tls_get_offset, @function\n"
+  "__interceptor___tls_get_offset:\n"
+#ifdef __s390x__
+  "la %r2, 0(%r2,%r12)\n"
+  "jg __tls_get_addr_hidden\n"
+#else
+  "basr %r3,0\n"
+  "0: la %r2,0(%r2,%r12)\n"
+  "l %r4,1f-0b(%r3)\n"
+  "b 0(%r4,%r3)\n"
+  "1: .long __tls_get_addr_hidden - 0b\n"
+#endif
+  ".size __interceptor___tls_get_offset, .-__interceptor___tls_get_offset\n"
+// Assembly wrapper to call REAL(__tls_get_offset)(arg)
+  ".type __tls_get_offset_wrapper, @function\n"
+  "__tls_get_offset_wrapper:\n"
+#ifdef __s390x__
+  "sgr %r2,%r12\n"
+#else
+  "sr %r2,%r12\n"
+#endif
+  "br %r3\n"
+  ".size __tls_get_offset_wrapper, .-__tls_get_offset_wrapper\n"
+);
+#endif // SANITIZER_S390
+#else
+#define INIT_TLS_GET_ADDR
+#endif
+
+#if SANITIZER_INTERCEPT_LISTXATTR
+INTERCEPTOR(SSIZE_T, listxattr, const char *path, char *list, SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, listxattr, path, list, size);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(listxattr)(path, list, size);
+  // Here and below, size == 0 is a special case where nothing is written to the
+  // buffer, and res contains the desired buffer size.
+  if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res);
+  return res;
+}
+INTERCEPTOR(SSIZE_T, llistxattr, const char *path, char *list, SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, llistxattr, path, list, size);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(llistxattr)(path, list, size);
+  if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res);
+  return res;
+}
+INTERCEPTOR(SSIZE_T, flistxattr, int fd, char *list, SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, flistxattr, fd, list, size);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(flistxattr)(fd, list, size);
+  if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res);
+  return res;
+}
+#define INIT_LISTXATTR                   \
+  COMMON_INTERCEPT_FUNCTION(listxattr);  \
+  COMMON_INTERCEPT_FUNCTION(llistxattr); \
+  COMMON_INTERCEPT_FUNCTION(flistxattr);
+#else
+#define INIT_LISTXATTR
+#endif
+
+#if SANITIZER_INTERCEPT_GETXATTR
+INTERCEPTOR(SSIZE_T, getxattr, const char *path, const char *name, char *value,
+            SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getxattr, path, name, value, size);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(getxattr)(path, name, value, size);
+  if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res);
+  return res;
+}
+INTERCEPTOR(SSIZE_T, lgetxattr, const char *path, const char *name, char *value,
+            SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lgetxattr, path, name, value, size);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(lgetxattr)(path, name, value, size);
+  if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res);
+  return res;
+}
+INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char *name, char *value,
+            SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetxattr, fd, name, value, size);
+  if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  SSIZE_T res = REAL(fgetxattr)(fd, name, value, size);
+  if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res);
+  return res;
+}
+#define INIT_GETXATTR                   \
+  COMMON_INTERCEPT_FUNCTION(getxattr);  \
+  COMMON_INTERCEPT_FUNCTION(lgetxattr); \
+  COMMON_INTERCEPT_FUNCTION(fgetxattr);
+#else
+#define INIT_GETXATTR
+#endif
+
+#if SANITIZER_INTERCEPT_GETRESID
+INTERCEPTOR(int, getresuid, void *ruid, void *euid, void *suid) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getresuid, ruid, euid, suid);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getresuid)(ruid, euid, suid);
+  if (res >= 0) {
+    if (ruid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ruid, uid_t_sz);
+    if (euid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, euid, uid_t_sz);
+    if (suid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, suid, uid_t_sz);
+  }
+  return res;
+}
+INTERCEPTOR(int, getresgid, void *rgid, void *egid, void *sgid) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getresgid, rgid, egid, sgid);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getresgid)(rgid, egid, sgid);
+  if (res >= 0) {
+    if (rgid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rgid, gid_t_sz);
+    if (egid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, egid, gid_t_sz);
+    if (sgid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sgid, gid_t_sz);
+  }
+  return res;
+}
+#define INIT_GETRESID                   \
+  COMMON_INTERCEPT_FUNCTION(getresuid); \
+  COMMON_INTERCEPT_FUNCTION(getresgid);
+#else
+#define INIT_GETRESID
+#endif
+
+#if SANITIZER_INTERCEPT_GETIFADDRS
+// As long as getifaddrs()/freeifaddrs() use calloc()/free(), we don't need to
+// intercept freeifaddrs(). If that ceases to be the case, we might need to
+// intercept it to poison the memory again.
+INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getifaddrs, ifap);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(getifaddrs)(ifap);
+  if (res == 0 && ifap) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifap, sizeof(void *));
+    __sanitizer_ifaddrs *p = *ifap;
+    while (p) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(__sanitizer_ifaddrs));
+      if (p->ifa_name)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_name,
+                                       REAL(strlen)(p->ifa_name) + 1);
+      if (p->ifa_addr)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_addr, struct_sockaddr_sz);
+      if (p->ifa_netmask)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_netmask, struct_sockaddr_sz);
+      // On Linux this is a union, but the other member also points to a
+      // struct sockaddr, so the following is sufficient.
+      if (p->ifa_dstaddr)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_dstaddr, struct_sockaddr_sz);
+      // FIXME(smatveev): Unpoison p->ifa_data as well.
+      p = p->ifa_next;
+    }
+  }
+  return res;
+}
+#define INIT_GETIFADDRS                  \
+  COMMON_INTERCEPT_FUNCTION(getifaddrs);
+#else
+#define INIT_GETIFADDRS
+#endif
+
+#if SANITIZER_INTERCEPT_IF_INDEXTONAME
+INTERCEPTOR(char *, if_indextoname, unsigned int ifindex, char* ifname) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, if_indextoname, ifindex, ifname);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  char *res = REAL(if_indextoname)(ifindex, ifname);
+  if (res && ifname)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
+  return res;
+}
+INTERCEPTOR(unsigned int, if_nametoindex, const char* ifname) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, if_nametoindex, ifname);
+  if (ifname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
+  return REAL(if_nametoindex)(ifname);
+}
+#define INIT_IF_INDEXTONAME                  \
+  COMMON_INTERCEPT_FUNCTION(if_indextoname); \
+  COMMON_INTERCEPT_FUNCTION(if_nametoindex);
+#else
+#define INIT_IF_INDEXTONAME
+#endif
+
+#if SANITIZER_INTERCEPT_CAPGET
+INTERCEPTOR(int, capget, void *hdrp, void *datap) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, capget, hdrp, datap);
+  if (hdrp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(capget)(hdrp, datap);
+  if (res == 0 && datap)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz);
+  // We can also return -1 and write to hdrp->version if the version passed in
+  // hdrp->version is unsupported. But that's not a trivial condition to check,
+  // and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent.
+  return res;
+}
+INTERCEPTOR(int, capset, void *hdrp, const void *datap) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap);
+  if (hdrp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);
+  if (datap)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, __user_cap_data_struct_sz);
+  return REAL(capset)(hdrp, datap);
+}
+#define INIT_CAPGET                  \
+  COMMON_INTERCEPT_FUNCTION(capget); \
+  COMMON_INTERCEPT_FUNCTION(capset);
+#else
+#define INIT_CAPGET
+#endif
+
+#if SANITIZER_INTERCEPT_AEABI_MEM
+INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+// Note the argument order.
+INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+#define INIT_AEABI_MEM                         \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memset8);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr);   \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4);  \
+  COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
+#else
+#define INIT_AEABI_MEM
+#endif  // SANITIZER_INTERCEPT_AEABI_MEM
+
+#if SANITIZER_INTERCEPT___BZERO
+INTERCEPTOR(void *, __bzero, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
+#else
+#define INIT___BZERO
+#endif  // SANITIZER_INTERCEPT___BZERO
+
+#if SANITIZER_INTERCEPT_BZERO
+INTERCEPTOR(void *, bzero, void *block, uptr size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
+#else
+#define INIT_BZERO
+#endif  // SANITIZER_INTERCEPT_BZERO
+
+#if SANITIZER_INTERCEPT_FTIME
+INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ftime, tp);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(ftime)(tp);
+  if (tp)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, sizeof(*tp));
+  return res;
+}
+#define INIT_FTIME COMMON_INTERCEPT_FUNCTION(ftime);
+#else
+#define INIT_FTIME
+#endif  // SANITIZER_INTERCEPT_FTIME
+
+#if SANITIZER_INTERCEPT_XDR
+INTERCEPTOR(void, xdrmem_create, __sanitizer_XDR *xdrs, uptr addr,
+            unsigned size, int op) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdrmem_create, xdrs, addr, size, op);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  REAL(xdrmem_create)(xdrs, addr, size, op);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs));
+  if (op == __sanitizer_XDR_ENCODE) {
+    // It's not obvious how much data individual xdr_ routines write.
+    // Simply unpoison the entire target buffer in advance.
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (void *)addr, size);
+  }
+}
+
+INTERCEPTOR(void, xdrstdio_create, __sanitizer_XDR *xdrs, void *file, int op) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdrstdio_create, xdrs, file, op);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  REAL(xdrstdio_create)(xdrs, file, op);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs));
+}
+
+// FIXME: under ASan the call below may write to freed memory and corrupt
+// its metadata. See
+// https://github.com/google/sanitizers/issues/321.
+#define XDR_INTERCEPTOR(F, T)                             \
+  INTERCEPTOR(int, F, __sanitizer_XDR *xdrs, T *p) {      \
+    void *ctx;                                            \
+    COMMON_INTERCEPTOR_ENTER(ctx, F, xdrs, p);            \
+    if (p && xdrs->x_op == __sanitizer_XDR_ENCODE)        \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));  \
+    int res = REAL(F)(xdrs, p);                           \
+    if (res && p && xdrs->x_op == __sanitizer_XDR_DECODE) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); \
+    return res;                                           \
+  }
+
+XDR_INTERCEPTOR(xdr_short, short)
+XDR_INTERCEPTOR(xdr_u_short, unsigned short)
+XDR_INTERCEPTOR(xdr_int, int)
+XDR_INTERCEPTOR(xdr_u_int, unsigned)
+XDR_INTERCEPTOR(xdr_long, long)
+XDR_INTERCEPTOR(xdr_u_long, unsigned long)
+XDR_INTERCEPTOR(xdr_hyper, long long)
+XDR_INTERCEPTOR(xdr_u_hyper, unsigned long long)
+XDR_INTERCEPTOR(xdr_longlong_t, long long)
+XDR_INTERCEPTOR(xdr_u_longlong_t, unsigned long long)
+XDR_INTERCEPTOR(xdr_int8_t, u8)
+XDR_INTERCEPTOR(xdr_uint8_t, u8)
+XDR_INTERCEPTOR(xdr_int16_t, u16)
+XDR_INTERCEPTOR(xdr_uint16_t, u16)
+XDR_INTERCEPTOR(xdr_int32_t, u32)
+XDR_INTERCEPTOR(xdr_uint32_t, u32)
+XDR_INTERCEPTOR(xdr_int64_t, u64)
+XDR_INTERCEPTOR(xdr_uint64_t, u64)
+XDR_INTERCEPTOR(xdr_quad_t, long long)
+XDR_INTERCEPTOR(xdr_u_quad_t, unsigned long long)
+XDR_INTERCEPTOR(xdr_bool, bool)
+XDR_INTERCEPTOR(xdr_enum, int)
+XDR_INTERCEPTOR(xdr_char, char)
+XDR_INTERCEPTOR(xdr_u_char, unsigned char)
+XDR_INTERCEPTOR(xdr_float, float)
+XDR_INTERCEPTOR(xdr_double, double)
+
+// FIXME: intercept xdr_array, opaque, union, vector, reference, pointer,
+// wrapstring, sizeof
+
+INTERCEPTOR(int, xdr_bytes, __sanitizer_XDR *xdrs, char **p, unsigned *sizep,
+            unsigned maxsize) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdr_bytes, xdrs, p, sizep, maxsize);
+  if (p && sizep && xdrs->x_op == __sanitizer_XDR_ENCODE) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sizep, sizeof(*sizep));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, *p, *sizep);
+  }
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(xdr_bytes)(xdrs, p, sizep, maxsize);
+  if (p && sizep && xdrs->x_op == __sanitizer_XDR_DECODE) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sizep, sizeof(*sizep));
+    if (res && *p && *sizep) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, *sizep);
+  }
+  return res;
+}
+
+INTERCEPTOR(int, xdr_string, __sanitizer_XDR *xdrs, char **p,
+            unsigned maxsize) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdr_string, xdrs, p, maxsize);
+  if (p && xdrs->x_op == __sanitizer_XDR_ENCODE) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+  }
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(xdr_string)(xdrs, p, maxsize);
+  if (p && xdrs->x_op == __sanitizer_XDR_DECODE) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+    if (res && *p)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *p, REAL(strlen)(*p) + 1);
+  }
+  return res;
+}
+
+#define INIT_XDR                               \
+  COMMON_INTERCEPT_FUNCTION(xdrmem_create);    \
+  COMMON_INTERCEPT_FUNCTION(xdrstdio_create);  \
+  COMMON_INTERCEPT_FUNCTION(xdr_short);        \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_short);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_int);          \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_int);        \
+  COMMON_INTERCEPT_FUNCTION(xdr_long);         \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_long);       \
+  COMMON_INTERCEPT_FUNCTION(xdr_hyper);        \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_hyper);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_longlong_t);   \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_longlong_t); \
+  COMMON_INTERCEPT_FUNCTION(xdr_int8_t);       \
+  COMMON_INTERCEPT_FUNCTION(xdr_uint8_t);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_int16_t);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_uint16_t);     \
+  COMMON_INTERCEPT_FUNCTION(xdr_int32_t);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_uint32_t);     \
+  COMMON_INTERCEPT_FUNCTION(xdr_int64_t);      \
+  COMMON_INTERCEPT_FUNCTION(xdr_uint64_t);     \
+  COMMON_INTERCEPT_FUNCTION(xdr_quad_t);       \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_quad_t);     \
+  COMMON_INTERCEPT_FUNCTION(xdr_bool);         \
+  COMMON_INTERCEPT_FUNCTION(xdr_enum);         \
+  COMMON_INTERCEPT_FUNCTION(xdr_char);         \
+  COMMON_INTERCEPT_FUNCTION(xdr_u_char);       \
+  COMMON_INTERCEPT_FUNCTION(xdr_float);        \
+  COMMON_INTERCEPT_FUNCTION(xdr_double);       \
+  COMMON_INTERCEPT_FUNCTION(xdr_bytes);        \
+  COMMON_INTERCEPT_FUNCTION(xdr_string);
+#else
+#define INIT_XDR
+#endif  // SANITIZER_INTERCEPT_XDR
+
+#if SANITIZER_INTERCEPT_TSEARCH
+INTERCEPTOR(void *, tsearch, void *key, void **rootp,
+            int (*compar)(const void *, const void *)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, tsearch, key, rootp, compar);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  void *res = REAL(tsearch)(key, rootp, compar);
+  if (res && *(void **)res == key)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(void *));
+  return res;
+}
+#define INIT_TSEARCH COMMON_INTERCEPT_FUNCTION(tsearch);
+#else
+#define INIT_TSEARCH
+#endif
+
+#if SANITIZER_INTERCEPT_LIBIO_INTERNALS || SANITIZER_INTERCEPT_FOPEN || \
+    SANITIZER_INTERCEPT_OPEN_MEMSTREAM
+void unpoison_file(__sanitizer_FILE *fp) {
+#if SANITIZER_HAS_STRUCT_FILE
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp, sizeof(*fp));
+#if SANITIZER_NETBSD
+  if (fp->_bf._base && fp->_bf._size > 0)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_bf._base,
+                                        fp->_bf._size);
+#else
+  if (fp->_IO_read_base && fp->_IO_read_base < fp->_IO_read_end)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_read_base,
+                                        fp->_IO_read_end - fp->_IO_read_base);
+#endif
+#endif  // SANITIZER_HAS_STRUCT_FILE
+}
+#endif
+
+#if SANITIZER_INTERCEPT_LIBIO_INTERNALS
+// These guys are called when a .c source is built with -O2.
+INTERCEPTOR(int, __uflow, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __uflow, fp);
+  int res = REAL(__uflow)(fp);
+  unpoison_file(fp);
+  return res;
+}
+INTERCEPTOR(int, __underflow, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __underflow, fp);
+  int res = REAL(__underflow)(fp);
+  unpoison_file(fp);
+  return res;
+}
+INTERCEPTOR(int, __overflow, __sanitizer_FILE *fp, int ch) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __overflow, fp, ch);
+  int res = REAL(__overflow)(fp, ch);
+  unpoison_file(fp);
+  return res;
+}
+INTERCEPTOR(int, __wuflow, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __wuflow, fp);
+  int res = REAL(__wuflow)(fp);
+  unpoison_file(fp);
+  return res;
+}
+INTERCEPTOR(int, __wunderflow, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __wunderflow, fp);
+  int res = REAL(__wunderflow)(fp);
+  unpoison_file(fp);
+  return res;
+}
+INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __woverflow, fp, ch);
+  int res = REAL(__woverflow)(fp, ch);
+  unpoison_file(fp);
+  return res;
+}
+#define INIT_LIBIO_INTERNALS               \
+  COMMON_INTERCEPT_FUNCTION(__uflow);      \
+  COMMON_INTERCEPT_FUNCTION(__underflow);  \
+  COMMON_INTERCEPT_FUNCTION(__overflow);   \
+  COMMON_INTERCEPT_FUNCTION(__wuflow);     \
+  COMMON_INTERCEPT_FUNCTION(__wunderflow); \
+  COMMON_INTERCEPT_FUNCTION(__woverflow);
+#else
+#define INIT_LIBIO_INTERNALS
+#endif
+
+#if SANITIZER_INTERCEPT_FOPEN
+INTERCEPTOR(__sanitizer_FILE *, fopen, const char *path, const char *mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+  __sanitizer_FILE *res = REAL(fopen)(path, mode);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
+  if (res) unpoison_file(res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_FILE *, fdopen, int fd, const char *mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fdopen, fd, mode);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+  __sanitizer_FILE *res = REAL(fdopen)(fd, mode);
+  if (res) unpoison_file(res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_FILE *, freopen, const char *path, const char *mode,
+            __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+  COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
+  __sanitizer_FILE *res = REAL(freopen)(path, mode, fp);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
+  if (res) unpoison_file(res);
+  return res;
+}
+#define INIT_FOPEN                   \
+  COMMON_INTERCEPT_FUNCTION(fopen);  \
+  COMMON_INTERCEPT_FUNCTION(fdopen); \
+  COMMON_INTERCEPT_FUNCTION(freopen);
+#else
+#define INIT_FOPEN
+#endif
+
+#if SANITIZER_INTERCEPT_FOPEN64
+INTERCEPTOR(__sanitizer_FILE *, fopen64, const char *path, const char *mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fopen64, path, mode);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+  __sanitizer_FILE *res = REAL(fopen64)(path, mode);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
+  if (res) unpoison_file(res);
+  return res;
+}
+INTERCEPTOR(__sanitizer_FILE *, freopen64, const char *path, const char *mode,
+            __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
+  COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
+  __sanitizer_FILE *res = REAL(freopen64)(path, mode, fp);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
+  if (res) unpoison_file(res);
+  return res;
+}
+#define INIT_FOPEN64                  \
+  COMMON_INTERCEPT_FUNCTION(fopen64); \
+  COMMON_INTERCEPT_FUNCTION(freopen64);
+#else
+#define INIT_FOPEN64
+#endif
+
+#if SANITIZER_INTERCEPT_OPEN_MEMSTREAM
+INTERCEPTOR(__sanitizer_FILE *, open_memstream, char **ptr, SIZE_T *sizeloc) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, open_memstream, ptr, sizeloc);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_FILE *res = REAL(open_memstream)(ptr, sizeloc);
+  if (res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, sizeof(*ptr));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sizeloc, sizeof(*sizeloc));
+    unpoison_file(res);
+    FileMetadata file = {ptr, sizeloc};
+    SetInterceptorMetadata(res, file);
+  }
+  return res;
+}
+INTERCEPTOR(__sanitizer_FILE *, open_wmemstream, wchar_t **ptr,
+            SIZE_T *sizeloc) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, open_wmemstream, ptr, sizeloc);
+  __sanitizer_FILE *res = REAL(open_wmemstream)(ptr, sizeloc);
+  if (res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, sizeof(*ptr));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sizeloc, sizeof(*sizeloc));
+    unpoison_file(res);
+    FileMetadata file = {(char **)ptr, sizeloc};
+    SetInterceptorMetadata(res, file);
+  }
+  return res;
+}
+INTERCEPTOR(__sanitizer_FILE *, fmemopen, void *buf, SIZE_T size,
+            const char *mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fmemopen, buf, size, mode);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  __sanitizer_FILE *res = REAL(fmemopen)(buf, size, mode);
+  if (res) unpoison_file(res);
+  return res;
+}
+#define INIT_OPEN_MEMSTREAM                   \
+  COMMON_INTERCEPT_FUNCTION(open_memstream);  \
+  COMMON_INTERCEPT_FUNCTION(open_wmemstream); \
+  COMMON_INTERCEPT_FUNCTION(fmemopen);
+#else
+#define INIT_OPEN_MEMSTREAM
+#endif
+
+#if SANITIZER_INTERCEPT_OBSTACK
+static void initialize_obstack(__sanitizer_obstack *obstack) {
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(obstack, sizeof(*obstack));
+  if (obstack->chunk)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(obstack->chunk,
+                                        sizeof(*obstack->chunk));
+}
+
+INTERCEPTOR(int, _obstack_begin_1, __sanitizer_obstack *obstack, int sz,
+            int align, void *(*alloc_fn)(uptr arg, uptr sz),
+            void (*free_fn)(uptr arg, void *p)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, _obstack_begin_1, obstack, sz, align, alloc_fn,
+                           free_fn);
+  int res = REAL(_obstack_begin_1)(obstack, sz, align, alloc_fn, free_fn);
+  if (res) initialize_obstack(obstack);
+  return res;
+}
+INTERCEPTOR(int, _obstack_begin, __sanitizer_obstack *obstack, int sz,
+            int align, void *(*alloc_fn)(uptr sz), void (*free_fn)(void *p)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, _obstack_begin, obstack, sz, align, alloc_fn,
+                           free_fn);
+  int res = REAL(_obstack_begin)(obstack, sz, align, alloc_fn, free_fn);
+  if (res) initialize_obstack(obstack);
+  return res;
+}
+INTERCEPTOR(void, _obstack_newchunk, __sanitizer_obstack *obstack, int length) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, _obstack_newchunk, obstack, length);
+  REAL(_obstack_newchunk)(obstack, length);
+  if (obstack->chunk)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(
+        obstack->chunk, obstack->next_free - (char *)obstack->chunk);
+}
+#define INIT_OBSTACK                           \
+  COMMON_INTERCEPT_FUNCTION(_obstack_begin_1); \
+  COMMON_INTERCEPT_FUNCTION(_obstack_begin);   \
+  COMMON_INTERCEPT_FUNCTION(_obstack_newchunk);
+#else
+#define INIT_OBSTACK
+#endif
+
+#if SANITIZER_INTERCEPT_FFLUSH
+INTERCEPTOR(int, fflush, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fflush, fp);
+  int res = REAL(fflush)(fp);
+  // FIXME: handle fp == NULL
+  if (fp) {
+    const FileMetadata *m = GetInterceptorMetadata(fp);
+    if (m) COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size);
+  }
+  return res;
+}
+#define INIT_FFLUSH COMMON_INTERCEPT_FUNCTION(fflush);
+#else
+#define INIT_FFLUSH
+#endif
+
+#if SANITIZER_INTERCEPT_FCLOSE
+INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fclose, fp);
+  COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
+  const FileMetadata *m = GetInterceptorMetadata(fp);
+  int res = REAL(fclose)(fp);
+  if (m) {
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size);
+    DeleteInterceptorMetadata(fp);
+  }
+  return res;
+}
+#define INIT_FCLOSE COMMON_INTERCEPT_FUNCTION(fclose);
+#else
+#define INIT_FCLOSE
+#endif
+
+#if SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
+INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
+  if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
+  COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);
+  void *res = REAL(dlopen)(filename, flag);
+  Symbolizer::GetOrInit()->InvalidateModuleList();
+  COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
+  return res;
+}
+
+INTERCEPTOR(int, dlclose, void *handle) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle);
+  int res = REAL(dlclose)(handle);
+  Symbolizer::GetOrInit()->InvalidateModuleList();
+  COMMON_INTERCEPTOR_LIBRARY_UNLOADED();
+  return res;
+}
+#define INIT_DLOPEN_DLCLOSE          \
+  COMMON_INTERCEPT_FUNCTION(dlopen); \
+  COMMON_INTERCEPT_FUNCTION(dlclose);
+#else
+#define INIT_DLOPEN_DLCLOSE
+#endif
+
+#if SANITIZER_INTERCEPT_GETPASS
+INTERCEPTOR(char *, getpass, const char *prompt) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getpass, prompt);
+  if (prompt)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, REAL(strlen)(prompt)+1);
+  char *res = REAL(getpass)(prompt);
+  if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res)+1);
+  return res;
+}
+
+#define INIT_GETPASS COMMON_INTERCEPT_FUNCTION(getpass);
+#else
+#define INIT_GETPASS
+#endif
+
+#if SANITIZER_INTERCEPT_TIMERFD
+INTERCEPTOR(int, timerfd_settime, int fd, int flags, void *new_value,
+            void *old_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timerfd_settime, fd, flags, new_value,
+                           old_value);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, struct_itimerspec_sz);
+  int res = REAL(timerfd_settime)(fd, flags, new_value, old_value);
+  if (res != -1 && old_value)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, struct_itimerspec_sz);
+  return res;
+}
+
+INTERCEPTOR(int, timerfd_gettime, int fd, void *curr_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timerfd_gettime, fd, curr_value);
+  int res = REAL(timerfd_gettime)(fd, curr_value);
+  if (res != -1 && curr_value)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, struct_itimerspec_sz);
+  return res;
+}
+#define INIT_TIMERFD                          \
+  COMMON_INTERCEPT_FUNCTION(timerfd_settime); \
+  COMMON_INTERCEPT_FUNCTION(timerfd_gettime);
+#else
+#define INIT_TIMERFD
+#endif
+
+#if SANITIZER_INTERCEPT_MLOCKX
+// Linux kernel has a bug that leads to kernel deadlock if a process
+// maps TBs of memory and then calls mlock().
+static void MlockIsUnsupported() {
+  static atomic_uint8_t printed;
+  if (atomic_exchange(&printed, 1, memory_order_relaxed))
+    return;
+  VPrintf(1, "%s ignores mlock/mlockall/munlock/munlockall\n",
+          SanitizerToolName);
+}
+
+INTERCEPTOR(int, mlock, const void *addr, uptr len) {
+  MlockIsUnsupported();
+  return 0;
+}
+
+INTERCEPTOR(int, munlock, const void *addr, uptr len) {
+  MlockIsUnsupported();
+  return 0;
+}
+
+INTERCEPTOR(int, mlockall, int flags) {
+  MlockIsUnsupported();
+  return 0;
+}
+
+INTERCEPTOR(int, munlockall, void) {
+  MlockIsUnsupported();
+  return 0;
+}
+
+#define INIT_MLOCKX                                                            \
+  COMMON_INTERCEPT_FUNCTION(mlock);                                            \
+  COMMON_INTERCEPT_FUNCTION(munlock);                                          \
+  COMMON_INTERCEPT_FUNCTION(mlockall);                                         \
+  COMMON_INTERCEPT_FUNCTION(munlockall);
+
+#else
+#define INIT_MLOCKX
+#endif  // SANITIZER_INTERCEPT_MLOCKX
+
+#if SANITIZER_INTERCEPT_FOPENCOOKIE
+struct WrappedCookie {
+  void *real_cookie;
+  __sanitizer_cookie_io_functions_t real_io_funcs;
+};
+
+static uptr wrapped_read(void *cookie, char *buf, uptr size) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedCookie *wrapped_cookie = (WrappedCookie *)cookie;
+  __sanitizer_cookie_io_read real_read = wrapped_cookie->real_io_funcs.read;
+  return real_read ? real_read(wrapped_cookie->real_cookie, buf, size) : 0;
+}
+
+static uptr wrapped_write(void *cookie, const char *buf, uptr size) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedCookie *wrapped_cookie = (WrappedCookie *)cookie;
+  __sanitizer_cookie_io_write real_write = wrapped_cookie->real_io_funcs.write;
+  return real_write ? real_write(wrapped_cookie->real_cookie, buf, size) : size;
+}
+
+static int wrapped_seek(void *cookie, u64 *offset, int whence) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(offset, sizeof(*offset));
+  WrappedCookie *wrapped_cookie = (WrappedCookie *)cookie;
+  __sanitizer_cookie_io_seek real_seek = wrapped_cookie->real_io_funcs.seek;
+  return real_seek ? real_seek(wrapped_cookie->real_cookie, offset, whence)
+                   : -1;
+}
+
+static int wrapped_close(void *cookie) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  WrappedCookie *wrapped_cookie = (WrappedCookie *)cookie;
+  __sanitizer_cookie_io_close real_close = wrapped_cookie->real_io_funcs.close;
+  int res = real_close ? real_close(wrapped_cookie->real_cookie) : 0;
+  InternalFree(wrapped_cookie);
+  return res;
+}
+
+INTERCEPTOR(__sanitizer_FILE *, fopencookie, void *cookie, const char *mode,
+            __sanitizer_cookie_io_functions_t io_funcs) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fopencookie, cookie, mode, io_funcs);
+  WrappedCookie *wrapped_cookie =
+      (WrappedCookie *)InternalAlloc(sizeof(WrappedCookie));
+  wrapped_cookie->real_cookie = cookie;
+  wrapped_cookie->real_io_funcs = io_funcs;
+  __sanitizer_FILE *res =
+      REAL(fopencookie)(wrapped_cookie, mode, {wrapped_read, wrapped_write,
+                                               wrapped_seek, wrapped_close});
+  return res;
+}
+
+#define INIT_FOPENCOOKIE COMMON_INTERCEPT_FUNCTION(fopencookie);
+#else
+#define INIT_FOPENCOOKIE
+#endif  // SANITIZER_INTERCEPT_FOPENCOOKIE
+
+#if SANITIZER_INTERCEPT_SEM
+INTERCEPTOR(int, sem_init, __sanitizer_sem_t *s, int pshared, unsigned value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value);
+  // Workaround a bug in glibc's "old" semaphore implementation by
+  // zero-initializing the sem_t contents. This has to be done here because
+  // interceptors bind to the lowest symbols version by default, hitting the
+  // buggy code path while the non-sanitized build of the same code works fine.
+  REAL(memset)(s, 0, sizeof(*s));
+  int res = REAL(sem_init)(s, pshared, value);
+  return res;
+}
+
+INTERCEPTOR(int, sem_destroy, __sanitizer_sem_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_destroy, s);
+  int res = REAL(sem_destroy)(s);
+  return res;
+}
+
+INTERCEPTOR(int, sem_wait, __sanitizer_sem_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_wait, s);
+  int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_wait)(s);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
+  }
+  return res;
+}
+
+INTERCEPTOR(int, sem_trywait, __sanitizer_sem_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_trywait, s);
+  int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_trywait)(s);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
+  }
+  return res;
+}
+
+INTERCEPTOR(int, sem_timedwait, __sanitizer_sem_t *s, void *abstime) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_timedwait, s, abstime);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, abstime, struct_timespec_sz);
+  int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_timedwait)(s, abstime);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
+  }
+  return res;
+}
+
+INTERCEPTOR(int, sem_post, __sanitizer_sem_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_post, s);
+  COMMON_INTERCEPTOR_RELEASE(ctx, (uptr)s);
+  int res = REAL(sem_post)(s);
+  return res;
+}
+
+INTERCEPTOR(int, sem_getvalue, __sanitizer_sem_t *s, int *sval) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sem_getvalue, s, sval);
+  int res = REAL(sem_getvalue)(s, sval);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sval, sizeof(*sval));
+  }
+  return res;
+}
+#define INIT_SEM                                                               \
+  COMMON_INTERCEPT_FUNCTION(sem_init);                                         \
+  COMMON_INTERCEPT_FUNCTION(sem_destroy);                                      \
+  COMMON_INTERCEPT_FUNCTION(sem_wait);                                         \
+  COMMON_INTERCEPT_FUNCTION(sem_trywait);                                      \
+  COMMON_INTERCEPT_FUNCTION(sem_timedwait);                                    \
+  COMMON_INTERCEPT_FUNCTION(sem_post);                                         \
+  COMMON_INTERCEPT_FUNCTION(sem_getvalue);
+#else
+#define INIT_SEM
+#endif // SANITIZER_INTERCEPT_SEM
+
+#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL
+INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcancelstate, state, oldstate);
+  int res = REAL(pthread_setcancelstate)(state, oldstate);
+  if (res == 0 && oldstate != nullptr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldstate, sizeof(*oldstate));
+  return res;
+}
+
+INTERCEPTOR(int, pthread_setcanceltype, int type, int *oldtype) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcanceltype, type, oldtype);
+  int res = REAL(pthread_setcanceltype)(type, oldtype);
+  if (res == 0 && oldtype != nullptr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldtype, sizeof(*oldtype));
+  return res;
+}
+#define INIT_PTHREAD_SETCANCEL                                                 \
+  COMMON_INTERCEPT_FUNCTION(pthread_setcancelstate);                           \
+  COMMON_INTERCEPT_FUNCTION(pthread_setcanceltype);
+#else
+#define INIT_PTHREAD_SETCANCEL
+#endif
+
+#if SANITIZER_INTERCEPT_MINCORE
+INTERCEPTOR(int, mincore, void *addr, uptr length, unsigned char *vec) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mincore, addr, length, vec);
+  int res = REAL(mincore)(addr, length, vec);
+  if (res == 0) {
+    uptr page_size = GetPageSizeCached();
+    uptr vec_size = ((length + page_size - 1) & (~(page_size - 1))) / page_size;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, vec, vec_size);
+  }
+  return res;
+}
+#define INIT_MINCORE COMMON_INTERCEPT_FUNCTION(mincore);
+#else
+#define INIT_MINCORE
+#endif
+
+#if SANITIZER_INTERCEPT_PROCESS_VM_READV
+INTERCEPTOR(SSIZE_T, process_vm_readv, int pid, __sanitizer_iovec *local_iov,
+            uptr liovcnt, __sanitizer_iovec *remote_iov, uptr riovcnt,
+            uptr flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, process_vm_readv, pid, local_iov, liovcnt,
+                           remote_iov, riovcnt, flags);
+  SSIZE_T res = REAL(process_vm_readv)(pid, local_iov, liovcnt, remote_iov,
+                                       riovcnt, flags);
+  if (res > 0)
+    write_iovec(ctx, local_iov, liovcnt, res);
+  return res;
+}
+
+INTERCEPTOR(SSIZE_T, process_vm_writev, int pid, __sanitizer_iovec *local_iov,
+            uptr liovcnt, __sanitizer_iovec *remote_iov, uptr riovcnt,
+            uptr flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, process_vm_writev, pid, local_iov, liovcnt,
+                           remote_iov, riovcnt, flags);
+  SSIZE_T res = REAL(process_vm_writev)(pid, local_iov, liovcnt, remote_iov,
+                                        riovcnt, flags);
+  if (res > 0)
+    read_iovec(ctx, local_iov, liovcnt, res);
+  return res;
+}
+#define INIT_PROCESS_VM_READV                                                  \
+  COMMON_INTERCEPT_FUNCTION(process_vm_readv);                                 \
+  COMMON_INTERCEPT_FUNCTION(process_vm_writev);
+#else
+#define INIT_PROCESS_VM_READV
+#endif
+
+#if SANITIZER_INTERCEPT_CTERMID
+INTERCEPTOR(char *, ctermid, char *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s);
+  char *res = REAL(ctermid)(s);
+  if (res) {
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+#define INIT_CTERMID COMMON_INTERCEPT_FUNCTION(ctermid);
+#else
+#define INIT_CTERMID
+#endif
+
+#if SANITIZER_INTERCEPT_CTERMID_R
+INTERCEPTOR(char *, ctermid_r, char *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s);
+  char *res = REAL(ctermid_r)(s);
+  if (res) {
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+  }
+  return res;
+}
+#define INIT_CTERMID_R COMMON_INTERCEPT_FUNCTION(ctermid_r);
+#else
+#define INIT_CTERMID_R
+#endif
+
+#if SANITIZER_INTERCEPT_RECV_RECVFROM
+INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, recv, fd, buf, len, flags);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SSIZE_T res = REAL(recv)(fd, buf, len, flags);
+  if (res > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
+  }
+  if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+
+INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,
+            void *srcaddr, int *addrlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, recvfrom, fd, buf, len, flags, srcaddr,
+                           addrlen);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SIZE_T srcaddr_sz;
+  if (srcaddr) srcaddr_sz = *addrlen;
+  (void)srcaddr_sz;  // prevent "set but not used" warning
+  SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, Min((SIZE_T)res, len));
+  if (res >= 0 && srcaddr)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(srcaddr,
+                                        Min((SIZE_T)*addrlen, srcaddr_sz));
+  return res;
+}
+#define INIT_RECV_RECVFROM          \
+  COMMON_INTERCEPT_FUNCTION(recv);  \
+  COMMON_INTERCEPT_FUNCTION(recvfrom);
+#else
+#define INIT_RECV_RECVFROM
+#endif
+
+#if SANITIZER_INTERCEPT_SEND_SENDTO
+INTERCEPTOR(SSIZE_T, send, int fd, void *buf, SIZE_T len, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, send, fd, buf, len, flags);
+  if (fd >= 0) {
+    COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+    COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  }
+  SSIZE_T res = REAL(send)(fd, buf, len, flags);
+  if (common_flags()->intercept_send && res > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
+  return res;
+}
+
+INTERCEPTOR(SSIZE_T, sendto, int fd, void *buf, SIZE_T len, int flags,
+            void *dstaddr, int addrlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sendto, fd, buf, len, flags, dstaddr, addrlen);
+  if (fd >= 0) {
+    COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+    COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  }
+  // Can't check dstaddr as it may have uninitialized padding at the end.
+  SSIZE_T res = REAL(sendto)(fd, buf, len, flags, dstaddr, addrlen);
+  if (common_flags()->intercept_send && res > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, Min((SIZE_T)res, len));
+  return res;
+}
+#define INIT_SEND_SENDTO           \
+  COMMON_INTERCEPT_FUNCTION(send); \
+  COMMON_INTERCEPT_FUNCTION(sendto);
+#else
+#define INIT_SEND_SENDTO
+#endif
+
+#if SANITIZER_INTERCEPT_EVENTFD_READ_WRITE
+INTERCEPTOR(int, eventfd_read, int fd, u64 *value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, eventfd_read, fd, value);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  int res = REAL(eventfd_read)(fd, value);
+  if (res == 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, sizeof(*value));
+    if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return res;
+}
+INTERCEPTOR(int, eventfd_write, int fd, u64 value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, eventfd_write, fd, value);
+  if (fd >= 0) {
+    COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+    COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd);
+  }
+  int res = REAL(eventfd_write)(fd, value);
+  return res;
+}
+#define INIT_EVENTFD_READ_WRITE            \
+  COMMON_INTERCEPT_FUNCTION(eventfd_read); \
+  COMMON_INTERCEPT_FUNCTION(eventfd_write)
+#else
+#define INIT_EVENTFD_READ_WRITE
+#endif
+
+#if SANITIZER_INTERCEPT_STAT
+INTERCEPTOR(int, stat, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, stat, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(stat)(path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz);
+  return res;
+}
+#define INIT_STAT COMMON_INTERCEPT_FUNCTION(stat)
+#else
+#define INIT_STAT
+#endif
+
+#if SANITIZER_INTERCEPT_LSTAT
+INTERCEPTOR(int, lstat, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, lstat, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(lstat)(path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz);
+  return res;
+}
+#define INIT_LSTAT COMMON_INTERCEPT_FUNCTION(lstat)
+#else
+#define INIT_LSTAT
+#endif
+
+#if SANITIZER_INTERCEPT___XSTAT
+INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __xstat, version, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(__xstat)(version, path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz);
+  return res;
+}
+#define INIT___XSTAT COMMON_INTERCEPT_FUNCTION(__xstat)
+#else
+#define INIT___XSTAT
+#endif
+
+#if SANITIZER_INTERCEPT___XSTAT64
+INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __xstat64, version, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(__xstat64)(version, path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
+  return res;
+}
+#define INIT___XSTAT64 COMMON_INTERCEPT_FUNCTION(__xstat64)
+#else
+#define INIT___XSTAT64
+#endif
+
+#if SANITIZER_INTERCEPT___LXSTAT
+INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __lxstat, version, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(__lxstat)(version, path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat_sz);
+  return res;
+}
+#define INIT___LXSTAT COMMON_INTERCEPT_FUNCTION(__lxstat)
+#else
+#define INIT___LXSTAT
+#endif
+
+#if SANITIZER_INTERCEPT___LXSTAT64
+INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __lxstat64, version, path, buf);
+  if (common_flags()->intercept_stat)
+    COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
+  int res = REAL(__lxstat64)(version, path, buf);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
+  return res;
+}
+#define INIT___LXSTAT64 COMMON_INTERCEPT_FUNCTION(__lxstat64)
+#else
+#define INIT___LXSTAT64
+#endif
+
+// FIXME: add other *stat interceptor
+
+#if SANITIZER_INTERCEPT_UTMP
+INTERCEPTOR(void *, getutent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutent, dummy);
+  void *res = REAL(getutent)(dummy);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutid, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutid, ut);
+  void *res = REAL(getutid)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutline, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutline, ut);
+  void *res = REAL(getutline)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmp_sz);
+  return res;
+}
+#define INIT_UTMP                      \
+  COMMON_INTERCEPT_FUNCTION(getutent); \
+  COMMON_INTERCEPT_FUNCTION(getutid);  \
+  COMMON_INTERCEPT_FUNCTION(getutline);
+#else
+#define INIT_UTMP
+#endif
+
+#if SANITIZER_INTERCEPT_UTMPX
+INTERCEPTOR(void *, getutxent, int dummy) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxent, dummy);
+  void *res = REAL(getutxent)(dummy);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutxid, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxid, ut);
+  void *res = REAL(getutxid)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+INTERCEPTOR(void *, getutxline, void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getutxline, ut);
+  void *res = REAL(getutxline)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+INTERCEPTOR(void *, pututxline, const void *ut) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pututxline, ut);
+  if (ut)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ut, __sanitizer::struct_utmpx_sz);
+  void *res = REAL(pututxline)(ut);
+  if (res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_utmpx_sz);
+  return res;
+}
+#define INIT_UTMPX                      \
+  COMMON_INTERCEPT_FUNCTION(getutxent); \
+  COMMON_INTERCEPT_FUNCTION(getutxid);  \
+  COMMON_INTERCEPT_FUNCTION(getutxline); \
+  COMMON_INTERCEPT_FUNCTION(pututxline);
+#else
+#define INIT_UTMPX
+#endif
+
+#if SANITIZER_INTERCEPT_GETLOADAVG
+INTERCEPTOR(int, getloadavg, double *loadavg, int nelem) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getloadavg, loadavg, nelem);
+  int res = REAL(getloadavg)(loadavg, nelem);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, loadavg, res * sizeof(*loadavg));
+  return res;
+}
+#define INIT_GETLOADAVG                      \
+  COMMON_INTERCEPT_FUNCTION(getloadavg);
+#else
+#define INIT_GETLOADAVG
+#endif
+
+#if SANITIZER_INTERCEPT_MCHECK_MPROBE
+INTERCEPTOR(int, mcheck, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
+
+INTERCEPTOR(int, mcheck_pedantic, void (*abortfunc)(int mstatus)) {
+  return 0;
+}
+
+INTERCEPTOR(int, mprobe, void *ptr) {
+  return 0;
+}
+#endif
+
+INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);
+  SIZE_T res = REAL(wcslen)(s);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (res + 1));
+  return res;
+}
+
+INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsnlen, s, n);
+  SIZE_T res = REAL(wcsnlen)(s, n);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * Min(res + 1, n));
+  return res;
+}
+#define INIT_WCSLEN                  \
+  COMMON_INTERCEPT_FUNCTION(wcslen); \
+  COMMON_INTERCEPT_FUNCTION(wcsnlen);
+
+#if SANITIZER_INTERCEPT_WCSCAT
+INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);
+  SIZE_T src_size = REAL(wcslen)(src);
+  SIZE_T dst_size = REAL(wcslen)(dst);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
+                                 (src_size + 1) * sizeof(wchar_t));
+  return REAL(wcscat)(dst, src);
+}
+
+INTERCEPTOR(wchar_t *, wcsncat, wchar_t *dst, const wchar_t *src, SIZE_T n) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);
+  SIZE_T src_size = REAL(wcsnlen)(src, n);
+  SIZE_T dst_size = REAL(wcslen)(dst);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, src,
+                                Min(src_size + 1, n) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
+                                 (src_size + 1) * sizeof(wchar_t));
+  return REAL(wcsncat)(dst, src, n);
+}
+#define INIT_WCSCAT                  \
+  COMMON_INTERCEPT_FUNCTION(wcscat); \
+  COMMON_INTERCEPT_FUNCTION(wcsncat);
+#else
+#define INIT_WCSCAT
+#endif
+
+#if SANITIZER_INTERCEPT_WCSDUP
+INTERCEPTOR(wchar_t *, wcsdup, wchar_t *s) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s);
+  SIZE_T len = REAL(wcslen)(s);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1));
+  wchar_t *result = REAL(wcsdup)(s);
+  if (result)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(wchar_t) * (len + 1));
+  return result;
+}
+
+#define INIT_WCSDUP COMMON_INTERCEPT_FUNCTION(wcsdup);
+#else
+#define INIT_WCSDUP
+#endif
+
+#if SANITIZER_INTERCEPT_STRXFRM
+static SIZE_T RealStrLen(const char *str) { return REAL(strlen)(str); }
+
+static SIZE_T RealStrLen(const wchar_t *str) { return REAL(wcslen)(str); }
+
+#define STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len, ...)             \
+  {                                                                        \
+    void *ctx;                                                             \
+    COMMON_INTERCEPTOR_ENTER(ctx, strxfrm, dest, src, len, ##__VA_ARGS__); \
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src,                                \
+                                  sizeof(*src) * (RealStrLen(src) + 1));   \
+    SIZE_T res = REAL(strxfrm)(dest, src, len, ##__VA_ARGS__);             \
+    if (res < len)                                                         \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, sizeof(*src) * (res + 1)); \
+    return res;                                                            \
+  }
+
+INTERCEPTOR(SIZE_T, strxfrm, char *dest, const char *src, SIZE_T len) {
+  STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len);
+}
+
+INTERCEPTOR(SIZE_T, strxfrm_l, char *dest, const char *src, SIZE_T len,
+            void *locale) {
+  STRXFRM_INTERCEPTOR_IMPL(strxfrm_l, dest, src, len, locale);
+}
+
+#define INIT_STRXFRM                  \
+  COMMON_INTERCEPT_FUNCTION(strxfrm); \
+  COMMON_INTERCEPT_FUNCTION(strxfrm_l);
+#else
+#define INIT_STRXFRM
+#endif
+
+#if SANITIZER_INTERCEPT___STRXFRM_L
+INTERCEPTOR(SIZE_T, __strxfrm_l, char *dest, const char *src, SIZE_T len,
+            void *locale) {
+  STRXFRM_INTERCEPTOR_IMPL(__strxfrm_l, dest, src, len, locale);
+}
+
+#define INIT___STRXFRM_L COMMON_INTERCEPT_FUNCTION(__strxfrm_l);
+#else
+#define INIT___STRXFRM_L
+#endif
+
+#if SANITIZER_INTERCEPT_WCSXFRM
+INTERCEPTOR(SIZE_T, wcsxfrm, wchar_t *dest, const wchar_t *src, SIZE_T len) {
+  STRXFRM_INTERCEPTOR_IMPL(wcsxfrm, dest, src, len);
+}
+
+INTERCEPTOR(SIZE_T, wcsxfrm_l, wchar_t *dest, const wchar_t *src, SIZE_T len,
+            void *locale) {
+  STRXFRM_INTERCEPTOR_IMPL(wcsxfrm_l, dest, src, len, locale);
+}
+
+#define INIT_WCSXFRM                  \
+  COMMON_INTERCEPT_FUNCTION(wcsxfrm); \
+  COMMON_INTERCEPT_FUNCTION(wcsxfrm_l);
+#else
+#define INIT_WCSXFRM
+#endif
+
+#if SANITIZER_INTERCEPT___WCSXFRM_L
+INTERCEPTOR(SIZE_T, __wcsxfrm_l, wchar_t *dest, const wchar_t *src, SIZE_T len,
+            void *locale) {
+  STRXFRM_INTERCEPTOR_IMPL(__wcsxfrm_l, dest, src, len, locale);
+}
+
+#define INIT___WCSXFRM_L COMMON_INTERCEPT_FUNCTION(__wcsxfrm_l);
+#else
+#define INIT___WCSXFRM_L
+#endif
+
+#if SANITIZER_INTERCEPT_ACCT
+INTERCEPTOR(int, acct, const char *file) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, acct, file);
+  if (file)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
+  return REAL(acct)(file);
+}
+#define INIT_ACCT COMMON_INTERCEPT_FUNCTION(acct)
+#else
+#define INIT_ACCT
+#endif
+
+#if SANITIZER_INTERCEPT_USER_FROM_UID
+INTERCEPTOR(const char *, user_from_uid, u32 uid, int nouser) {
+  void *ctx;
+  const char *user;
+  COMMON_INTERCEPTOR_ENTER(ctx, user_from_uid, uid, nouser);
+  user = REAL(user_from_uid)(uid, nouser);
+  if (user)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, REAL(strlen)(user) + 1);
+  return user;
+}
+#define INIT_USER_FROM_UID COMMON_INTERCEPT_FUNCTION(user_from_uid)
+#else
+#define INIT_USER_FROM_UID
+#endif
+
+#if SANITIZER_INTERCEPT_UID_FROM_USER
+INTERCEPTOR(int, uid_from_user, const char *name, u32 *uid) {
+  void *ctx;
+  int res;
+  COMMON_INTERCEPTOR_ENTER(ctx, uid_from_user, name, uid);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  res = REAL(uid_from_user)(name, uid);
+  if (uid)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, uid, sizeof(*uid));
+  return res;
+}
+#define INIT_UID_FROM_USER COMMON_INTERCEPT_FUNCTION(uid_from_user)
+#else
+#define INIT_UID_FROM_USER
+#endif
+
+#if SANITIZER_INTERCEPT_GROUP_FROM_GID
+INTERCEPTOR(const char *, group_from_gid, u32 gid, int nogroup) {
+  void *ctx;
+  const char *group;
+  COMMON_INTERCEPTOR_ENTER(ctx, group_from_gid, gid, nogroup);
+  group = REAL(group_from_gid)(gid, nogroup);
+  if (group)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, REAL(strlen)(group) + 1);
+  return group;
+}
+#define INIT_GROUP_FROM_GID COMMON_INTERCEPT_FUNCTION(group_from_gid)
+#else
+#define INIT_GROUP_FROM_GID
+#endif
+
+#if SANITIZER_INTERCEPT_GID_FROM_GROUP
+INTERCEPTOR(int, gid_from_group, const char *group, u32 *gid) {
+  void *ctx;
+  int res;
+  COMMON_INTERCEPTOR_ENTER(ctx, gid_from_group, group, gid);
+  if (group)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, group, REAL(strlen)(group) + 1);
+  res = REAL(gid_from_group)(group, gid);
+  if (gid)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, gid, sizeof(*gid));
+  return res;
+}
+#define INIT_GID_FROM_GROUP COMMON_INTERCEPT_FUNCTION(gid_from_group)
+#else
+#define INIT_GID_FROM_GROUP
+#endif
+
+#if SANITIZER_INTERCEPT_ACCESS
+INTERCEPTOR(int, access, const char *path, int mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, access, path, mode);
+  if (path)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  return REAL(access)(path, mode);
+}
+#define INIT_ACCESS COMMON_INTERCEPT_FUNCTION(access)
+#else
+#define INIT_ACCESS
+#endif
+
+#if SANITIZER_INTERCEPT_FACCESSAT
+INTERCEPTOR(int, faccessat, int fd, const char *path, int mode, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, faccessat, fd, path, mode, flags);
+  if (path)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  return REAL(faccessat)(fd, path, mode, flags);
+}
+#define INIT_FACCESSAT COMMON_INTERCEPT_FUNCTION(faccessat)
+#else
+#define INIT_FACCESSAT
+#endif
+
+#if SANITIZER_INTERCEPT_GETGROUPLIST
+INTERCEPTOR(int, getgrouplist, const char *name, u32 basegid, u32 *groups,
+            int *ngroups) {
+  void *ctx;
+  int res;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgrouplist, name, basegid, groups, ngroups);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  if (ngroups)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ngroups, sizeof(*ngroups));
+  res = REAL(getgrouplist)(name, basegid, groups, ngroups);
+  if (!res && groups && ngroups) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(*groups) * (*ngroups));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ngroups, sizeof(*ngroups));
+  }
+  return res;
+}
+
+#define INIT_GETGROUPLIST COMMON_INTERCEPT_FUNCTION(getgrouplist);
+#else
+#define INIT_GETGROUPLIST
+#endif
+
+#if SANITIZER_INTERCEPT_GETGROUPMEMBERSHIP
+INTERCEPTOR(int, getgroupmembership, const char *name, u32 basegid, u32 *groups,
+            int maxgrp, int *ngroups) {
+  void *ctx;
+  int res;
+  COMMON_INTERCEPTOR_ENTER(ctx, getgroupmembership, name, basegid, groups,
+                           maxgrp, ngroups);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  res = REAL(getgroupmembership)(name, basegid, groups, maxgrp, ngroups);
+  if (!res && groups && ngroups) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(*groups) * (*ngroups));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ngroups, sizeof(*ngroups));
+  }
+  return res;
+}
+
+#define INIT_GETGROUPMEMBERSHIP COMMON_INTERCEPT_FUNCTION(getgroupmembership);
+#else
+#define INIT_GETGROUPMEMBERSHIP
+#endif
+
+#if SANITIZER_INTERCEPT_READLINK
+INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) {
+  void* ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readlink, path, buf, bufsiz);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
+  return res;
+}
+
+#define INIT_READLINK COMMON_INTERCEPT_FUNCTION(readlink)
+#else
+#define INIT_READLINK
+#endif
+
+#if SANITIZER_INTERCEPT_READLINKAT
+INTERCEPTOR(SSIZE_T, readlinkat, int dirfd, const char *path, char *buf,
+            SIZE_T bufsiz) {
+  void* ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, readlinkat, dirfd, path, buf, bufsiz);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  SSIZE_T res = REAL(readlinkat)(dirfd, path, buf, bufsiz);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
+  return res;
+}
+
+#define INIT_READLINKAT COMMON_INTERCEPT_FUNCTION(readlinkat)
+#else
+#define INIT_READLINKAT
+#endif
+
+#if SANITIZER_INTERCEPT_NAME_TO_HANDLE_AT
+INTERCEPTOR(int, name_to_handle_at, int dirfd, const char *pathname,
+            struct file_handle *handle, int *mount_id, int flags) {
+  void* ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, name_to_handle_at, dirfd, pathname, handle,
+                           mount_id, flags);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, REAL(strlen)(pathname) + 1);
+
+  __sanitizer_file_handle *sanitizer_handle =
+      reinterpret_cast<__sanitizer_file_handle*>(handle);
+  COMMON_INTERCEPTOR_READ_RANGE(
+      ctx, &sanitizer_handle->handle_bytes,
+      sizeof(sanitizer_handle->handle_bytes));
+
+  int res = REAL(name_to_handle_at)(dirfd, pathname, handle, mount_id, flags);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(
+        ctx, &sanitizer_handle->handle_bytes,
+        sizeof(sanitizer_handle->handle_bytes));
+    COMMON_INTERCEPTOR_WRITE_RANGE(
+        ctx, &sanitizer_handle->handle_type,
+        sizeof(sanitizer_handle->handle_type));
+    COMMON_INTERCEPTOR_WRITE_RANGE(
+        ctx, &sanitizer_handle->f_handle, sanitizer_handle->handle_bytes);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mount_id, sizeof(*mount_id));
+  }
+  return res;
+}
+
+#define INIT_NAME_TO_HANDLE_AT COMMON_INTERCEPT_FUNCTION(name_to_handle_at)
+#else
+#define INIT_NAME_TO_HANDLE_AT
+#endif
+
+#if SANITIZER_INTERCEPT_OPEN_BY_HANDLE_AT
+INTERCEPTOR(int, open_by_handle_at, int mount_fd, struct file_handle* handle,
+            int flags) {
+  void* ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, open_by_handle_at, mount_fd, handle, flags);
+
+  __sanitizer_file_handle *sanitizer_handle =
+      reinterpret_cast<__sanitizer_file_handle*>(handle);
+  COMMON_INTERCEPTOR_READ_RANGE(
+      ctx, &sanitizer_handle->handle_bytes,
+      sizeof(sanitizer_handle->handle_bytes));
+  COMMON_INTERCEPTOR_READ_RANGE(
+      ctx, &sanitizer_handle->handle_type,
+      sizeof(sanitizer_handle->handle_type));
+  COMMON_INTERCEPTOR_READ_RANGE(
+      ctx, &sanitizer_handle->f_handle, sanitizer_handle->handle_bytes);
+
+  return REAL(open_by_handle_at)(mount_fd, handle, flags);
+}
+
+#define INIT_OPEN_BY_HANDLE_AT COMMON_INTERCEPT_FUNCTION(open_by_handle_at)
+#else
+#define INIT_OPEN_BY_HANDLE_AT
+#endif
+
+#if SANITIZER_INTERCEPT_STRLCPY
+INTERCEPTOR(SIZE_T, strlcpy, char *dst, char *src, SIZE_T size) {
+  void *ctx;
+  SIZE_T res;
+  COMMON_INTERCEPTOR_ENTER(ctx, strlcpy, dst, src, size);
+  if (src) {
+    // Keep strnlen as macro argument, as macro may ignore it.
+    COMMON_INTERCEPTOR_READ_STRING(
+        ctx, src, Min(internal_strnlen(src, size), size - 1) + 1);
+  }
+  res = REAL(strlcpy)(dst, src, size);
+  COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, REAL(strlen)(dst) + 1);
+  return res;
+}
+
+INTERCEPTOR(SIZE_T, strlcat, char *dst, char *src, SIZE_T size) {
+  void *ctx;
+  SIZE_T len = 0;
+  COMMON_INTERCEPTOR_ENTER(ctx, strlcat, dst, src, size);
+  // src is checked in the strlcpy() interceptor
+  if (dst) {
+    len = internal_strnlen(dst, size);
+    COMMON_INTERCEPTOR_READ_STRING(ctx, dst, Min(len, size - 1) + 1);
+  }
+  // Reuse the rest of the code in the strlcpy() interceptor
+  return WRAP(strlcpy)(dst + len, src, size - len) + len;
+}
+#define INIT_STRLCPY \
+  COMMON_INTERCEPT_FUNCTION(strlcpy); \
+  COMMON_INTERCEPT_FUNCTION(strlcat);
+#else
+#define INIT_STRLCPY
+#endif
+
+#if SANITIZER_INTERCEPT_MMAP
+INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd,
+            OFF_T off) {
+  void *ctx;
+  if (common_flags()->detect_write_exec)
+    ReportMmapWriteExec(prot);
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
+  COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
+  COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off);
+}
+
+INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
+  void *ctx;
+  if (common_flags()->detect_write_exec)
+    ReportMmapWriteExec(prot);
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return (int)internal_mprotect(addr, sz, prot);
+  COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);
+  MprotectMallocZones(addr, prot);
+  return REAL(mprotect)(addr, sz, prot);
+}
+#define INIT_MMAP                                                              \
+  COMMON_INTERCEPT_FUNCTION(mmap);                                             \
+  COMMON_INTERCEPT_FUNCTION(mprotect);
+#else
+#define INIT_MMAP
+#endif
+
+#if SANITIZER_INTERCEPT_MMAP64
+INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags, int fd,
+            OFF64_T off) {
+  void *ctx;
+  if (common_flags()->detect_write_exec)
+    ReportMmapWriteExec(prot);
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
+  COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
+  COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap64, addr, sz, prot, flags, fd, off);
+}
+#define INIT_MMAP64 COMMON_INTERCEPT_FUNCTION(mmap64);
+#else
+#define INIT_MMAP64
+#endif
+
+#if SANITIZER_INTERCEPT_DEVNAME
+INTERCEPTOR(char *, devname, u64 dev, u32 type) {
+  void *ctx;
+  char *name;
+  COMMON_INTERCEPTOR_ENTER(ctx, devname, dev, type);
+  name = REAL(devname)(dev, type);
+  if (name)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  return name;
+}
+#define INIT_DEVNAME COMMON_INTERCEPT_FUNCTION(devname);
+#else
+#define INIT_DEVNAME
+#endif
+
+#if SANITIZER_INTERCEPT_DEVNAME_R
+#if SANITIZER_NETBSD
+#define DEVNAME_R_RETTYPE int
+#define DEVNAME_R_SUCCESS(x) (!(x))
+#else
+#define DEVNAME_R_RETTYPE char*
+#define DEVNAME_R_SUCCESS(x) (x)
+#endif
+INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path,
+            uptr len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len);
+  DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len);
+  if (DEVNAME_R_SUCCESS(res))
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  return res;
+}
+#define INIT_DEVNAME_R COMMON_INTERCEPT_FUNCTION(devname_r);
+#else
+#define INIT_DEVNAME_R
+#endif
+
+#if SANITIZER_INTERCEPT_FGETLN
+INTERCEPTOR(char *, fgetln, __sanitizer_FILE *stream, SIZE_T *len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetln, stream, len);
+  char *str = REAL(fgetln)(stream, len);
+  if (str && len) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, str, *len);
+  }
+  return str;
+}
+#define INIT_FGETLN COMMON_INTERCEPT_FUNCTION(fgetln)
+#else
+#define INIT_FGETLN
+#endif
+
+#if SANITIZER_INTERCEPT_STRMODE
+INTERCEPTOR(void, strmode, u32 mode, char *bp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strmode, mode, bp);
+  REAL(strmode)(mode, bp);
+  if (bp)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, REAL(strlen)(bp) + 1);
+}
+#define INIT_STRMODE COMMON_INTERCEPT_FUNCTION(strmode)
+#else
+#define INIT_STRMODE
+#endif
+
+#if SANITIZER_INTERCEPT_TTYENT
+INTERCEPTOR(struct __sanitizer_ttyent *, getttyent, void) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getttyent);
+  struct __sanitizer_ttyent *ttyent = REAL(getttyent)();
+  if (ttyent)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);
+  return ttyent;
+}
+INTERCEPTOR(struct __sanitizer_ttyent *, getttynam, char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getttynam, name);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  struct __sanitizer_ttyent *ttyent = REAL(getttynam)(name);
+  if (ttyent)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);
+  return ttyent;
+}
+INTERCEPTOR(int, setttyentpath, char *path) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setttyentpath, path);
+  if (path)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  return REAL(setttyentpath)(path);
+}
+#define INIT_TTYENT \
+  COMMON_INTERCEPT_FUNCTION(getttyent); \
+  COMMON_INTERCEPT_FUNCTION(getttynam); \
+  COMMON_INTERCEPT_FUNCTION(setttyentpath)
+#else
+#define INIT_TTYENT
+#endif
+
+#if SANITIZER_INTERCEPT_PROTOENT
+static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
+
+  SIZE_T pp_size = 1; // One handles the trailing \0
+
+  for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, REAL(strlen)(*pp) + 1);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
+                                  pp_size * sizeof(char **));
+}
+
+INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
+  struct __sanitizer_protoent *p = REAL(getprotoent)();
+  if (p)
+    write_protoent(ctx, p);
+  return p;
+}
+
+INTERCEPTOR(struct __sanitizer_protoent *, getprotobyname, const char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname, name);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
+  if (p)
+    write_protoent(ctx, p);
+  return p;
+}
+
+INTERCEPTOR(struct __sanitizer_protoent *, getprotobynumber, int proto) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber, proto);
+  struct __sanitizer_protoent *p = REAL(getprotobynumber)(proto);
+  if (p)
+    write_protoent(ctx, p);
+  return p;
+}
+#define INIT_PROTOENT \
+  COMMON_INTERCEPT_FUNCTION(getprotoent); \
+  COMMON_INTERCEPT_FUNCTION(getprotobyname); \
+  COMMON_INTERCEPT_FUNCTION(getprotobynumber)
+#else
+#define INIT_PROTOENT
+#endif
+
+#if SANITIZER_INTERCEPT_PROTOENT_R
+INTERCEPTOR(int, getprotoent_r, struct __sanitizer_protoent *result_buf,
+            char *buf, SIZE_T buflen, struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent_r, result_buf, buf, buflen,
+                           result);
+  int res = REAL(getprotoent_r)(result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+INTERCEPTOR(int, getprotobyname_r, const char *name,
+            struct __sanitizer_protoent *result_buf, char *buf, SIZE_T buflen,
+            struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
+                           buflen, result);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+INTERCEPTOR(int, getprotobynumber_r, int num,
+            struct __sanitizer_protoent *result_buf, char *buf,
+            SIZE_T buflen, struct __sanitizer_protoent **result) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotobynumber_r, num, result_buf, buf,
+                           buflen, result);
+  int res = REAL(getprotobynumber_r)(num, result_buf, buf, buflen, result);
+
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
+  if (!res && *result)
+    write_protoent(ctx, *result);
+  return res;
+}
+
+#define INIT_PROTOENT_R \
+  COMMON_INTERCEPT_FUNCTION(getprotoent_r); \
+  COMMON_INTERCEPT_FUNCTION(getprotobyname_r); \
+  COMMON_INTERCEPT_FUNCTION(getprotobynumber_r);
+#else
+#define INIT_PROTOENT_R
+#endif
+
+#if SANITIZER_INTERCEPT_NETENT
+INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getnetent);
+  struct __sanitizer_netent *n = REAL(getnetent)();
+  if (n) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+
+    SIZE_T nn_size = 1; // One handles the trailing \0
+
+    for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
+                                   nn_size * sizeof(char **));
+  }
+  return n;
+}
+
+INTERCEPTOR(struct __sanitizer_netent *, getnetbyname, const char *name) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getnetbyname, name);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+  struct __sanitizer_netent *n = REAL(getnetbyname)(name);
+  if (n) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+
+    SIZE_T nn_size = 1; // One handles the trailing \0
+
+    for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
+                                   nn_size * sizeof(char **));
+  }
+  return n;
+}
+
+INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getnetbyaddr, net, type);
+  struct __sanitizer_netent *n = REAL(getnetbyaddr)(net, type);
+  if (n) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
+
+    SIZE_T nn_size = 1; // One handles the trailing \0
+
+    for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, REAL(strlen)(*nn) + 1);
+
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
+                                   nn_size * sizeof(char **));
+  }
+  return n;
+}
+#define INIT_NETENT \
+  COMMON_INTERCEPT_FUNCTION(getnetent); \
+  COMMON_INTERCEPT_FUNCTION(getnetbyname); \
+  COMMON_INTERCEPT_FUNCTION(getnetbyaddr)
+#else
+#define INIT_NETENT
+#endif
+
+#if SANITIZER_INTERCEPT_GETMNTINFO
+INTERCEPTOR(int, getmntinfo, void **mntbufp, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getmntinfo, mntbufp, flags);
+  int cnt = REAL(getmntinfo)(mntbufp, flags);
+  if (cnt > 0 && mntbufp) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mntbufp, sizeof(void *));
+    if (*mntbufp)
+#if SANITIZER_NETBSD
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statvfs_sz);
+#else
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statfs_sz);
+#endif
+  }
+  return cnt;
+}
+#define INIT_GETMNTINFO COMMON_INTERCEPT_FUNCTION(getmntinfo)
+#else
+#define INIT_GETMNTINFO
+#endif
+
+#if SANITIZER_INTERCEPT_MI_VECTOR_HASH
+INTERCEPTOR(void, mi_vector_hash, const void *key, SIZE_T len, u32 seed,
+            u32 hashes[3]) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mi_vector_hash, key, len, seed, hashes);
+  if (key)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, key, len);
+  REAL(mi_vector_hash)(key, len, seed, hashes);
+  if (hashes)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hashes, sizeof(hashes[0]) * 3);
+}
+#define INIT_MI_VECTOR_HASH COMMON_INTERCEPT_FUNCTION(mi_vector_hash)
+#else
+#define INIT_MI_VECTOR_HASH
+#endif
+
+#if SANITIZER_INTERCEPT_SETVBUF
+INTERCEPTOR(int, setvbuf, __sanitizer_FILE *stream, char *buf, int mode,
+  SIZE_T size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setvbuf, stream, buf, mode, size);
+  int ret = REAL(setvbuf)(stream, buf, mode, size);
+  if (buf)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
+  if (stream)
+      unpoison_file(stream);
+  return ret;
+}
+
+INTERCEPTOR(void, setbuf, __sanitizer_FILE *stream, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setbuf, stream, buf);
+  REAL(setbuf)(stream, buf);
+  if (buf) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);
+  }
+  if (stream)
+      unpoison_file(stream);
+}
+
+INTERCEPTOR(void, setbuffer, __sanitizer_FILE *stream, char *buf, int mode) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode);
+  REAL(setbuffer)(stream, buf, mode);
+  if (buf) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);
+  }
+  if (stream)
+    unpoison_file(stream);
+}
+
+INTERCEPTOR(void, setlinebuf, __sanitizer_FILE *stream) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, setlinebuf, stream);
+  REAL(setlinebuf)(stream);
+  if (stream)
+    unpoison_file(stream);
+}
+#define INIT_SETVBUF COMMON_INTERCEPT_FUNCTION(setvbuf); \
+    COMMON_INTERCEPT_FUNCTION(setbuf); \
+    COMMON_INTERCEPT_FUNCTION(setbuffer); \
+    COMMON_INTERCEPT_FUNCTION(setlinebuf)
+#else
+#define INIT_SETVBUF
+#endif
+
+#if SANITIZER_INTERCEPT_GETVFSSTAT
+INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getvfsstat, buf, bufsize, flags);
+  int ret = REAL(getvfsstat)(buf, bufsize, flags);
+  if (buf && ret > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, ret * struct_statvfs_sz);
+  return ret;
+}
+#define INIT_GETVFSSTAT COMMON_INTERCEPT_FUNCTION(getvfsstat)
+#else
+#define INIT_GETVFSSTAT
+#endif
+
+#if SANITIZER_INTERCEPT_REGEX
+INTERCEPTOR(int, regcomp, void *preg, const char *pattern, int cflags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags);
+  if (pattern)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1);
+  int res = REAL(regcomp)(preg, pattern, cflags);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz);
+  return res;
+}
+INTERCEPTOR(int, regexec, const void *preg, const char *string, SIZE_T nmatch,
+            struct __sanitizer_regmatch *pmatch[], int eflags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regexec, preg, string, nmatch, pmatch, eflags);
+  if (preg)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
+  if (string)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1);
+  int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags);
+  if (!res && pmatch)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz);
+  return res;
+}
+INTERCEPTOR(SIZE_T, regerror, int errcode, const void *preg, char *errbuf,
+            SIZE_T errbuf_size) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regerror, errcode, preg, errbuf, errbuf_size);
+  if (preg)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
+  SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size);
+  if (errbuf)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1);
+  return res;
+}
+INTERCEPTOR(void, regfree, const void *preg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regfree, preg);
+  if (preg)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
+  REAL(regfree)(preg);
+}
+#define INIT_REGEX                                                             \
+  COMMON_INTERCEPT_FUNCTION(regcomp);                                          \
+  COMMON_INTERCEPT_FUNCTION(regexec);                                          \
+  COMMON_INTERCEPT_FUNCTION(regerror);                                         \
+  COMMON_INTERCEPT_FUNCTION(regfree);
+#else
+#define INIT_REGEX
+#endif
+
+#if SANITIZER_INTERCEPT_REGEXSUB
+INTERCEPTOR(SSIZE_T, regnsub, char *buf, SIZE_T bufsiz, const char *sub,
+            const struct __sanitizer_regmatch *rm, const char *str) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str);
+  if (sub)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
+  // The implementation demands and hardcodes 10 elements
+  if (rm)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
+  if (str)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
+  SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str);
+  if (res > 0 && buf)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+  return res;
+}
+INTERCEPTOR(SSIZE_T, regasub, char **buf, const char *sub,
+            const struct __sanitizer_regmatch *rm, const char *sstr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr);
+  if (sub)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
+  // Hardcode 10 elements as this is hardcoded size
+  if (rm)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
+  if (sstr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1);
+  SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr);
+  if (res > 0 && buf) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *buf, REAL(strlen)(*buf) + 1);
+  }
+  return res;
+}
+
+#define INIT_REGEXSUB                                                          \
+  COMMON_INTERCEPT_FUNCTION(regnsub);                                          \
+  COMMON_INTERCEPT_FUNCTION(regasub);
+#else
+#define INIT_REGEXSUB
+#endif
+
+#if SANITIZER_INTERCEPT_FTS
+INTERCEPTOR(void *, fts_open, char *const *path_argv, int options,
+            int (*compar)(void **, void **)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fts_open, path_argv, options, compar);
+  if (path_argv) {
+    for (char *const *pa = path_argv; ; ++pa) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
+      if (!*pa)
+        break;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+    }
+  }
+  // TODO(kamil): handle compar callback
+  void *fts = REAL(fts_open)(path_argv, options, compar);
+  if (fts)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, fts, struct_FTS_sz);
+  return fts;
+}
+
+INTERCEPTOR(void *, fts_read, void *ftsp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fts_read, ftsp);
+  if (ftsp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz);
+  void *ftsent = REAL(fts_read)(ftsp);
+  if (ftsent)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz);
+  return ftsent;
+}
+
+INTERCEPTOR(void *, fts_children, void *ftsp, int options) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fts_children, ftsp, options);
+  if (ftsp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz);
+  void *ftsent = REAL(fts_children)(ftsp, options);
+  if (ftsent)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ftsent, struct_FTSENT_sz);
+  return ftsent;
+}
+
+INTERCEPTOR(int, fts_set, void *ftsp, void *f, int options) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fts_set, ftsp, f, options);
+  if (ftsp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz);
+  if (f)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, f, struct_FTSENT_sz);
+  return REAL(fts_set)(ftsp, f, options);
+}
+
+INTERCEPTOR(int, fts_close, void *ftsp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fts_close, ftsp);
+  if (ftsp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, ftsp, struct_FTS_sz);
+  return REAL(fts_close)(ftsp);
+}
+#define INIT_FTS                                                               \
+  COMMON_INTERCEPT_FUNCTION(fts_open);                                         \
+  COMMON_INTERCEPT_FUNCTION(fts_read);                                         \
+  COMMON_INTERCEPT_FUNCTION(fts_children);                                     \
+  COMMON_INTERCEPT_FUNCTION(fts_set);                                          \
+  COMMON_INTERCEPT_FUNCTION(fts_close);
+#else
+#define INIT_FTS
+#endif
+
+#if SANITIZER_INTERCEPT_SYSCTL
+INTERCEPTOR(int, sysctl, int *name, unsigned int namelen, void *oldp,
+            SIZE_T *oldlenp, void *newp, SIZE_T newlen) {
+  void *ctx;
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_sysctl(name, namelen, oldp, oldlenp, newp, newlen);
+  COMMON_INTERCEPTOR_ENTER(ctx, sysctl, name, namelen, oldp, oldlenp, newp,
+                           newlen);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, namelen * sizeof(*name));
+  if (oldlenp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));
+  if (newp && newlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen);
+  int res = REAL(sysctl)(name, namelen, oldp, oldlenp, newp, newlen);
+  if (!res) {
+    if (oldlenp) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp));
+      if (oldp)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp);
+    }
+  }
+  return res;
+}
+
+INTERCEPTOR(int, sysctlbyname, char *sname, void *oldp, SIZE_T *oldlenp,
+            void *newp, SIZE_T newlen) {
+  void *ctx;
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_sysctlbyname(sname, oldp, oldlenp, newp, newlen);
+  COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp,
+                           newlen);
+  if (sname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+  if (oldlenp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));
+  if (newp && newlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, newp, newlen);
+  int res = REAL(sysctlbyname)(sname, oldp, oldlenp, newp, newlen);
+  if (!res) {
+    if (oldlenp) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldlenp, sizeof(*oldlenp));
+      if (oldp)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldp, *oldlenp);
+    }
+  }
+  return res;
+}
+
+INTERCEPTOR(int, sysctlnametomib, const char *sname, int *name,
+            SIZE_T *namelenp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp);
+  if (sname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+  if (namelenp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
+  int res = REAL(sysctlnametomib)(sname, name, namelenp);
+  if (!res) {
+    if (namelenp) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp));
+      if (name)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name));
+    }
+  }
+  return res;
+}
+
+#define INIT_SYSCTL                        \
+  COMMON_INTERCEPT_FUNCTION(sysctl);       \
+  COMMON_INTERCEPT_FUNCTION(sysctlbyname); \
+  COMMON_INTERCEPT_FUNCTION(sysctlnametomib);
+#else
+#define INIT_SYSCTL
+#endif
+
+#if SANITIZER_INTERCEPT_ASYSCTL
+INTERCEPTOR(void *, asysctl, const int *name, SIZE_T namelen, SIZE_T *len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, asysctl, name, namelen, len);
+  if (name)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, name, sizeof(*name) * namelen);
+  void *res = REAL(asysctl)(name, namelen, len);
+  if (res && len) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len);
+  }
+  return res;
+}
+
+INTERCEPTOR(void *, asysctlbyname, const char *sname, SIZE_T *len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len);
+  if (sname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+  void *res = REAL(asysctlbyname)(sname, len);
+  if (res && len) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, *len);
+  }
+  return res;
+}
+#define INIT_ASYSCTL                           \
+  COMMON_INTERCEPT_FUNCTION(asysctl);          \
+  COMMON_INTERCEPT_FUNCTION(asysctlbyname);
+#else
+#define INIT_ASYSCTL
+#endif
+
+#if SANITIZER_INTERCEPT_SYSCTLGETMIBINFO
+INTERCEPTOR(int, sysctlgetmibinfo, char *sname, int *name,
+            unsigned int *namelenp, char *cname, SIZE_T *csz, void **rnode,
+            int v) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname,
+                           csz, rnode, v);
+  if (sname)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
+  if (namelenp)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
+  if (csz)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, csz, sizeof(*csz));
+  // Skip rnode, it's rarely used and not trivial to sanitize
+  // It's also used mostly internally
+  int res = REAL(sysctlgetmibinfo)(sname, name, namelenp, cname, csz, rnode, v);
+  if (!res) {
+    if (namelenp) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelenp, sizeof(*namelenp));
+      if (name)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, *namelenp * sizeof(*name));
+    }
+    if (csz) {
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, csz, sizeof(*csz));
+      if (cname)
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cname, *csz);
+    }
+  }
+  return res;
+}
+#define INIT_SYSCTLGETMIBINFO                  \
+  COMMON_INTERCEPT_FUNCTION(sysctlgetmibinfo);
+#else
+#define INIT_SYSCTLGETMIBINFO
+#endif
+
+#if SANITIZER_INTERCEPT_NL_LANGINFO
+INTERCEPTOR(char *, nl_langinfo, long item) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item);
+  char *ret = REAL(nl_langinfo)(item);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
+  return ret;
+}
+#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo)
+#else
+#define INIT_NL_LANGINFO
+#endif
+
+#if SANITIZER_INTERCEPT_MODCTL
+INTERCEPTOR(int, modctl, int operation, void *argp) {
+  void *ctx;
+  int ret;
+  COMMON_INTERCEPTOR_ENTER(ctx, modctl, operation, argp);
+
+  if (operation == modctl_load) {
+    if (argp) {
+      __sanitizer_modctl_load_t *ml = (__sanitizer_modctl_load_t *)argp;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml));
+      if (ml->ml_filename)
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename,
+                                      REAL(strlen)(ml->ml_filename) + 1);
+      if (ml->ml_props)
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen);
+    }
+    ret = REAL(modctl)(operation, argp);
+  } else if (operation == modctl_unload) {
+    if (argp) {
+      const char *name = (const char *)argp;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+    }
+    ret = REAL(modctl)(operation, argp);
+  } else if (operation == modctl_stat) {
+    uptr iov_len;
+    struct __sanitizer_iovec *iov = (struct __sanitizer_iovec *)argp;
+    if (iov) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, iov, sizeof(*iov));
+      iov_len = iov->iov_len;
+    }
+    ret = REAL(modctl)(operation, argp);
+    if (iov)
+      COMMON_INTERCEPTOR_WRITE_RANGE(
+          ctx, iov->iov_base, Min(iov_len,  iov->iov_len));
+  } else if (operation == modctl_exists) {
+    ret = REAL(modctl)(operation, argp);
+  } else {
+    ret = REAL(modctl)(operation, argp);
+  }
+
+  return ret;
+}
+#define INIT_MODCTL COMMON_INTERCEPT_FUNCTION(modctl)
+#else
+#define INIT_MODCTL
+#endif
+
+#if SANITIZER_INTERCEPT_STRTONUM
+INTERCEPTOR(long long, strtonum, const char *nptr, long long minval,
+            long long maxval, const char **errstr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtonum, nptr, minval, maxval, errstr);
+
+  // TODO(kamil): Implement strtoll as a common inteceptor
+  char *real_endptr;
+  long long ret = (long long)REAL(strtoimax)(nptr, &real_endptr, 10);
+  StrtolFixAndCheck(ctx, nptr, nullptr, real_endptr, 10);
+
+  ret = REAL(strtonum)(nptr, minval, maxval, errstr);
+  if (errstr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *));
+     if (*errstr)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *errstr, REAL(strlen)(*errstr) + 1);
+  }
+  return ret;
+}
+#define INIT_STRTONUM COMMON_INTERCEPT_FUNCTION(strtonum)
+#else
+#define INIT_STRTONUM
+#endif
+
+#if SANITIZER_INTERCEPT_FPARSELN
+INTERCEPTOR(char *, fparseln, __sanitizer_FILE *stream, SIZE_T *len,
+            SIZE_T *lineno, const char delim[3], int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fparseln, stream, len, lineno, delim, flags);
+  if (lineno)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, lineno, sizeof(*lineno));
+  if (delim)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3);
+  char *ret = REAL(fparseln)(stream, len, lineno, delim, flags);
+  if (ret) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
+    if (len)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
+    if (lineno)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineno, sizeof(*lineno));
+  }
+  return ret;
+}
+#define INIT_FPARSELN COMMON_INTERCEPT_FUNCTION(fparseln)
+#else
+#define INIT_FPARSELN
+#endif
+
+#if SANITIZER_INTERCEPT_STATVFS1
+INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  int res = REAL(statvfs1)(path, buf, flags);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
+  return res;
+}
+INTERCEPTOR(int, fstatvfs1, int fd, void *buf, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs1, fd, buf, flags);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  int res = REAL(fstatvfs1)(fd, buf, flags);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
+    if (fd >= 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return res;
+}
+#define INIT_STATVFS1                  \
+  COMMON_INTERCEPT_FUNCTION(statvfs1);  \
+  COMMON_INTERCEPT_FUNCTION(fstatvfs1);
+#else
+#define INIT_STATVFS1
+#endif
+
+#if SANITIZER_INTERCEPT_STRTOI
+INTERCEPTOR(INTMAX_T, strtoi, const char *nptr, char **endptr, int base,
+            INTMAX_T low, INTMAX_T high, int *rstatus) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtoi, nptr, endptr, base, low, high, rstatus);
+  char *real_endptr;
+  INTMAX_T ret = REAL(strtoi)(nptr, &real_endptr, base, low, high, rstatus);
+  StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+  if (rstatus)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus));
+  return ret;
+}
+
+INTERCEPTOR(UINTMAX_T, strtou, const char *nptr, char **endptr, int base,
+            UINTMAX_T low, UINTMAX_T high, int *rstatus) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strtou, nptr, endptr, base, low, high, rstatus);
+  char *real_endptr;
+  UINTMAX_T ret = REAL(strtou)(nptr, &real_endptr, base, low, high, rstatus);
+  StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+  if (rstatus)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rstatus, sizeof(*rstatus));
+  return ret;
+}
+#define INIT_STRTOI                                                            \
+  COMMON_INTERCEPT_FUNCTION(strtoi);                                           \
+  COMMON_INTERCEPT_FUNCTION(strtou)
+#else
+#define INIT_STRTOI
+#endif
+
+#if SANITIZER_INTERCEPT_CAPSICUM
+#define CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights, ...)          \
+  {                                                                        \
+    void *ctx;                                                             \
+    COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_init, rights, ##__VA_ARGS__); \
+    if (rights)                                                            \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));         \
+    __sanitizer_cap_rights_t *ret =                                        \
+        REAL(cap_rights_init)(rights, ##__VA_ARGS__);                      \
+    if (ret)                                                               \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret));              \
+    return ret;                                                            \
+  }
+
+#define CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights, ...)           \
+  {                                                                       \
+    void *ctx;                                                            \
+    COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_set, rights, ##__VA_ARGS__); \
+    if (rights)                                                           \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));        \
+    __sanitizer_cap_rights_t *ret =                                       \
+        REAL(cap_rights_set)(rights, ##__VA_ARGS__);                      \
+    if (ret)                                                              \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret));             \
+    return ret;                                                           \
+  }
+
+#define CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights, ...)         \
+  {                                                                         \
+    void *ctx;                                                              \
+    COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_clear, rights, ##__VA_ARGS__); \
+    if (rights)                                                             \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));          \
+    __sanitizer_cap_rights_t *ret =                                         \
+        REAL(cap_rights_clear)(rights, ##__VA_ARGS__);                      \
+    if (ret)                                                                \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret));               \
+    return ret;                                                             \
+  }
+
+#define CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights, ...)        \
+  {                                                                          \
+    void *ctx;                                                               \
+    COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_set, rights, ##__VA_ARGS__); \
+    if (rights)                                                              \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));           \
+    return REAL(cap_rights_is_set)(rights, ##__VA_ARGS__);                   \
+  }
+
+INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_init,
+            __sanitizer_cap_rights_t *rights) {
+  CAP_RIGHTS_INIT_INTERCEPTOR(cap_rights_init, rights);
+}
+
+INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_set,
+            __sanitizer_cap_rights_t *rights) {
+  CAP_RIGHTS_SET_INTERCEPTOR(cap_rights_set, rights);
+}
+
+INTERCEPTOR(__sanitizer_cap_rights_t *, cap_rights_clear,
+            __sanitizer_cap_rights_t *rights) {
+  CAP_RIGHTS_CLEAR_INTERCEPTOR(cap_rights_clear, rights);
+}
+
+INTERCEPTOR(bool, cap_rights_is_set,
+            __sanitizer_cap_rights_t *rights) {
+  CAP_RIGHTS_IS_SET_INTERCEPTOR(cap_rights_is_set, rights);
+}
+
+INTERCEPTOR(int, cap_rights_limit, int fd,
+            const __sanitizer_cap_rights_t *rights) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_limit, fd, rights);
+  if (rights)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));
+
+  return REAL(cap_rights_limit)(fd, rights);
+}
+
+INTERCEPTOR(int, cap_rights_get, int fd, __sanitizer_cap_rights_t *rights) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_get, fd, rights);
+  int ret = REAL(cap_rights_get)(fd, rights);
+  if (!ret && rights)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rights, sizeof(*rights));
+
+  return ret;
+}
+
+INTERCEPTOR(bool, cap_rights_is_valid, const __sanitizer_cap_rights_t *rights) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_is_valid, rights);
+  if (rights)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, rights, sizeof(*rights));
+
+  return REAL(cap_rights_is_valid(rights));
+}
+
+INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_merge,
+  __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_merge, dst, src);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+
+  __sanitizer_cap_rights *ret = REAL(cap_rights_merge)(dst, src);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+
+  return ret;
+}
+
+INTERCEPTOR(__sanitizer_cap_rights *, cap_rights_remove,
+  __sanitizer_cap_rights *dst, const __sanitizer_cap_rights *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_remove, dst, src);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, sizeof(*src));
+
+  __sanitizer_cap_rights *ret = REAL(cap_rights_remove)(dst, src);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+
+  return ret;
+}
+
+INTERCEPTOR(bool, cap_rights_contains, const __sanitizer_cap_rights *big,
+  const __sanitizer_cap_rights *little) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_rights_contains, big, little);
+  if (little)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, little, sizeof(*little));
+  if (big)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, big, sizeof(*big));
+
+  return REAL(cap_rights_contains)(big, little);
+}
+
+INTERCEPTOR(int, cap_ioctls_limit, int fd, const uptr *cmds, SIZE_T ncmds) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_limit, fd, cmds, ncmds);
+  if (cmds)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cmds, sizeof(*cmds) * ncmds);
+
+  return REAL(cap_ioctls_limit)(fd, cmds, ncmds);
+}
+
+INTERCEPTOR(int, cap_ioctls_get, int fd, uptr *cmds, SIZE_T maxcmds) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cap_ioctls_get, fd, cmds, maxcmds);
+  int ret = REAL(cap_ioctls_get)(fd, cmds, maxcmds);
+  if (!ret && cmds)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cmds, sizeof(*cmds) * maxcmds);
+
+  return ret;
+}
+#define INIT_CAPSICUM                          \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_init); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_set); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_clear); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_is_set); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_get);   \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_limit); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_contains); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_remove); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_merge); \
+  COMMON_INTERCEPT_FUNCTION(cap_rights_is_valid); \
+  COMMON_INTERCEPT_FUNCTION(cap_ioctls_get);   \
+  COMMON_INTERCEPT_FUNCTION(cap_ioctls_limit)
+#else
+#define INIT_CAPSICUM
+#endif
+
+#if SANITIZER_INTERCEPT_SHA1
+INTERCEPTOR(void, SHA1Init, void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1Init, context);
+  REAL(SHA1Init)(context);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz);
+}
+INTERCEPTOR(void, SHA1Update, void *context, const u8 *data, unsigned len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1Update, context, data, len);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz);
+  REAL(SHA1Update)(context, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA1_CTX_sz);
+}
+INTERCEPTOR(void, SHA1Final, u8 digest[20], void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1Final, digest, context);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz);
+  REAL(SHA1Final)(digest, context);
+  if (digest)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20);
+}
+INTERCEPTOR(void, SHA1Transform, u32 state[5], u8 buffer[64]) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1Transform, state, buffer);
+  if (state)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5);
+  if (buffer)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u8) * 64);
+  REAL(SHA1Transform)(state, buffer);
+  if (state)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5);
+}
+INTERCEPTOR(char *, SHA1End, void *context, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1End, context, buf);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA1_CTX_sz);
+  char *ret = REAL(SHA1End)(context, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, SHA1File, char *filename, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(SHA1File)(filename, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, SHA1FileChunk, char *filename, char *buf, OFF_T offset,
+  OFF_T length) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, SHA1Data, u8 *data, SIZE_T len, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, SHA1Data, data, len, buf);
+  if (data)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  char *ret = REAL(SHA1Data)(data, len, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
+  return ret;
+}
+#define INIT_SHA1                                                              \
+  COMMON_INTERCEPT_FUNCTION(SHA1Init);                                         \
+  COMMON_INTERCEPT_FUNCTION(SHA1Update);                                       \
+  COMMON_INTERCEPT_FUNCTION(SHA1Final);                                        \
+  COMMON_INTERCEPT_FUNCTION(SHA1Transform);                                    \
+  COMMON_INTERCEPT_FUNCTION(SHA1End);                                          \
+  COMMON_INTERCEPT_FUNCTION(SHA1File);                                         \
+  COMMON_INTERCEPT_FUNCTION(SHA1FileChunk);                                    \
+  COMMON_INTERCEPT_FUNCTION(SHA1Data)
+#else
+#define INIT_SHA1
+#endif
+
+#if SANITIZER_INTERCEPT_MD4
+INTERCEPTOR(void, MD4Init, void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4Init, context);
+  REAL(MD4Init)(context);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz);
+}
+
+INTERCEPTOR(void, MD4Update, void *context, const unsigned char *data,
+            unsigned int len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4Update, context, data, len);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz);
+  REAL(MD4Update)(context, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD4_CTX_sz);
+}
+
+INTERCEPTOR(void, MD4Final, unsigned char digest[16], void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4Final, digest, context);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz);
+  REAL(MD4Final)(digest, context);
+  if (digest)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16);
+}
+
+INTERCEPTOR(char *, MD4End, void *context, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4End, context, buf);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD4_CTX_sz);
+  char *ret = REAL(MD4End)(context, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD4File, const char *filename, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(MD4File)(filename, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD4Data, const unsigned char *data, unsigned int len,
+            char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD4Data, data, len, buf);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  char *ret = REAL(MD4Data)(data, len, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
+  return ret;
+}
+
+#define INIT_MD4                                                               \
+  COMMON_INTERCEPT_FUNCTION(MD4Init);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD4Update);                                        \
+  COMMON_INTERCEPT_FUNCTION(MD4Final);                                         \
+  COMMON_INTERCEPT_FUNCTION(MD4End);                                           \
+  COMMON_INTERCEPT_FUNCTION(MD4File);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD4Data)
+#else
+#define INIT_MD4
+#endif
+
+#if SANITIZER_INTERCEPT_RMD160
+INTERCEPTOR(void, RMD160Init, void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160Init, context);
+  REAL(RMD160Init)(context);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz);
+}
+INTERCEPTOR(void, RMD160Update, void *context, const u8 *data, unsigned len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160Update, context, data, len);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz);
+  REAL(RMD160Update)(context, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, RMD160_CTX_sz);
+}
+INTERCEPTOR(void, RMD160Final, u8 digest[20], void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160Final, digest, context);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz);
+  REAL(RMD160Final)(digest, context);
+  if (digest)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(u8) * 20);
+}
+INTERCEPTOR(void, RMD160Transform, u32 state[5], u16 buffer[16]) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160Transform, state, buffer);
+  if (state)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, state, sizeof(u32) * 5);
+  if (buffer)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, sizeof(u32) * 16);
+  REAL(RMD160Transform)(state, buffer);
+  if (state)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, state, sizeof(u32) * 5);
+}
+INTERCEPTOR(char *, RMD160End, void *context, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160End, context, buf);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, RMD160_CTX_sz);
+  char *ret = REAL(RMD160End)(context, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, RMD160File, char *filename, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(RMD160File)(filename, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, RMD160FileChunk, char *filename, char *buf, OFF_T offset,
+  OFF_T length) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
+  return ret;
+}
+INTERCEPTOR(char *, RMD160Data, u8 *data, SIZE_T len, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, RMD160Data, data, len, buf);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  char *ret = REAL(RMD160Data)(data, len, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
+  return ret;
+}
+#define INIT_RMD160                                                            \
+  COMMON_INTERCEPT_FUNCTION(RMD160Init);                                       \
+  COMMON_INTERCEPT_FUNCTION(RMD160Update);                                     \
+  COMMON_INTERCEPT_FUNCTION(RMD160Final);                                      \
+  COMMON_INTERCEPT_FUNCTION(RMD160Transform);                                  \
+  COMMON_INTERCEPT_FUNCTION(RMD160End);                                        \
+  COMMON_INTERCEPT_FUNCTION(RMD160File);                                       \
+  COMMON_INTERCEPT_FUNCTION(RMD160FileChunk);                                  \
+  COMMON_INTERCEPT_FUNCTION(RMD160Data)
+#else
+#define INIT_RMD160
+#endif
+
+#if SANITIZER_INTERCEPT_MD5
+INTERCEPTOR(void, MD5Init, void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context);
+  REAL(MD5Init)(context);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
+}
+
+INTERCEPTOR(void, MD5Update, void *context, const unsigned char *data,
+            unsigned int len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
+  REAL(MD5Update)(context, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
+}
+
+INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
+  REAL(MD5Final)(digest, context);
+  if (digest)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16);
+}
+
+INTERCEPTOR(char *, MD5End, void *context, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
+  char *ret = REAL(MD5End)(context, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD5File, const char *filename, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(MD5File)(filename, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD5Data, const unsigned char *data, unsigned int len,
+            char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  char *ret = REAL(MD5Data)(data, len, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
+  return ret;
+}
+
+#define INIT_MD5                                                               \
+  COMMON_INTERCEPT_FUNCTION(MD5Init);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD5Update);                                        \
+  COMMON_INTERCEPT_FUNCTION(MD5Final);                                         \
+  COMMON_INTERCEPT_FUNCTION(MD5End);                                           \
+  COMMON_INTERCEPT_FUNCTION(MD5File);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD5Data)
+#else
+#define INIT_MD5
+#endif
+
+#if SANITIZER_INTERCEPT_FSEEK
+INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fseek, stream, offset, whence);
+  return REAL(fseek)(stream, offset, whence);
+}
+INTERCEPTOR(int, fseeko, __sanitizer_FILE *stream, OFF_T offset, int whence) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fseeko, stream, offset, whence);
+  return REAL(fseeko)(stream, offset, whence);
+}
+INTERCEPTOR(long int, ftell, __sanitizer_FILE *stream) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ftell, stream);
+  return REAL(ftell)(stream);
+}
+INTERCEPTOR(OFF_T, ftello, __sanitizer_FILE *stream) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, ftello, stream);
+  return REAL(ftello)(stream);
+}
+INTERCEPTOR(void, rewind, __sanitizer_FILE *stream) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, rewind, stream);
+  return REAL(rewind)(stream);
+}
+INTERCEPTOR(int, fgetpos, __sanitizer_FILE *stream, void *pos) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fgetpos, stream, pos);
+  int ret = REAL(fgetpos)(stream, pos);
+  if (pos && !ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pos, fpos_t_sz);
+  return ret;
+}
+INTERCEPTOR(int, fsetpos, __sanitizer_FILE *stream, const void *pos) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fsetpos, stream, pos);
+  if (pos)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, pos, fpos_t_sz);
+  return REAL(fsetpos)(stream, pos);
+}
+#define INIT_FSEEK \
+  COMMON_INTERCEPT_FUNCTION(fseek); \
+  COMMON_INTERCEPT_FUNCTION(fseeko); \
+  COMMON_INTERCEPT_FUNCTION(ftell); \
+  COMMON_INTERCEPT_FUNCTION(ftello); \
+  COMMON_INTERCEPT_FUNCTION(rewind); \
+  COMMON_INTERCEPT_FUNCTION(fgetpos); \
+  COMMON_INTERCEPT_FUNCTION(fsetpos)
+#else
+#define INIT_FSEEK
+#endif
+
+#if SANITIZER_INTERCEPT_MD2
+INTERCEPTOR(void, MD2Init, void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2Init, context);
+  REAL(MD2Init)(context);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz);
+}
+
+INTERCEPTOR(void, MD2Update, void *context, const unsigned char *data,
+            unsigned int len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2Update, context, data, len);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz);
+  REAL(MD2Update)(context, data, len);
+  if (context)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD2_CTX_sz);
+}
+
+INTERCEPTOR(void, MD2Final, unsigned char digest[16], void *context) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2Final, digest, context);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz);
+  REAL(MD2Final)(digest, context);
+  if (digest)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16);
+}
+
+INTERCEPTOR(char *, MD2End, void *context, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2End, context, buf);
+  if (context)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD2_CTX_sz);
+  char *ret = REAL(MD2End)(context, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD2File, const char *filename, char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf);
+  if (filename)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
+  char *ret = REAL(MD2File)(filename, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
+  return ret;
+}
+
+INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len,
+            char *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, MD2Data, data, len, buf);
+  if (data && len > 0)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
+  char *ret = REAL(MD2Data)(data, len, buf);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
+  return ret;
+}
+
+#define INIT_MD2                                                               \
+  COMMON_INTERCEPT_FUNCTION(MD2Init);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD2Update);                                        \
+  COMMON_INTERCEPT_FUNCTION(MD2Final);                                         \
+  COMMON_INTERCEPT_FUNCTION(MD2End);                                           \
+  COMMON_INTERCEPT_FUNCTION(MD2File);                                          \
+  COMMON_INTERCEPT_FUNCTION(MD2Data)
+#else
+#define INIT_MD2
+#endif
+
+#if SANITIZER_INTERCEPT_SHA2
+#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \
+  INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \
+    REAL(SHA##LEN##_Init)(context); \
+    if (context) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
+  } \
+  INTERCEPTOR(void, SHA##LEN##_Update, void *context, \
+              const u8 *data, SIZE_T len) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \
+    if (data && len > 0) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
+    if (context) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
+    REAL(SHA##LEN##_Update)(context, data, len); \
+    if (context) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
+  } \
+  INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \
+  void *context) { \
+    void *ctx; \
+    CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \
+    if (context) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
+    REAL(SHA##LEN##_Final)(digest, context); \
+    if (digest) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \
+                                     sizeof(digest[0]) * \
+  SHA##LEN##_digest_length); \
+  } \
+  INTERCEPTOR(char *, SHA##LEN##_End, void *context, char *buf) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \
+    if (context) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
+    char *ret = REAL(SHA##LEN##_End)(context, buf); \
+    if (ret) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
+    return ret; \
+  } \
+  INTERCEPTOR(char *, SHA##LEN##_File, const char *filename, char *buf) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
+    if (filename) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
+    char *ret = REAL(SHA##LEN##_File)(filename, buf); \
+    if (ret) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
+    return ret; \
+  } \
+  INTERCEPTOR(char *, SHA##LEN##_FileChunk, const char *filename, char *buf, \
+              OFF_T offset, OFF_T length) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
+  length); \
+    if (filename) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
+    char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
+    if (ret) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
+    return ret; \
+  } \
+  INTERCEPTOR(char *, SHA##LEN##_Data, u8 *data, SIZE_T len, char *buf) { \
+    void *ctx; \
+    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \
+    if (data && len > 0) \
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
+    char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \
+    if (ret) \
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
+    return ret; \
+  }
+
+SHA2_INTERCEPTORS(224, u32);
+SHA2_INTERCEPTORS(256, u32);
+SHA2_INTERCEPTORS(384, u64);
+SHA2_INTERCEPTORS(512, u64);
+
+#define INIT_SHA2_INTECEPTORS(LEN) \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \
+  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data)
+
+#define INIT_SHA2 \
+  INIT_SHA2_INTECEPTORS(224); \
+  INIT_SHA2_INTECEPTORS(256); \
+  INIT_SHA2_INTECEPTORS(384); \
+  INIT_SHA2_INTECEPTORS(512)
+#undef SHA2_INTERCEPTORS
+#else
+#define INIT_SHA2
+#endif
+
+#if SANITIZER_INTERCEPT_VIS
+INTERCEPTOR(char *, vis, char *dst, int c, int flag, int nextc) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, vis, dst, c, flag, nextc);
+  char *end = REAL(vis)(dst, c, flag, nextc);
+  // dst is NULL terminated and end points to the NULL char
+  if (dst && end)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
+  return end;
+}
+INTERCEPTOR(char *, nvis, char *dst, SIZE_T dlen, int c, int flag, int nextc) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, nvis, dst, dlen, c, flag, nextc);
+  char *end = REAL(nvis)(dst, dlen, c, flag, nextc);
+  // nvis cannot make sure the dst is NULL terminated
+  if (dst && end)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
+  return end;
+}
+INTERCEPTOR(int, strvis, char *dst, const char *src, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int len = REAL(strvis)(dst, src, flag);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
+  return len;
+}
+INTERCEPTOR(int, stravis, char **dst, const char *src, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int len = REAL(stravis)(dst, src, flag);
+  if (dst) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *));
+    if (*dst)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *dst, len + 1);
+  }
+  return len;
+}
+INTERCEPTOR(int, strnvis, char *dst, SIZE_T dlen, const char *src, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int len = REAL(strnvis)(dst, dlen, src, flag);
+  // The interface will be valid even if there is no space for NULL char
+  if (dst && len > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
+  return len;
+}
+INTERCEPTOR(int, strvisx, char *dst, const char *src, SIZE_T len, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strvisx, dst, src, len, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  int ret = REAL(strvisx)(dst, src, len, flag);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len,
+            int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strnvisx, dst, dlen, src, len, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  int ret = REAL(strnvisx)(dst, dlen, src, len, flag);
+  if (dst && ret >= 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strenvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len,
+            int flag, int *cerr_ptr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strenvisx, dst, dlen, src, len, flag, cerr_ptr);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold
+  // according to the implementation
+  if (cerr_ptr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int));
+  int ret = REAL(strenvisx)(dst, dlen, src, len, flag, cerr_ptr);
+  if (dst && ret >= 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  if (cerr_ptr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int));
+  return ret;
+}
+INTERCEPTOR(char *, svis, char *dst, int c, int flag, int nextc,
+            const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  char *end = REAL(svis)(dst, c, flag, nextc, extra);
+  if (dst && end)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
+  return end;
+}
+INTERCEPTOR(char *, snvis, char *dst, SIZE_T dlen, int c, int flag, int nextc,
+            const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra);
+  if (dst && end)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst,
+                                   Min((SIZE_T)(end - dst + 1), dlen));
+  return end;
+}
+INTERCEPTOR(int, strsvis, char *dst, const char *src, int flag,
+            const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  int len = REAL(strsvis)(dst, src, flag, extra);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
+  return len;
+}
+INTERCEPTOR(int, strsnvis, char *dst, SIZE_T dlen, const char *src, int flag,
+            const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  int len = REAL(strsnvis)(dst, dlen, src, flag, extra);
+  // The interface will be valid even if there is no space for NULL char
+  if (dst && len >= 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
+  return len;
+}
+INTERCEPTOR(int, strsvisx, char *dst, const char *src, SIZE_T len, int flag,
+            const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strsvisx, dst, src, len, flag, extra);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  int ret = REAL(strsvisx)(dst, src, len, flag, extra);
+  if (dst)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strsnvisx, char *dst, SIZE_T dlen, const char *src, SIZE_T len,
+            int flag, const char *extra) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strsnvisx, dst, dlen, src, len, flag, extra);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra);
+  if (dst && ret >= 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strsenvisx, char *dst, SIZE_T dlen, const char *src,
+            SIZE_T len, int flag, const char *extra, int *cerr_ptr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strsenvisx, dst, dlen, src, len, flag, extra,
+                           cerr_ptr);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
+  if (extra)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
+  // FIXME: only need to be checked when "flag | VIS_NOLOCALE" doesn't hold
+  // according to the implementation
+  if (cerr_ptr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cerr_ptr, sizeof(int));
+  int ret = REAL(strsenvisx)(dst, dlen, src, len, flag, extra, cerr_ptr);
+  if (dst && ret >= 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  if (cerr_ptr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cerr_ptr, sizeof(int));
+  return ret;
+}
+INTERCEPTOR(int, unvis, char *cp, int c, int *astate, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, unvis, cp, c, astate, flag);
+  if (astate)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, astate, sizeof(*astate));
+  int ret = REAL(unvis)(cp, c, astate, flag);
+  if (ret == unvis_valid || ret == unvis_validpush) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cp, sizeof(*cp));
+  }
+  return ret;
+}
+INTERCEPTOR(int, strunvis, char *dst, const char *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int ret = REAL(strunvis)(dst, src);
+  if (ret != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strnunvis, char *dst, SIZE_T dlen, const char *src) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int ret = REAL(strnunvis)(dst, dlen, src);
+  if (ret != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strunvisx, char *dst, const char *src, int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int ret = REAL(strunvisx)(dst, src, flag);
+  if (ret != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+INTERCEPTOR(int, strnunvisx, char *dst, SIZE_T dlen, const char *src,
+            int flag) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag);
+  if (src)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
+  int ret = REAL(strnunvisx)(dst, dlen, src, flag);
+  if (ret != -1)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
+  return ret;
+}
+#define INIT_VIS                                                               \
+  COMMON_INTERCEPT_FUNCTION(vis);                                              \
+  COMMON_INTERCEPT_FUNCTION(nvis);                                             \
+  COMMON_INTERCEPT_FUNCTION(strvis);                                           \
+  COMMON_INTERCEPT_FUNCTION(stravis);                                          \
+  COMMON_INTERCEPT_FUNCTION(strnvis);                                          \
+  COMMON_INTERCEPT_FUNCTION(strvisx);                                          \
+  COMMON_INTERCEPT_FUNCTION(strnvisx);                                         \
+  COMMON_INTERCEPT_FUNCTION(strenvisx);                                        \
+  COMMON_INTERCEPT_FUNCTION(svis);                                             \
+  COMMON_INTERCEPT_FUNCTION(snvis);                                            \
+  COMMON_INTERCEPT_FUNCTION(strsvis);                                          \
+  COMMON_INTERCEPT_FUNCTION(strsnvis);                                         \
+  COMMON_INTERCEPT_FUNCTION(strsvisx);                                         \
+  COMMON_INTERCEPT_FUNCTION(strsnvisx);                                        \
+  COMMON_INTERCEPT_FUNCTION(strsenvisx);                                       \
+  COMMON_INTERCEPT_FUNCTION(unvis);                                            \
+  COMMON_INTERCEPT_FUNCTION(strunvis);                                         \
+  COMMON_INTERCEPT_FUNCTION(strnunvis);                                        \
+  COMMON_INTERCEPT_FUNCTION(strunvisx);                                        \
+  COMMON_INTERCEPT_FUNCTION(strnunvisx)
+#else
+#define INIT_VIS
+#endif
+
+#if SANITIZER_INTERCEPT_CDB
+INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open, const char *path, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags);
+  if (path)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags);
+  if (cdbr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));
+  return cdbr;
+}
+
+INTERCEPTOR(struct __sanitizer_cdbr *, cdbr_open_mem, void *base, SIZE_T size,
+  int flags, void (*unmap)(void *, void *, SIZE_T), void *cookie) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open_mem, base, size, flags, unmap,
+    cookie);
+  if (base && size)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, base, size);
+  struct __sanitizer_cdbr *cdbr =
+    REAL(cdbr_open_mem)(base, size, flags, unmap, cookie);
+  if (cdbr)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));
+  return cdbr;
+}
+
+INTERCEPTOR(u32, cdbr_entries, struct __sanitizer_cdbr *cdbr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_entries, cdbr);
+  if (cdbr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr));
+  return REAL(cdbr_entries)(cdbr);
+}
+
+INTERCEPTOR(int, cdbr_get, struct __sanitizer_cdbr *cdbr, u32 index,
+            const void **data, SIZE_T *datalen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_get, cdbr, index, data, datalen);
+  if (cdbr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr));
+  int ret = REAL(cdbr_get)(cdbr, index, data, datalen);
+  if (!ret) {
+    if (data)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data));
+    if (datalen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen));
+    if (data && datalen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen);
+  }
+  return ret;
+}
+
+INTERCEPTOR(int, cdbr_find, struct __sanitizer_cdbr *cdbr, const void *key,
+            SIZE_T keylen, const void **data, SIZE_T *datalen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_find, cdbr, key, keylen, data, datalen);
+  if (cdbr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr));
+  if (key)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen);
+  int ret = REAL(cdbr_find)(cdbr, key, keylen, data, datalen);
+  if (!ret) {
+    if (data)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(*data));
+    if (datalen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datalen, sizeof(*datalen));
+    if (data && datalen)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *data, *datalen);
+  }
+  return ret;
+}
+
+INTERCEPTOR(void, cdbr_close, struct __sanitizer_cdbr *cdbr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbr_close, cdbr);
+  if (cdbr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbr, sizeof(*cdbr));
+  REAL(cdbr_close)(cdbr);
+}
+
+INTERCEPTOR(struct __sanitizer_cdbw *, cdbw_open) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_open);
+  struct __sanitizer_cdbw *ret = REAL(cdbw_open)();
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, sizeof(*ret));
+  return ret;
+}
+
+INTERCEPTOR(int, cdbw_put, struct __sanitizer_cdbw *cdbw, const void *key,
+  SIZE_T keylen, const void *data, SIZE_T datalen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put, cdbw, key, keylen, data, datalen);
+  if (cdbw)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw));
+  if (data && datalen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen);
+  if (key && keylen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen);
+  int ret = REAL(cdbw_put)(cdbw, key, keylen, data, datalen);
+  if (!ret && cdbw)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw));
+  return ret;
+}
+
+INTERCEPTOR(int, cdbw_put_data, struct __sanitizer_cdbw *cdbw, const void *data,
+  SIZE_T datalen, u32 *index) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_data, cdbw, data, datalen, index);
+  if (cdbw)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw));
+  if (data && datalen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, datalen);
+  int ret = REAL(cdbw_put_data)(cdbw, data, datalen, index);
+  if (!ret) {
+    if (index)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, index, sizeof(*index));
+    if (cdbw)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw));
+  }
+  return ret;
+}
+
+INTERCEPTOR(int, cdbw_put_key, struct __sanitizer_cdbw *cdbw, const void *key,
+  SIZE_T keylen, u32 index) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_put_key, cdbw, key, keylen, index);
+  if (cdbw)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw));
+  if (key && keylen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, key, keylen);
+  int ret = REAL(cdbw_put_key)(cdbw, key, keylen, index);
+  if (!ret && cdbw)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw));
+  return ret;
+}
+
+INTERCEPTOR(int, cdbw_output, struct __sanitizer_cdbw *cdbw, int output,
+  const char descr[16], u32 (*seedgen)(void)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_output, cdbw, output, descr, seedgen);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, output);
+  if (cdbw)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw));
+  if (descr)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, descr, internal_strnlen(descr, 16));
+  if (seedgen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)seedgen, sizeof(seedgen));
+  int ret = REAL(cdbw_output)(cdbw, output, descr, seedgen);
+  if (!ret) {
+    if (cdbw)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbw, sizeof(*cdbw));
+    if (output >= 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, output);
+  }
+  return ret;
+}
+
+INTERCEPTOR(void, cdbw_close, struct __sanitizer_cdbw *cdbw) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, cdbw_close, cdbw);
+  if (cdbw)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, cdbw, sizeof(*cdbw));
+  REAL(cdbw_close)(cdbw);
+}
+
+#define INIT_CDB \
+  COMMON_INTERCEPT_FUNCTION(cdbr_open); \
+  COMMON_INTERCEPT_FUNCTION(cdbr_open_mem); \
+  COMMON_INTERCEPT_FUNCTION(cdbr_entries); \
+  COMMON_INTERCEPT_FUNCTION(cdbr_get); \
+  COMMON_INTERCEPT_FUNCTION(cdbr_find); \
+  COMMON_INTERCEPT_FUNCTION(cdbr_close); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_open); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_put); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_put_data); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_put_key); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_output); \
+  COMMON_INTERCEPT_FUNCTION(cdbw_close)
+#else
+#define INIT_CDB
+#endif
+
+#if SANITIZER_INTERCEPT_GETFSENT
+INTERCEPTOR(void *, getfsent) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getfsent);
+  void *ret = REAL(getfsent)();
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
+  return ret;
+}
+
+INTERCEPTOR(void *, getfsspec, const char *spec) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec);
+  if (spec)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1);
+  void *ret = REAL(getfsspec)(spec);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
+  return ret;
+}
+
+INTERCEPTOR(void *, getfsfile, const char *file) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file);
+  if (file)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
+  void *ret = REAL(getfsfile)(file);
+  if (ret)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
+  return ret;
+}
+
+#define INIT_GETFSENT \
+  COMMON_INTERCEPT_FUNCTION(getfsent); \
+  COMMON_INTERCEPT_FUNCTION(getfsspec); \
+  COMMON_INTERCEPT_FUNCTION(getfsfile);
+#else
+#define INIT_GETFSENT
+#endif
+
+#if SANITIZER_INTERCEPT_ARC4RANDOM
+INTERCEPTOR(void, arc4random_buf, void *buf, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, arc4random_buf, buf, len);
+  REAL(arc4random_buf)(buf, len);
+  if (buf && len)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, len);
+}
+
+INTERCEPTOR(void, arc4random_addrandom, u8 *dat, int datlen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, arc4random_addrandom, dat, datlen);
+  if (dat && datlen)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, dat, datlen);
+  REAL(arc4random_addrandom)(dat, datlen);
+}
+
+#define INIT_ARC4RANDOM \
+  COMMON_INTERCEPT_FUNCTION(arc4random_buf); \
+  COMMON_INTERCEPT_FUNCTION(arc4random_addrandom);
+#else
+#define INIT_ARC4RANDOM
+#endif
+
+#if SANITIZER_INTERCEPT_POPEN
+INTERCEPTOR(__sanitizer_FILE *, popen, const char *command, const char *type) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type);
+  if (command)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1);
+  if (type)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
+  __sanitizer_FILE *res = REAL(popen)(command, type);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
+  if (res) unpoison_file(res);
+  return res;
+}
+#define INIT_POPEN COMMON_INTERCEPT_FUNCTION(popen)
+#else
+#define INIT_POPEN
+#endif
+
+#if SANITIZER_INTERCEPT_POPENVE
+INTERCEPTOR(__sanitizer_FILE *, popenve, const char *path,
+            char *const *argv, char *const *envp, const char *type) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type);
+  if (path)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  if (argv) {
+    for (char *const *pa = argv; ; ++pa) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
+      if (!*pa)
+        break;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+    }
+  }
+  if (envp) {
+    for (char *const *pa = envp; ; ++pa) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
+      if (!*pa)
+        break;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, *pa, REAL(strlen)(*pa) + 1);
+    }
+  }
+  if (type)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
+  __sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type);
+  COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
+  if (res) unpoison_file(res);
+  return res;
+}
+#define INIT_POPENVE COMMON_INTERCEPT_FUNCTION(popenve)
+#else
+#define INIT_POPENVE
+#endif
+
+#if SANITIZER_INTERCEPT_PCLOSE
+INTERCEPTOR(int, pclose, __sanitizer_FILE *fp) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, pclose, fp);
+  COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
+  const FileMetadata *m = GetInterceptorMetadata(fp);
+  int res = REAL(pclose)(fp);
+  if (m) {
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size);
+    DeleteInterceptorMetadata(fp);
+  }
+  return res;
+}
+#define INIT_PCLOSE COMMON_INTERCEPT_FUNCTION(pclose);
+#else
+#define INIT_PCLOSE
+#endif
+
+#if SANITIZER_INTERCEPT_FUNOPEN
+typedef int (*funopen_readfn)(void *cookie, char *buf, int len);
+typedef int (*funopen_writefn)(void *cookie, const char *buf, int len);
+typedef OFF_T (*funopen_seekfn)(void *cookie, OFF_T offset, int whence);
+typedef int (*funopen_closefn)(void *cookie);
+
+struct WrappedFunopenCookie {
+  void *real_cookie;
+  funopen_readfn real_read;
+  funopen_writefn real_write;
+  funopen_seekfn real_seek;
+  funopen_closefn real_close;
+};
+
+static int wrapped_funopen_read(void *cookie, char *buf, int len) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie;
+  funopen_readfn real_read = wrapped_cookie->real_read;
+  return real_read(wrapped_cookie->real_cookie, buf, len);
+}
+
+static int wrapped_funopen_write(void *cookie, const char *buf, int len) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie;
+  funopen_writefn real_write = wrapped_cookie->real_write;
+  return real_write(wrapped_cookie->real_cookie, buf, len);
+}
+
+static OFF_T wrapped_funopen_seek(void *cookie, OFF_T offset, int whence) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie;
+  funopen_seekfn real_seek = wrapped_cookie->real_seek;
+  return real_seek(wrapped_cookie->real_cookie, offset, whence);
+}
+
+static int wrapped_funopen_close(void *cookie) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  WrappedFunopenCookie *wrapped_cookie = (WrappedFunopenCookie *)cookie;
+  funopen_closefn real_close = wrapped_cookie->real_close;
+  int res = real_close(wrapped_cookie->real_cookie);
+  InternalFree(wrapped_cookie);
+  return res;
+}
+
+INTERCEPTOR(__sanitizer_FILE *, funopen, void *cookie, funopen_readfn readfn,
+            funopen_writefn writefn, funopen_seekfn seekfn,
+            funopen_closefn closefn) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, funopen, cookie, readfn, writefn, seekfn,
+                           closefn);
+
+  WrappedFunopenCookie *wrapped_cookie =
+      (WrappedFunopenCookie *)InternalAlloc(sizeof(WrappedFunopenCookie));
+  wrapped_cookie->real_cookie = cookie;
+  wrapped_cookie->real_read = readfn;
+  wrapped_cookie->real_write = writefn;
+  wrapped_cookie->real_seek = seekfn;
+  wrapped_cookie->real_close = closefn;
+
+  __sanitizer_FILE *res =
+      REAL(funopen)(wrapped_cookie,
+                    readfn  ? wrapped_funopen_read  : nullptr,
+                    writefn ? wrapped_funopen_write : nullptr,
+                    seekfn  ? wrapped_funopen_seek  : nullptr,
+                    closefn ? wrapped_funopen_close : nullptr);
+  if (res)
+    unpoison_file(res);
+  return res;
+}
+#define INIT_FUNOPEN COMMON_INTERCEPT_FUNCTION(funopen)
+#else
+#define INIT_FUNOPEN
+#endif
+
+#if SANITIZER_INTERCEPT_FUNOPEN2
+typedef SSIZE_T (*funopen2_readfn)(void *cookie, void *buf, SIZE_T len);
+typedef SSIZE_T (*funopen2_writefn)(void *cookie, const void *buf, SIZE_T len);
+typedef OFF_T (*funopen2_seekfn)(void *cookie, OFF_T offset, int whence);
+typedef int (*funopen2_flushfn)(void *cookie);
+typedef int (*funopen2_closefn)(void *cookie);
+
+struct WrappedFunopen2Cookie {
+  void *real_cookie;
+  funopen2_readfn real_read;
+  funopen2_writefn real_write;
+  funopen2_seekfn real_seek;
+  funopen2_flushfn real_flush;
+  funopen2_closefn real_close;
+};
+
+static SSIZE_T wrapped_funopen2_read(void *cookie, void *buf, SIZE_T len) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie;
+  funopen2_readfn real_read = wrapped_cookie->real_read;
+  return real_read(wrapped_cookie->real_cookie, buf, len);
+}
+
+static SSIZE_T wrapped_funopen2_write(void *cookie, const void *buf,
+                                      SIZE_T len) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie;
+  funopen2_writefn real_write = wrapped_cookie->real_write;
+  return real_write(wrapped_cookie->real_cookie, buf, len);
+}
+
+static OFF_T wrapped_funopen2_seek(void *cookie, OFF_T offset, int whence) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie;
+  funopen2_seekfn real_seek = wrapped_cookie->real_seek;
+  return real_seek(wrapped_cookie->real_cookie, offset, whence);
+}
+
+static int wrapped_funopen2_flush(void *cookie) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie;
+  funopen2_flushfn real_flush = wrapped_cookie->real_flush;
+  return real_flush(wrapped_cookie->real_cookie);
+}
+
+static int wrapped_funopen2_close(void *cookie) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
+  WrappedFunopen2Cookie *wrapped_cookie = (WrappedFunopen2Cookie *)cookie;
+  funopen2_closefn real_close = wrapped_cookie->real_close;
+  int res = real_close(wrapped_cookie->real_cookie);
+  InternalFree(wrapped_cookie);
+  return res;
+}
+
+INTERCEPTOR(__sanitizer_FILE *, funopen2, void *cookie, funopen2_readfn readfn,
+            funopen2_writefn writefn, funopen2_seekfn seekfn,
+            funopen2_flushfn flushfn, funopen2_closefn closefn) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, funopen2, cookie, readfn, writefn, seekfn,
+                           flushfn, closefn);
+
+  WrappedFunopen2Cookie *wrapped_cookie =
+      (WrappedFunopen2Cookie *)InternalAlloc(sizeof(WrappedFunopen2Cookie));
+  wrapped_cookie->real_cookie = cookie;
+  wrapped_cookie->real_read = readfn;
+  wrapped_cookie->real_write = writefn;
+  wrapped_cookie->real_seek = seekfn;
+  wrapped_cookie->real_flush = flushfn;
+  wrapped_cookie->real_close = closefn;
+
+  __sanitizer_FILE *res =
+      REAL(funopen2)(wrapped_cookie,
+                     readfn  ? wrapped_funopen2_read  : nullptr,
+                     writefn ? wrapped_funopen2_write : nullptr,
+                     seekfn  ? wrapped_funopen2_seek  : nullptr,
+                     flushfn ? wrapped_funopen2_flush : nullptr,
+                     closefn ? wrapped_funopen2_close : nullptr);
+  if (res)
+    unpoison_file(res);
+  return res;
+}
+#define INIT_FUNOPEN2 COMMON_INTERCEPT_FUNCTION(funopen2)
+#else
+#define INIT_FUNOPEN2
+#endif
+
+#if SANITIZER_INTERCEPT_FDEVNAME
+INTERCEPTOR(char *, fdevname,  int fd) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fdevname, fd);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  char *name = REAL(fdevname)(fd);
+  if (name) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+    if (fd > 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return name;
+}
+
+INTERCEPTOR(char *, fdevname_r,  int fd, char *buf, SIZE_T len) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fdevname_r, fd, buf, len);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  char *name = REAL(fdevname_r)(fd, buf, len);
+  if (name && buf && len > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
+    if (fd > 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return name;
+}
+
+#define INIT_FDEVNAME \
+  COMMON_INTERCEPT_FUNCTION(fdevname); \
+  COMMON_INTERCEPT_FUNCTION(fdevname_r);
+#else
+#define INIT_FDEVNAME
+#endif
+
+#if SANITIZER_INTERCEPT_GETUSERSHELL
+INTERCEPTOR(char *, getusershell) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
+  char *res = REAL(getusershell)();
+  if (res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+
+#define INIT_GETUSERSHELL COMMON_INTERCEPT_FUNCTION(getusershell);
+#else
+#define INIT_GETUSERSHELL
+#endif
+
+#if SANITIZER_INTERCEPT_SL_INIT
+INTERCEPTOR(void *, sl_init) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sl_init);
+  void *res = REAL(sl_init)();
+  if (res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer::struct_StringList_sz);
+  return res;
+}
+
+INTERCEPTOR(int, sl_add, void *sl, char *item) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sl_add, sl, item);
+  if (sl)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
+  if (item)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
+  int res = REAL(sl_add)(sl, item);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
+  return res;
+}
+
+INTERCEPTOR(char *, sl_find, void *sl, const char *item) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sl_find, sl, item);
+  if (sl)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
+  if (item)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
+  char *res = REAL(sl_find)(sl, item);
+  if (res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+  return res;
+}
+
+INTERCEPTOR(void, sl_free, void *sl, int freeall) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sl_free, sl, freeall);
+  if (sl)
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
+  REAL(sl_free)(sl, freeall);
+}
+
+#define INIT_SL_INIT                  \
+  COMMON_INTERCEPT_FUNCTION(sl_init); \
+  COMMON_INTERCEPT_FUNCTION(sl_add);  \
+  COMMON_INTERCEPT_FUNCTION(sl_find); \
+  COMMON_INTERCEPT_FUNCTION(sl_free);
+#else
+#define INIT_SL_INIT
+#endif
+
+#if SANITIZER_INTERCEPT_GETRANDOM
+INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
+  SSIZE_T n = REAL(getrandom)(buf, buflen, flags);
+  if (n > 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, n);
+  }
+  return n;
+}
+#define INIT_GETRANDOM COMMON_INTERCEPT_FUNCTION(getrandom)
+#else
+#define INIT_GETRANDOM
+#endif
+
+#if SANITIZER_INTERCEPT_CRYPT
+INTERCEPTOR(char *, crypt, char *key, char *salt) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
+  char *res = REAL(crypt)(key, salt);
+  if (res != nullptr)
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
+  return res;
+}
+#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt);
+#else
+#define INIT_CRYPT
+#endif
+
+#if SANITIZER_INTERCEPT_CRYPT_R
+INTERCEPTOR(char *, crypt_r, char *key, char *salt, void *data) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
+  char *res = REAL(crypt_r)(key, salt, data);
+  if (res != nullptr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data,
+                                   __sanitizer::struct_crypt_data_sz);
+    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
+  }
+  return res;
+}
+#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r);
+#else
+#define INIT_CRYPT_R
+#endif
+
+#if SANITIZER_INTERCEPT_GETENTROPY
+INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getentropy, buf, buflen);
+  int r = REAL(getentropy)(buf, buflen);
+  if (r == 0) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, buflen);
+  }
+  return r;
+}
+#define INIT_GETENTROPY COMMON_INTERCEPT_FUNCTION(getentropy)
+#else
+#define INIT_GETENTROPY
+#endif
+
+#if SANITIZER_INTERCEPT_QSORT
+// Glibc qsort uses a temporary buffer allocated either on stack or on heap.
+// Poisoned memory from there may get copied into the comparator arguments,
+// where it needs to be dealt with. But even that is not enough - the results of
+// the sort may be copied into the input/output array based on the results of
+// the comparator calls, but directly from the temp memory, bypassing the
+// unpoisoning done in wrapped_qsort_compar. We deal with this by, again,
+// unpoisoning the entire array after the sort is done.
+//
+// We can not check that the entire array is initialized at the beginning. IMHO,
+// it's fine for parts of the sorted objects to contain uninitialized memory,
+// ex. as padding in structs.
+typedef int (*qsort_compar_f)(const void *, const void *);
+static THREADLOCAL qsort_compar_f qsort_compar;
+static THREADLOCAL SIZE_T qsort_size;
+int wrapped_qsort_compar(const void *a, const void *b) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_size);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_size);
+  return qsort_compar(a, b);
+}
+
+INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
+            qsort_compar_f compar) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, qsort, base, nmemb, size, compar);
+  // Run the comparator over all array elements to detect any memory issues.
+  if (nmemb > 1) {
+    for (SIZE_T i = 0; i < nmemb - 1; ++i) {
+      void *p = (void *)((char *)base + i * size);
+      void *q = (void *)((char *)base + (i + 1) * size);
+      COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
+      compar(p, q);
+    }
+  }
+  qsort_compar_f old_compar = qsort_compar;
+  qsort_compar = compar;
+  SIZE_T old_size = qsort_size;
+  qsort_size = size;
+  REAL(qsort)(base, nmemb, size, wrapped_qsort_compar);
+  qsort_compar = old_compar;
+  qsort_size = old_size;
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
+}
+#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
+#else
+#define INIT_QSORT
+#endif
+
+#if SANITIZER_INTERCEPT_QSORT_R
+typedef int (*qsort_r_compar_f)(const void *, const void *, void *);
+static THREADLOCAL qsort_r_compar_f qsort_r_compar;
+static THREADLOCAL SIZE_T qsort_r_size;
+int wrapped_qsort_r_compar(const void *a, const void *b, void *arg) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_r_size);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_r_size);
+  return qsort_r_compar(a, b, arg);
+}
+
+INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
+            qsort_r_compar_f compar, void *arg) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
+  // Run the comparator over all array elements to detect any memory issues.
+  if (nmemb > 1) {
+    for (SIZE_T i = 0; i < nmemb - 1; ++i) {
+      void *p = (void *)((char *)base + i * size);
+      void *q = (void *)((char *)base + (i + 1) * size);
+      COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+      compar(p, q, arg);
+    }
+  }
+  qsort_r_compar_f old_compar = qsort_r_compar;
+  qsort_r_compar = compar;
+  SIZE_T old_size = qsort_r_size;
+  qsort_r_size = size;
+  REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
+  qsort_r_compar = old_compar;
+  qsort_r_size = old_size;
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
+}
+#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
+#else
+#define INIT_QSORT_R
+#endif
+
+#if SANITIZER_INTERCEPT_SIGALTSTACK
+INTERCEPTOR(int, sigaltstack, void *ss, void *oss) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, sigaltstack, ss, oss);
+  int r = REAL(sigaltstack)(ss, oss);
+  if (r == 0 && oss != nullptr) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oss, struct_stack_t_sz);
+  }
+  return r;
+}
+#define INIT_SIGALTSTACK COMMON_INTERCEPT_FUNCTION(sigaltstack)
+#else
+#define INIT_SIGALTSTACK
+#endif
+
+#if SANITIZER_INTERCEPT_UNAME
+INTERCEPTOR(int, uname, struct utsname *utsname) {
+#if SANITIZER_LINUX
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_uname(utsname);
+#endif
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, uname, utsname);
+  int res = REAL(uname)(utsname);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, utsname,
+                                   __sanitizer::struct_utsname_sz);
+  return res;
+}
+#define INIT_UNAME COMMON_INTERCEPT_FUNCTION(uname)
+#else
+#define INIT_UNAME
+#endif
+
+#if SANITIZER_INTERCEPT___XUNAME
+// FreeBSD's <sys/utsname.h> define uname() as
+// static __inline int uname(struct utsname *name) {
+//   return __xuname(SYS_NMLN, (void*)name);
+// }
+INTERCEPTOR(int, __xuname, int size, void *utsname) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __xuname, size, utsname);
+  int res = REAL(__xuname)(size, utsname);
+  if (!res)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, utsname,
+                                   __sanitizer::struct_utsname_sz);
+  return res;
+}
+#define INIT___XUNAME COMMON_INTERCEPT_FUNCTION(__xuname)
+#else
+#define INIT___XUNAME
+#endif
+
+#include "sanitizer_common_interceptors_netbsd_compat.inc"
+
+static void InitializeCommonInterceptors() {
+#if SI_POSIX
+  static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
+  interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();
+#endif
+
+  INIT_MMAP;
+  INIT_MMAP64;
+  INIT_TEXTDOMAIN;
+  INIT_STRLEN;
+  INIT_STRNLEN;
+  INIT_STRNDUP;
+  INIT___STRNDUP;
+  INIT_STRCMP;
+  INIT_STRNCMP;
+  INIT_STRCASECMP;
+  INIT_STRNCASECMP;
+  INIT_STRSTR;
+  INIT_STRCASESTR;
+  INIT_STRCHR;
+  INIT_STRCHRNUL;
+  INIT_STRRCHR;
+  INIT_STRSPN;
+  INIT_STRTOK;
+  INIT_STRPBRK;
+  INIT_STRXFRM;
+  INIT___STRXFRM_L;
+  INIT_MEMSET;
+  INIT_MEMMOVE;
+  INIT_MEMCPY;
+  INIT_MEMCHR;
+  INIT_MEMCMP;
+  INIT_BCMP;
+  INIT_MEMRCHR;
+  INIT_MEMMEM;
+  INIT_READ;
+  INIT_FREAD;
+  INIT_PREAD;
+  INIT_PREAD64;
+  INIT_READV;
+  INIT_PREADV;
+  INIT_PREADV64;
+  INIT_WRITE;
+  INIT_FWRITE;
+  INIT_PWRITE;
+  INIT_PWRITE64;
+  INIT_WRITEV;
+  INIT_PWRITEV;
+  INIT_PWRITEV64;
+  INIT_FGETS;
+  INIT_FPUTS;
+  INIT_PUTS;
+  INIT_PRCTL;
+  INIT_LOCALTIME_AND_FRIENDS;
+  INIT_STRPTIME;
+  INIT_SCANF;
+  INIT_ISOC99_SCANF;
+  INIT_PRINTF;
+  INIT_PRINTF_L;
+  INIT_ISOC99_PRINTF;
+  INIT_FREXP;
+  INIT_FREXPF_FREXPL;
+  INIT_GETPWNAM_AND_FRIENDS;
+  INIT_GETPWNAM_R_AND_FRIENDS;
+  INIT_GETPWENT;
+  INIT_FGETPWENT;
+  INIT_GETPWENT_R;
+  INIT_FGETPWENT_R;
+  INIT_FGETGRENT_R;
+  INIT_SETPWENT;
+  INIT_CLOCK_GETTIME;
+  INIT_CLOCK_GETCPUCLOCKID;
+  INIT_GETITIMER;
+  INIT_TIME;
+  INIT_GLOB;
+  INIT_GLOB64;
+  INIT_WAIT;
+  INIT_WAIT4;
+  INIT_INET;
+  INIT_PTHREAD_GETSCHEDPARAM;
+  INIT_GETADDRINFO;
+  INIT_GETNAMEINFO;
+  INIT_GETSOCKNAME;
+  INIT_GETHOSTBYNAME;
+  INIT_GETHOSTBYNAME2;
+  INIT_GETHOSTBYNAME_R;
+  INIT_GETHOSTBYNAME2_R;
+  INIT_GETHOSTBYADDR_R;
+  INIT_GETHOSTENT_R;
+  INIT_GETSOCKOPT;
+  INIT_ACCEPT;
+  INIT_ACCEPT4;
+  INIT_PACCEPT;
+  INIT_MODF;
+  INIT_RECVMSG;
+  INIT_SENDMSG;
+  INIT_RECVMMSG;
+  INIT_SENDMMSG;
+  INIT_SYSMSG;
+  INIT_GETPEERNAME;
+  INIT_IOCTL;
+  INIT_INET_ATON;
+  INIT_SYSINFO;
+  INIT_READDIR;
+  INIT_READDIR64;
+  INIT_PTRACE;
+  INIT_SETLOCALE;
+  INIT_GETCWD;
+  INIT_GET_CURRENT_DIR_NAME;
+  INIT_STRTOIMAX;
+  INIT_MBSTOWCS;
+  INIT_MBSNRTOWCS;
+  INIT_WCSTOMBS;
+  INIT_WCSNRTOMBS;
+  INIT_WCRTOMB;
+  INIT_WCTOMB;
+  INIT_TCGETATTR;
+  INIT_REALPATH;
+  INIT_CANONICALIZE_FILE_NAME;
+  INIT_CONFSTR;
+  INIT_SCHED_GETAFFINITY;
+  INIT_SCHED_GETPARAM;
+  INIT_STRERROR;
+  INIT_STRERROR_R;
+  INIT_XPG_STRERROR_R;
+  INIT_SCANDIR;
+  INIT_SCANDIR64;
+  INIT_GETGROUPS;
+  INIT_POLL;
+  INIT_PPOLL;
+  INIT_WORDEXP;
+  INIT_SIGWAIT;
+  INIT_SIGWAITINFO;
+  INIT_SIGTIMEDWAIT;
+  INIT_SIGSETOPS;
+  INIT_SIGPENDING;
+  INIT_SIGPROCMASK;
+  INIT_PTHREAD_SIGMASK;
+  INIT_BACKTRACE;
+  INIT__EXIT;
+  INIT_PTHREAD_MUTEX_LOCK;
+  INIT_PTHREAD_MUTEX_UNLOCK;
+  INIT___PTHREAD_MUTEX_LOCK;
+  INIT___PTHREAD_MUTEX_UNLOCK;
+  INIT___LIBC_MUTEX_LOCK;
+  INIT___LIBC_MUTEX_UNLOCK;
+  INIT___LIBC_THR_SETCANCELSTATE;
+  INIT_GETMNTENT;
+  INIT_GETMNTENT_R;
+  INIT_STATFS;
+  INIT_STATFS64;
+  INIT_STATVFS;
+  INIT_STATVFS64;
+  INIT_INITGROUPS;
+  INIT_ETHER_NTOA_ATON;
+  INIT_ETHER_HOST;
+  INIT_ETHER_R;
+  INIT_SHMCTL;
+  INIT_RANDOM_R;
+  INIT_PTHREAD_ATTR_GET;
+  INIT_PTHREAD_ATTR_GET_SCHED;
+  INIT_PTHREAD_ATTR_GETINHERITSCHED;
+  INIT_PTHREAD_ATTR_GETAFFINITY_NP;
+  INIT_PTHREAD_MUTEXATTR_GETPSHARED;
+  INIT_PTHREAD_MUTEXATTR_GETTYPE;
+  INIT_PTHREAD_MUTEXATTR_GETPROTOCOL;
+  INIT_PTHREAD_MUTEXATTR_GETPRIOCEILING;
+  INIT_PTHREAD_MUTEXATTR_GETROBUST;
+  INIT_PTHREAD_MUTEXATTR_GETROBUST_NP;
+  INIT_PTHREAD_RWLOCKATTR_GETPSHARED;
+  INIT_PTHREAD_RWLOCKATTR_GETKIND_NP;
+  INIT_PTHREAD_CONDATTR_GETPSHARED;
+  INIT_PTHREAD_CONDATTR_GETCLOCK;
+  INIT_PTHREAD_BARRIERATTR_GETPSHARED;
+  INIT_TMPNAM;
+  INIT_TMPNAM_R;
+  INIT_TTYNAME;
+  INIT_TTYNAME_R;
+  INIT_TEMPNAM;
+  INIT_PTHREAD_SETNAME_NP;
+  INIT_PTHREAD_GETNAME_NP;
+  INIT_SINCOS;
+  INIT_REMQUO;
+  INIT_REMQUOL;
+  INIT_LGAMMA;
+  INIT_LGAMMAL;
+  INIT_LGAMMA_R;
+  INIT_LGAMMAL_R;
+  INIT_DRAND48_R;
+  INIT_RAND_R;
+  INIT_GETLINE;
+  INIT_ICONV;
+  INIT_TIMES;
+  INIT_TLS_GET_ADDR;
+  INIT_LISTXATTR;
+  INIT_GETXATTR;
+  INIT_GETRESID;
+  INIT_GETIFADDRS;
+  INIT_IF_INDEXTONAME;
+  INIT_CAPGET;
+  INIT_AEABI_MEM;
+  INIT___BZERO;
+  INIT_BZERO;
+  INIT_FTIME;
+  INIT_XDR;
+  INIT_TSEARCH;
+  INIT_LIBIO_INTERNALS;
+  INIT_FOPEN;
+  INIT_FOPEN64;
+  INIT_OPEN_MEMSTREAM;
+  INIT_OBSTACK;
+  INIT_FFLUSH;
+  INIT_FCLOSE;
+  INIT_DLOPEN_DLCLOSE;
+  INIT_GETPASS;
+  INIT_TIMERFD;
+  INIT_MLOCKX;
+  INIT_FOPENCOOKIE;
+  INIT_SEM;
+  INIT_PTHREAD_SETCANCEL;
+  INIT_MINCORE;
+  INIT_PROCESS_VM_READV;
+  INIT_CTERMID;
+  INIT_CTERMID_R;
+  INIT_RECV_RECVFROM;
+  INIT_SEND_SENDTO;
+  INIT_STAT;
+  INIT_EVENTFD_READ_WRITE;
+  INIT_LSTAT;
+  INIT___XSTAT;
+  INIT___XSTAT64;
+  INIT___LXSTAT;
+  INIT___LXSTAT64;
+  // FIXME: add other *stat interceptors.
+  INIT_UTMP;
+  INIT_UTMPX;
+  INIT_GETLOADAVG;
+  INIT_WCSLEN;
+  INIT_WCSCAT;
+  INIT_WCSDUP;
+  INIT_WCSXFRM;
+  INIT___WCSXFRM_L;
+  INIT_ACCT;
+  INIT_USER_FROM_UID;
+  INIT_UID_FROM_USER;
+  INIT_GROUP_FROM_GID;
+  INIT_GID_FROM_GROUP;
+  INIT_ACCESS;
+  INIT_FACCESSAT;
+  INIT_GETGROUPLIST;
+  INIT_GETGROUPMEMBERSHIP;
+  INIT_READLINK;
+  INIT_READLINKAT;
+  INIT_NAME_TO_HANDLE_AT;
+  INIT_OPEN_BY_HANDLE_AT;
+  INIT_STRLCPY;
+  INIT_DEVNAME;
+  INIT_DEVNAME_R;
+  INIT_FGETLN;
+  INIT_STRMODE;
+  INIT_TTYENT;
+  INIT_PROTOENT;
+  INIT_PROTOENT_R;
+  INIT_NETENT;
+  INIT_GETMNTINFO;
+  INIT_MI_VECTOR_HASH;
+  INIT_SETVBUF;
+  INIT_GETVFSSTAT;
+  INIT_REGEX;
+  INIT_REGEXSUB;
+  INIT_FTS;
+  INIT_SYSCTL;
+  INIT_ASYSCTL;
+  INIT_SYSCTLGETMIBINFO;
+  INIT_NL_LANGINFO;
+  INIT_MODCTL;
+  INIT_STRTONUM;
+  INIT_FPARSELN;
+  INIT_STATVFS1;
+  INIT_STRTOI;
+  INIT_CAPSICUM;
+  INIT_SHA1;
+  INIT_MD4;
+  INIT_RMD160;
+  INIT_MD5;
+  INIT_FSEEK;
+  INIT_MD2;
+  INIT_SHA2;
+  INIT_VIS;
+  INIT_CDB;
+  INIT_GETFSENT;
+  INIT_ARC4RANDOM;
+  INIT_POPEN;
+  INIT_POPENVE;
+  INIT_PCLOSE;
+  INIT_FUNOPEN;
+  INIT_FUNOPEN2;
+  INIT_FDEVNAME;
+  INIT_GETUSERSHELL;
+  INIT_SL_INIT;
+  INIT_GETRANDOM;
+  INIT_CRYPT;
+  INIT_CRYPT_R;
+  INIT_GETENTROPY;
+  INIT_QSORT;
+  INIT_QSORT_R;
+  INIT_SIGALTSTACK;
+  INIT_UNAME;
+  INIT___XUNAME;
+
+  INIT___PRINTF_CHK;
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc
new file mode 100644
index 0000000000..bbbedda8fb
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc
@@ -0,0 +1,562 @@
+//===-- sanitizer_common_interceptors_format.inc ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Scanf/printf implementation for use in *Sanitizer interceptors.
+// Follows http://pubs.opengroup.org/onlinepubs/9699919799/functions/fscanf.html
+// and http://pubs.opengroup.org/onlinepubs/9699919799/functions/fprintf.html
+// with a few common GNU extensions.
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdarg.h>
+
+static const char *parse_number(const char *p, int *out) {
+  *out = internal_atoll(p);
+  while (*p >= '0' && *p <= '9')
+    ++p;
+  return p;
+}
+
+static const char *maybe_parse_param_index(const char *p, int *out) {
+  // n$
+  if (*p >= '0' && *p <= '9') {
+    int number;
+    const char *q = parse_number(p, &number);
+    CHECK(q);
+    if (*q == '$') {
+      *out = number;
+      p = q + 1;
+    }
+  }
+
+  // Otherwise, do not change p. This will be re-parsed later as the field
+  // width.
+  return p;
+}
+
+static bool char_is_one_of(char c, const char *s) {
+  return !!internal_strchr(s, c);
+}
+
+static const char *maybe_parse_length_modifier(const char *p, char ll[2]) {
+  if (char_is_one_of(*p, "jztLq")) {
+    ll[0] = *p;
+    ++p;
+  } else if (*p == 'h') {
+    ll[0] = 'h';
+    ++p;
+    if (*p == 'h') {
+      ll[1] = 'h';
+      ++p;
+    }
+  } else if (*p == 'l') {
+    ll[0] = 'l';
+    ++p;
+    if (*p == 'l') {
+      ll[1] = 'l';
+      ++p;
+    }
+  }
+  return p;
+}
+
+// Returns true if the character is an integer conversion specifier.
+static bool format_is_integer_conv(char c) {
+  return char_is_one_of(c, "diouxXn");
+}
+
+// Returns true if the character is an floating point conversion specifier.
+static bool format_is_float_conv(char c) {
+  return char_is_one_of(c, "aAeEfFgG");
+}
+
+// Returns string output character size for string-like conversions,
+// or 0 if the conversion is invalid.
+static int format_get_char_size(char convSpecifier,
+                                const char lengthModifier[2]) {
+  if (char_is_one_of(convSpecifier, "CS")) {
+    return sizeof(wchar_t);
+  }
+
+  if (char_is_one_of(convSpecifier, "cs[")) {
+    if (lengthModifier[0] == 'l' && lengthModifier[1] == '\0')
+      return sizeof(wchar_t);
+    else if (lengthModifier[0] == '\0')
+      return sizeof(char);
+  }
+
+  return 0;
+}
+
+enum FormatStoreSize {
+  // Store size not known in advance; can be calculated as wcslen() of the
+  // destination buffer.
+  FSS_WCSLEN = -2,
+  // Store size not known in advance; can be calculated as strlen() of the
+  // destination buffer.
+  FSS_STRLEN = -1,
+  // Invalid conversion specifier.
+  FSS_INVALID = 0
+};
+
+// Returns the memory size of a format directive (if >0), or a value of
+// FormatStoreSize.
+static int format_get_value_size(char convSpecifier,
+                                 const char lengthModifier[2],
+                                 bool promote_float) {
+  if (format_is_integer_conv(convSpecifier)) {
+    switch (lengthModifier[0]) {
+    case 'h':
+      return lengthModifier[1] == 'h' ? sizeof(char) : sizeof(short);
+    case 'l':
+      return lengthModifier[1] == 'l' ? sizeof(long long) : sizeof(long);
+    case 'q':
+      return sizeof(long long);
+    case 'L':
+      return sizeof(long long);
+    case 'j':
+      return sizeof(INTMAX_T);
+    case 'z':
+      return sizeof(SIZE_T);
+    case 't':
+      return sizeof(PTRDIFF_T);
+    case 0:
+      return sizeof(int);
+    default:
+      return FSS_INVALID;
+    }
+  }
+
+  if (format_is_float_conv(convSpecifier)) {
+    switch (lengthModifier[0]) {
+    case 'L':
+    case 'q':
+      return sizeof(long double);
+    case 'l':
+      return lengthModifier[1] == 'l' ? sizeof(long double)
+                                           : sizeof(double);
+    case 0:
+      // Printf promotes floats to doubles but scanf does not
+      return promote_float ? sizeof(double) : sizeof(float);
+    default:
+      return FSS_INVALID;
+    }
+  }
+
+  if (convSpecifier == 'p') {
+    if (lengthModifier[0] != 0)
+      return FSS_INVALID;
+    return sizeof(void *);
+  }
+
+  return FSS_INVALID;
+}
+
+struct ScanfDirective {
+  int argIdx; // argument index, or -1 if not specified ("%n$")
+  int fieldWidth;
+  const char *begin;
+  const char *end;
+  bool suppressed; // suppress assignment ("*")
+  bool allocate;   // allocate space ("m")
+  char lengthModifier[2];
+  char convSpecifier;
+  bool maybeGnuMalloc;
+};
+
+// Parse scanf format string. If a valid directive in encountered, it is
+// returned in dir. This function returns the pointer to the first
+// unprocessed character, or 0 in case of error.
+// In case of the end-of-string, a pointer to the closing \0 is returned.
+static const char *scanf_parse_next(const char *p, bool allowGnuMalloc,
+                                    ScanfDirective *dir) {
+  internal_memset(dir, 0, sizeof(*dir));
+  dir->argIdx = -1;
+
+  while (*p) {
+    if (*p != '%') {
+      ++p;
+      continue;
+    }
+    dir->begin = p;
+    ++p;
+    // %%
+    if (*p == '%') {
+      ++p;
+      continue;
+    }
+    if (*p == '\0') {
+      return nullptr;
+    }
+    // %n$
+    p = maybe_parse_param_index(p, &dir->argIdx);
+    CHECK(p);
+    // *
+    if (*p == '*') {
+      dir->suppressed = true;
+      ++p;
+    }
+    // Field width
+    if (*p >= '0' && *p <= '9') {
+      p = parse_number(p, &dir->fieldWidth);
+      CHECK(p);
+      if (dir->fieldWidth <= 0)  // Width if at all must be non-zero
+        return nullptr;
+    }
+    // m
+    if (*p == 'm') {
+      dir->allocate = true;
+      ++p;
+    }
+    // Length modifier.
+    p = maybe_parse_length_modifier(p, dir->lengthModifier);
+    // Conversion specifier.
+    dir->convSpecifier = *p++;
+    // Consume %[...] expression.
+    if (dir->convSpecifier == '[') {
+      if (*p == '^')
+        ++p;
+      if (*p == ']')
+        ++p;
+      while (*p && *p != ']')
+        ++p;
+      if (*p == 0)
+        return nullptr; // unexpected end of string
+                        // Consume the closing ']'.
+      ++p;
+    }
+    // This is unfortunately ambiguous between old GNU extension
+    // of %as, %aS and %a[...] and newer POSIX %a followed by
+    // letters s, S or [.
+    if (allowGnuMalloc && dir->convSpecifier == 'a' &&
+        !dir->lengthModifier[0]) {
+      if (*p == 's' || *p == 'S') {
+        dir->maybeGnuMalloc = true;
+        ++p;
+      } else if (*p == '[') {
+        // Watch for %a[h-j%d], if % appears in the
+        // [...] range, then we need to give up, we don't know
+        // if scanf will parse it as POSIX %a [h-j %d ] or
+        // GNU allocation of string with range dh-j plus %.
+        const char *q = p + 1;
+        if (*q == '^')
+          ++q;
+        if (*q == ']')
+          ++q;
+        while (*q && *q != ']' && *q != '%')
+          ++q;
+        if (*q == 0 || *q == '%')
+          return nullptr;
+        p = q + 1; // Consume the closing ']'.
+        dir->maybeGnuMalloc = true;
+      }
+    }
+    dir->end = p;
+    break;
+  }
+  return p;
+}
+
+static int scanf_get_value_size(ScanfDirective *dir) {
+  if (dir->allocate) {
+    if (!char_is_one_of(dir->convSpecifier, "cCsS["))
+      return FSS_INVALID;
+    return sizeof(char *);
+  }
+
+  if (dir->maybeGnuMalloc) {
+    if (dir->convSpecifier != 'a' || dir->lengthModifier[0])
+      return FSS_INVALID;
+    // This is ambiguous, so check the smaller size of char * (if it is
+    // a GNU extension of %as, %aS or %a[...]) and float (if it is
+    // POSIX %a followed by s, S or [ letters).
+    return sizeof(char *) < sizeof(float) ? sizeof(char *) : sizeof(float);
+  }
+
+  if (char_is_one_of(dir->convSpecifier, "cCsS[")) {
+    bool needsTerminator = char_is_one_of(dir->convSpecifier, "sS[");
+    unsigned charSize =
+        format_get_char_size(dir->convSpecifier, dir->lengthModifier);
+    if (charSize == 0)
+      return FSS_INVALID;
+    if (dir->fieldWidth == 0) {
+      if (!needsTerminator)
+        return charSize;
+      return (charSize == sizeof(char)) ? FSS_STRLEN : FSS_WCSLEN;
+    }
+    return (dir->fieldWidth + needsTerminator) * charSize;
+  }
+
+  return format_get_value_size(dir->convSpecifier, dir->lengthModifier, false);
+}
+
+// Common part of *scanf interceptors.
+// Process format string and va_list, and report all store ranges.
+// Stops when "consuming" n_inputs input items.
+static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
+                         const char *format, va_list aq) {
+  CHECK_GT(n_inputs, 0);
+  const char *p = format;
+
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, format, internal_strlen(format) + 1);
+
+  while (*p) {
+    ScanfDirective dir;
+    p = scanf_parse_next(p, allowGnuMalloc, &dir);
+    if (!p)
+      break;
+    if (dir.convSpecifier == 0) {
+      // This can only happen at the end of the format string.
+      CHECK_EQ(*p, 0);
+      break;
+    }
+    // Here the directive is valid. Do what it says.
+    if (dir.argIdx != -1) {
+      // Unsupported.
+      break;
+    }
+    if (dir.suppressed)
+      continue;
+    int size = scanf_get_value_size(&dir);
+    if (size == FSS_INVALID) {
+      Report("%s: WARNING: unexpected format specifier in scanf interceptor: ",
+             SanitizerToolName, "%.*s\n", dir.end - dir.begin, dir.begin);
+      break;
+    }
+    void *argp = va_arg(aq, void *);
+    if (dir.convSpecifier != 'n')
+      --n_inputs;
+    if (n_inputs < 0)
+      break;
+    if (size == FSS_STRLEN) {
+      size = internal_strlen((const char *)argp) + 1;
+    } else if (size == FSS_WCSLEN) {
+      // FIXME: actually use wcslen() to calculate it.
+      size = 0;
+    }
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
+  }
+}
+
+#if SANITIZER_INTERCEPT_PRINTF
+
+struct PrintfDirective {
+  int fieldWidth;
+  int fieldPrecision;
+  int argIdx; // width argument index, or -1 if not specified ("%*n$")
+  int precisionIdx; // precision argument index, or -1 if not specified (".*n$")
+  const char *begin;
+  const char *end;
+  bool starredWidth;
+  bool starredPrecision;
+  char lengthModifier[2];
+  char convSpecifier;
+};
+
+static const char *maybe_parse_number(const char *p, int *out) {
+  if (*p >= '0' && *p <= '9')
+    p = parse_number(p, out);
+  return p;
+}
+
+static const char *maybe_parse_number_or_star(const char *p, int *out,
+                                              bool *star) {
+  if (*p == '*') {
+    *star = true;
+    ++p;
+  } else {
+    *star = false;
+    p = maybe_parse_number(p, out);
+  }
+  return p;
+}
+
+// Parse printf format string. Same as scanf_parse_next.
+static const char *printf_parse_next(const char *p, PrintfDirective *dir) {
+  internal_memset(dir, 0, sizeof(*dir));
+  dir->argIdx = -1;
+  dir->precisionIdx = -1;
+
+  while (*p) {
+    if (*p != '%') {
+      ++p;
+      continue;
+    }
+    dir->begin = p;
+    ++p;
+    // %%
+    if (*p == '%') {
+      ++p;
+      continue;
+    }
+    if (*p == '\0') {
+      return nullptr;
+    }
+    // %n$
+    p = maybe_parse_param_index(p, &dir->precisionIdx);
+    CHECK(p);
+    // Flags
+    while (char_is_one_of(*p, "'-+ #0")) {
+      ++p;
+    }
+    // Field width
+    p = maybe_parse_number_or_star(p, &dir->fieldWidth,
+                                   &dir->starredWidth);
+    if (!p)
+      return nullptr;
+    // Precision
+    if (*p == '.') {
+      ++p;
+      // Actual precision is optional (surprise!)
+      p = maybe_parse_number_or_star(p, &dir->fieldPrecision,
+                                     &dir->starredPrecision);
+      if (!p)
+        return nullptr;
+      // m$
+      if (dir->starredPrecision) {
+        p = maybe_parse_param_index(p, &dir->precisionIdx);
+        CHECK(p);
+      }
+    }
+    // Length modifier.
+    p = maybe_parse_length_modifier(p, dir->lengthModifier);
+    // Conversion specifier.
+    dir->convSpecifier = *p++;
+    dir->end = p;
+    break;
+  }
+  return p;
+}
+
+static int printf_get_value_size(PrintfDirective *dir) {
+  if (char_is_one_of(dir->convSpecifier, "cCsS")) {
+    unsigned charSize =
+        format_get_char_size(dir->convSpecifier, dir->lengthModifier);
+    if (charSize == 0)
+      return FSS_INVALID;
+    if (char_is_one_of(dir->convSpecifier, "sS")) {
+      return (charSize == sizeof(char)) ? FSS_STRLEN : FSS_WCSLEN;
+    }
+    return charSize;
+  }
+
+  return format_get_value_size(dir->convSpecifier, dir->lengthModifier, true);
+}
+
+#define SKIP_SCALAR_ARG(aq, convSpecifier, size)                   \
+  do {                                                             \
+    if (format_is_float_conv(convSpecifier)) {                     \
+      switch (size) {                                              \
+      case 8:                                                      \
+        va_arg(*aq, double);                                       \
+        break;                                                     \
+      case 12:                                                     \
+        va_arg(*aq, long double);                                  \
+        break;                                                     \
+      case 16:                                                     \
+        va_arg(*aq, long double);                                  \
+        break;                                                     \
+      default:                                                     \
+        Report("WARNING: unexpected floating-point arg size"       \
+               " in printf interceptor: %d\n", size);              \
+        return;                                                    \
+      }                                                            \
+    } else {                                                       \
+      switch (size) {                                              \
+      case 1:                                                      \
+      case 2:                                                      \
+      case 4:                                                      \
+        va_arg(*aq, u32);                                          \
+        break;                                                     \
+      case 8:                                                      \
+        va_arg(*aq, u64);                                          \
+        break;                                                     \
+      default:                                                     \
+        Report("WARNING: unexpected arg size"                      \
+               " in printf interceptor: %d\n", size);              \
+        return;                                                    \
+      }                                                            \
+    }                                                              \
+  } while (0)
+
+// Common part of *printf interceptors.
+// Process format string and va_list, and report all load ranges.
+static void printf_common(void *ctx, const char *format, va_list aq) {
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, format, internal_strlen(format) + 1);
+
+  const char *p = format;
+
+  while (*p) {
+    PrintfDirective dir;
+    p = printf_parse_next(p, &dir);
+    if (!p)
+      break;
+    if (dir.convSpecifier == 0) {
+      // This can only happen at the end of the format string.
+      CHECK_EQ(*p, 0);
+      break;
+    }
+    // Here the directive is valid. Do what it says.
+    if (dir.argIdx != -1 || dir.precisionIdx != -1) {
+      // Unsupported.
+      break;
+    }
+    if (dir.starredWidth) {
+      // Dynamic width
+      SKIP_SCALAR_ARG(&aq, 'd', sizeof(int));
+    }
+    if (dir.starredPrecision) {
+      // Dynamic precision
+      SKIP_SCALAR_ARG(&aq, 'd', sizeof(int));
+    }
+    // %m does not require an argument: strlen(errno).
+    if (dir.convSpecifier == 'm')
+      continue;
+    int size = printf_get_value_size(&dir);
+    if (size == FSS_INVALID) {
+      static int ReportedOnce;
+      if (!ReportedOnce++)
+        Report(
+            "%s: WARNING: unexpected format specifier in printf "
+            "interceptor: %.*s (reported once per process)\n",
+            SanitizerToolName, dir.end - dir.begin, dir.begin);
+      break;
+    }
+    if (dir.convSpecifier == 'n') {
+      void *argp = va_arg(aq, void *);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
+      continue;
+    } else if (size == FSS_STRLEN) {
+      if (void *argp = va_arg(aq, void *)) {
+        if (dir.starredPrecision) {
+          // FIXME: properly support starred precision for strings.
+          size = 0;
+        } else if (dir.fieldPrecision > 0) {
+          // Won't read more than "precision" symbols.
+          size = internal_strnlen((const char *)argp, dir.fieldPrecision);
+          if (size < dir.fieldPrecision) size++;
+        } else {
+          // Whole string will be accessed.
+          size = internal_strlen((const char *)argp) + 1;
+        }
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
+      }
+    } else if (size == FSS_WCSLEN) {
+      if (void *argp = va_arg(aq, void *)) {
+        // FIXME: Properly support wide-character strings (via wcsrtombs).
+        size = 0;
+        COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size);
+      }
+    } else {
+      // Skip non-pointer args
+      SKIP_SCALAR_ARG(&aq, dir.convSpecifier, size);
+    }
+  }
+}
+
+#endif // SANITIZER_INTERCEPT_PRINTF
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
new file mode 100644
index 0000000000..490a04b218
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
@@ -0,0 +1,609 @@
+//===-- sanitizer_common_interceptors_ioctl.inc -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Ioctl handling in common sanitizer interceptors.
+//===----------------------------------------------------------------------===//
+
+#if !SANITIZER_NETBSD
+
+#include "sanitizer_flags.h"
+
+struct ioctl_desc {
+  unsigned req;
+  // FIXME: support read+write arguments. Currently READWRITE and WRITE do the
+  // same thing.
+  // XXX: The declarations below may use WRITE instead of READWRITE, unless
+  // explicitly noted.
+  enum {
+    NONE,
+    READ,
+    WRITE,
+    READWRITE,
+    CUSTOM
+  } type : 3;
+  unsigned size : 29;
+  const char* name;
+};
+
+const unsigned ioctl_table_max = 500;
+static ioctl_desc ioctl_table[ioctl_table_max];
+static unsigned ioctl_table_size = 0;
+
+// This can not be declared as a global, because references to struct_*_sz
+// require a global initializer. And this table must be available before global
+// initializers are run.
+static void ioctl_table_fill() {
+#define _(rq, tp, sz)                                    \
+  if (IOCTL_##rq != IOCTL_NOT_PRESENT) {                 \
+    CHECK(ioctl_table_size < ioctl_table_max);           \
+    ioctl_table[ioctl_table_size].req = IOCTL_##rq;      \
+    ioctl_table[ioctl_table_size].type = ioctl_desc::tp; \
+    ioctl_table[ioctl_table_size].size = sz;             \
+    ioctl_table[ioctl_table_size].name = #rq;            \
+    ++ioctl_table_size;                                  \
+  }
+
+  _(FIOASYNC, READ, sizeof(int));
+  _(FIOCLEX, NONE, 0);
+  _(FIOGETOWN, WRITE, sizeof(int));
+  _(FIONBIO, READ, sizeof(int));
+  _(FIONCLEX, NONE, 0);
+  _(FIOSETOWN, READ, sizeof(int));
+  _(SIOCATMARK, WRITE, sizeof(int));
+  _(SIOCGIFCONF, CUSTOM, 0);
+  _(SIOCGPGRP, WRITE, sizeof(int));
+  _(SIOCSPGRP, READ, sizeof(int));
+#if !SANITIZER_SOLARIS
+  _(TIOCCONS, NONE, 0);
+#endif
+  _(TIOCEXCL, NONE, 0);
+  _(TIOCGETD, WRITE, sizeof(int));
+  _(TIOCGPGRP, WRITE, pid_t_sz);
+  _(TIOCGWINSZ, WRITE, struct_winsize_sz);
+  _(TIOCMBIC, READ, sizeof(int));
+  _(TIOCMBIS, READ, sizeof(int));
+  _(TIOCMGET, WRITE, sizeof(int));
+  _(TIOCMSET, READ, sizeof(int));
+  _(TIOCNOTTY, NONE, 0);
+  _(TIOCNXCL, NONE, 0);
+  _(TIOCOUTQ, WRITE, sizeof(int));
+  _(TIOCPKT, READ, sizeof(int));
+  _(TIOCSCTTY, NONE, 0);
+  _(TIOCSETD, READ, sizeof(int));
+  _(TIOCSPGRP, READ, pid_t_sz);
+  _(TIOCSTI, READ, sizeof(char));
+  _(TIOCSWINSZ, READ, struct_winsize_sz);
+
+#if !SANITIZER_IOS
+  _(SIOCADDMULTI, READ, struct_ifreq_sz);
+  _(SIOCDELMULTI, READ, struct_ifreq_sz);
+  _(SIOCGIFADDR, WRITE, struct_ifreq_sz);
+  _(SIOCGIFBRDADDR, WRITE, struct_ifreq_sz);
+  _(SIOCGIFDSTADDR, WRITE, struct_ifreq_sz);
+  _(SIOCGIFFLAGS, WRITE, struct_ifreq_sz);
+  _(SIOCGIFMETRIC, WRITE, struct_ifreq_sz);
+  _(SIOCGIFMTU, WRITE, struct_ifreq_sz);
+  _(SIOCGIFNETMASK, WRITE, struct_ifreq_sz);
+  _(SIOCSIFADDR, READ, struct_ifreq_sz);
+  _(SIOCSIFBRDADDR, READ, struct_ifreq_sz);
+  _(SIOCSIFDSTADDR, READ, struct_ifreq_sz);
+  _(SIOCSIFFLAGS, READ, struct_ifreq_sz);
+  _(SIOCSIFMETRIC, READ, struct_ifreq_sz);
+  _(SIOCSIFMTU, READ, struct_ifreq_sz);
+  _(SIOCSIFNETMASK, READ, struct_ifreq_sz);
+#endif
+
+#if (SANITIZER_LINUX && !SANITIZER_ANDROID)
+  _(SIOCGETSGCNT, WRITE, struct_sioc_sg_req_sz);
+  _(SIOCGETVIFCNT, WRITE, struct_sioc_vif_req_sz);
+#endif
+
+#if SANITIZER_LINUX
+  // Conflicting request ids.
+  // _(CDROMAUDIOBUFSIZ, NONE, 0);
+  // _(SNDCTL_TMR_CONTINUE, NONE, 0);
+  // _(SNDCTL_TMR_START, NONE, 0);
+  // _(SNDCTL_TMR_STOP, NONE, 0);
+  // _(SOUND_MIXER_READ_LOUD, WRITE, sizeof(int)); // same as ...READ_ENHANCE
+  // _(SOUND_MIXER_READ_MUTE, WRITE, sizeof(int)); // same as ...READ_ENHANCE
+  // _(SOUND_MIXER_WRITE_LOUD, WRITE, sizeof(int)); // same as ...WRITE_ENHANCE
+  // _(SOUND_MIXER_WRITE_MUTE, WRITE, sizeof(int)); // same as ...WRITE_ENHANCE
+  _(BLKFLSBUF, NONE, 0);
+  _(BLKGETSIZE, WRITE, sizeof(uptr));
+  _(BLKRAGET, WRITE, sizeof(int));
+  _(BLKRASET, NONE, 0);
+  _(BLKROGET, WRITE, sizeof(int));
+  _(BLKROSET, READ, sizeof(int));
+  _(BLKRRPART, NONE, 0);
+  _(CDROMEJECT, NONE, 0);
+  _(CDROMEJECT_SW, NONE, 0);
+  _(CDROMMULTISESSION, WRITE, struct_cdrom_multisession_sz);
+  _(CDROMPAUSE, NONE, 0);
+  _(CDROMPLAYMSF, READ, struct_cdrom_msf_sz);
+  _(CDROMPLAYTRKIND, READ, struct_cdrom_ti_sz);
+  _(CDROMREADAUDIO, READ, struct_cdrom_read_audio_sz);
+  _(CDROMREADCOOKED, READ, struct_cdrom_msf_sz);
+  _(CDROMREADMODE1, READ, struct_cdrom_msf_sz);
+  _(CDROMREADMODE2, READ, struct_cdrom_msf_sz);
+  _(CDROMREADRAW, READ, struct_cdrom_msf_sz);
+  _(CDROMREADTOCENTRY, WRITE, struct_cdrom_tocentry_sz);
+  _(CDROMREADTOCHDR, WRITE, struct_cdrom_tochdr_sz);
+  _(CDROMRESET, NONE, 0);
+  _(CDROMRESUME, NONE, 0);
+  _(CDROMSEEK, READ, struct_cdrom_msf_sz);
+  _(CDROMSTART, NONE, 0);
+  _(CDROMSTOP, NONE, 0);
+  _(CDROMSUBCHNL, WRITE, struct_cdrom_subchnl_sz);
+  _(CDROMVOLCTRL, READ, struct_cdrom_volctrl_sz);
+  _(CDROMVOLREAD, WRITE, struct_cdrom_volctrl_sz);
+  _(CDROM_GET_UPC, WRITE, 8);
+  _(EVIOCGABS, WRITE, struct_input_absinfo_sz); // fixup
+  _(EVIOCGBIT, WRITE, struct_input_id_sz); // fixup
+  _(EVIOCGEFFECTS, WRITE, sizeof(int));
+  _(EVIOCGID, WRITE, struct_input_id_sz);
+  _(EVIOCGKEY, WRITE, 0);
+  _(EVIOCGKEYCODE, WRITE, sizeof(int) * 2);
+  _(EVIOCGLED, WRITE, 0);
+  _(EVIOCGNAME, WRITE, 0);
+  _(EVIOCGPHYS, WRITE, 0);
+  _(EVIOCGRAB, READ, sizeof(int));
+  _(EVIOCGREP, WRITE, sizeof(int) * 2);
+  _(EVIOCGSND, WRITE, 0);
+  _(EVIOCGSW, WRITE, 0);
+  _(EVIOCGUNIQ, WRITE, 0);
+  _(EVIOCGVERSION, WRITE, sizeof(int));
+  _(EVIOCRMFF, READ, sizeof(int));
+  _(EVIOCSABS, READ, struct_input_absinfo_sz); // fixup
+  _(EVIOCSFF, READ, struct_ff_effect_sz);
+  _(EVIOCSKEYCODE, READ, sizeof(int) * 2);
+  _(EVIOCSREP, READ, sizeof(int) * 2);
+  _(FDCLRPRM, NONE, 0);
+  _(FDDEFPRM, READ, struct_floppy_struct_sz);
+  _(FDFLUSH, NONE, 0);
+  _(FDFMTBEG, NONE, 0);
+  _(FDFMTEND, NONE, 0);
+  _(FDFMTTRK, READ, struct_format_descr_sz);
+  _(FDGETDRVPRM, WRITE, struct_floppy_drive_params_sz);
+  _(FDGETDRVSTAT, WRITE, struct_floppy_drive_struct_sz);
+  _(FDGETDRVTYP, WRITE, 16);
+  _(FDGETFDCSTAT, WRITE, struct_floppy_fdc_state_sz);
+  _(FDGETMAXERRS, WRITE, struct_floppy_max_errors_sz);
+  _(FDGETPRM, WRITE, struct_floppy_struct_sz);
+  _(FDMSGOFF, NONE, 0);
+  _(FDMSGON, NONE, 0);
+  _(FDPOLLDRVSTAT, WRITE, struct_floppy_drive_struct_sz);
+  _(FDRAWCMD, WRITE, struct_floppy_raw_cmd_sz);
+  _(FDRESET, NONE, 0);
+  _(FDSETDRVPRM, READ, struct_floppy_drive_params_sz);
+  _(FDSETEMSGTRESH, NONE, 0);
+  _(FDSETMAXERRS, READ, struct_floppy_max_errors_sz);
+  _(FDSETPRM, READ, struct_floppy_struct_sz);
+  _(FDTWADDLE, NONE, 0);
+  _(FDWERRORCLR, NONE, 0);
+  _(FDWERRORGET, WRITE, struct_floppy_write_errors_sz);
+  _(HDIO_DRIVE_CMD, WRITE, sizeof(int));
+  _(HDIO_GETGEO, WRITE, struct_hd_geometry_sz);
+  _(HDIO_GET_32BIT, WRITE, sizeof(int));
+  _(HDIO_GET_DMA, WRITE, sizeof(int));
+  _(HDIO_GET_IDENTITY, WRITE, struct_hd_driveid_sz);
+  _(HDIO_GET_KEEPSETTINGS, WRITE, sizeof(int));
+  _(HDIO_GET_MULTCOUNT, WRITE, sizeof(int));
+  _(HDIO_GET_NOWERR, WRITE, sizeof(int));
+  _(HDIO_GET_UNMASKINTR, WRITE, sizeof(int));
+  _(HDIO_SET_32BIT, NONE, 0);
+  _(HDIO_SET_DMA, NONE, 0);
+  _(HDIO_SET_KEEPSETTINGS, NONE, 0);
+  _(HDIO_SET_MULTCOUNT, NONE, 0);
+  _(HDIO_SET_NOWERR, NONE, 0);
+  _(HDIO_SET_UNMASKINTR, NONE, 0);
+  _(MTIOCGET, WRITE, struct_mtget_sz);
+  _(MTIOCPOS, WRITE, struct_mtpos_sz);
+  _(MTIOCTOP, READ, struct_mtop_sz);
+  _(PPPIOCGASYNCMAP, WRITE, sizeof(int));
+  _(PPPIOCGDEBUG, WRITE, sizeof(int));
+  _(PPPIOCGFLAGS, WRITE, sizeof(int));
+  _(PPPIOCGUNIT, WRITE, sizeof(int));
+  _(PPPIOCGXASYNCMAP, WRITE, sizeof(int) * 8);
+  _(PPPIOCSASYNCMAP, READ, sizeof(int));
+  _(PPPIOCSDEBUG, READ, sizeof(int));
+  _(PPPIOCSFLAGS, READ, sizeof(int));
+  _(PPPIOCSMAXCID, READ, sizeof(int));
+  _(PPPIOCSMRU, READ, sizeof(int));
+  _(PPPIOCSXASYNCMAP, READ, sizeof(int) * 8);
+  _(SIOCADDRT, READ, struct_rtentry_sz);
+  _(SIOCDARP, READ, struct_arpreq_sz);
+  _(SIOCDELRT, READ, struct_rtentry_sz);
+  _(SIOCDRARP, READ, struct_arpreq_sz);
+  _(SIOCGARP, WRITE, struct_arpreq_sz);
+  _(SIOCGIFENCAP, WRITE, sizeof(int));
+  _(SIOCGIFHWADDR, WRITE, struct_ifreq_sz);
+  _(SIOCGIFMAP, WRITE, struct_ifreq_sz);
+  _(SIOCGIFMEM, WRITE, struct_ifreq_sz);
+  _(SIOCGIFNAME, NONE, 0);
+  _(SIOCGIFSLAVE, NONE, 0);
+  _(SIOCGRARP, WRITE, struct_arpreq_sz);
+  _(SIOCGSTAMP, WRITE, timeval_sz);
+  _(SIOCSARP, READ, struct_arpreq_sz);
+  _(SIOCSIFENCAP, READ, sizeof(int));
+  _(SIOCSIFHWADDR, READ, struct_ifreq_sz);
+  _(SIOCSIFLINK, NONE, 0);
+  _(SIOCSIFMAP, READ, struct_ifreq_sz);
+  _(SIOCSIFMEM, READ, struct_ifreq_sz);
+  _(SIOCSIFSLAVE, NONE, 0);
+  _(SIOCSRARP, READ, struct_arpreq_sz);
+  _(SNDCTL_COPR_HALT, WRITE, struct_copr_debug_buf_sz);
+  _(SNDCTL_COPR_LOAD, READ, struct_copr_buffer_sz);
+  _(SNDCTL_COPR_RCODE, WRITE, struct_copr_debug_buf_sz);
+  _(SNDCTL_COPR_RCVMSG, WRITE, struct_copr_msg_sz);
+  _(SNDCTL_COPR_RDATA, WRITE, struct_copr_debug_buf_sz);
+  _(SNDCTL_COPR_RESET, NONE, 0);
+  _(SNDCTL_COPR_RUN, WRITE, struct_copr_debug_buf_sz);
+  _(SNDCTL_COPR_SENDMSG, READ, struct_copr_msg_sz);
+  _(SNDCTL_COPR_WCODE, READ, struct_copr_debug_buf_sz);
+  _(SNDCTL_COPR_WDATA, READ, struct_copr_debug_buf_sz);
+  _(SNDCTL_DSP_GETBLKSIZE, WRITE, sizeof(int));
+  _(SNDCTL_DSP_GETFMTS, WRITE, sizeof(int));
+  _(SNDCTL_DSP_NONBLOCK, NONE, 0);
+  _(SNDCTL_DSP_POST, NONE, 0);
+  _(SNDCTL_DSP_RESET, NONE, 0);
+  _(SNDCTL_DSP_SETFMT, WRITE, sizeof(int));
+  _(SNDCTL_DSP_SETFRAGMENT, WRITE, sizeof(int));
+  _(SNDCTL_DSP_SPEED, WRITE, sizeof(int));
+  _(SNDCTL_DSP_STEREO, WRITE, sizeof(int));
+  _(SNDCTL_DSP_SUBDIVIDE, WRITE, sizeof(int));
+  _(SNDCTL_DSP_SYNC, NONE, 0);
+  _(SNDCTL_FM_4OP_ENABLE, READ, sizeof(int));
+  _(SNDCTL_FM_LOAD_INSTR, READ, struct_sbi_instrument_sz);
+  _(SNDCTL_MIDI_INFO, WRITE, struct_midi_info_sz);
+  _(SNDCTL_MIDI_PRETIME, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_CTRLRATE, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_GETINCOUNT, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_GETOUTCOUNT, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_NRMIDIS, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_NRSYNTHS, WRITE, sizeof(int));
+  _(SNDCTL_SEQ_OUTOFBAND, READ, struct_seq_event_rec_sz);
+  _(SNDCTL_SEQ_PANIC, NONE, 0);
+  _(SNDCTL_SEQ_PERCMODE, NONE, 0);
+  _(SNDCTL_SEQ_RESET, NONE, 0);
+  _(SNDCTL_SEQ_RESETSAMPLES, READ, sizeof(int));
+  _(SNDCTL_SEQ_SYNC, NONE, 0);
+  _(SNDCTL_SEQ_TESTMIDI, READ, sizeof(int));
+  _(SNDCTL_SEQ_THRESHOLD, READ, sizeof(int));
+  _(SNDCTL_SYNTH_INFO, WRITE, struct_synth_info_sz);
+  _(SNDCTL_SYNTH_MEMAVL, WRITE, sizeof(int));
+  _(SNDCTL_TMR_METRONOME, READ, sizeof(int));
+  _(SNDCTL_TMR_SELECT, WRITE, sizeof(int));
+  _(SNDCTL_TMR_SOURCE, WRITE, sizeof(int));
+  _(SNDCTL_TMR_TEMPO, WRITE, sizeof(int));
+  _(SNDCTL_TMR_TIMEBASE, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_ALTPCM, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_BASS, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_CAPS, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_CD, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_DEVMASK, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_ENHANCE, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_IGAIN, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_IMIX, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_LINE, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_LINE1, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_LINE2, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_LINE3, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_MIC, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_OGAIN, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_PCM, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_RECLEV, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_RECMASK, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_RECSRC, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_SPEAKER, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_STEREODEVS, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_SYNTH, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_TREBLE, WRITE, sizeof(int));
+  _(SOUND_MIXER_READ_VOLUME, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_ALTPCM, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_BASS, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_CD, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_ENHANCE, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_IGAIN, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_IMIX, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_LINE, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_LINE1, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_LINE2, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_LINE3, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_MIC, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_OGAIN, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_PCM, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_RECLEV, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_RECSRC, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_SPEAKER, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_SYNTH, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_TREBLE, WRITE, sizeof(int));
+  _(SOUND_MIXER_WRITE_VOLUME, WRITE, sizeof(int));
+  _(SOUND_PCM_READ_BITS, WRITE, sizeof(int));
+  _(SOUND_PCM_READ_CHANNELS, WRITE, sizeof(int));
+  _(SOUND_PCM_READ_FILTER, WRITE, sizeof(int));
+  _(SOUND_PCM_READ_RATE, WRITE, sizeof(int));
+  _(SOUND_PCM_WRITE_CHANNELS, WRITE, sizeof(int));
+  _(SOUND_PCM_WRITE_FILTER, WRITE, sizeof(int));
+  _(TCFLSH, NONE, 0);
+  _(TCGETA, WRITE, struct_termio_sz);
+  _(TCGETS, WRITE, struct_termios_sz);
+  _(TCSBRK, NONE, 0);
+  _(TCSBRKP, NONE, 0);
+  _(TCSETA, READ, struct_termio_sz);
+  _(TCSETAF, READ, struct_termio_sz);
+  _(TCSETAW, READ, struct_termio_sz);
+  _(TCSETS, READ, struct_termios_sz);
+  _(TCSETSF, READ, struct_termios_sz);
+  _(TCSETSW, READ, struct_termios_sz);
+  _(TCXONC, NONE, 0);
+  _(TIOCGLCKTRMIOS, WRITE, struct_termios_sz);
+  _(TIOCGSOFTCAR, WRITE, sizeof(int));
+  _(TIOCINQ, WRITE, sizeof(int));
+  _(TIOCLINUX, READ, sizeof(char));
+  _(TIOCSERCONFIG, NONE, 0);
+  _(TIOCSERGETLSR, WRITE, sizeof(int));
+  _(TIOCSERGWILD, WRITE, sizeof(int));
+  _(TIOCSERSWILD, READ, sizeof(int));
+  _(TIOCSLCKTRMIOS, READ, struct_termios_sz);
+  _(TIOCSSOFTCAR, READ, sizeof(int));
+  _(VT_ACTIVATE, NONE, 0);
+  _(VT_DISALLOCATE, NONE, 0);
+  _(VT_GETMODE, WRITE, struct_vt_mode_sz);
+  _(VT_GETSTATE, WRITE, struct_vt_stat_sz);
+  _(VT_OPENQRY, WRITE, sizeof(int));
+  _(VT_RELDISP, NONE, 0);
+  _(VT_RESIZE, READ, struct_vt_sizes_sz);
+  _(VT_RESIZEX, READ, struct_vt_consize_sz);
+  _(VT_SENDSIG, NONE, 0);
+  _(VT_SETMODE, READ, struct_vt_mode_sz);
+  _(VT_WAITACTIVE, NONE, 0);
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  // _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE
+  _(CYGETDEFTHRESH, WRITE, sizeof(int));
+  _(CYGETDEFTIMEOUT, WRITE, sizeof(int));
+  _(CYGETMON, WRITE, struct_cyclades_monitor_sz);
+  _(CYGETTHRESH, WRITE, sizeof(int));
+  _(CYGETTIMEOUT, WRITE, sizeof(int));
+  _(CYSETDEFTHRESH, NONE, 0);
+  _(CYSETDEFTIMEOUT, NONE, 0);
+  _(CYSETTHRESH, NONE, 0);
+  _(CYSETTIMEOUT, NONE, 0);
+  _(EQL_EMANCIPATE, WRITE, struct_ifreq_sz);
+  _(EQL_ENSLAVE, WRITE, struct_ifreq_sz);
+  _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz);
+  _(EQL_GETSLAVECFG, WRITE, struct_ifreq_sz);
+  _(EQL_SETMASTRCFG, WRITE, struct_ifreq_sz);
+  _(EQL_SETSLAVECFG, WRITE, struct_ifreq_sz);
+  _(EVIOCGKEYCODE_V2, WRITE, struct_input_keymap_entry_sz);
+  _(EVIOCGPROP, WRITE, 0);
+  _(EVIOCSKEYCODE_V2, READ, struct_input_keymap_entry_sz);
+  _(FS_IOC_GETFLAGS, WRITE, sizeof(int));
+  _(FS_IOC_GETVERSION, WRITE, sizeof(int));
+  _(FS_IOC_SETFLAGS, READ, sizeof(int));
+  _(FS_IOC_SETVERSION, READ, sizeof(int));
+  _(GIO_CMAP, WRITE, 48);
+  _(GIO_FONT, WRITE, 8192);
+  _(GIO_SCRNMAP, WRITE, e_tabsz);
+  _(GIO_UNIMAP, WRITE, struct_unimapdesc_sz);
+  _(GIO_UNISCRNMAP, WRITE, sizeof(short) * e_tabsz);
+  _(KDADDIO, NONE, 0);
+  _(KDDELIO, NONE, 0);
+  _(KDDISABIO, NONE, 0);
+  _(KDENABIO, NONE, 0);
+  _(KDGETKEYCODE, WRITE, struct_kbkeycode_sz);
+  _(KDGETLED, WRITE, 1);
+  _(KDGETMODE, WRITE, sizeof(int));
+  _(KDGKBDIACR, WRITE, struct_kbdiacrs_sz);
+  _(KDGKBENT, WRITE, struct_kbentry_sz);
+  _(KDGKBLED, WRITE, sizeof(int));
+  _(KDGKBMETA, WRITE, sizeof(int));
+  _(KDGKBMODE, WRITE, sizeof(int));
+  _(KDGKBSENT, WRITE, struct_kbsentry_sz);
+  _(KDGKBTYPE, WRITE, 1);
+  _(KDMAPDISP, NONE, 0);
+  _(KDMKTONE, NONE, 0);
+  _(KDSETKEYCODE, READ, struct_kbkeycode_sz);
+  _(KDSETLED, NONE, 0);
+  _(KDSETMODE, NONE, 0);
+  _(KDSIGACCEPT, NONE, 0);
+  _(KDSKBDIACR, READ, struct_kbdiacrs_sz);
+  _(KDSKBENT, READ, struct_kbentry_sz);
+  _(KDSKBLED, NONE, 0);
+  _(KDSKBMETA, NONE, 0);
+  _(KDSKBMODE, NONE, 0);
+  _(KDSKBSENT, READ, struct_kbsentry_sz);
+  _(KDUNMAPDISP, NONE, 0);
+  _(KIOCSOUND, NONE, 0);
+  _(LPABORT, NONE, 0);
+  _(LPABORTOPEN, NONE, 0);
+  _(LPCAREFUL, NONE, 0);
+  _(LPCHAR, NONE, 0);
+  _(LPGETIRQ, WRITE, sizeof(int));
+  _(LPGETSTATUS, WRITE, sizeof(int));
+  _(LPRESET, NONE, 0);
+  _(LPSETIRQ, NONE, 0);
+  _(LPTIME, NONE, 0);
+  _(LPWAIT, NONE, 0);
+  _(MTIOCGETCONFIG, WRITE, struct_mtconfiginfo_sz);
+  _(MTIOCSETCONFIG, READ, struct_mtconfiginfo_sz);
+  _(PIO_CMAP, NONE, 0);
+  _(PIO_FONT, READ, 8192);
+  _(PIO_SCRNMAP, READ, e_tabsz);
+  _(PIO_UNIMAP, READ, struct_unimapdesc_sz);
+  _(PIO_UNIMAPCLR, READ, struct_unimapinit_sz);
+  _(PIO_UNISCRNMAP, READ, sizeof(short) * e_tabsz);
+  _(SCSI_IOCTL_PROBE_HOST, READ, sizeof(int));
+  _(SCSI_IOCTL_TAGGED_DISABLE, NONE, 0);
+  _(SCSI_IOCTL_TAGGED_ENABLE, NONE, 0);
+  _(SNDCTL_DSP_GETISPACE, WRITE, struct_audio_buf_info_sz);
+  _(SNDCTL_DSP_GETOSPACE, WRITE, struct_audio_buf_info_sz);
+  _(TIOCGSERIAL, WRITE, struct_serial_struct_sz);
+  _(TIOCSERGETMULTI, WRITE, struct_serial_multiport_struct_sz);
+  _(TIOCSERSETMULTI, READ, struct_serial_multiport_struct_sz);
+  _(TIOCSSERIAL, READ, struct_serial_struct_sz);
+
+  // The following ioctl requests are shared between AX25, IPX, netrom and
+  // mrouted.
+  // _(SIOCAIPXITFCRT, READ, sizeof(char));
+  // _(SIOCAX25GETUID, READ, struct_sockaddr_ax25_sz);
+  // _(SIOCNRGETPARMS, WRITE, struct_nr_parms_struct_sz);
+  // _(SIOCAIPXPRISLT, READ, sizeof(char));
+  // _(SIOCNRSETPARMS, READ, struct_nr_parms_struct_sz);
+  // _(SIOCAX25ADDUID, READ, struct_sockaddr_ax25_sz);
+  // _(SIOCNRDECOBS, NONE, 0);
+  // _(SIOCAX25DELUID, READ, struct_sockaddr_ax25_sz);
+  // _(SIOCIPXCFGDATA, WRITE, struct_ipx_config_data_sz);
+  // _(SIOCAX25NOUID, READ, sizeof(int));
+  // _(SIOCNRRTCTL, READ, sizeof(int));
+  // _(SIOCAX25DIGCTL, READ, sizeof(int));
+  // _(SIOCAX25GETPARMS, WRITE, struct_ax25_parms_struct_sz);
+  // _(SIOCAX25SETPARMS, READ, struct_ax25_parms_struct_sz);
+#endif
+#undef _
+}
+
+static bool ioctl_initialized = false;
+
+struct ioctl_desc_compare {
+  bool operator()(const ioctl_desc& left, const ioctl_desc& right) const {
+    return left.req < right.req;
+  }
+};
+
+static void ioctl_init() {
+  ioctl_table_fill();
+  Sort(ioctl_table, ioctl_table_size, ioctl_desc_compare());
+
+  bool bad = false;
+  for (unsigned i = 0; i < ioctl_table_size - 1; ++i) {
+    if (ioctl_table[i].req >= ioctl_table[i + 1].req) {
+      Printf("Duplicate or unsorted ioctl request id %x >= %x (%s vs %s)\n",
+             ioctl_table[i].req, ioctl_table[i + 1].req, ioctl_table[i].name,
+             ioctl_table[i + 1].name);
+      bad = true;
+    }
+  }
+
+  if (bad) Die();
+
+  ioctl_initialized = true;
+}
+
+// Handle the most evil ioctls that encode argument value as part of request id.
+static unsigned ioctl_request_fixup(unsigned req) {
+#if SANITIZER_LINUX
+  // Strip size and event number.
+  const unsigned kEviocgbitMask =
+      (IOC_SIZEMASK << IOC_SIZESHIFT) | EVIOC_EV_MAX;
+  if ((req & ~kEviocgbitMask) == IOCTL_EVIOCGBIT)
+    return IOCTL_EVIOCGBIT;
+  // Strip absolute axis number.
+  if ((req & ~EVIOC_ABS_MAX) == IOCTL_EVIOCGABS)
+    return IOCTL_EVIOCGABS;
+  if ((req & ~EVIOC_ABS_MAX) == IOCTL_EVIOCSABS)
+    return IOCTL_EVIOCSABS;
+#endif
+  return req;
+}
+
+static const ioctl_desc *ioctl_table_lookup(unsigned req) {
+  int left = 0;
+  int right = ioctl_table_size;
+  while (left < right) {
+    int mid = (left + right) / 2;
+    if (ioctl_table[mid].req < req)
+      left = mid + 1;
+    else
+      right = mid;
+  }
+  if (left == right && ioctl_table[left].req == req)
+    return ioctl_table + left;
+  else
+    return nullptr;
+}
+
+static bool ioctl_decode(unsigned req, ioctl_desc *desc) {
+  CHECK(desc);
+  desc->req = req;
+  desc->name = "<DECODED_IOCTL>";
+  desc->size = IOC_SIZE(req);
+  // Sanity check.
+  if (desc->size > 0xFFFF) return false;
+  unsigned dir = IOC_DIR(req);
+  switch (dir) {
+    case IOC_NONE:
+      desc->type = ioctl_desc::NONE;
+      break;
+    case IOC_READ | IOC_WRITE:
+      desc->type = ioctl_desc::READWRITE;
+      break;
+    case IOC_READ:
+      desc->type = ioctl_desc::WRITE;
+      break;
+    case IOC_WRITE:
+      desc->type = ioctl_desc::READ;
+      break;
+    default:
+      return false;
+  }
+  // Size can be 0 iff type is NONE.
+  if ((desc->type == IOC_NONE) != (desc->size == 0)) return false;
+  // Sanity check.
+  if (IOC_TYPE(req) == 0) return false;
+  return true;
+}
+
+static const ioctl_desc *ioctl_lookup(unsigned req) {
+  req = ioctl_request_fixup(req);
+  const ioctl_desc *desc = ioctl_table_lookup(req);
+  if (desc) return desc;
+
+  // Try stripping access size from the request id.
+  desc = ioctl_table_lookup(req & ~(IOC_SIZEMASK << IOC_SIZESHIFT));
+  // Sanity check: requests that encode access size are either read or write and
+  // have size of 0 in the table.
+  if (desc && desc->size == 0 &&
+      (desc->type == ioctl_desc::READWRITE || desc->type == ioctl_desc::WRITE ||
+       desc->type == ioctl_desc::READ))
+    return desc;
+  return nullptr;
+}
+
+static void ioctl_common_pre(void *ctx, const ioctl_desc *desc, int d,
+                             unsigned request, void *arg) {
+  if (desc->type == ioctl_desc::READ || desc->type == ioctl_desc::READWRITE) {
+    unsigned size = desc->size ? desc->size : IOC_SIZE(request);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, arg, size);
+  }
+  if (desc->type != ioctl_desc::CUSTOM)
+    return;
+  if (request == IOCTL_SIOCGIFCONF) {
+    struct __sanitizer_ifconf *ifc = (__sanitizer_ifconf *)arg;
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, (char*)&ifc->ifc_len,
+                                  sizeof(ifc->ifc_len));
+  }
+}
+
+static void ioctl_common_post(void *ctx, const ioctl_desc *desc, int res, int d,
+                              unsigned request, void *arg) {
+  if (desc->type == ioctl_desc::WRITE || desc->type == ioctl_desc::READWRITE) {
+    // FIXME: add verbose output
+    unsigned size = desc->size ? desc->size : IOC_SIZE(request);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg, size);
+  }
+  if (desc->type != ioctl_desc::CUSTOM)
+    return;
+  if (request == IOCTL_SIOCGIFCONF) {
+    struct __sanitizer_ifconf *ifc = (__sanitizer_ifconf *)arg;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifc->ifc_ifcu.ifcu_req, ifc->ifc_len);
+  }
+}
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc
new file mode 100644
index 0000000000..6aa73ec8c6
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc
@@ -0,0 +1,128 @@
+//===-- sanitizer_common_interceptors_netbsd_compat.inc ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common function interceptors for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// Interceptors for NetBSD old function calls that have been versioned.
+//
+// NetBSD minimal version supported 9.0.
+// NetBSD current version supported 9.99.26.
+//
+//===----------------------------------------------------------------------===//
+
+#if SANITIZER_NETBSD
+
+// First undef all mangled symbols.
+// Next, define compat interceptors.
+// Finally, undef INIT_ and redefine it.
+// This allows to avoid preprocessor issues.
+
+#undef fstatvfs
+#undef fstatvfs1
+#undef getmntinfo
+#undef getvfsstat
+#undef statvfs
+#undef statvfs1
+
+INTERCEPTOR(int, statvfs, char *path, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(statvfs)(path, buf);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
+  return res;
+}
+
+INTERCEPTOR(int, fstatvfs, int fd, void *buf) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  // FIXME: under ASan the call below may write to freed memory and corrupt
+  // its metadata. See
+  // https://github.com/google/sanitizers/issues/321.
+  int res = REAL(fstatvfs)(fd, buf);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
+    if (fd >= 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return res;
+}
+
+#undef INIT_STATVFS
+#define INIT_STATVFS \
+  COMMON_INTERCEPT_FUNCTION(statvfs); \
+  COMMON_INTERCEPT_FUNCTION(fstatvfs); \
+  COMMON_INTERCEPT_FUNCTION(__statvfs90); \
+  COMMON_INTERCEPT_FUNCTION(__fstatvfs90)
+
+INTERCEPTOR(int, __getmntinfo13, void **mntbufp, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, __getmntinfo13, mntbufp, flags);
+  int cnt = REAL(__getmntinfo13)(mntbufp, flags);
+  if (cnt > 0 && mntbufp) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mntbufp, sizeof(void *));
+    if (*mntbufp)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *mntbufp, cnt * struct_statvfs90_sz);
+  }
+  return cnt;
+}
+
+#undef INIT_GETMNTINFO
+#define INIT_GETMNTINFO \
+  COMMON_INTERCEPT_FUNCTION(__getmntinfo13); \
+  COMMON_INTERCEPT_FUNCTION(__getmntinfo90)
+
+INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, getvfsstat, buf, bufsize, flags);
+  int ret = REAL(getvfsstat)(buf, bufsize, flags);
+  if (buf && ret > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, ret * struct_statvfs90_sz);
+  return ret;
+}
+
+#undef INIT_GETVFSSTAT
+#define INIT_GETVFSSTAT \
+  COMMON_INTERCEPT_FUNCTION(getvfsstat); \
+  COMMON_INTERCEPT_FUNCTION(__getvfsstat90)
+
+INTERCEPTOR(int, statvfs1, const char *path, void *buf, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
+  if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+  int res = REAL(statvfs1)(path, buf, flags);
+  if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
+  return res;
+}
+
+INTERCEPTOR(int, fstatvfs1, int fd, void *buf, int flags) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs1, fd, buf, flags);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  int res = REAL(fstatvfs1)(fd, buf, flags);
+  if (!res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
+    if (fd >= 0)
+      COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  }
+  return res;
+}
+
+#undef INIT_STATVFS1
+#define INIT_STATVFS1 \
+  COMMON_INTERCEPT_FUNCTION(statvfs1); \
+  COMMON_INTERCEPT_FUNCTION(fstatvfs1); \
+  COMMON_INTERCEPT_FUNCTION(__statvfs190); \
+  COMMON_INTERCEPT_FUNCTION(__fstatvfs190)
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interface.inc b/lib/tsan/sanitizer_common/sanitizer_common_interface.inc
new file mode 100644
index 0000000000..c78b6e10b6
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interface.inc
@@ -0,0 +1,41 @@
+//===-- sanitizer_common_interface.inc ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Common interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
+INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
+INTERFACE_FUNCTION(__sanitizer_set_death_callback)
+INTERFACE_FUNCTION(__sanitizer_set_report_path)
+INTERFACE_FUNCTION(__sanitizer_set_report_fd)
+INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
+INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
+INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
+INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
+// Sanitizer weak hooks
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strncmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strstr)
+// Stacktrace interface.
+INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc)
+INTERFACE_FUNCTION(__sanitizer_symbolize_global)
+INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
+// Allocator interface.
+INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
+INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
+INTERFACE_FUNCTION(__sanitizer_get_heap_size)
+INTERFACE_FUNCTION(__sanitizer_get_ownership)
+INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes)
+INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks)
+INTERFACE_FUNCTION(__sanitizer_purge_allocator)
+INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
+INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
+INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc b/lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc
new file mode 100644
index 0000000000..38f9531148
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc
@@ -0,0 +1,13 @@
+//===-- sanitizer_common_interface_posix.inc ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Common interface list only available for Posix systems.
+//===----------------------------------------------------------------------===//
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
+INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc b/lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc
new file mode 100644
index 0000000000..532ac9ead3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc
@@ -0,0 +1,2914 @@
+//===-- sanitizer_common_syscalls.inc ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common syscalls handlers for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// This file should be included into the tool's interceptor file,
+// which has to define it's own macros:
+//   COMMON_SYSCALL_PRE_READ_RANGE
+//          Called in prehook for regions that will be read by the kernel and
+//          must be initialized.
+//   COMMON_SYSCALL_PRE_WRITE_RANGE
+//          Called in prehook for regions that will be written to by the kernel
+//          and must be addressable. The actual write range may be smaller than
+//          reported in the prehook. See POST_WRITE_RANGE.
+//   COMMON_SYSCALL_POST_READ_RANGE
+//          Called in posthook for regions that were read by the kernel. Does
+//          not make much sense.
+//   COMMON_SYSCALL_POST_WRITE_RANGE
+//          Called in posthook for regions that were written to by the kernel
+//          and are now initialized.
+//   COMMON_SYSCALL_ACQUIRE(addr)
+//          Acquire memory visibility from addr.
+//   COMMON_SYSCALL_RELEASE(addr)
+//          Release memory visibility to addr.
+//   COMMON_SYSCALL_FD_CLOSE(fd)
+//          Called before closing file descriptor fd.
+//   COMMON_SYSCALL_FD_ACQUIRE(fd)
+//          Acquire memory visibility from fd.
+//   COMMON_SYSCALL_FD_RELEASE(fd)
+//          Release memory visibility to fd.
+//   COMMON_SYSCALL_PRE_FORK()
+//          Called before fork syscall.
+//   COMMON_SYSCALL_POST_FORK(long res)
+//          Called after fork syscall.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_LINUX
+
+#include "sanitizer_libc.h"
+
+#define PRE_SYSCALL(name)                                                      \
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
+#define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
+#define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
+
+#define POST_SYSCALL(name)                                                     \
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
+#define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
+#define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
+
+#ifndef COMMON_SYSCALL_ACQUIRE
+# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
+#endif
+
+#ifndef COMMON_SYSCALL_RELEASE
+# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_CLOSE
+# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_ACQUIRE
+# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_RELEASE
+# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_PRE_FORK
+# define COMMON_SYSCALL_PRE_FORK() {}
+#endif
+
+#ifndef COMMON_SYSCALL_POST_FORK
+# define COMMON_SYSCALL_POST_FORK(res) {}
+#endif
+
+// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
+
+extern "C" {
+struct sanitizer_kernel_iovec {
+  void *iov_base;
+  unsigned long iov_len;
+};
+
+struct sanitizer_kernel_msghdr {
+  void *msg_name;
+  int msg_namelen;
+  struct sanitizer_kernel_iovec *msg_iov;
+  unsigned long msg_iovlen;
+  void *msg_control;
+  unsigned long msg_controllen;
+  unsigned msg_flags;
+};
+
+struct sanitizer_kernel_mmsghdr {
+  struct sanitizer_kernel_msghdr msg_hdr;
+  unsigned msg_len;
+};
+
+struct sanitizer_kernel_timespec {
+  long tv_sec;
+  long tv_nsec;
+};
+
+struct sanitizer_kernel_timeval {
+  long tv_sec;
+  long tv_usec;
+};
+
+struct sanitizer_kernel_rusage {
+  struct sanitizer_kernel_timeval ru_timeval[2];
+  long ru_long[14];
+};
+
+struct sanitizer_kernel_sockaddr {
+  unsigned short sa_family;
+  char sa_data[14];
+};
+
+// Real sigset size is always passed as a syscall argument.
+// Declare it "void" to catch sizeof(kernel_sigset_t).
+typedef void kernel_sigset_t;
+
+static void kernel_write_iovec(const __sanitizer_iovec *iovec,
+                        SIZE_T iovlen, SIZE_T maxlen) {
+  for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
+    SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
+    POST_WRITE(iovec[i].iov_base, sz);
+    maxlen -= sz;
+  }
+}
+
+// This functions uses POST_READ, because it needs to run after syscall to know
+// the real read range.
+static void kernel_read_iovec(const __sanitizer_iovec *iovec,
+                       SIZE_T iovlen, SIZE_T maxlen) {
+  POST_READ(iovec, sizeof(*iovec) * iovlen);
+  for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
+    SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
+    POST_READ(iovec[i].iov_base, sz);
+    maxlen -= sz;
+  }
+}
+
+PRE_SYSCALL(recvmsg)(long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
+  PRE_READ(msg, sizeof(*msg));
+}
+
+POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
+                      long flags) {
+  if (res >= 0) {
+    if (msg) {
+      for (unsigned long i = 0; i < msg->msg_iovlen; ++i) {
+        POST_WRITE(msg->msg_iov[i].iov_base, msg->msg_iov[i].iov_len);
+      }
+      POST_WRITE(msg->msg_control, msg->msg_controllen);
+    }
+  }
+}
+
+PRE_SYSCALL(recvmmsg)(long fd, sanitizer_kernel_mmsghdr *msg, long vlen,
+                      long flags, void *timeout) {
+  PRE_READ(msg, vlen * sizeof(*msg));
+}
+
+POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
+                       long vlen, long flags, void *timeout) {
+  if (res >= 0) {
+    if (msg) {
+      for (unsigned long i = 0; i < msg->msg_hdr.msg_iovlen; ++i) {
+        POST_WRITE(msg->msg_hdr.msg_iov[i].iov_base,
+                   msg->msg_hdr.msg_iov[i].iov_len);
+      }
+      POST_WRITE(msg->msg_hdr.msg_control, msg->msg_hdr.msg_controllen);
+      POST_WRITE(&msg->msg_len, sizeof(msg->msg_len));
+    }
+    if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(read)(long fd, void *buf, uptr count) {
+  if (buf) {
+    PRE_WRITE(buf, count);
+  }
+}
+
+POST_SYSCALL(read)(long res, long fd, void *buf, uptr count) {
+  if (res > 0 && buf) {
+    POST_WRITE(buf, res);
+  }
+}
+
+PRE_SYSCALL(time)(void *tloc) {}
+
+POST_SYSCALL(time)(long res, void *tloc) {
+  if (res >= 0) {
+    if (tloc) POST_WRITE(tloc, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(stime)(void *tptr) {}
+
+POST_SYSCALL(stime)(long res, void *tptr) {
+  if (res >= 0) {
+    if (tptr) POST_WRITE(tptr, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(gettimeofday)(void *tv, void *tz) {}
+
+POST_SYSCALL(gettimeofday)(long res, void *tv, void *tz) {
+  if (res >= 0) {
+    if (tv) POST_WRITE(tv, timeval_sz);
+    if (tz) POST_WRITE(tz, struct_timezone_sz);
+  }
+}
+
+PRE_SYSCALL(settimeofday)(void *tv, void *tz) {}
+
+POST_SYSCALL(settimeofday)(long res, void *tv, void *tz) {
+  if (res >= 0) {
+    if (tv) POST_WRITE(tv, timeval_sz);
+    if (tz) POST_WRITE(tz, struct_timezone_sz);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(adjtimex)(void *txc_p) {}
+
+POST_SYSCALL(adjtimex)(long res, void *txc_p) {
+  if (res >= 0) {
+    if (txc_p) POST_WRITE(txc_p, struct_timex_sz);
+  }
+}
+#endif
+
+PRE_SYSCALL(times)(void *tbuf) {}
+
+POST_SYSCALL(times)(long res, void *tbuf) {
+  if (res >= 0) {
+    if (tbuf) POST_WRITE(tbuf, struct_tms_sz);
+  }
+}
+
+PRE_SYSCALL(gettid)() {}
+
+POST_SYSCALL(gettid)(long res) {}
+
+PRE_SYSCALL(nanosleep)(void *rqtp, void *rmtp) {}
+
+POST_SYSCALL(nanosleep)(long res, void *rqtp, void *rmtp) {
+  if (res >= 0) {
+    if (rqtp) POST_WRITE(rqtp, struct_timespec_sz);
+    if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(alarm)(long seconds) {}
+
+POST_SYSCALL(alarm)(long res, long seconds) {}
+
+PRE_SYSCALL(getpid)() {}
+
+POST_SYSCALL(getpid)(long res) {}
+
+PRE_SYSCALL(getppid)() {}
+
+POST_SYSCALL(getppid)(long res) {}
+
+PRE_SYSCALL(getuid)() {}
+
+POST_SYSCALL(getuid)(long res) {}
+
+PRE_SYSCALL(geteuid)() {}
+
+POST_SYSCALL(geteuid)(long res) {}
+
+PRE_SYSCALL(getgid)() {}
+
+POST_SYSCALL(getgid)(long res) {}
+
+PRE_SYSCALL(getegid)() {}
+
+POST_SYSCALL(getegid)(long res) {}
+
+PRE_SYSCALL(getresuid)(void *ruid, void *euid, void *suid) {}
+
+POST_SYSCALL(getresuid)(long res, void *ruid, void *euid, void *suid) {
+  if (res >= 0) {
+    if (ruid) POST_WRITE(ruid, sizeof(unsigned));
+    if (euid) POST_WRITE(euid, sizeof(unsigned));
+    if (suid) POST_WRITE(suid, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(getresgid)(void *rgid, void *egid, void *sgid) {}
+
+POST_SYSCALL(getresgid)(long res, void *rgid, void *egid, void *sgid) {
+  if (res >= 0) {
+    if (rgid) POST_WRITE(rgid, sizeof(unsigned));
+    if (egid) POST_WRITE(egid, sizeof(unsigned));
+    if (sgid) POST_WRITE(sgid, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(getpgid)(long pid) {}
+
+POST_SYSCALL(getpgid)(long res, long pid) {}
+
+PRE_SYSCALL(getpgrp)() {}
+
+POST_SYSCALL(getpgrp)(long res) {}
+
+PRE_SYSCALL(getsid)(long pid) {}
+
+POST_SYSCALL(getsid)(long res, long pid) {}
+
+PRE_SYSCALL(getgroups)(long gidsetsize, void *grouplist) {}
+
+POST_SYSCALL(getgroups)(long res, long gidsetsize,
+                        __sanitizer___kernel_gid_t *grouplist) {
+  if (res >= 0) {
+    if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+  }
+}
+
+PRE_SYSCALL(setregid)(long rgid, long egid) {}
+
+POST_SYSCALL(setregid)(long res, long rgid, long egid) {}
+
+PRE_SYSCALL(setgid)(long gid) {}
+
+POST_SYSCALL(setgid)(long res, long gid) {}
+
+PRE_SYSCALL(setreuid)(long ruid, long euid) {}
+
+POST_SYSCALL(setreuid)(long res, long ruid, long euid) {}
+
+PRE_SYSCALL(setuid)(long uid) {}
+
+POST_SYSCALL(setuid)(long res, long uid) {}
+
+PRE_SYSCALL(setresuid)(long ruid, long euid, long suid) {}
+
+POST_SYSCALL(setresuid)(long res, long ruid, long euid, long suid) {}
+
+PRE_SYSCALL(setresgid)(long rgid, long egid, long sgid) {}
+
+POST_SYSCALL(setresgid)(long res, long rgid, long egid, long sgid) {}
+
+PRE_SYSCALL(setfsuid)(long uid) {}
+
+POST_SYSCALL(setfsuid)(long res, long uid) {}
+
+PRE_SYSCALL(setfsgid)(long gid) {}
+
+POST_SYSCALL(setfsgid)(long res, long gid) {}
+
+PRE_SYSCALL(setpgid)(long pid, long pgid) {}
+
+POST_SYSCALL(setpgid)(long res, long pid, long pgid) {}
+
+PRE_SYSCALL(setsid)() {}
+
+POST_SYSCALL(setsid)(long res) {}
+
+PRE_SYSCALL(setgroups)(long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
+  if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+}
+
+POST_SYSCALL(setgroups)(long res, long gidsetsize,
+                        __sanitizer___kernel_gid_t *grouplist) {}
+
+PRE_SYSCALL(acct)(const void *name) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(acct)(long res, const void *name) {}
+
+PRE_SYSCALL(capget)(void *header, void *dataptr) {
+  if (header) PRE_READ(header, __user_cap_header_struct_sz);
+}
+
+POST_SYSCALL(capget)(long res, void *header, void *dataptr) {
+  if (res >= 0)
+    if (dataptr) POST_WRITE(dataptr, __user_cap_data_struct_sz);
+}
+
+PRE_SYSCALL(capset)(void *header, const void *data) {
+  if (header) PRE_READ(header, __user_cap_header_struct_sz);
+  if (data) PRE_READ(data, __user_cap_data_struct_sz);
+}
+
+POST_SYSCALL(capset)(long res, void *header, const void *data) {}
+
+PRE_SYSCALL(personality)(long personality) {}
+
+POST_SYSCALL(personality)(long res, long personality) {}
+
+PRE_SYSCALL(sigpending)(void *set) {}
+
+POST_SYSCALL(sigpending)(long res, void *set) {
+  if (res >= 0) {
+    if (set) POST_WRITE(set, old_sigset_t_sz);
+  }
+}
+
+PRE_SYSCALL(sigprocmask)(long how, void *set, void *oset) {}
+
+POST_SYSCALL(sigprocmask)(long res, long how, void *set, void *oset) {
+  if (res >= 0) {
+    if (set) POST_WRITE(set, old_sigset_t_sz);
+    if (oset) POST_WRITE(oset, old_sigset_t_sz);
+  }
+}
+
+PRE_SYSCALL(getitimer)(long which, void *value) {}
+
+POST_SYSCALL(getitimer)(long res, long which, void *value) {
+  if (res >= 0) {
+    if (value) POST_WRITE(value, struct_itimerval_sz);
+  }
+}
+
+PRE_SYSCALL(setitimer)(long which, void *value, void *ovalue) {}
+
+POST_SYSCALL(setitimer)(long res, long which, void *value, void *ovalue) {
+  if (res >= 0) {
+    if (value) POST_WRITE(value, struct_itimerval_sz);
+    if (ovalue) POST_WRITE(ovalue, struct_itimerval_sz);
+  }
+}
+
+PRE_SYSCALL(timer_create)(long which_clock, void *timer_event_spec,
+                          void *created_timer_id) {}
+
+POST_SYSCALL(timer_create)(long res, long which_clock, void *timer_event_spec,
+                           void *created_timer_id) {
+  if (res >= 0) {
+    if (timer_event_spec) POST_WRITE(timer_event_spec, struct_sigevent_sz);
+    if (created_timer_id) POST_WRITE(created_timer_id, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(timer_gettime)(long timer_id, void *setting) {}
+
+POST_SYSCALL(timer_gettime)(long res, long timer_id, void *setting) {
+  if (res >= 0) {
+    if (setting) POST_WRITE(setting, struct_itimerspec_sz);
+  }
+}
+
+PRE_SYSCALL(timer_getoverrun)(long timer_id) {}
+
+POST_SYSCALL(timer_getoverrun)(long res, long timer_id) {}
+
+PRE_SYSCALL(timer_settime)(long timer_id, long flags, const void *new_setting,
+                           void *old_setting) {
+  if (new_setting) PRE_READ(new_setting, struct_itimerspec_sz);
+}
+
+POST_SYSCALL(timer_settime)(long res, long timer_id, long flags,
+                            const void *new_setting, void *old_setting) {
+  if (res >= 0) {
+    if (old_setting) POST_WRITE(old_setting, struct_itimerspec_sz);
+  }
+}
+
+PRE_SYSCALL(timer_delete)(long timer_id) {}
+
+POST_SYSCALL(timer_delete)(long res, long timer_id) {}
+
+PRE_SYSCALL(clock_settime)(long which_clock, const void *tp) {
+  if (tp) PRE_READ(tp, struct_timespec_sz);
+}
+
+POST_SYSCALL(clock_settime)(long res, long which_clock, const void *tp) {}
+
+PRE_SYSCALL(clock_gettime)(long which_clock, void *tp) {}
+
+POST_SYSCALL(clock_gettime)(long res, long which_clock, void *tp) {
+  if (res >= 0) {
+    if (tp) POST_WRITE(tp, struct_timespec_sz);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(clock_adjtime)(long which_clock, void *tx) {}
+
+POST_SYSCALL(clock_adjtime)(long res, long which_clock, void *tx) {
+  if (res >= 0) {
+    if (tx) POST_WRITE(tx, struct_timex_sz);
+  }
+}
+#endif
+
+PRE_SYSCALL(clock_getres)(long which_clock, void *tp) {}
+
+POST_SYSCALL(clock_getres)(long res, long which_clock, void *tp) {
+  if (res >= 0) {
+    if (tp) POST_WRITE(tp, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(clock_nanosleep)(long which_clock, long flags, const void *rqtp,
+                             void *rmtp) {
+  if (rqtp) PRE_READ(rqtp, struct_timespec_sz);
+}
+
+POST_SYSCALL(clock_nanosleep)(long res, long which_clock, long flags,
+                              const void *rqtp, void *rmtp) {
+  if (res >= 0) {
+    if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(nice)(long increment) {}
+
+POST_SYSCALL(nice)(long res, long increment) {}
+
+PRE_SYSCALL(sched_setscheduler)(long pid, long policy, void *param) {}
+
+POST_SYSCALL(sched_setscheduler)(long res, long pid, long policy, void *param) {
+  if (res >= 0) {
+    if (param) POST_WRITE(param, struct_sched_param_sz);
+  }
+}
+
+PRE_SYSCALL(sched_setparam)(long pid, void *param) {
+  if (param) PRE_READ(param, struct_sched_param_sz);
+}
+
+POST_SYSCALL(sched_setparam)(long res, long pid, void *param) {}
+
+PRE_SYSCALL(sched_getscheduler)(long pid) {}
+
+POST_SYSCALL(sched_getscheduler)(long res, long pid) {}
+
+PRE_SYSCALL(sched_getparam)(long pid, void *param) {}
+
+POST_SYSCALL(sched_getparam)(long res, long pid, void *param) {
+  if (res >= 0) {
+    if (param) POST_WRITE(param, struct_sched_param_sz);
+  }
+}
+
+PRE_SYSCALL(sched_setaffinity)(long pid, long len, void *user_mask_ptr) {
+  if (user_mask_ptr) PRE_READ(user_mask_ptr, len);
+}
+
+POST_SYSCALL(sched_setaffinity)(long res, long pid, long len,
+                                void *user_mask_ptr) {}
+
+PRE_SYSCALL(sched_getaffinity)(long pid, long len, void *user_mask_ptr) {}
+
+POST_SYSCALL(sched_getaffinity)(long res, long pid, long len,
+                                void *user_mask_ptr) {
+  if (res >= 0) {
+    if (user_mask_ptr) POST_WRITE(user_mask_ptr, len);
+  }
+}
+
+PRE_SYSCALL(sched_yield)() {}
+
+POST_SYSCALL(sched_yield)(long res) {}
+
+PRE_SYSCALL(sched_get_priority_max)(long policy) {}
+
+POST_SYSCALL(sched_get_priority_max)(long res, long policy) {}
+
+PRE_SYSCALL(sched_get_priority_min)(long policy) {}
+
+POST_SYSCALL(sched_get_priority_min)(long res, long policy) {}
+
+PRE_SYSCALL(sched_rr_get_interval)(long pid, void *interval) {}
+
+POST_SYSCALL(sched_rr_get_interval)(long res, long pid, void *interval) {
+  if (res >= 0) {
+    if (interval) POST_WRITE(interval, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(setpriority)(long which, long who, long niceval) {}
+
+POST_SYSCALL(setpriority)(long res, long which, long who, long niceval) {}
+
+PRE_SYSCALL(getpriority)(long which, long who) {}
+
+POST_SYSCALL(getpriority)(long res, long which, long who) {}
+
+PRE_SYSCALL(shutdown)(long arg0, long arg1) {}
+
+POST_SYSCALL(shutdown)(long res, long arg0, long arg1) {}
+
+PRE_SYSCALL(reboot)(long magic1, long magic2, long cmd, void *arg) {}
+
+POST_SYSCALL(reboot)(long res, long magic1, long magic2, long cmd, void *arg) {}
+
+PRE_SYSCALL(restart_syscall)() {}
+
+POST_SYSCALL(restart_syscall)(long res) {}
+
+PRE_SYSCALL(kexec_load)(long entry, long nr_segments, void *segments,
+                        long flags) {}
+
+POST_SYSCALL(kexec_load)(long res, long entry, long nr_segments, void *segments,
+                         long flags) {
+  if (res >= 0) {
+    if (segments) POST_WRITE(segments, struct_kexec_segment_sz);
+  }
+}
+
+PRE_SYSCALL(exit)(long error_code) {}
+
+POST_SYSCALL(exit)(long res, long error_code) {}
+
+PRE_SYSCALL(exit_group)(long error_code) {}
+
+POST_SYSCALL(exit_group)(long res, long error_code) {}
+
+PRE_SYSCALL(wait4)(long pid, void *stat_addr, long options, void *ru) {}
+
+POST_SYSCALL(wait4)(long res, long pid, void *stat_addr, long options,
+                    void *ru) {
+  if (res >= 0) {
+    if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
+    if (ru) POST_WRITE(ru, struct_rusage_sz);
+  }
+}
+
+PRE_SYSCALL(waitid)(long which, long pid, void *infop, long options, void *ru) {
+}
+
+POST_SYSCALL(waitid)(long res, long which, long pid, void *infop, long options,
+                     void *ru) {
+  if (res >= 0) {
+    if (infop) POST_WRITE(infop, siginfo_t_sz);
+    if (ru) POST_WRITE(ru, struct_rusage_sz);
+  }
+}
+
+PRE_SYSCALL(waitpid)(long pid, void *stat_addr, long options) {}
+
+POST_SYSCALL(waitpid)(long res, long pid, void *stat_addr, long options) {
+  if (res >= 0) {
+    if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
+  }
+}
+
+PRE_SYSCALL(set_tid_address)(void *tidptr) {}
+
+POST_SYSCALL(set_tid_address)(long res, void *tidptr) {
+  if (res >= 0) {
+    if (tidptr) POST_WRITE(tidptr, sizeof(int));
+  }
+}
+
+PRE_SYSCALL(init_module)(void *umod, long len, const void *uargs) {
+  if (uargs)
+    PRE_READ(uargs, __sanitizer::internal_strlen((const char *)uargs) + 1);
+}
+
+POST_SYSCALL(init_module)(long res, void *umod, long len, const void *uargs) {}
+
+PRE_SYSCALL(delete_module)(const void *name_user, long flags) {
+  if (name_user)
+    PRE_READ(name_user,
+             __sanitizer::internal_strlen((const char *)name_user) + 1);
+}
+
+POST_SYSCALL(delete_module)(long res, const void *name_user, long flags) {}
+
+PRE_SYSCALL(rt_sigprocmask)(long how, void *set, void *oset, long sigsetsize) {}
+
+POST_SYSCALL(rt_sigprocmask)(long res, long how, kernel_sigset_t *set,
+                             kernel_sigset_t *oset, long sigsetsize) {
+  if (res >= 0) {
+    if (set) POST_WRITE(set, sigsetsize);
+    if (oset) POST_WRITE(oset, sigsetsize);
+  }
+}
+
+PRE_SYSCALL(rt_sigpending)(void *set, long sigsetsize) {}
+
+POST_SYSCALL(rt_sigpending)(long res, kernel_sigset_t *set, long sigsetsize) {
+  if (res >= 0) {
+    if (set) POST_WRITE(set, sigsetsize);
+  }
+}
+
+PRE_SYSCALL(rt_sigtimedwait)(const kernel_sigset_t *uthese, void *uinfo,
+                             const void *uts, long sigsetsize) {
+  if (uthese) PRE_READ(uthese, sigsetsize);
+  if (uts) PRE_READ(uts, struct_timespec_sz);
+}
+
+POST_SYSCALL(rt_sigtimedwait)(long res, const void *uthese, void *uinfo,
+                              const void *uts, long sigsetsize) {
+  if (res >= 0) {
+    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+  }
+}
+
+PRE_SYSCALL(rt_tgsigqueueinfo)(long tgid, long pid, long sig, void *uinfo) {}
+
+POST_SYSCALL(rt_tgsigqueueinfo)(long res, long tgid, long pid, long sig,
+                                void *uinfo) {
+  if (res >= 0) {
+    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+  }
+}
+
+PRE_SYSCALL(kill)(long pid, long sig) {}
+
+POST_SYSCALL(kill)(long res, long pid, long sig) {}
+
+PRE_SYSCALL(tgkill)(long tgid, long pid, long sig) {}
+
+POST_SYSCALL(tgkill)(long res, long tgid, long pid, long sig) {}
+
+PRE_SYSCALL(tkill)(long pid, long sig) {}
+
+POST_SYSCALL(tkill)(long res, long pid, long sig) {}
+
+PRE_SYSCALL(rt_sigqueueinfo)(long pid, long sig, void *uinfo) {}
+
+POST_SYSCALL(rt_sigqueueinfo)(long res, long pid, long sig, void *uinfo) {
+  if (res >= 0) {
+    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+  }
+}
+
+PRE_SYSCALL(sgetmask)() {}
+
+POST_SYSCALL(sgetmask)(long res) {}
+
+PRE_SYSCALL(ssetmask)(long newmask) {}
+
+POST_SYSCALL(ssetmask)(long res, long newmask) {}
+
+PRE_SYSCALL(signal)(long sig, long handler) {}
+
+POST_SYSCALL(signal)(long res, long sig, long handler) {}
+
+PRE_SYSCALL(pause)() {}
+
+POST_SYSCALL(pause)(long res) {}
+
+PRE_SYSCALL(sync)() {}
+
+POST_SYSCALL(sync)(long res) {}
+
+PRE_SYSCALL(fsync)(long fd) {}
+
+POST_SYSCALL(fsync)(long res, long fd) {}
+
+PRE_SYSCALL(fdatasync)(long fd) {}
+
+POST_SYSCALL(fdatasync)(long res, long fd) {}
+
+PRE_SYSCALL(bdflush)(long func, long data) {}
+
+POST_SYSCALL(bdflush)(long res, long func, long data) {}
+
+PRE_SYSCALL(mount)(void *dev_name, void *dir_name, void *type, long flags,
+                   void *data) {}
+
+POST_SYSCALL(mount)(long res, void *dev_name, void *dir_name, void *type,
+                    long flags, void *data) {
+  if (res >= 0) {
+    if (dev_name)
+      POST_WRITE(dev_name,
+                 __sanitizer::internal_strlen((const char *)dev_name) + 1);
+    if (dir_name)
+      POST_WRITE(dir_name,
+                 __sanitizer::internal_strlen((const char *)dir_name) + 1);
+    if (type)
+      POST_WRITE(type, __sanitizer::internal_strlen((const char *)type) + 1);
+  }
+}
+
+PRE_SYSCALL(umount)(void *name, long flags) {}
+
+POST_SYSCALL(umount)(long res, void *name, long flags) {
+  if (res >= 0) {
+    if (name)
+      POST_WRITE(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  }
+}
+
+PRE_SYSCALL(oldumount)(void *name) {}
+
+POST_SYSCALL(oldumount)(long res, void *name) {
+  if (res >= 0) {
+    if (name)
+      POST_WRITE(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  }
+}
+
+PRE_SYSCALL(truncate)(const void *path, long length) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(truncate)(long res, const void *path, long length) {}
+
+PRE_SYSCALL(ftruncate)(long fd, long length) {}
+
+POST_SYSCALL(ftruncate)(long res, long fd, long length) {}
+
+PRE_SYSCALL(stat)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(stat)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(statfs)(const void *path, void *buf) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(statfs)(long res, const void *path, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, struct_statfs_sz);
+  }
+}
+
+PRE_SYSCALL(statfs64)(const void *path, long sz, void *buf) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(statfs64)(long res, const void *path, long sz, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, struct_statfs64_sz);
+  }
+}
+
+PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
+
+POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, struct_statfs_sz);
+  }
+}
+
+PRE_SYSCALL(fstatfs64)(long fd, long sz, void *buf) {}
+
+POST_SYSCALL(fstatfs64)(long res, long fd, long sz, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, struct_statfs64_sz);
+  }
+}
+#endif // !SANITIZER_ANDROID
+
+PRE_SYSCALL(lstat)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(lstat)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+  }
+}
+
+PRE_SYSCALL(fstat)(long fd, void *statbuf) {}
+
+POST_SYSCALL(fstat)(long res, long fd, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+  }
+}
+
+PRE_SYSCALL(newstat)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(newstat)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+  }
+}
+
+PRE_SYSCALL(newlstat)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(newlstat)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+  }
+}
+
+PRE_SYSCALL(newfstat)(long fd, void *statbuf) {}
+
+POST_SYSCALL(newfstat)(long res, long fd, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(ustat)(long dev, void *ubuf) {}
+
+POST_SYSCALL(ustat)(long res, long dev, void *ubuf) {
+  if (res >= 0) {
+    if (ubuf) POST_WRITE(ubuf, struct_ustat_sz);
+  }
+}
+#endif  // !SANITIZER_ANDROID
+
+PRE_SYSCALL(stat64)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(stat64)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+  }
+}
+
+PRE_SYSCALL(fstat64)(long fd, void *statbuf) {}
+
+POST_SYSCALL(fstat64)(long res, long fd, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+  }
+}
+
+PRE_SYSCALL(lstat64)(const void *filename, void *statbuf) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(lstat64)(long res, const void *filename, void *statbuf) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+  }
+}
+
+PRE_SYSCALL(setxattr)(const void *path, const void *name, const void *value,
+                      long size, long flags) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  if (value) PRE_READ(value, size);
+}
+
+POST_SYSCALL(setxattr)(long res, const void *path, const void *name,
+                       const void *value, long size, long flags) {}
+
+PRE_SYSCALL(lsetxattr)(const void *path, const void *name, const void *value,
+                       long size, long flags) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  if (value) PRE_READ(value, size);
+}
+
+POST_SYSCALL(lsetxattr)(long res, const void *path, const void *name,
+                        const void *value, long size, long flags) {}
+
+PRE_SYSCALL(fsetxattr)(long fd, const void *name, const void *value, long size,
+                       long flags) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  if (value) PRE_READ(value, size);
+}
+
+POST_SYSCALL(fsetxattr)(long res, long fd, const void *name, const void *value,
+                        long size, long flags) {}
+
+PRE_SYSCALL(getxattr)(const void *path, const void *name, void *value,
+                      long size) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(getxattr)(long res, const void *path, const void *name,
+                       void *value, long size) {
+  if (size && res > 0) {
+    if (value) POST_WRITE(value, res);
+  }
+}
+
+PRE_SYSCALL(lgetxattr)(const void *path, const void *name, void *value,
+                       long size) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(lgetxattr)(long res, const void *path, const void *name,
+                        void *value, long size) {
+  if (size && res > 0) {
+    if (value) POST_WRITE(value, res);
+  }
+}
+
+PRE_SYSCALL(fgetxattr)(long fd, const void *name, void *value, long size) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(fgetxattr)(long res, long fd, const void *name, void *value,
+                        long size) {
+  if (size && res > 0) {
+    if (value) POST_WRITE(value, res);
+  }
+}
+
+PRE_SYSCALL(listxattr)(const void *path, void *list, long size) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(listxattr)(long res, const void *path, void *list, long size) {
+  if (size && res > 0) {
+    if (list) POST_WRITE(list, res);
+  }
+}
+
+PRE_SYSCALL(llistxattr)(const void *path, void *list, long size) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(llistxattr)(long res, const void *path, void *list, long size) {
+  if (size && res > 0) {
+    if (list) POST_WRITE(list, res);
+  }
+}
+
+PRE_SYSCALL(flistxattr)(long fd, void *list, long size) {}
+
+POST_SYSCALL(flistxattr)(long res, long fd, void *list, long size) {
+  if (size && res > 0) {
+    if (list) POST_WRITE(list, res);
+  }
+}
+
+PRE_SYSCALL(removexattr)(const void *path, const void *name) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(removexattr)(long res, const void *path, const void *name) {}
+
+PRE_SYSCALL(lremovexattr)(const void *path, const void *name) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(lremovexattr)(long res, const void *path, const void *name) {}
+
+PRE_SYSCALL(fremovexattr)(long fd, const void *name) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(fremovexattr)(long res, long fd, const void *name) {}
+
+PRE_SYSCALL(brk)(long brk) {}
+
+POST_SYSCALL(brk)(long res, long brk) {}
+
+PRE_SYSCALL(mprotect)(long start, long len, long prot) {}
+
+POST_SYSCALL(mprotect)(long res, long start, long len, long prot) {}
+
+PRE_SYSCALL(mremap)(long addr, long old_len, long new_len, long flags,
+                    long new_addr) {}
+
+POST_SYSCALL(mremap)(long res, long addr, long old_len, long new_len,
+                     long flags, long new_addr) {}
+
+PRE_SYSCALL(remap_file_pages)(long start, long size, long prot, long pgoff,
+                              long flags) {}
+
+POST_SYSCALL(remap_file_pages)(long res, long start, long size, long prot,
+                               long pgoff, long flags) {}
+
+PRE_SYSCALL(msync)(long start, long len, long flags) {}
+
+POST_SYSCALL(msync)(long res, long start, long len, long flags) {}
+
+PRE_SYSCALL(munmap)(long addr, long len) {}
+
+POST_SYSCALL(munmap)(long res, long addr, long len) {}
+
+PRE_SYSCALL(mlock)(long start, long len) {}
+
+POST_SYSCALL(mlock)(long res, long start, long len) {}
+
+PRE_SYSCALL(munlock)(long start, long len) {}
+
+POST_SYSCALL(munlock)(long res, long start, long len) {}
+
+PRE_SYSCALL(mlockall)(long flags) {}
+
+POST_SYSCALL(mlockall)(long res, long flags) {}
+
+PRE_SYSCALL(munlockall)() {}
+
+POST_SYSCALL(munlockall)(long res) {}
+
+PRE_SYSCALL(madvise)(long start, long len, long behavior) {}
+
+POST_SYSCALL(madvise)(long res, long start, long len, long behavior) {}
+
+PRE_SYSCALL(mincore)(long start, long len, void *vec) {}
+
+POST_SYSCALL(mincore)(long res, long start, long len, void *vec) {
+  if (res >= 0) {
+    if (vec) {
+      POST_WRITE(vec, (len + GetPageSizeCached() - 1) / GetPageSizeCached());
+    }
+  }
+}
+
+PRE_SYSCALL(pivot_root)(const void *new_root, const void *put_old) {
+  if (new_root)
+    PRE_READ(new_root,
+             __sanitizer::internal_strlen((const char *)new_root) + 1);
+  if (put_old)
+    PRE_READ(put_old, __sanitizer::internal_strlen((const char *)put_old) + 1);
+}
+
+POST_SYSCALL(pivot_root)(long res, const void *new_root, const void *put_old) {}
+
+PRE_SYSCALL(chroot)(const void *filename) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(chroot)(long res, const void *filename) {}
+
+PRE_SYSCALL(mknod)(const void *filename, long mode, long dev) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(mknod)(long res, const void *filename, long mode, long dev) {}
+
+PRE_SYSCALL(link)(const void *oldname, const void *newname) {
+  if (oldname)
+    PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
+  if (newname)
+    PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
+}
+
+POST_SYSCALL(link)(long res, const void *oldname, const void *newname) {}
+
+PRE_SYSCALL(symlink)(const void *old, const void *new_) {
+  if (old) PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
+  if (new_)
+    PRE_READ(new_, __sanitizer::internal_strlen((const char *)new_) + 1);
+}
+
+POST_SYSCALL(symlink)(long res, const void *old, const void *new_) {}
+
+PRE_SYSCALL(unlink)(const void *pathname) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(unlink)(long res, const void *pathname) {}
+
+PRE_SYSCALL(rename)(const void *oldname, const void *newname) {
+  if (oldname)
+    PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
+  if (newname)
+    PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
+}
+
+POST_SYSCALL(rename)(long res, const void *oldname, const void *newname) {}
+
+PRE_SYSCALL(chmod)(const void *filename, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(chmod)(long res, const void *filename, long mode) {}
+
+PRE_SYSCALL(fchmod)(long fd, long mode) {}
+
+POST_SYSCALL(fchmod)(long res, long fd, long mode) {}
+
+PRE_SYSCALL(fcntl)(long fd, long cmd, long arg) {}
+
+POST_SYSCALL(fcntl)(long res, long fd, long cmd, long arg) {}
+
+PRE_SYSCALL(fcntl64)(long fd, long cmd, long arg) {}
+
+POST_SYSCALL(fcntl64)(long res, long fd, long cmd, long arg) {}
+
+PRE_SYSCALL(pipe)(void *fildes) {}
+
+POST_SYSCALL(pipe)(long res, void *fildes) {
+  if (res >= 0)
+    if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+}
+
+PRE_SYSCALL(pipe2)(void *fildes, long flags) {}
+
+POST_SYSCALL(pipe2)(long res, void *fildes, long flags) {
+  if (res >= 0)
+    if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+}
+
+PRE_SYSCALL(dup)(long fildes) {}
+
+POST_SYSCALL(dup)(long res, long fildes) {}
+
+PRE_SYSCALL(dup2)(long oldfd, long newfd) {}
+
+POST_SYSCALL(dup2)(long res, long oldfd, long newfd) {}
+
+PRE_SYSCALL(dup3)(long oldfd, long newfd, long flags) {}
+
+POST_SYSCALL(dup3)(long res, long oldfd, long newfd, long flags) {}
+
+PRE_SYSCALL(ioperm)(long from, long num, long on) {}
+
+POST_SYSCALL(ioperm)(long res, long from, long num, long on) {}
+
+PRE_SYSCALL(ioctl)(long fd, long cmd, long arg) {}
+
+POST_SYSCALL(ioctl)(long res, long fd, long cmd, long arg) {}
+
+PRE_SYSCALL(flock)(long fd, long cmd) {}
+
+POST_SYSCALL(flock)(long res, long fd, long cmd) {}
+
+PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
+  if (ctx) PRE_WRITE(ctx, sizeof(*ctx));
+}
+
+POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
+  if (res >= 0) {
+    if (ctx) POST_WRITE(ctx, sizeof(*ctx));
+    // (*ctx) is actually a pointer to a kernel mapped page, and there are
+    // people out there who are crazy enough to peek into that page's 32-byte
+    // header.
+    if (*ctx) POST_WRITE(*ctx, 32);
+  }
+}
+
+PRE_SYSCALL(io_destroy)(long ctx) {}
+
+POST_SYSCALL(io_destroy)(long res, long ctx) {}
+
+PRE_SYSCALL(io_getevents)(long ctx_id, long min_nr, long nr,
+                          __sanitizer_io_event *ioevpp, void *timeout) {
+  if (timeout) PRE_READ(timeout, struct_timespec_sz);
+}
+
+POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
+                           __sanitizer_io_event *ioevpp, void *timeout) {
+  if (res >= 0) {
+    if (ioevpp) POST_WRITE(ioevpp, res * sizeof(*ioevpp));
+    if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+  }
+  for (long i = 0; i < res; i++) {
+    // We synchronize io_submit -> io_getevents/io_cancel using the
+    // user-provided data context. Data is not necessary a pointer, it can be
+    // an int, 0 or whatever; acquire/release will correctly handle this.
+    // This scheme can lead to false negatives, e.g. when all operations
+    // synchronize on 0. But there does not seem to be a better solution
+    // (except wrapping all operations in own context, which is unreliable).
+    // We can not reliably extract fildes in io_getevents.
+    COMMON_SYSCALL_ACQUIRE((void*)ioevpp[i].data);
+  }
+}
+
+PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
+  for (long i = 0; i < nr; ++i) {
+    uptr op = iocbpp[i]->aio_lio_opcode;
+    void *data = (void*)iocbpp[i]->aio_data;
+    void *buf = (void*)iocbpp[i]->aio_buf;
+    uptr len = (uptr)iocbpp[i]->aio_nbytes;
+    if (op == iocb_cmd_pwrite && buf && len) {
+      PRE_READ(buf, len);
+    } else if (op == iocb_cmd_pread && buf && len) {
+      POST_WRITE(buf, len);
+    } else if (op == iocb_cmd_pwritev) {
+      __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+      for (uptr v = 0; v < len; v++)
+        PRE_READ(iovec[v].iov_base, iovec[v].iov_len);
+    } else if (op == iocb_cmd_preadv) {
+      __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+      for (uptr v = 0; v < len; v++)
+        POST_WRITE(iovec[v].iov_base, iovec[v].iov_len);
+    }
+    // See comment in io_getevents.
+    COMMON_SYSCALL_RELEASE(data);
+  }
+}
+
+POST_SYSCALL(io_submit)(long res, long ctx_id, long nr,
+    __sanitizer_iocb **iocbpp) {}
+
+PRE_SYSCALL(io_cancel)(long ctx_id, __sanitizer_iocb *iocb,
+    __sanitizer_io_event *result) {
+}
+
+POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
+    __sanitizer_io_event *result) {
+  if (res == 0) {
+    if (result) {
+      // See comment in io_getevents.
+      COMMON_SYSCALL_ACQUIRE((void*)result->data);
+      POST_WRITE(result, sizeof(*result));
+    }
+    if (iocb)
+      POST_WRITE(iocb, sizeof(*iocb));
+  }
+}
+
+PRE_SYSCALL(sendfile)(long out_fd, long in_fd, void *offset, long count) {}
+
+POST_SYSCALL(sendfile)(long res, long out_fd, long in_fd,
+                       __sanitizer___kernel_off_t *offset, long count) {
+  if (res >= 0) {
+    if (offset) POST_WRITE(offset, sizeof(*offset));
+  }
+}
+
+PRE_SYSCALL(sendfile64)(long out_fd, long in_fd, void *offset, long count) {}
+
+POST_SYSCALL(sendfile64)(long res, long out_fd, long in_fd,
+                         __sanitizer___kernel_loff_t *offset, long count) {
+  if (res >= 0) {
+    if (offset) POST_WRITE(offset, sizeof(*offset));
+  }
+}
+
+PRE_SYSCALL(readlink)(const void *path, void *buf, long bufsiz) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(readlink)(long res, const void *path, void *buf, long bufsiz) {
+  if (res >= 0) {
+    if (buf)
+      POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
+  }
+}
+
+PRE_SYSCALL(creat)(const void *pathname, long mode) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(creat)(long res, const void *pathname, long mode) {}
+
+PRE_SYSCALL(open)(const void *filename, long flags, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(open)(long res, const void *filename, long flags, long mode) {}
+
+PRE_SYSCALL(close)(long fd) {
+  COMMON_SYSCALL_FD_CLOSE((int)fd);
+}
+
+POST_SYSCALL(close)(long res, long fd) {}
+
+PRE_SYSCALL(access)(const void *filename, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(access)(long res, const void *filename, long mode) {}
+
+PRE_SYSCALL(vhangup)() {}
+
+POST_SYSCALL(vhangup)(long res) {}
+
+PRE_SYSCALL(chown)(const void *filename, long user, long group) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(chown)(long res, const void *filename, long user, long group) {}
+
+PRE_SYSCALL(lchown)(const void *filename, long user, long group) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(lchown)(long res, const void *filename, long user, long group) {}
+
+PRE_SYSCALL(fchown)(long fd, long user, long group) {}
+
+POST_SYSCALL(fchown)(long res, long fd, long user, long group) {}
+
+#if SANITIZER_USES_UID16_SYSCALLS
+PRE_SYSCALL(chown16)(const void *filename, long user, long group) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(chown16)(long res, const void *filename, long user, long group) {}
+
+PRE_SYSCALL(lchown16)(const void *filename, long user, long group) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(lchown16)(long res, const void *filename, long user, long group) {}
+
+PRE_SYSCALL(fchown16)(long fd, long user, long group) {}
+
+POST_SYSCALL(fchown16)(long res, long fd, long user, long group) {}
+
+PRE_SYSCALL(setregid16)(long rgid, long egid) {}
+
+POST_SYSCALL(setregid16)(long res, long rgid, long egid) {}
+
+PRE_SYSCALL(setgid16)(long gid) {}
+
+POST_SYSCALL(setgid16)(long res, long gid) {}
+
+PRE_SYSCALL(setreuid16)(long ruid, long euid) {}
+
+POST_SYSCALL(setreuid16)(long res, long ruid, long euid) {}
+
+PRE_SYSCALL(setuid16)(long uid) {}
+
+POST_SYSCALL(setuid16)(long res, long uid) {}
+
+PRE_SYSCALL(setresuid16)(long ruid, long euid, long suid) {}
+
+POST_SYSCALL(setresuid16)(long res, long ruid, long euid, long suid) {}
+
+PRE_SYSCALL(getresuid16)(void *ruid, void *euid, void *suid) {}
+
+POST_SYSCALL(getresuid16)(long res, __sanitizer___kernel_old_uid_t *ruid,
+                          __sanitizer___kernel_old_uid_t *euid,
+                          __sanitizer___kernel_old_uid_t *suid) {
+  if (res >= 0) {
+    if (ruid) POST_WRITE(ruid, sizeof(*ruid));
+    if (euid) POST_WRITE(euid, sizeof(*euid));
+    if (suid) POST_WRITE(suid, sizeof(*suid));
+  }
+}
+
+PRE_SYSCALL(setresgid16)(long rgid, long egid, long sgid) {}
+
+POST_SYSCALL(setresgid16)(long res, long rgid, long egid, long sgid) {}
+
+PRE_SYSCALL(getresgid16)(void *rgid, void *egid, void *sgid) {}
+
+POST_SYSCALL(getresgid16)(long res, __sanitizer___kernel_old_gid_t *rgid,
+                          __sanitizer___kernel_old_gid_t *egid,
+                          __sanitizer___kernel_old_gid_t *sgid) {
+  if (res >= 0) {
+    if (rgid) POST_WRITE(rgid, sizeof(*rgid));
+    if (egid) POST_WRITE(egid, sizeof(*egid));
+    if (sgid) POST_WRITE(sgid, sizeof(*sgid));
+  }
+}
+
+PRE_SYSCALL(setfsuid16)(long uid) {}
+
+POST_SYSCALL(setfsuid16)(long res, long uid) {}
+
+PRE_SYSCALL(setfsgid16)(long gid) {}
+
+POST_SYSCALL(setfsgid16)(long res, long gid) {}
+
+PRE_SYSCALL(getgroups16)(long gidsetsize,
+                         __sanitizer___kernel_old_gid_t *grouplist) {}
+
+POST_SYSCALL(getgroups16)(long res, long gidsetsize,
+                          __sanitizer___kernel_old_gid_t *grouplist) {
+  if (res >= 0) {
+    if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+  }
+}
+
+PRE_SYSCALL(setgroups16)(long gidsetsize,
+                         __sanitizer___kernel_old_gid_t *grouplist) {
+  if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+}
+
+POST_SYSCALL(setgroups16)(long res, long gidsetsize,
+                          __sanitizer___kernel_old_gid_t *grouplist) {}
+
+PRE_SYSCALL(getuid16)() {}
+
+POST_SYSCALL(getuid16)(long res) {}
+
+PRE_SYSCALL(geteuid16)() {}
+
+POST_SYSCALL(geteuid16)(long res) {}
+
+PRE_SYSCALL(getgid16)() {}
+
+POST_SYSCALL(getgid16)(long res) {}
+
+PRE_SYSCALL(getegid16)() {}
+
+POST_SYSCALL(getegid16)(long res) {}
+#endif // SANITIZER_USES_UID16_SYSCALLS
+
+PRE_SYSCALL(utime)(void *filename, void *times) {}
+
+POST_SYSCALL(utime)(long res, void *filename, void *times) {
+  if (res >= 0) {
+    if (filename)
+      POST_WRITE(filename,
+                 __sanitizer::internal_strlen((const char *)filename) + 1);
+    if (times) POST_WRITE(times, struct_utimbuf_sz);
+  }
+}
+
+PRE_SYSCALL(utimes)(void *filename, void *utimes) {}
+
+POST_SYSCALL(utimes)(long res, void *filename, void *utimes) {
+  if (res >= 0) {
+    if (filename)
+      POST_WRITE(filename,
+                 __sanitizer::internal_strlen((const char *)filename) + 1);
+    if (utimes) POST_WRITE(utimes, timeval_sz);
+  }
+}
+
+PRE_SYSCALL(lseek)(long fd, long offset, long origin) {}
+
+POST_SYSCALL(lseek)(long res, long fd, long offset, long origin) {}
+
+PRE_SYSCALL(llseek)(long fd, long offset_high, long offset_low, void *result,
+                    long origin) {}
+
+POST_SYSCALL(llseek)(long res, long fd, long offset_high, long offset_low,
+                     void *result, long origin) {
+  if (res >= 0) {
+    if (result) POST_WRITE(result, sizeof(long long));
+  }
+}
+
+PRE_SYSCALL(readv)(long fd, const __sanitizer_iovec *vec, long vlen) {}
+
+POST_SYSCALL(readv)(long res, long fd, const __sanitizer_iovec *vec,
+                    long vlen) {
+  if (res >= 0) {
+    if (vec) kernel_write_iovec(vec, vlen, res);
+  }
+}
+
+PRE_SYSCALL(write)(long fd, const void *buf, long count) {
+  if (buf) PRE_READ(buf, count);
+}
+
+POST_SYSCALL(write)(long res, long fd, const void *buf, long count) {}
+
+PRE_SYSCALL(writev)(long fd, const __sanitizer_iovec *vec, long vlen) {}
+
+POST_SYSCALL(writev)(long res, long fd, const __sanitizer_iovec *vec,
+                     long vlen) {
+  if (res >= 0) {
+    if (vec) kernel_read_iovec(vec, vlen, res);
+  }
+}
+
+#ifdef _LP64
+PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos) {}
+
+POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, res);
+  }
+}
+
+PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos) {
+  if (buf) PRE_READ(buf, count);
+}
+
+POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
+                       long pos) {}
+#else
+PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos0, long pos1) {}
+
+POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos0,
+                      long pos1) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, res);
+  }
+}
+
+PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos0,
+                      long pos1) {
+  if (buf) PRE_READ(buf, count);
+}
+
+POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
+                       long pos0, long pos1) {}
+#endif
+
+PRE_SYSCALL(preadv)(long fd, const __sanitizer_iovec *vec, long vlen,
+                    long pos_l, long pos_h) {}
+
+POST_SYSCALL(preadv)(long res, long fd, const __sanitizer_iovec *vec, long vlen,
+                     long pos_l, long pos_h) {
+  if (res >= 0) {
+    if (vec) kernel_write_iovec(vec, vlen, res);
+  }
+}
+
+PRE_SYSCALL(pwritev)(long fd, const __sanitizer_iovec *vec, long vlen,
+                     long pos_l, long pos_h) {}
+
+POST_SYSCALL(pwritev)(long res, long fd, const __sanitizer_iovec *vec,
+                      long vlen, long pos_l, long pos_h) {
+  if (res >= 0) {
+    if (vec) kernel_read_iovec(vec, vlen, res);
+  }
+}
+
+PRE_SYSCALL(getcwd)(void *buf, long size) {}
+
+POST_SYSCALL(getcwd)(long res, void *buf, long size) {
+  if (res >= 0) {
+    if (buf)
+      POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
+  }
+}
+
+PRE_SYSCALL(mkdir)(const void *pathname, long mode) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(mkdir)(long res, const void *pathname, long mode) {}
+
+PRE_SYSCALL(chdir)(const void *filename) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(chdir)(long res, const void *filename) {}
+
+PRE_SYSCALL(fchdir)(long fd) {}
+
+POST_SYSCALL(fchdir)(long res, long fd) {}
+
+PRE_SYSCALL(rmdir)(const void *pathname) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(rmdir)(long res, const void *pathname) {}
+
+PRE_SYSCALL(lookup_dcookie)(u64 cookie64, void *buf, long len) {}
+
+POST_SYSCALL(lookup_dcookie)(long res, u64 cookie64, void *buf, long len) {
+  if (res >= 0) {
+    if (buf)
+      POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
+  }
+}
+
+PRE_SYSCALL(quotactl)(long cmd, const void *special, long id, void *addr) {
+  if (special)
+    PRE_READ(special, __sanitizer::internal_strlen((const char *)special) + 1);
+}
+
+POST_SYSCALL(quotactl)(long res, long cmd, const void *special, long id,
+                       void *addr) {}
+
+PRE_SYSCALL(getdents)(long fd, void *dirent, long count) {}
+
+POST_SYSCALL(getdents)(long res, long fd, void *dirent, long count) {
+  if (res >= 0) {
+    if (dirent) POST_WRITE(dirent, res);
+  }
+}
+
+PRE_SYSCALL(getdents64)(long fd, void *dirent, long count) {}
+
+POST_SYSCALL(getdents64)(long res, long fd, void *dirent, long count) {
+  if (res >= 0) {
+    if (dirent) POST_WRITE(dirent, res);
+  }
+}
+
+PRE_SYSCALL(setsockopt)(long fd, long level, long optname, void *optval,
+                        long optlen) {}
+
+POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
+                         void *optval, long optlen) {
+  if (res >= 0) {
+    if (optval)
+      POST_WRITE(optval,
+                 __sanitizer::internal_strlen((const char *)optval) + 1);
+  }
+}
+
+PRE_SYSCALL(getsockopt)(long fd, long level, long optname, void *optval,
+                        void *optlen) {}
+
+POST_SYSCALL(getsockopt)(long res, long fd, long level, long optname,
+                         void *optval, void *optlen) {
+  if (res >= 0) {
+    if (optval)
+      POST_WRITE(optval,
+                 __sanitizer::internal_strlen((const char *)optval) + 1);
+    if (optlen) POST_WRITE(optlen, sizeof(int));
+  }
+}
+
+PRE_SYSCALL(bind)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
+
+POST_SYSCALL(bind)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                   long arg2) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+  }
+}
+
+PRE_SYSCALL(connect)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
+
+POST_SYSCALL(connect)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                      long arg2) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+  }
+}
+
+PRE_SYSCALL(accept)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
+
+POST_SYSCALL(accept)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                     void *arg2) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(accept4)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2,
+                     long arg3) {}
+
+POST_SYSCALL(accept4)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                      void *arg2, long arg3) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(getsockname)(long arg0, sanitizer_kernel_sockaddr *arg1,
+                         void *arg2) {}
+
+POST_SYSCALL(getsockname)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                          void *arg2) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(getpeername)(long arg0, sanitizer_kernel_sockaddr *arg1,
+                         void *arg2) {}
+
+POST_SYSCALL(getpeername)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+                          void *arg2) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+  }
+}
+
+PRE_SYSCALL(send)(long arg0, void *arg1, long arg2, long arg3) {}
+
+POST_SYSCALL(send)(long res, long arg0, void *arg1, long arg2, long arg3) {
+  if (res) {
+    if (arg1) POST_READ(arg1, res);
+  }
+}
+
+PRE_SYSCALL(sendto)(long arg0, void *arg1, long arg2, long arg3,
+                    sanitizer_kernel_sockaddr *arg4, long arg5) {}
+
+POST_SYSCALL(sendto)(long res, long arg0, void *arg1, long arg2, long arg3,
+                     sanitizer_kernel_sockaddr *arg4, long arg5) {
+  if (res >= 0) {
+    if (arg1) POST_READ(arg1, res);
+    if (arg4) POST_WRITE(arg4, sizeof(*arg4));
+  }
+}
+
+PRE_SYSCALL(sendmsg)(long fd, void *msg, long flags) {}
+
+POST_SYSCALL(sendmsg)(long res, long fd, void *msg, long flags) {
+  // FIXME: POST_READ
+}
+
+PRE_SYSCALL(sendmmsg)(long fd, void *msg, long vlen, long flags) {}
+
+POST_SYSCALL(sendmmsg)(long res, long fd, void *msg, long vlen, long flags) {
+  // FIXME: POST_READ
+}
+
+PRE_SYSCALL(recv)(long arg0, void *buf, long len, long flags) {}
+
+POST_SYSCALL(recv)(long res, void *buf, long len, long flags) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, res);
+  }
+}
+
+PRE_SYSCALL(recvfrom)(long arg0, void *buf, long len, long flags,
+                      sanitizer_kernel_sockaddr *arg4, void *arg5) {}
+
+POST_SYSCALL(recvfrom)(long res, long arg0, void *buf, long len, long flags,
+                       sanitizer_kernel_sockaddr *arg4, void *arg5) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, res);
+    if (arg4) POST_WRITE(arg4, sizeof(*arg4));
+    if (arg5) POST_WRITE(arg5, sizeof(int));
+  }
+}
+
+PRE_SYSCALL(socket)(long arg0, long arg1, long arg2) {}
+
+POST_SYSCALL(socket)(long res, long arg0, long arg1, long arg2) {}
+
+PRE_SYSCALL(socketpair)(long arg0, long arg1, long arg2, int *sv) {}
+
+POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, int *sv) {
+  if (res >= 0)
+    if (sv) POST_WRITE(sv, sizeof(int) * 2);
+}
+
+PRE_SYSCALL(socketcall)(long call, void *args) {}
+
+POST_SYSCALL(socketcall)(long res, long call, void *args) {
+  if (res >= 0) {
+    if (args) POST_WRITE(args, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(listen)(long arg0, long arg1) {}
+
+POST_SYSCALL(listen)(long res, long arg0, long arg1) {}
+
+PRE_SYSCALL(poll)(void *ufds, long nfds, long timeout) {}
+
+POST_SYSCALL(poll)(long res, __sanitizer_pollfd *ufds, long nfds,
+                   long timeout) {
+  if (res >= 0) {
+    if (ufds) POST_WRITE(ufds, nfds * sizeof(*ufds));
+  }
+}
+
+PRE_SYSCALL(select)(long n, __sanitizer___kernel_fd_set *inp,
+                    __sanitizer___kernel_fd_set *outp,
+                    __sanitizer___kernel_fd_set *exp, void *tvp) {}
+
+POST_SYSCALL(select)(long res, long n, __sanitizer___kernel_fd_set *inp,
+                     __sanitizer___kernel_fd_set *outp,
+                     __sanitizer___kernel_fd_set *exp, void *tvp) {
+  if (res >= 0) {
+    if (inp) POST_WRITE(inp, sizeof(*inp));
+    if (outp) POST_WRITE(outp, sizeof(*outp));
+    if (exp) POST_WRITE(exp, sizeof(*exp));
+    if (tvp) POST_WRITE(tvp, timeval_sz);
+  }
+}
+
+PRE_SYSCALL(old_select)(void *arg) {}
+
+POST_SYSCALL(old_select)(long res, void *arg) {}
+
+PRE_SYSCALL(epoll_create)(long size) {}
+
+POST_SYSCALL(epoll_create)(long res, long size) {}
+
+PRE_SYSCALL(epoll_create1)(long flags) {}
+
+POST_SYSCALL(epoll_create1)(long res, long flags) {}
+
+PRE_SYSCALL(epoll_ctl)(long epfd, long op, long fd, void *event) {}
+
+POST_SYSCALL(epoll_ctl)(long res, long epfd, long op, long fd, void *event) {
+  if (res >= 0) {
+    if (event) POST_WRITE(event, struct_epoll_event_sz);
+  }
+}
+
+PRE_SYSCALL(epoll_wait)(long epfd, void *events, long maxevents, long timeout) {
+}
+
+POST_SYSCALL(epoll_wait)(long res, long epfd, void *events, long maxevents,
+                         long timeout) {
+  if (res >= 0) {
+    if (events) POST_WRITE(events, struct_epoll_event_sz);
+  }
+}
+
+PRE_SYSCALL(epoll_pwait)(long epfd, void *events, long maxevents, long timeout,
+                         const kernel_sigset_t *sigmask, long sigsetsize) {
+  if (sigmask) PRE_READ(sigmask, sigsetsize);
+}
+
+POST_SYSCALL(epoll_pwait)(long res, long epfd, void *events, long maxevents,
+                          long timeout, const void *sigmask, long sigsetsize) {
+  if (res >= 0) {
+    if (events) POST_WRITE(events, struct_epoll_event_sz);
+  }
+}
+
+PRE_SYSCALL(gethostname)(void *name, long len) {}
+
+POST_SYSCALL(gethostname)(long res, void *name, long len) {
+  if (res >= 0) {
+    if (name)
+      POST_WRITE(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  }
+}
+
+PRE_SYSCALL(sethostname)(void *name, long len) {}
+
+POST_SYSCALL(sethostname)(long res, void *name, long len) {
+  if (res >= 0) {
+    if (name)
+      POST_WRITE(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  }
+}
+
+PRE_SYSCALL(setdomainname)(void *name, long len) {}
+
+POST_SYSCALL(setdomainname)(long res, void *name, long len) {
+  if (res >= 0) {
+    if (name)
+      POST_WRITE(name, __sanitizer::internal_strlen((const char *)name) + 1);
+  }
+}
+
+PRE_SYSCALL(newuname)(void *name) {}
+
+POST_SYSCALL(newuname)(long res, void *name) {
+  if (res >= 0) {
+    if (name) POST_WRITE(name, struct_new_utsname_sz);
+  }
+}
+
+PRE_SYSCALL(uname)(void *arg0) {}
+
+POST_SYSCALL(uname)(long res, void *arg0) {
+  if (res >= 0) {
+    if (arg0) POST_WRITE(arg0, struct_old_utsname_sz);
+  }
+}
+
+PRE_SYSCALL(olduname)(void *arg0) {}
+
+POST_SYSCALL(olduname)(long res, void *arg0) {
+  if (res >= 0) {
+    if (arg0) POST_WRITE(arg0, struct_oldold_utsname_sz);
+  }
+}
+
+PRE_SYSCALL(getrlimit)(long resource, void *rlim) {}
+
+POST_SYSCALL(getrlimit)(long res, long resource, void *rlim) {
+  if (res >= 0) {
+    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+  }
+}
+
+PRE_SYSCALL(old_getrlimit)(long resource, void *rlim) {}
+
+POST_SYSCALL(old_getrlimit)(long res, long resource, void *rlim) {
+  if (res >= 0) {
+    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+  }
+}
+
+PRE_SYSCALL(setrlimit)(long resource, void *rlim) {}
+
+POST_SYSCALL(setrlimit)(long res, long resource, void *rlim) {
+  if (res >= 0) {
+    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(prlimit64)(long pid, long resource, const void *new_rlim,
+                       void *old_rlim) {
+  if (new_rlim) PRE_READ(new_rlim, struct_rlimit64_sz);
+}
+
+POST_SYSCALL(prlimit64)(long res, long pid, long resource, const void *new_rlim,
+                        void *old_rlim) {
+  if (res >= 0) {
+    if (old_rlim) POST_WRITE(old_rlim, struct_rlimit64_sz);
+  }
+}
+#endif
+
+PRE_SYSCALL(getrusage)(long who, void *ru) {}
+
+POST_SYSCALL(getrusage)(long res, long who, void *ru) {
+  if (res >= 0) {
+    if (ru) POST_WRITE(ru, struct_rusage_sz);
+  }
+}
+
+PRE_SYSCALL(umask)(long mask) {}
+
+POST_SYSCALL(umask)(long res, long mask) {}
+
+PRE_SYSCALL(msgget)(long key, long msgflg) {}
+
+POST_SYSCALL(msgget)(long res, long key, long msgflg) {}
+
+PRE_SYSCALL(msgsnd)(long msqid, void *msgp, long msgsz, long msgflg) {
+  if (msgp) PRE_READ(msgp, msgsz);
+}
+
+POST_SYSCALL(msgsnd)(long res, long msqid, void *msgp, long msgsz,
+                     long msgflg) {}
+
+PRE_SYSCALL(msgrcv)(long msqid, void *msgp, long msgsz, long msgtyp,
+                    long msgflg) {}
+
+POST_SYSCALL(msgrcv)(long res, long msqid, void *msgp, long msgsz, long msgtyp,
+                     long msgflg) {
+  if (res >= 0) {
+    if (msgp) POST_WRITE(msgp, res);
+  }
+}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(msgctl)(long msqid, long cmd, void *buf) {}
+
+POST_SYSCALL(msgctl)(long res, long msqid, long cmd, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, struct_msqid_ds_sz);
+  }
+}
+#endif
+
+PRE_SYSCALL(semget)(long key, long nsems, long semflg) {}
+
+POST_SYSCALL(semget)(long res, long key, long nsems, long semflg) {}
+
+PRE_SYSCALL(semop)(long semid, void *sops, long nsops) {}
+
+POST_SYSCALL(semop)(long res, long semid, void *sops, long nsops) {}
+
+PRE_SYSCALL(semctl)(long semid, long semnum, long cmd, void *arg) {}
+
+POST_SYSCALL(semctl)(long res, long semid, long semnum, long cmd, void *arg) {}
+
+PRE_SYSCALL(semtimedop)(long semid, void *sops, long nsops,
+                        const void *timeout) {
+  if (timeout) PRE_READ(timeout, struct_timespec_sz);
+}
+
+POST_SYSCALL(semtimedop)(long res, long semid, void *sops, long nsops,
+                         const void *timeout) {}
+
+PRE_SYSCALL(shmat)(long shmid, void *shmaddr, long shmflg) {}
+
+POST_SYSCALL(shmat)(long res, long shmid, void *shmaddr, long shmflg) {
+  if (res >= 0) {
+    if (shmaddr)
+      POST_WRITE(shmaddr,
+                 __sanitizer::internal_strlen((const char *)shmaddr) + 1);
+  }
+}
+
+PRE_SYSCALL(shmget)(long key, long size, long flag) {}
+
+POST_SYSCALL(shmget)(long res, long key, long size, long flag) {}
+
+PRE_SYSCALL(shmdt)(void *shmaddr) {}
+
+POST_SYSCALL(shmdt)(long res, void *shmaddr) {
+  if (res >= 0) {
+    if (shmaddr)
+      POST_WRITE(shmaddr,
+                 __sanitizer::internal_strlen((const char *)shmaddr) + 1);
+  }
+}
+
+PRE_SYSCALL(ipc)(long call, long first, long second, long third, void *ptr,
+                 long fifth) {}
+
+POST_SYSCALL(ipc)(long res, long call, long first, long second, long third,
+                  void *ptr, long fifth) {}
+
+#if !SANITIZER_ANDROID
+PRE_SYSCALL(shmctl)(long shmid, long cmd, void *buf) {}
+
+POST_SYSCALL(shmctl)(long res, long shmid, long cmd, void *buf) {
+  if (res >= 0) {
+    if (buf) POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
+  }
+}
+
+PRE_SYSCALL(mq_open)(const void *name, long oflag, long mode, void *attr) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(mq_open)(long res, const void *name, long oflag, long mode,
+                      void *attr) {
+  if (res >= 0) {
+    if (attr) POST_WRITE(attr, struct_mq_attr_sz);
+  }
+}
+
+PRE_SYSCALL(mq_unlink)(const void *name) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(mq_unlink)(long res, const void *name) {}
+
+PRE_SYSCALL(mq_timedsend)(long mqdes, const void *msg_ptr, long msg_len,
+                          long msg_prio, const void *abs_timeout) {
+  if (msg_ptr) PRE_READ(msg_ptr, msg_len);
+  if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+}
+
+POST_SYSCALL(mq_timedsend)(long res, long mqdes, const void *msg_ptr,
+                           long msg_len, long msg_prio,
+                           const void *abs_timeout) {}
+
+PRE_SYSCALL(mq_timedreceive)(long mqdes, void *msg_ptr, long msg_len,
+                             void *msg_prio, const void *abs_timeout) {
+  if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+}
+
+POST_SYSCALL(mq_timedreceive)(long res, long mqdes, void *msg_ptr, long msg_len,
+                              int *msg_prio, const void *abs_timeout) {
+  if (res >= 0) {
+    if (msg_ptr) POST_WRITE(msg_ptr, res);
+    if (msg_prio) POST_WRITE(msg_prio, sizeof(*msg_prio));
+  }
+}
+
+PRE_SYSCALL(mq_notify)(long mqdes, const void *notification) {
+  if (notification) PRE_READ(notification, struct_sigevent_sz);
+}
+
+POST_SYSCALL(mq_notify)(long res, long mqdes, const void *notification) {}
+
+PRE_SYSCALL(mq_getsetattr)(long mqdes, const void *mqstat, void *omqstat) {
+  if (mqstat) PRE_READ(mqstat, struct_mq_attr_sz);
+}
+
+POST_SYSCALL(mq_getsetattr)(long res, long mqdes, const void *mqstat,
+                            void *omqstat) {
+  if (res >= 0) {
+    if (omqstat) POST_WRITE(omqstat, struct_mq_attr_sz);
+  }
+}
+#endif  // SANITIZER_ANDROID
+
+PRE_SYSCALL(pciconfig_iobase)(long which, long bus, long devfn) {}
+
+POST_SYSCALL(pciconfig_iobase)(long res, long which, long bus, long devfn) {}
+
+PRE_SYSCALL(pciconfig_read)(long bus, long dfn, long off, long len, void *buf) {
+}
+
+POST_SYSCALL(pciconfig_read)(long res, long bus, long dfn, long off, long len,
+                             void *buf) {}
+
+PRE_SYSCALL(pciconfig_write)(long bus, long dfn, long off, long len,
+                             void *buf) {}
+
+POST_SYSCALL(pciconfig_write)(long res, long bus, long dfn, long off, long len,
+                              void *buf) {}
+
+PRE_SYSCALL(swapon)(const void *specialfile, long swap_flags) {
+  if (specialfile)
+    PRE_READ(specialfile,
+             __sanitizer::internal_strlen((const char *)specialfile) + 1);
+}
+
+POST_SYSCALL(swapon)(long res, const void *specialfile, long swap_flags) {}
+
+PRE_SYSCALL(swapoff)(const void *specialfile) {
+  if (specialfile)
+    PRE_READ(specialfile,
+             __sanitizer::internal_strlen((const char *)specialfile) + 1);
+}
+
+POST_SYSCALL(swapoff)(long res, const void *specialfile) {}
+
+PRE_SYSCALL(sysctl)(__sanitizer___sysctl_args *args) {
+  if (args) {
+    if (args->name) PRE_READ(args->name, args->nlen * sizeof(*args->name));
+    if (args->newval) PRE_READ(args->name, args->newlen);
+  }
+}
+
+POST_SYSCALL(sysctl)(long res, __sanitizer___sysctl_args *args) {
+  if (res >= 0) {
+    if (args && args->oldval && args->oldlenp) {
+      POST_WRITE(args->oldlenp, sizeof(*args->oldlenp));
+      POST_WRITE(args->oldval, *args->oldlenp);
+    }
+  }
+}
+
+PRE_SYSCALL(sysinfo)(void *info) {}
+
+POST_SYSCALL(sysinfo)(long res, void *info) {
+  if (res >= 0) {
+    if (info) POST_WRITE(info, struct_sysinfo_sz);
+  }
+}
+
+PRE_SYSCALL(sysfs)(long option, long arg1, long arg2) {}
+
+POST_SYSCALL(sysfs)(long res, long option, long arg1, long arg2) {}
+
+PRE_SYSCALL(syslog)(long type, void *buf, long len) {}
+
+POST_SYSCALL(syslog)(long res, long type, void *buf, long len) {
+  if (res >= 0) {
+    if (buf)
+      POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
+  }
+}
+
+PRE_SYSCALL(uselib)(const void *library) {
+  if (library)
+    PRE_READ(library, __sanitizer::internal_strlen((const char *)library) + 1);
+}
+
+POST_SYSCALL(uselib)(long res, const void *library) {}
+
+PRE_SYSCALL(ni_syscall)() {}
+
+POST_SYSCALL(ni_syscall)(long res) {}
+
+PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
+#if !SANITIZER_ANDROID && \
+    (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+     defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+  if (data) {
+    if (request == ptrace_setregs) {
+      PRE_READ((void *)data, struct_user_regs_struct_sz);
+    } else if (request == ptrace_setfpregs) {
+      PRE_READ((void *)data, struct_user_fpregs_struct_sz);
+    } else if (request == ptrace_setfpxregs) {
+      PRE_READ((void *)data, struct_user_fpxregs_struct_sz);
+    } else if (request == ptrace_setsiginfo) {
+      PRE_READ((void *)data, siginfo_t_sz);
+    } else if (request == ptrace_setregset) {
+      __sanitizer_iovec *iov = (__sanitizer_iovec *)data;
+      PRE_READ(iov->iov_base, iov->iov_len);
+    }
+  }
+#endif
+}
+
+POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
+#if !SANITIZER_ANDROID && \
+    (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+     defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+  if (res >= 0 && data) {
+    // Note that this is different from the interceptor in
+    // sanitizer_common_interceptors.inc.
+    // PEEK* requests return resulting values through data pointer.
+    if (request == ptrace_getregs) {
+      POST_WRITE((void *)data, struct_user_regs_struct_sz);
+    } else if (request == ptrace_getfpregs) {
+      POST_WRITE((void *)data, struct_user_fpregs_struct_sz);
+    } else if (request == ptrace_getfpxregs) {
+      POST_WRITE((void *)data, struct_user_fpxregs_struct_sz);
+    } else if (request == ptrace_getsiginfo) {
+      POST_WRITE((void *)data, siginfo_t_sz);
+    } else if (request == ptrace_getregset) {
+      __sanitizer_iovec *iov = (__sanitizer_iovec *)data;
+      POST_WRITE(iov->iov_base, iov->iov_len);
+    } else if (request == ptrace_peekdata || request == ptrace_peektext ||
+               request == ptrace_peekuser) {
+      POST_WRITE((void *)data, sizeof(void *));
+    }
+  }
+#endif
+}
+
+PRE_SYSCALL(add_key)(const void *_type, const void *_description,
+                     const void *_payload, long plen, long destringid) {
+  if (_type)
+    PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
+  if (_description)
+    PRE_READ(_description,
+             __sanitizer::internal_strlen((const char *)_description) + 1);
+}
+
+POST_SYSCALL(add_key)(long res, const void *_type, const void *_description,
+                      const void *_payload, long plen, long destringid) {}
+
+PRE_SYSCALL(request_key)(const void *_type, const void *_description,
+                         const void *_callout_info, long destringid) {
+  if (_type)
+    PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
+  if (_description)
+    PRE_READ(_description,
+             __sanitizer::internal_strlen((const char *)_description) + 1);
+  if (_callout_info)
+    PRE_READ(_callout_info,
+             __sanitizer::internal_strlen((const char *)_callout_info) + 1);
+}
+
+POST_SYSCALL(request_key)(long res, const void *_type, const void *_description,
+                          const void *_callout_info, long destringid) {}
+
+PRE_SYSCALL(keyctl)(long cmd, long arg2, long arg3, long arg4, long arg5) {}
+
+POST_SYSCALL(keyctl)(long res, long cmd, long arg2, long arg3, long arg4,
+                     long arg5) {}
+
+PRE_SYSCALL(ioprio_set)(long which, long who, long ioprio) {}
+
+POST_SYSCALL(ioprio_set)(long res, long which, long who, long ioprio) {}
+
+PRE_SYSCALL(ioprio_get)(long which, long who) {}
+
+POST_SYSCALL(ioprio_get)(long res, long which, long who) {}
+
+PRE_SYSCALL(set_mempolicy)(long mode, void *nmask, long maxnode) {}
+
+POST_SYSCALL(set_mempolicy)(long res, long mode, void *nmask, long maxnode) {
+  if (res >= 0) {
+    if (nmask) POST_WRITE(nmask, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(migrate_pages)(long pid, long maxnode, const void *from,
+                           const void *to) {
+  if (from) PRE_READ(from, sizeof(long));
+  if (to) PRE_READ(to, sizeof(long));
+}
+
+POST_SYSCALL(migrate_pages)(long res, long pid, long maxnode, const void *from,
+                            const void *to) {}
+
+PRE_SYSCALL(move_pages)(long pid, long nr_pages, const void **pages,
+                        const int *nodes, int *status, long flags) {
+  if (pages) PRE_READ(pages, nr_pages * sizeof(*pages));
+  if (nodes) PRE_READ(nodes, nr_pages * sizeof(*nodes));
+}
+
+POST_SYSCALL(move_pages)(long res, long pid, long nr_pages, const void **pages,
+                         const int *nodes, int *status, long flags) {
+  if (res >= 0) {
+    if (status) POST_WRITE(status, nr_pages * sizeof(*status));
+  }
+}
+
+PRE_SYSCALL(mbind)(long start, long len, long mode, void *nmask, long maxnode,
+                   long flags) {}
+
+POST_SYSCALL(mbind)(long res, long start, long len, long mode, void *nmask,
+                    long maxnode, long flags) {
+  if (res >= 0) {
+    if (nmask) POST_WRITE(nmask, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(get_mempolicy)(void *policy, void *nmask, long maxnode, long addr,
+                           long flags) {}
+
+POST_SYSCALL(get_mempolicy)(long res, void *policy, void *nmask, long maxnode,
+                            long addr, long flags) {
+  if (res >= 0) {
+    if (policy) POST_WRITE(policy, sizeof(int));
+    if (nmask) POST_WRITE(nmask, sizeof(long));
+  }
+}
+
+PRE_SYSCALL(inotify_init)() {}
+
+POST_SYSCALL(inotify_init)(long res) {}
+
+PRE_SYSCALL(inotify_init1)(long flags) {}
+
+POST_SYSCALL(inotify_init1)(long res, long flags) {}
+
+PRE_SYSCALL(inotify_add_watch)(long fd, const void *path, long mask) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(inotify_add_watch)(long res, long fd, const void *path,
+                                long mask) {}
+
+PRE_SYSCALL(inotify_rm_watch)(long fd, long wd) {}
+
+POST_SYSCALL(inotify_rm_watch)(long res, long fd, long wd) {}
+
+PRE_SYSCALL(spu_run)(long fd, void *unpc, void *ustatus) {}
+
+POST_SYSCALL(spu_run)(long res, long fd, unsigned *unpc, unsigned *ustatus) {
+  if (res >= 0) {
+    if (unpc) POST_WRITE(unpc, sizeof(*unpc));
+    if (ustatus) POST_WRITE(ustatus, sizeof(*ustatus));
+  }
+}
+
+PRE_SYSCALL(spu_create)(const void *name, long flags, long mode, long fd) {
+  if (name)
+    PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
+}
+
+POST_SYSCALL(spu_create)(long res, const void *name, long flags, long mode,
+                         long fd) {}
+
+PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(mknodat)(long res, long dfd, const void *filename, long mode,
+                      long dev) {}
+
+PRE_SYSCALL(mkdirat)(long dfd, const void *pathname, long mode) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(mkdirat)(long res, long dfd, const void *pathname, long mode) {}
+
+PRE_SYSCALL(unlinkat)(long dfd, const void *pathname, long flag) {
+  if (pathname)
+    PRE_READ(pathname,
+             __sanitizer::internal_strlen((const char *)pathname) + 1);
+}
+
+POST_SYSCALL(unlinkat)(long res, long dfd, const void *pathname, long flag) {}
+
+PRE_SYSCALL(symlinkat)(const void *oldname, long newdfd, const void *newname) {
+  if (oldname)
+    PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
+  if (newname)
+    PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
+}
+
+POST_SYSCALL(symlinkat)(long res, const void *oldname, long newdfd,
+                        const void *newname) {}
+
+PRE_SYSCALL(linkat)(long olddfd, const void *oldname, long newdfd,
+                    const void *newname, long flags) {
+  if (oldname)
+    PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
+  if (newname)
+    PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
+}
+
+POST_SYSCALL(linkat)(long res, long olddfd, const void *oldname, long newdfd,
+                     const void *newname, long flags) {}
+
+PRE_SYSCALL(renameat)(long olddfd, const void *oldname, long newdfd,
+                      const void *newname) {
+  if (oldname)
+    PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
+  if (newname)
+    PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
+}
+
+POST_SYSCALL(renameat)(long res, long olddfd, const void *oldname, long newdfd,
+                       const void *newname) {}
+
+PRE_SYSCALL(futimesat)(long dfd, const void *filename, void *utimes) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(futimesat)(long res, long dfd, const void *filename,
+                        void *utimes) {
+  if (res >= 0) {
+    if (utimes) POST_WRITE(utimes, timeval_sz);
+  }
+}
+
+PRE_SYSCALL(faccessat)(long dfd, const void *filename, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(faccessat)(long res, long dfd, const void *filename, long mode) {}
+
+PRE_SYSCALL(fchmodat)(long dfd, const void *filename, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(fchmodat)(long res, long dfd, const void *filename, long mode) {}
+
+PRE_SYSCALL(fchownat)(long dfd, const void *filename, long user, long group,
+                      long flag) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(fchownat)(long res, long dfd, const void *filename, long user,
+                       long group, long flag) {}
+
+PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(openat)(long res, long dfd, const void *filename, long flags,
+                     long mode) {}
+
+PRE_SYSCALL(newfstatat)(long dfd, const void *filename, void *statbuf,
+                        long flag) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(newfstatat)(long res, long dfd, const void *filename,
+                         void *statbuf, long flag) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+  }
+}
+
+PRE_SYSCALL(fstatat64)(long dfd, const void *filename, void *statbuf,
+                       long flag) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(fstatat64)(long res, long dfd, const void *filename, void *statbuf,
+                        long flag) {
+  if (res >= 0) {
+    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+  }
+}
+
+PRE_SYSCALL(readlinkat)(long dfd, const void *path, void *buf, long bufsiz) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
+
+POST_SYSCALL(readlinkat)(long res, long dfd, const void *path, void *buf,
+                         long bufsiz) {
+  if (res >= 0) {
+    if (buf)
+      POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
+  }
+}
+
+PRE_SYSCALL(utimensat)(long dfd, const void *filename, void *utimes,
+                       long flags) {
+  if (filename)
+    PRE_READ(filename,
+             __sanitizer::internal_strlen((const char *)filename) + 1);
+}
+
+POST_SYSCALL(utimensat)(long res, long dfd, const void *filename, void *utimes,
+                        long flags) {
+  if (res >= 0) {
+    if (utimes) POST_WRITE(utimes, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(unshare)(long unshare_flags) {}
+
+POST_SYSCALL(unshare)(long res, long unshare_flags) {}
+
+PRE_SYSCALL(splice)(long fd_in, void *off_in, long fd_out, void *off_out,
+                    long len, long flags) {}
+
+POST_SYSCALL(splice)(long res, long fd_in, void *off_in, long fd_out,
+                     void *off_out, long len, long flags) {
+  if (res >= 0) {
+    if (off_in) POST_WRITE(off_in, sizeof(long long));
+    if (off_out) POST_WRITE(off_out, sizeof(long long));
+  }
+}
+
+PRE_SYSCALL(vmsplice)(long fd, const __sanitizer_iovec *iov, long nr_segs,
+                      long flags) {}
+
+POST_SYSCALL(vmsplice)(long res, long fd, const __sanitizer_iovec *iov,
+                       long nr_segs, long flags) {
+  if (res >= 0) {
+    if (iov) kernel_read_iovec(iov, nr_segs, res);
+  }
+}
+
+PRE_SYSCALL(tee)(long fdin, long fdout, long len, long flags) {}
+
+POST_SYSCALL(tee)(long res, long fdin, long fdout, long len, long flags) {}
+
+PRE_SYSCALL(get_robust_list)(long pid, void *head_ptr, void *len_ptr) {}
+
+POST_SYSCALL(get_robust_list)(long res, long pid, void *head_ptr,
+                              void *len_ptr) {}
+
+PRE_SYSCALL(set_robust_list)(void *head, long len) {}
+
+POST_SYSCALL(set_robust_list)(long res, void *head, long len) {}
+
+PRE_SYSCALL(getcpu)(void *cpu, void *node, void *cache) {}
+
+POST_SYSCALL(getcpu)(long res, void *cpu, void *node, void *cache) {
+  if (res >= 0) {
+    if (cpu) POST_WRITE(cpu, sizeof(unsigned));
+    if (node) POST_WRITE(node, sizeof(unsigned));
+    // The third argument to this system call is nowadays unused.
+  }
+}
+
+PRE_SYSCALL(signalfd)(long ufd, void *user_mask, long sizemask) {}
+
+POST_SYSCALL(signalfd)(long res, long ufd, kernel_sigset_t *user_mask,
+                       long sizemask) {
+  if (res >= 0) {
+    if (user_mask) POST_WRITE(user_mask, sizemask);
+  }
+}
+
+PRE_SYSCALL(signalfd4)(long ufd, void *user_mask, long sizemask, long flags) {}
+
+POST_SYSCALL(signalfd4)(long res, long ufd, kernel_sigset_t *user_mask,
+                        long sizemask, long flags) {
+  if (res >= 0) {
+    if (user_mask) POST_WRITE(user_mask, sizemask);
+  }
+}
+
+PRE_SYSCALL(timerfd_create)(long clockid, long flags) {}
+
+POST_SYSCALL(timerfd_create)(long res, long clockid, long flags) {}
+
+PRE_SYSCALL(timerfd_settime)(long ufd, long flags, const void *utmr,
+                             void *otmr) {
+  if (utmr) PRE_READ(utmr, struct_itimerspec_sz);
+}
+
+POST_SYSCALL(timerfd_settime)(long res, long ufd, long flags, const void *utmr,
+                              void *otmr) {
+  if (res >= 0) {
+    if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+  }
+}
+
+PRE_SYSCALL(timerfd_gettime)(long ufd, void *otmr) {}
+
+POST_SYSCALL(timerfd_gettime)(long res, long ufd, void *otmr) {
+  if (res >= 0) {
+    if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+  }
+}
+
+PRE_SYSCALL(eventfd)(long count) {}
+
+POST_SYSCALL(eventfd)(long res, long count) {}
+
+PRE_SYSCALL(eventfd2)(long count, long flags) {}
+
+POST_SYSCALL(eventfd2)(long res, long count, long flags) {}
+
+PRE_SYSCALL(old_readdir)(long arg0, void *arg1, long arg2) {}
+
+POST_SYSCALL(old_readdir)(long res, long arg0, void *arg1, long arg2) {
+  // Missing definition of 'struct old_linux_dirent'.
+}
+
+PRE_SYSCALL(pselect6)(long arg0, __sanitizer___kernel_fd_set *arg1,
+                      __sanitizer___kernel_fd_set *arg2,
+                      __sanitizer___kernel_fd_set *arg3, void *arg4,
+                      void *arg5) {}
+
+POST_SYSCALL(pselect6)(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
+                       __sanitizer___kernel_fd_set *arg2,
+                       __sanitizer___kernel_fd_set *arg3, void *arg4,
+                       void *arg5) {
+  if (res >= 0) {
+    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2) POST_WRITE(arg2, sizeof(*arg2));
+    if (arg3) POST_WRITE(arg3, sizeof(*arg3));
+    if (arg4) POST_WRITE(arg4, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(ppoll)(__sanitizer_pollfd *arg0, long arg1, void *arg2,
+                   const kernel_sigset_t *arg3, long arg4) {
+  if (arg3) PRE_READ(arg3, arg4);
+}
+
+POST_SYSCALL(ppoll)(long res, __sanitizer_pollfd *arg0, long arg1, void *arg2,
+                    const void *arg3, long arg4) {
+  if (res >= 0) {
+    if (arg0) POST_WRITE(arg0, sizeof(*arg0));
+    if (arg2) POST_WRITE(arg2, struct_timespec_sz);
+  }
+}
+
+PRE_SYSCALL(syncfs)(long fd) {}
+
+POST_SYSCALL(syncfs)(long res, long fd) {}
+
+PRE_SYSCALL(perf_event_open)(__sanitizer_perf_event_attr *attr_uptr, long pid,
+                             long cpu, long group_fd, long flags) {
+  if (attr_uptr) PRE_READ(attr_uptr, attr_uptr->size);
+}
+
+POST_SYSCALL(perf_event_open)(long res, __sanitizer_perf_event_attr *attr_uptr,
+                              long pid, long cpu, long group_fd, long flags) {}
+
+PRE_SYSCALL(mmap_pgoff)(long addr, long len, long prot, long flags, long fd,
+                        long pgoff) {}
+
+POST_SYSCALL(mmap_pgoff)(long res, long addr, long len, long prot, long flags,
+                         long fd, long pgoff) {}
+
+PRE_SYSCALL(old_mmap)(void *arg) {}
+
+POST_SYSCALL(old_mmap)(long res, void *arg) {}
+
+PRE_SYSCALL(name_to_handle_at)(long dfd, const void *name, void *handle,
+                               void *mnt_id, long flag) {}
+
+POST_SYSCALL(name_to_handle_at)(long res, long dfd, const void *name,
+                                void *handle, void *mnt_id, long flag) {}
+
+PRE_SYSCALL(open_by_handle_at)(long mountdirfd, void *handle, long flags) {}
+
+POST_SYSCALL(open_by_handle_at)(long res, long mountdirfd, void *handle,
+                                long flags) {}
+
+PRE_SYSCALL(setns)(long fd, long nstype) {}
+
+POST_SYSCALL(setns)(long res, long fd, long nstype) {}
+
+PRE_SYSCALL(process_vm_readv)(long pid, const __sanitizer_iovec *lvec,
+                              long liovcnt, const void *rvec, long riovcnt,
+                              long flags) {}
+
+POST_SYSCALL(process_vm_readv)(long res, long pid,
+                               const __sanitizer_iovec *lvec, long liovcnt,
+                               const void *rvec, long riovcnt, long flags) {
+  if (res >= 0) {
+    if (lvec) kernel_write_iovec(lvec, liovcnt, res);
+  }
+}
+
+PRE_SYSCALL(process_vm_writev)(long pid, const __sanitizer_iovec *lvec,
+                               long liovcnt, const void *rvec, long riovcnt,
+                               long flags) {}
+
+POST_SYSCALL(process_vm_writev)(long res, long pid,
+                                const __sanitizer_iovec *lvec, long liovcnt,
+                                const void *rvec, long riovcnt, long flags) {
+  if (res >= 0) {
+    if (lvec) kernel_read_iovec(lvec, liovcnt, res);
+  }
+}
+
+PRE_SYSCALL(fork)() {
+  COMMON_SYSCALL_PRE_FORK();
+}
+
+POST_SYSCALL(fork)(long res) {
+  COMMON_SYSCALL_POST_FORK(res);
+}
+
+PRE_SYSCALL(vfork)() {
+  COMMON_SYSCALL_PRE_FORK();
+}
+
+POST_SYSCALL(vfork)(long res) {
+  COMMON_SYSCALL_POST_FORK(res);
+}
+
+PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
+                       __sanitizer_kernel_sigaction_t *oldact) {
+  if (act) {
+    PRE_READ(&act->sigaction, sizeof(act->sigaction));
+    PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
+    PRE_READ(&act->sa_mask, sizeof(act->sa_mask));
+  }
+}
+
+POST_SYSCALL(sigaction)(long res, long signum,
+                        const __sanitizer_kernel_sigaction_t *act,
+                        __sanitizer_kernel_sigaction_t *oldact) {
+  if (res >= 0 && oldact) POST_WRITE(oldact, sizeof(*oldact));
+}
+
+PRE_SYSCALL(rt_sigaction)(long signum,
+                          const __sanitizer_kernel_sigaction_t *act,
+                          __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+  if (act) {
+    PRE_READ(&act->sigaction, sizeof(act->sigaction));
+    PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
+    PRE_READ(&act->sa_mask, sz);
+  }
+}
+
+POST_SYSCALL(rt_sigaction)(long res, long signum,
+                           const __sanitizer_kernel_sigaction_t *act,
+                           __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+  if (res >= 0 && oldact) {
+    SIZE_T oldact_sz = ((char *)&oldact->sa_mask) - ((char *)oldact) + sz;
+    POST_WRITE(oldact, oldact_sz);
+  }
+}
+
+PRE_SYSCALL(getrandom)(void *buf, uptr count, long flags) {
+  if (buf) {
+    PRE_WRITE(buf, count);
+  }
+}
+
+POST_SYSCALL(getrandom)(long res, void *buf, uptr count, long flags) {
+  if (res > 0 && buf) {
+    POST_WRITE(buf, res);
+  }
+}
+
+PRE_SYSCALL(sigaltstack)(const void *ss, void *oss) {
+  if (ss != nullptr) {
+    PRE_READ(ss, struct_stack_t_sz);
+  }
+  if (oss != nullptr) {
+    PRE_WRITE(oss, struct_stack_t_sz);
+  }
+}
+
+POST_SYSCALL(sigaltstack)(long res, void *ss, void *oss) {
+  if (res == 0) {
+    if (oss != nullptr) {
+      POST_WRITE(oss, struct_stack_t_sz);
+    }
+  }
+}
+}  // extern "C"
+
+#undef PRE_SYSCALL
+#undef PRE_READ
+#undef PRE_WRITE
+#undef POST_SYSCALL
+#undef POST_READ
+#undef POST_WRITE
+
+#endif  // SANITIZER_LINUX
diff --git a/lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc b/lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc
new file mode 100644
index 0000000000..d7ab0c3d98
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc
@@ -0,0 +1,33 @@
+//===-- sanitizer_coverage_interface.inc ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Sanitizer Coverage interface list.
+//===----------------------------------------------------------------------===//
+INTERFACE_FUNCTION(__sanitizer_cov_dump)
+INTERFACE_FUNCTION(__sanitizer_cov_reset)
+INTERFACE_FUNCTION(__sanitizer_dump_coverage)
+INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage)
+INTERFACE_WEAK_FUNCTION(__sancov_default_options)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init)
diff --git a/lib/tsan/sanitizer_common/sanitizer_dbghelp.h b/lib/tsan/sanitizer_common/sanitizer_dbghelp.h
new file mode 100644
index 0000000000..00a5399800
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_dbghelp.h
@@ -0,0 +1,41 @@
+//===-- sanitizer_dbghelp.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Wrappers for lazy loaded dbghelp.dll. Provides function pointers and a
+// callback to initialize them.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_SYMBOLIZER_WIN_H
+#define SANITIZER_SYMBOLIZER_WIN_H
+
+#if !SANITIZER_WINDOWS
+#error "sanitizer_dbghelp.h is a Windows-only header"
+#endif
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <dbghelp.h>
+
+namespace __sanitizer {
+
+extern decltype(::StackWalk64) *StackWalk64;
+extern decltype(::SymCleanup) *SymCleanup;
+extern decltype(::SymFromAddr) *SymFromAddr;
+extern decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
+extern decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
+extern decltype(::SymGetModuleBase64) *SymGetModuleBase64;
+extern decltype(::SymGetSearchPathW) *SymGetSearchPathW;
+extern decltype(::SymInitialize) *SymInitialize;
+extern decltype(::SymSetOptions) *SymSetOptions;
+extern decltype(::SymSetSearchPathW) *SymSetSearchPathW;
+extern decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_WIN_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h
new file mode 100644
index 0000000000..b80cff460e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h
@@ -0,0 +1,410 @@
+//===-- sanitizer_deadlock_detector.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime.
+// The deadlock detector maintains a directed graph of lock acquisitions.
+// When a lock event happens, the detector checks if the locks already held by
+// the current thread are reachable from the newly acquired lock.
+//
+// The detector can handle only a fixed amount of simultaneously live locks
+// (a lock is alive if it has been locked at least once and has not been
+// destroyed). When the maximal number of locks is reached the entire graph
+// is flushed and the new lock epoch is started. The node ids from the old
+// epochs can not be used with any of the detector methods except for
+// nodeBelongsToCurrentEpoch().
+//
+// FIXME: this is work in progress, nothing really works yet.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_DEADLOCK_DETECTOR_H
+#define SANITIZER_DEADLOCK_DETECTOR_H
+
+#include "sanitizer_bvgraph.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+// Thread-local state for DeadlockDetector.
+// It contains the locks currently held by the owning thread.
+template <class BV>
+class DeadlockDetectorTLS {
+ public:
+  // No CTOR.
+  void clear() {
+    bv_.clear();
+    epoch_ = 0;
+    n_recursive_locks = 0;
+    n_all_locks_ = 0;
+  }
+
+  bool empty() const { return bv_.empty(); }
+
+  void ensureCurrentEpoch(uptr current_epoch) {
+    if (epoch_ == current_epoch) return;
+    bv_.clear();
+    epoch_ = current_epoch;
+    n_recursive_locks = 0;
+    n_all_locks_ = 0;
+  }
+
+  uptr getEpoch() const { return epoch_; }
+
+  // Returns true if this is the first (non-recursive) acquisition of this lock.
+  bool addLock(uptr lock_id, uptr current_epoch, u32 stk) {
+    CHECK_EQ(epoch_, current_epoch);
+    if (!bv_.setBit(lock_id)) {
+      // The lock is already held by this thread, it must be recursive.
+      CHECK_LT(n_recursive_locks, ARRAY_SIZE(recursive_locks));
+      recursive_locks[n_recursive_locks++] = lock_id;
+      return false;
+    }
+    CHECK_LT(n_all_locks_, ARRAY_SIZE(all_locks_with_contexts_));
+    // lock_id < BV::kSize, can cast to a smaller int.
+    u32 lock_id_short = static_cast<u32>(lock_id);
+    LockWithContext l = {lock_id_short, stk};
+    all_locks_with_contexts_[n_all_locks_++] = l;
+    return true;
+  }
+
+  void removeLock(uptr lock_id) {
+    if (n_recursive_locks) {
+      for (sptr i = n_recursive_locks - 1; i >= 0; i--) {
+        if (recursive_locks[i] == lock_id) {
+          n_recursive_locks--;
+          Swap(recursive_locks[i], recursive_locks[n_recursive_locks]);
+          return;
+        }
+      }
+    }
+    if (!bv_.clearBit(lock_id))
+      return;  // probably addLock happened before flush
+    if (n_all_locks_) {
+      for (sptr i = n_all_locks_ - 1; i >= 0; i--) {
+        if (all_locks_with_contexts_[i].lock == static_cast<u32>(lock_id)) {
+          Swap(all_locks_with_contexts_[i],
+               all_locks_with_contexts_[n_all_locks_ - 1]);
+          n_all_locks_--;
+          break;
+        }
+      }
+    }
+  }
+
+  u32 findLockContext(uptr lock_id) {
+    for (uptr i = 0; i < n_all_locks_; i++)
+      if (all_locks_with_contexts_[i].lock == static_cast<u32>(lock_id))
+        return all_locks_with_contexts_[i].stk;
+    return 0;
+  }
+
+  const BV &getLocks(uptr current_epoch) const {
+    CHECK_EQ(epoch_, current_epoch);
+    return bv_;
+  }
+
+  uptr getNumLocks() const { return n_all_locks_; }
+  uptr getLock(uptr idx) const { return all_locks_with_contexts_[idx].lock; }
+
+ private:
+  BV bv_;
+  uptr epoch_;
+  uptr recursive_locks[64];
+  uptr n_recursive_locks;
+  struct LockWithContext {
+    u32 lock;
+    u32 stk;
+  };
+  LockWithContext all_locks_with_contexts_[64];
+  uptr n_all_locks_;
+};
+
+// DeadlockDetector.
+// For deadlock detection to work we need one global DeadlockDetector object
+// and one DeadlockDetectorTLS object per evey thread.
+// This class is not thread safe, all concurrent accesses should be guarded
+// by an external lock.
+// Most of the methods of this class are not thread-safe (i.e. should
+// be protected by an external lock) unless explicitly told otherwise.
+template <class BV>
+class DeadlockDetector {
+ public:
+  typedef BV BitVector;
+
+  uptr size() const { return g_.size(); }
+
+  // No CTOR.
+  void clear() {
+    current_epoch_ = 0;
+    available_nodes_.clear();
+    recycled_nodes_.clear();
+    g_.clear();
+    n_edges_ = 0;
+  }
+
+  // Allocate new deadlock detector node.
+  // If we are out of available nodes first try to recycle some.
+  // If there is nothing to recycle, flush the graph and increment the epoch.
+  // Associate 'data' (opaque user's object) with the new node.
+  uptr newNode(uptr data) {
+    if (!available_nodes_.empty())
+      return getAvailableNode(data);
+    if (!recycled_nodes_.empty()) {
+      for (sptr i = n_edges_ - 1; i >= 0; i--) {
+        if (recycled_nodes_.getBit(edges_[i].from) ||
+            recycled_nodes_.getBit(edges_[i].to)) {
+          Swap(edges_[i], edges_[n_edges_ - 1]);
+          n_edges_--;
+        }
+      }
+      CHECK(available_nodes_.empty());
+      // removeEdgesFrom was called in removeNode.
+      g_.removeEdgesTo(recycled_nodes_);
+      available_nodes_.setUnion(recycled_nodes_);
+      recycled_nodes_.clear();
+      return getAvailableNode(data);
+    }
+    // We are out of vacant nodes. Flush and increment the current_epoch_.
+    current_epoch_ += size();
+    recycled_nodes_.clear();
+    available_nodes_.setAll();
+    g_.clear();
+    n_edges_ = 0;
+    return getAvailableNode(data);
+  }
+
+  // Get data associated with the node created by newNode().
+  uptr getData(uptr node) const { return data_[nodeToIndex(node)]; }
+
+  bool nodeBelongsToCurrentEpoch(uptr node) {
+    return node && (node / size() * size()) == current_epoch_;
+  }
+
+  void removeNode(uptr node) {
+    uptr idx = nodeToIndex(node);
+    CHECK(!available_nodes_.getBit(idx));
+    CHECK(recycled_nodes_.setBit(idx));
+    g_.removeEdgesFrom(idx);
+  }
+
+  void ensureCurrentEpoch(DeadlockDetectorTLS<BV> *dtls) {
+    dtls->ensureCurrentEpoch(current_epoch_);
+  }
+
+  // Returns true if there is a cycle in the graph after this lock event.
+  // Ideally should be called before the lock is acquired so that we can
+  // report a deadlock before a real deadlock happens.
+  bool onLockBefore(DeadlockDetectorTLS<BV> *dtls, uptr cur_node) {
+    ensureCurrentEpoch(dtls);
+    uptr cur_idx = nodeToIndex(cur_node);
+    return g_.isReachable(cur_idx, dtls->getLocks(current_epoch_));
+  }
+
+  u32 findLockContext(DeadlockDetectorTLS<BV> *dtls, uptr node) {
+    return dtls->findLockContext(nodeToIndex(node));
+  }
+
+  // Add cur_node to the set of locks held currently by dtls.
+  void onLockAfter(DeadlockDetectorTLS<BV> *dtls, uptr cur_node, u32 stk = 0) {
+    ensureCurrentEpoch(dtls);
+    uptr cur_idx = nodeToIndex(cur_node);
+    dtls->addLock(cur_idx, current_epoch_, stk);
+  }
+
+  // Experimental *racy* fast path function.
+  // Returns true if all edges from the currently held locks to cur_node exist.
+  bool hasAllEdges(DeadlockDetectorTLS<BV> *dtls, uptr cur_node) {
+    uptr local_epoch = dtls->getEpoch();
+    // Read from current_epoch_ is racy.
+    if (cur_node && local_epoch == current_epoch_ &&
+        local_epoch == nodeToEpoch(cur_node)) {
+      uptr cur_idx = nodeToIndexUnchecked(cur_node);
+      for (uptr i = 0, n = dtls->getNumLocks(); i < n; i++) {
+        if (!g_.hasEdge(dtls->getLock(i), cur_idx))
+          return false;
+      }
+      return true;
+    }
+    return false;
+  }
+
+  // Adds edges from currently held locks to cur_node,
+  // returns the number of added edges, and puts the sources of added edges
+  // into added_edges[].
+  // Should be called before onLockAfter.
+  uptr addEdges(DeadlockDetectorTLS<BV> *dtls, uptr cur_node, u32 stk,
+                int unique_tid) {
+    ensureCurrentEpoch(dtls);
+    uptr cur_idx = nodeToIndex(cur_node);
+    uptr added_edges[40];
+    uptr n_added_edges = g_.addEdges(dtls->getLocks(current_epoch_), cur_idx,
+                                     added_edges, ARRAY_SIZE(added_edges));
+    for (uptr i = 0; i < n_added_edges; i++) {
+      if (n_edges_ < ARRAY_SIZE(edges_)) {
+        Edge e = {(u16)added_edges[i], (u16)cur_idx,
+                  dtls->findLockContext(added_edges[i]), stk,
+                  unique_tid};
+        edges_[n_edges_++] = e;
+      }
+    }
+    return n_added_edges;
+  }
+
+  bool findEdge(uptr from_node, uptr to_node, u32 *stk_from, u32 *stk_to,
+                int *unique_tid) {
+    uptr from_idx = nodeToIndex(from_node);
+    uptr to_idx = nodeToIndex(to_node);
+    for (uptr i = 0; i < n_edges_; i++) {
+      if (edges_[i].from == from_idx && edges_[i].to == to_idx) {
+        *stk_from = edges_[i].stk_from;
+        *stk_to = edges_[i].stk_to;
+        *unique_tid = edges_[i].unique_tid;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // Test-only function. Handles the before/after lock events,
+  // returns true if there is a cycle.
+  bool onLock(DeadlockDetectorTLS<BV> *dtls, uptr cur_node, u32 stk = 0) {
+    ensureCurrentEpoch(dtls);
+    bool is_reachable = !isHeld(dtls, cur_node) && onLockBefore(dtls, cur_node);
+    addEdges(dtls, cur_node, stk, 0);
+    onLockAfter(dtls, cur_node, stk);
+    return is_reachable;
+  }
+
+  // Handles the try_lock event, returns false.
+  // When a try_lock event happens (i.e. a try_lock call succeeds) we need
+  // to add this lock to the currently held locks, but we should not try to
+  // change the lock graph or to detect a cycle.  We may want to investigate
+  // whether a more aggressive strategy is possible for try_lock.
+  bool onTryLock(DeadlockDetectorTLS<BV> *dtls, uptr cur_node, u32 stk = 0) {
+    ensureCurrentEpoch(dtls);
+    uptr cur_idx = nodeToIndex(cur_node);
+    dtls->addLock(cur_idx, current_epoch_, stk);
+    return false;
+  }
+
+  // Returns true iff dtls is empty (no locks are currently held) and we can
+  // add the node to the currently held locks w/o chanding the global state.
+  // This operation is thread-safe as it only touches the dtls.
+  bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
+    if (!dtls->empty()) return false;
+    if (dtls->getEpoch() && dtls->getEpoch() == nodeToEpoch(node)) {
+      dtls->addLock(nodeToIndexUnchecked(node), nodeToEpoch(node), stk);
+      return true;
+    }
+    return false;
+  }
+
+  // Finds a path between the lock 'cur_node' (currently not held in dtls)
+  // and some currently held lock, returns the length of the path
+  // or 0 on failure.
+  uptr findPathToLock(DeadlockDetectorTLS<BV> *dtls, uptr cur_node, uptr *path,
+                      uptr path_size) {
+    tmp_bv_.copyFrom(dtls->getLocks(current_epoch_));
+    uptr idx = nodeToIndex(cur_node);
+    CHECK(!tmp_bv_.getBit(idx));
+    uptr res = g_.findShortestPath(idx, tmp_bv_, path, path_size);
+    for (uptr i = 0; i < res; i++)
+      path[i] = indexToNode(path[i]);
+    if (res)
+      CHECK_EQ(path[0], cur_node);
+    return res;
+  }
+
+  // Handle the unlock event.
+  // This operation is thread-safe as it only touches the dtls.
+  void onUnlock(DeadlockDetectorTLS<BV> *dtls, uptr node) {
+    if (dtls->getEpoch() == nodeToEpoch(node))
+      dtls->removeLock(nodeToIndexUnchecked(node));
+  }
+
+  // Tries to handle the lock event w/o writing to global state.
+  // Returns true on success.
+  // This operation is thread-safe as it only touches the dtls
+  // (modulo racy nature of hasAllEdges).
+  bool onLockFast(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
+    if (hasAllEdges(dtls, node)) {
+      dtls->addLock(nodeToIndexUnchecked(node), nodeToEpoch(node), stk);
+      return true;
+    }
+    return false;
+  }
+
+  bool isHeld(DeadlockDetectorTLS<BV> *dtls, uptr node) const {
+    return dtls->getLocks(current_epoch_).getBit(nodeToIndex(node));
+  }
+
+  uptr testOnlyGetEpoch() const { return current_epoch_; }
+  bool testOnlyHasEdge(uptr l1, uptr l2) {
+    return g_.hasEdge(nodeToIndex(l1), nodeToIndex(l2));
+  }
+  // idx1 and idx2 are raw indices to g_, not lock IDs.
+  bool testOnlyHasEdgeRaw(uptr idx1, uptr idx2) {
+    return g_.hasEdge(idx1, idx2);
+  }
+
+  void Print() {
+    for (uptr from = 0; from < size(); from++)
+      for (uptr to = 0; to < size(); to++)
+        if (g_.hasEdge(from, to))
+          Printf("  %zx => %zx\n", from, to);
+  }
+
+ private:
+  void check_idx(uptr idx) const { CHECK_LT(idx, size()); }
+
+  void check_node(uptr node) const {
+    CHECK_GE(node, size());
+    CHECK_EQ(current_epoch_, nodeToEpoch(node));
+  }
+
+  uptr indexToNode(uptr idx) const {
+    check_idx(idx);
+    return idx + current_epoch_;
+  }
+
+  uptr nodeToIndexUnchecked(uptr node) const { return node % size(); }
+
+  uptr nodeToIndex(uptr node) const {
+    check_node(node);
+    return nodeToIndexUnchecked(node);
+  }
+
+  uptr nodeToEpoch(uptr node) const { return node / size() * size(); }
+
+  uptr getAvailableNode(uptr data) {
+    uptr idx = available_nodes_.getAndClearFirstOne();
+    data_[idx] = data;
+    return indexToNode(idx);
+  }
+
+  struct Edge {
+    u16 from;
+    u16 to;
+    u32 stk_from;
+    u32 stk_to;
+    int unique_tid;
+  };
+
+  uptr current_epoch_;
+  BV available_nodes_;
+  BV recycled_nodes_;
+  BV tmp_bv_;
+  BVGraph<BV> g_;
+  uptr data_[BV::kSize];
+  Edge edges_[BV::kSize * 32];
+  uptr n_edges_;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_DEADLOCK_DETECTOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_deadlock_detector1.cpp b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector1.cpp
new file mode 100644
index 0000000000..d4a325bea4
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector1.cpp
@@ -0,0 +1,194 @@
+//===-- sanitizer_deadlock_detector1.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Deadlock detector implementation based on NxN adjacency bit matrix.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_deadlock_detector_interface.h"
+#include "sanitizer_deadlock_detector.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_mutex.h"
+
+#if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1
+
+namespace __sanitizer {
+
+typedef TwoLevelBitVector<> DDBV;  // DeadlockDetector's bit vector.
+
+struct DDPhysicalThread {
+};
+
+struct DDLogicalThread {
+  u64 ctx;
+  DeadlockDetectorTLS<DDBV> dd;
+  DDReport rep;
+  bool report_pending;
+};
+
+struct DD : public DDetector {
+  SpinMutex mtx;
+  DeadlockDetector<DDBV> dd;
+  DDFlags flags;
+
+  explicit DD(const DDFlags *flags);
+
+  DDPhysicalThread *CreatePhysicalThread() override;
+  void DestroyPhysicalThread(DDPhysicalThread *pt) override;
+
+  DDLogicalThread *CreateLogicalThread(u64 ctx) override;
+  void DestroyLogicalThread(DDLogicalThread *lt) override;
+
+  void MutexInit(DDCallback *cb, DDMutex *m) override;
+  void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) override;
+  void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
+                      bool trylock) override;
+  void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) override;
+  void MutexDestroy(DDCallback *cb, DDMutex *m) override;
+
+  DDReport *GetReport(DDCallback *cb) override;
+
+  void MutexEnsureID(DDLogicalThread *lt, DDMutex *m);
+  void ReportDeadlock(DDCallback *cb, DDMutex *m);
+};
+
+DDetector *DDetector::Create(const DDFlags *flags) {
+  (void)flags;
+  void *mem = MmapOrDie(sizeof(DD), "deadlock detector");
+  return new(mem) DD(flags);
+}
+
+DD::DD(const DDFlags *flags)
+    : flags(*flags) {
+  dd.clear();
+}
+
+DDPhysicalThread* DD::CreatePhysicalThread() {
+  return nullptr;
+}
+
+void DD::DestroyPhysicalThread(DDPhysicalThread *pt) {
+}
+
+DDLogicalThread* DD::CreateLogicalThread(u64 ctx) {
+  DDLogicalThread *lt = (DDLogicalThread*)InternalAlloc(sizeof(*lt));
+  lt->ctx = ctx;
+  lt->dd.clear();
+  lt->report_pending = false;
+  return lt;
+}
+
+void DD::DestroyLogicalThread(DDLogicalThread *lt) {
+  lt->~DDLogicalThread();
+  InternalFree(lt);
+}
+
+void DD::MutexInit(DDCallback *cb, DDMutex *m) {
+  m->id = 0;
+  m->stk = cb->Unwind();
+}
+
+void DD::MutexEnsureID(DDLogicalThread *lt, DDMutex *m) {
+  if (!dd.nodeBelongsToCurrentEpoch(m->id))
+    m->id = dd.newNode(reinterpret_cast<uptr>(m));
+  dd.ensureCurrentEpoch(&lt->dd);
+}
+
+void DD::MutexBeforeLock(DDCallback *cb,
+    DDMutex *m, bool wlock) {
+  DDLogicalThread *lt = cb->lt;
+  if (lt->dd.empty()) return;  // This will be the first lock held by lt.
+  if (dd.hasAllEdges(&lt->dd, m->id)) return;  // We already have all edges.
+  SpinMutexLock lk(&mtx);
+  MutexEnsureID(lt, m);
+  if (dd.isHeld(&lt->dd, m->id))
+    return;  // FIXME: allow this only for recursive locks.
+  if (dd.onLockBefore(&lt->dd, m->id)) {
+    // Actually add this edge now so that we have all the stack traces.
+    dd.addEdges(&lt->dd, m->id, cb->Unwind(), cb->UniqueTid());
+    ReportDeadlock(cb, m);
+  }
+}
+
+void DD::ReportDeadlock(DDCallback *cb, DDMutex *m) {
+  DDLogicalThread *lt = cb->lt;
+  uptr path[20];
+  uptr len = dd.findPathToLock(&lt->dd, m->id, path, ARRAY_SIZE(path));
+  if (len == 0U) {
+    // A cycle of 20+ locks? Well, that's a bit odd...
+    Printf("WARNING: too long mutex cycle found\n");
+    return;
+  }
+  CHECK_EQ(m->id, path[0]);
+  lt->report_pending = true;
+  len = Min<uptr>(len, DDReport::kMaxLoopSize);
+  DDReport *rep = &lt->rep;
+  rep->n = len;
+  for (uptr i = 0; i < len; i++) {
+    uptr from = path[i];
+    uptr to = path[(i + 1) % len];
+    DDMutex *m0 = (DDMutex*)dd.getData(from);
+    DDMutex *m1 = (DDMutex*)dd.getData(to);
+
+    u32 stk_from = -1U, stk_to = -1U;
+    int unique_tid = 0;
+    dd.findEdge(from, to, &stk_from, &stk_to, &unique_tid);
+    // Printf("Edge: %zd=>%zd: %u/%u T%d\n", from, to, stk_from, stk_to,
+    //    unique_tid);
+    rep->loop[i].thr_ctx = unique_tid;
+    rep->loop[i].mtx_ctx0 = m0->ctx;
+    rep->loop[i].mtx_ctx1 = m1->ctx;
+    rep->loop[i].stk[0] = stk_to;
+    rep->loop[i].stk[1] = stk_from;
+  }
+}
+
+void DD::MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock, bool trylock) {
+  DDLogicalThread *lt = cb->lt;
+  u32 stk = 0;
+  if (flags.second_deadlock_stack)
+    stk = cb->Unwind();
+  // Printf("T%p MutexLock:   %zx stk %u\n", lt, m->id, stk);
+  if (dd.onFirstLock(&lt->dd, m->id, stk))
+    return;
+  if (dd.onLockFast(&lt->dd, m->id, stk))
+    return;
+
+  SpinMutexLock lk(&mtx);
+  MutexEnsureID(lt, m);
+  if (wlock)  // Only a recursive rlock may be held.
+    CHECK(!dd.isHeld(&lt->dd, m->id));
+  if (!trylock)
+    dd.addEdges(&lt->dd, m->id, stk ? stk : cb->Unwind(), cb->UniqueTid());
+  dd.onLockAfter(&lt->dd, m->id, stk);
+}
+
+void DD::MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {
+  // Printf("T%p MutexUnLock: %zx\n", cb->lt, m->id);
+  dd.onUnlock(&cb->lt->dd, m->id);
+}
+
+void DD::MutexDestroy(DDCallback *cb,
+    DDMutex *m) {
+  if (!m->id) return;
+  SpinMutexLock lk(&mtx);
+  if (dd.nodeBelongsToCurrentEpoch(m->id))
+    dd.removeNode(m->id);
+  m->id = 0;
+}
+
+DDReport *DD::GetReport(DDCallback *cb) {
+  if (!cb->lt->report_pending)
+    return nullptr;
+  cb->lt->report_pending = false;
+  return &cb->lt->rep;
+}
+
+} // namespace __sanitizer
+#endif // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1
diff --git a/lib/tsan/sanitizer_common/sanitizer_deadlock_detector2.cpp b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector2.cpp
new file mode 100644
index 0000000000..4026739d4e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector2.cpp
@@ -0,0 +1,423 @@
+//===-- sanitizer_deadlock_detector2.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Deadlock detector implementation based on adjacency lists.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_deadlock_detector_interface.h"
+#include "sanitizer_common.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_mutex.h"
+
+#if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2
+
+namespace __sanitizer {
+
+const int kMaxNesting = 64;
+const u32 kNoId = -1;
+const u32 kEndId = -2;
+const int kMaxLink = 8;
+const int kL1Size = 1024;
+const int kL2Size = 1024;
+const int kMaxMutex = kL1Size * kL2Size;
+
+struct Id {
+  u32 id;
+  u32 seq;
+
+  explicit Id(u32 id = 0, u32 seq = 0)
+      : id(id)
+      , seq(seq) {
+  }
+};
+
+struct Link {
+  u32 id;
+  u32 seq;
+  u32 tid;
+  u32 stk0;
+  u32 stk1;
+
+  explicit Link(u32 id = 0, u32 seq = 0, u32 tid = 0, u32 s0 = 0, u32 s1 = 0)
+      : id(id)
+      , seq(seq)
+      , tid(tid)
+      , stk0(s0)
+      , stk1(s1) {
+  }
+};
+
+struct DDPhysicalThread {
+  DDReport rep;
+  bool report_pending;
+  bool visited[kMaxMutex];
+  Link pending[kMaxMutex];
+  Link path[kMaxMutex];
+};
+
+struct ThreadMutex {
+  u32 id;
+  u32 stk;
+};
+
+struct DDLogicalThread {
+  u64         ctx;
+  ThreadMutex locked[kMaxNesting];
+  int         nlocked;
+};
+
+struct Mutex {
+  StaticSpinMutex mtx;
+  u32 seq;
+  int nlink;
+  Link link[kMaxLink];
+};
+
+struct DD : public DDetector {
+  explicit DD(const DDFlags *flags);
+
+  DDPhysicalThread* CreatePhysicalThread();
+  void DestroyPhysicalThread(DDPhysicalThread *pt);
+
+  DDLogicalThread* CreateLogicalThread(u64 ctx);
+  void DestroyLogicalThread(DDLogicalThread *lt);
+
+  void MutexInit(DDCallback *cb, DDMutex *m);
+  void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock);
+  void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
+      bool trylock);
+  void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock);
+  void MutexDestroy(DDCallback *cb, DDMutex *m);
+
+  DDReport *GetReport(DDCallback *cb);
+
+  void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx);
+  void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath);
+  u32 allocateId(DDCallback *cb);
+  Mutex *getMutex(u32 id);
+  u32 getMutexId(Mutex *m);
+
+  DDFlags flags;
+
+  Mutex* mutex[kL1Size];
+
+  SpinMutex mtx;
+  InternalMmapVector<u32> free_id;
+  int id_gen = 0;
+};
+
+DDetector *DDetector::Create(const DDFlags *flags) {
+  (void)flags;
+  void *mem = MmapOrDie(sizeof(DD), "deadlock detector");
+  return new(mem) DD(flags);
+}
+
+DD::DD(const DDFlags *flags) : flags(*flags) { free_id.reserve(1024); }
+
+DDPhysicalThread* DD::CreatePhysicalThread() {
+  DDPhysicalThread *pt = (DDPhysicalThread*)MmapOrDie(sizeof(DDPhysicalThread),
+      "deadlock detector (physical thread)");
+  return pt;
+}
+
+void DD::DestroyPhysicalThread(DDPhysicalThread *pt) {
+  pt->~DDPhysicalThread();
+  UnmapOrDie(pt, sizeof(DDPhysicalThread));
+}
+
+DDLogicalThread* DD::CreateLogicalThread(u64 ctx) {
+  DDLogicalThread *lt = (DDLogicalThread*)InternalAlloc(
+      sizeof(DDLogicalThread));
+  lt->ctx = ctx;
+  lt->nlocked = 0;
+  return lt;
+}
+
+void DD::DestroyLogicalThread(DDLogicalThread *lt) {
+  lt->~DDLogicalThread();
+  InternalFree(lt);
+}
+
+void DD::MutexInit(DDCallback *cb, DDMutex *m) {
+  VPrintf(2, "#%llu: DD::MutexInit(%p)\n", cb->lt->ctx, m);
+  m->id = kNoId;
+  m->recursion = 0;
+  atomic_store(&m->owner, 0, memory_order_relaxed);
+}
+
+Mutex *DD::getMutex(u32 id) {
+  return &mutex[id / kL2Size][id % kL2Size];
+}
+
+u32 DD::getMutexId(Mutex *m) {
+  for (int i = 0; i < kL1Size; i++) {
+    Mutex *tab = mutex[i];
+    if (tab == 0)
+      break;
+    if (m >= tab && m < tab + kL2Size)
+      return i * kL2Size + (m - tab);
+  }
+  return -1;
+}
+
+u32 DD::allocateId(DDCallback *cb) {
+  u32 id = -1;
+  SpinMutexLock l(&mtx);
+  if (free_id.size() > 0) {
+    id = free_id.back();
+    free_id.pop_back();
+  } else {
+    CHECK_LT(id_gen, kMaxMutex);
+    if ((id_gen % kL2Size) == 0) {
+      mutex[id_gen / kL2Size] = (Mutex*)MmapOrDie(kL2Size * sizeof(Mutex),
+          "deadlock detector (mutex table)");
+    }
+    id = id_gen++;
+  }
+  CHECK_LE(id, kMaxMutex);
+  VPrintf(3, "#%llu: DD::allocateId assign id %d\n", cb->lt->ctx, id);
+  return id;
+}
+
+void DD::MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) {
+  VPrintf(2, "#%llu: DD::MutexBeforeLock(%p, wlock=%d) nlocked=%d\n",
+      cb->lt->ctx, m, wlock, cb->lt->nlocked);
+  DDPhysicalThread *pt = cb->pt;
+  DDLogicalThread *lt = cb->lt;
+
+  uptr owner = atomic_load(&m->owner, memory_order_relaxed);
+  if (owner == (uptr)cb->lt) {
+    VPrintf(3, "#%llu: DD::MutexBeforeLock recursive\n",
+        cb->lt->ctx);
+    return;
+  }
+
+  CHECK_LE(lt->nlocked, kMaxNesting);
+
+  // FIXME(dvyukov): don't allocate id if lt->nlocked == 0?
+  if (m->id == kNoId)
+    m->id = allocateId(cb);
+
+  ThreadMutex *tm = &lt->locked[lt->nlocked++];
+  tm->id = m->id;
+  if (flags.second_deadlock_stack)
+    tm->stk = cb->Unwind();
+  if (lt->nlocked == 1) {
+    VPrintf(3, "#%llu: DD::MutexBeforeLock first mutex\n",
+        cb->lt->ctx);
+    return;
+  }
+
+  bool added = false;
+  Mutex *mtx = getMutex(m->id);
+  for (int i = 0; i < lt->nlocked - 1; i++) {
+    u32 id1 = lt->locked[i].id;
+    u32 stk1 = lt->locked[i].stk;
+    Mutex *mtx1 = getMutex(id1);
+    SpinMutexLock l(&mtx1->mtx);
+    if (mtx1->nlink == kMaxLink) {
+      // FIXME(dvyukov): check stale links
+      continue;
+    }
+    int li = 0;
+    for (; li < mtx1->nlink; li++) {
+      Link *link = &mtx1->link[li];
+      if (link->id == m->id) {
+        if (link->seq != mtx->seq) {
+          link->seq = mtx->seq;
+          link->tid = lt->ctx;
+          link->stk0 = stk1;
+          link->stk1 = cb->Unwind();
+          added = true;
+          VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n",
+              cb->lt->ctx, getMutexId(mtx1), m->id);
+        }
+        break;
+      }
+    }
+    if (li == mtx1->nlink) {
+      // FIXME(dvyukov): check stale links
+      Link *link = &mtx1->link[mtx1->nlink++];
+      link->id = m->id;
+      link->seq = mtx->seq;
+      link->tid = lt->ctx;
+      link->stk0 = stk1;
+      link->stk1 = cb->Unwind();
+      added = true;
+      VPrintf(3, "#%llu: DD::MutexBeforeLock added %d->%d link\n",
+          cb->lt->ctx, getMutexId(mtx1), m->id);
+    }
+  }
+
+  if (!added || mtx->nlink == 0) {
+    VPrintf(3, "#%llu: DD::MutexBeforeLock don't check\n",
+        cb->lt->ctx);
+    return;
+  }
+
+  CycleCheck(pt, lt, m);
+}
+
+void DD::MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
+    bool trylock) {
+  VPrintf(2, "#%llu: DD::MutexAfterLock(%p, wlock=%d, try=%d) nlocked=%d\n",
+      cb->lt->ctx, m, wlock, trylock, cb->lt->nlocked);
+  DDLogicalThread *lt = cb->lt;
+
+  uptr owner = atomic_load(&m->owner, memory_order_relaxed);
+  if (owner == (uptr)cb->lt) {
+    VPrintf(3, "#%llu: DD::MutexAfterLock recursive\n", cb->lt->ctx);
+    CHECK(wlock);
+    m->recursion++;
+    return;
+  }
+  CHECK_EQ(owner, 0);
+  if (wlock) {
+    VPrintf(3, "#%llu: DD::MutexAfterLock set owner\n", cb->lt->ctx);
+    CHECK_EQ(m->recursion, 0);
+    m->recursion = 1;
+    atomic_store(&m->owner, (uptr)cb->lt, memory_order_relaxed);
+  }
+
+  if (!trylock)
+    return;
+
+  CHECK_LE(lt->nlocked, kMaxNesting);
+  if (m->id == kNoId)
+    m->id = allocateId(cb);
+  ThreadMutex *tm = &lt->locked[lt->nlocked++];
+  tm->id = m->id;
+  if (flags.second_deadlock_stack)
+    tm->stk = cb->Unwind();
+}
+
+void DD::MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {
+  VPrintf(2, "#%llu: DD::MutexBeforeUnlock(%p, wlock=%d) nlocked=%d\n",
+      cb->lt->ctx, m, wlock, cb->lt->nlocked);
+  DDLogicalThread *lt = cb->lt;
+
+  uptr owner = atomic_load(&m->owner, memory_order_relaxed);
+  if (owner == (uptr)cb->lt) {
+    VPrintf(3, "#%llu: DD::MutexBeforeUnlock recursive\n", cb->lt->ctx);
+    if (--m->recursion > 0)
+      return;
+    VPrintf(3, "#%llu: DD::MutexBeforeUnlock reset owner\n", cb->lt->ctx);
+    atomic_store(&m->owner, 0, memory_order_relaxed);
+  }
+  CHECK_NE(m->id, kNoId);
+  int last = lt->nlocked - 1;
+  for (int i = last; i >= 0; i--) {
+    if (cb->lt->locked[i].id == m->id) {
+      lt->locked[i] = lt->locked[last];
+      lt->nlocked--;
+      break;
+    }
+  }
+}
+
+void DD::MutexDestroy(DDCallback *cb, DDMutex *m) {
+  VPrintf(2, "#%llu: DD::MutexDestroy(%p)\n",
+      cb->lt->ctx, m);
+  DDLogicalThread *lt = cb->lt;
+
+  if (m->id == kNoId)
+    return;
+
+  // Remove the mutex from lt->locked if there.
+  int last = lt->nlocked - 1;
+  for (int i = last; i >= 0; i--) {
+    if (lt->locked[i].id == m->id) {
+      lt->locked[i] = lt->locked[last];
+      lt->nlocked--;
+      break;
+    }
+  }
+
+  // Clear and invalidate the mutex descriptor.
+  {
+    Mutex *mtx = getMutex(m->id);
+    SpinMutexLock l(&mtx->mtx);
+    mtx->seq++;
+    mtx->nlink = 0;
+  }
+
+  // Return id to cache.
+  {
+    SpinMutexLock l(&mtx);
+    free_id.push_back(m->id);
+  }
+}
+
+void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt,
+    DDMutex *m) {
+  internal_memset(pt->visited, 0, sizeof(pt->visited));
+  int npath = 0;
+  int npending = 0;
+  {
+    Mutex *mtx = getMutex(m->id);
+    SpinMutexLock l(&mtx->mtx);
+    for (int li = 0; li < mtx->nlink; li++)
+      pt->pending[npending++] = mtx->link[li];
+  }
+  while (npending > 0) {
+    Link link = pt->pending[--npending];
+    if (link.id == kEndId) {
+      npath--;
+      continue;
+    }
+    if (pt->visited[link.id])
+      continue;
+    Mutex *mtx1 = getMutex(link.id);
+    SpinMutexLock l(&mtx1->mtx);
+    if (mtx1->seq != link.seq)
+      continue;
+    pt->visited[link.id] = true;
+    if (mtx1->nlink == 0)
+      continue;
+    pt->path[npath++] = link;
+    pt->pending[npending++] = Link(kEndId);
+    if (link.id == m->id)
+      return Report(pt, lt, npath);  // Bingo!
+    for (int li = 0; li < mtx1->nlink; li++) {
+      Link *link1 = &mtx1->link[li];
+      // Mutex *mtx2 = getMutex(link->id);
+      // FIXME(dvyukov): fast seq check
+      // FIXME(dvyukov): fast nlink != 0 check
+      // FIXME(dvyukov): fast pending check?
+      // FIXME(dvyukov): npending can be larger than kMaxMutex
+      pt->pending[npending++] = *link1;
+    }
+  }
+}
+
+void DD::Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath) {
+  DDReport *rep = &pt->rep;
+  rep->n = npath;
+  for (int i = 0; i < npath; i++) {
+    Link *link = &pt->path[i];
+    Link *link0 = &pt->path[i ? i - 1 : npath - 1];
+    rep->loop[i].thr_ctx = link->tid;
+    rep->loop[i].mtx_ctx0 = link0->id;
+    rep->loop[i].mtx_ctx1 = link->id;
+    rep->loop[i].stk[0] = flags.second_deadlock_stack ? link->stk0 : 0;
+    rep->loop[i].stk[1] = link->stk1;
+  }
+  pt->report_pending = true;
+}
+
+DDReport *DD::GetReport(DDCallback *cb) {
+  if (!cb->pt->report_pending)
+    return 0;
+  cb->pt->report_pending = false;
+  return &cb->pt->rep;
+}
+
+}  // namespace __sanitizer
+#endif  // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 2
diff --git a/lib/tsan/sanitizer_common/sanitizer_deadlock_detector_interface.h b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector_interface.h
new file mode 100644
index 0000000000..a4722b080e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_deadlock_detector_interface.h
@@ -0,0 +1,92 @@
+//===-- sanitizer_deadlock_detector_interface.h -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime.
+// Abstract deadlock detector interface.
+// FIXME: this is work in progress, nothing really works yet.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_DEADLOCK_DETECTOR_INTERFACE_H
+#define SANITIZER_DEADLOCK_DETECTOR_INTERFACE_H
+
+#ifndef SANITIZER_DEADLOCK_DETECTOR_VERSION
+# define SANITIZER_DEADLOCK_DETECTOR_VERSION 1
+#endif
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_atomic.h"
+
+namespace __sanitizer {
+
+// dd - deadlock detector.
+// lt - logical (user) thread.
+// pt - physical (OS) thread.
+
+struct DDPhysicalThread;
+struct DDLogicalThread;
+
+struct DDMutex {
+#if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1
+  uptr id;
+  u32  stk;  // creation stack
+#elif SANITIZER_DEADLOCK_DETECTOR_VERSION == 2
+  u32              id;
+  u32              recursion;
+  atomic_uintptr_t owner;
+#else
+# error "BAD SANITIZER_DEADLOCK_DETECTOR_VERSION"
+#endif
+  u64  ctx;
+};
+
+struct DDFlags {
+  bool second_deadlock_stack;
+};
+
+struct DDReport {
+  enum { kMaxLoopSize = 20 };
+  int n;  // number of entries in loop
+  struct {
+    u64 thr_ctx;   // user thread context
+    u64 mtx_ctx0;  // user mutex context, start of the edge
+    u64 mtx_ctx1;  // user mutex context, end of the edge
+    u32 stk[2];  // stack ids for the edge
+  } loop[kMaxLoopSize];
+};
+
+struct DDCallback {
+  DDPhysicalThread *pt;
+  DDLogicalThread  *lt;
+
+  virtual u32 Unwind() { return 0; }
+  virtual int UniqueTid() { return 0; }
+};
+
+struct DDetector {
+  static DDetector *Create(const DDFlags *flags);
+
+  virtual DDPhysicalThread* CreatePhysicalThread() { return nullptr; }
+  virtual void DestroyPhysicalThread(DDPhysicalThread *pt) {}
+
+  virtual DDLogicalThread* CreateLogicalThread(u64 ctx) { return nullptr; }
+  virtual void DestroyLogicalThread(DDLogicalThread *lt) {}
+
+  virtual void MutexInit(DDCallback *cb, DDMutex *m) {}
+  virtual void MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) {}
+  virtual void MutexAfterLock(DDCallback *cb, DDMutex *m, bool wlock,
+      bool trylock) {}
+  virtual void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {}
+  virtual void MutexDestroy(DDCallback *cb, DDMutex *m) {}
+
+  virtual DDReport *GetReport(DDCallback *cb) { return nullptr; }
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_DEADLOCK_DETECTOR_INTERFACE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_errno.cpp b/lib/tsan/sanitizer_common/sanitizer_errno.cpp
new file mode 100644
index 0000000000..cbadf4d924
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_errno.cpp
@@ -0,0 +1,34 @@
+//===-- sanitizer_errno.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into other
+// files (e.g. interceptors are not supposed to include any system headers).
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_internal_defs.h"
+
+#include <errno.h>
+
+namespace __sanitizer {
+
+COMPILER_CHECK(errno_ENOMEM == ENOMEM);
+COMPILER_CHECK(errno_EBUSY == EBUSY);
+COMPILER_CHECK(errno_EINVAL == EINVAL);
+
+// EOWNERDEAD is not present in some older platforms.
+#if defined(EOWNERDEAD)
+extern const int errno_EOWNERDEAD = EOWNERDEAD;
+#else
+extern const int errno_EOWNERDEAD = -1;
+#endif
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_errno.h b/lib/tsan/sanitizer_common/sanitizer_errno.h
new file mode 100644
index 0000000000..584e66e4a8
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_errno.h
@@ -0,0 +1,39 @@
+//===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into sensitive
+// files (e.g. interceptors are not supposed to include any system headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_H
+#define SANITIZER_ERRNO_H
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+#  define __errno_location __error
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
+  SANITIZER_RTEMS
+#  define __errno_location __errno
+#elif SANITIZER_SOLARIS
+#  define __errno_location ___errno
+#elif SANITIZER_WINDOWS
+#  define __errno_location _errno
+#endif
+
+extern "C" int *__errno_location();
+
+#define errno (*__errno_location())
+
+#endif  // SANITIZER_ERRNO_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_errno_codes.h b/lib/tsan/sanitizer_common/sanitizer_errno_codes.h
new file mode 100644
index 0000000000..f388d0d364
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_errno_codes.h
@@ -0,0 +1,33 @@
+//===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno codes to avoid including errno.h and its dependencies into
+// sensitive files (e.g. interceptors are not supposed to include any system
+// headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_CODES_H
+#define SANITIZER_ERRNO_CODES_H
+
+namespace __sanitizer {
+
+#define errno_ENOMEM 12
+#define errno_EBUSY 16
+#define errno_EINVAL 22
+
+// Those might not present or their value differ on different platforms.
+extern const int errno_EOWNERDEAD;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ERRNO_CODES_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_file.cpp b/lib/tsan/sanitizer_common/sanitizer_file.cpp
new file mode 100644
index 0000000000..79930d7942
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_file.cpp
@@ -0,0 +1,215 @@
+//===-- sanitizer_file.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.  It defines filesystem-related interfaces.  This
+// is separate from sanitizer_common.cpp so that it's simpler to disable
+// all the filesystem support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+
+namespace __sanitizer {
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+  WriteToFile(kStderrFd, buffer, length);
+}
+
+StaticSpinMutex report_file_mu;
+ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
+
+void RawWrite(const char *buffer) {
+  report_file.Write(buffer, internal_strlen(buffer));
+}
+
+void ReportFile::ReopenIfNecessary() {
+  mu->CheckLocked();
+  if (fd == kStdoutFd || fd == kStderrFd) return;
+
+  uptr pid = internal_getpid();
+  // If in tracer, use the parent's file.
+  if (pid == stoptheworld_tracer_pid)
+    pid = stoptheworld_tracer_ppid;
+  if (fd != kInvalidFd) {
+    // If the report file is already opened by the current process,
+    // do nothing. Otherwise the report file was opened by the parent
+    // process, close it now.
+    if (fd_pid == pid)
+      return;
+    else
+      CloseFile(fd);
+  }
+
+  const char *exe_name = GetProcessName();
+  if (common_flags()->log_exe_name && exe_name) {
+    internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix,
+                      exe_name, pid);
+  } else {
+    internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
+  }
+  fd = OpenFile(full_path, WrOnly);
+  if (fd == kInvalidFd) {
+    const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
+    WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
+    WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
+    Die();
+  }
+  fd_pid = pid;
+}
+
+void ReportFile::SetReportPath(const char *path) {
+  if (!path)
+    return;
+  uptr len = internal_strlen(path);
+  if (len > sizeof(path_prefix) - 100) {
+    Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
+           path[0], path[1], path[2], path[3],
+           path[4], path[5], path[6], path[7]);
+    Die();
+  }
+
+  SpinMutexLock l(mu);
+  if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
+    CloseFile(fd);
+  fd = kInvalidFd;
+  if (internal_strcmp(path, "stdout") == 0) {
+    fd = kStdoutFd;
+  } else if (internal_strcmp(path, "stderr") == 0) {
+    fd = kStderrFd;
+  } else {
+    internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
+  }
+}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+                      uptr *read_len, uptr max_len, error_t *errno_p) {
+  *buff = nullptr;
+  *buff_size = 0;
+  *read_len = 0;
+  if (!max_len)
+    return true;
+  uptr PageSize = GetPageSizeCached();
+  uptr kMinFileLen = Min(PageSize, max_len);
+
+  // The files we usually open are not seekable, so try different buffer sizes.
+  for (uptr size = kMinFileLen;; size = Min(size * 2, max_len)) {
+    UnmapOrDie(*buff, *buff_size);
+    *buff = (char*)MmapOrDie(size, __func__);
+    *buff_size = size;
+    fd_t fd = OpenFile(file_name, RdOnly, errno_p);
+    if (fd == kInvalidFd) {
+      UnmapOrDie(*buff, *buff_size);
+      return false;
+    }
+    *read_len = 0;
+    // Read up to one page at a time.
+    bool reached_eof = false;
+    while (*read_len < size) {
+      uptr just_read;
+      if (!ReadFromFile(fd, *buff + *read_len, size - *read_len, &just_read,
+                        errno_p)) {
+        UnmapOrDie(*buff, *buff_size);
+        CloseFile(fd);
+        return false;
+      }
+      *read_len += just_read;
+      if (just_read == 0 || *read_len == max_len) {
+        reached_eof = true;
+        break;
+      }
+    }
+    CloseFile(fd);
+    if (reached_eof)  // We've read the whole file.
+      break;
+  }
+  return true;
+}
+
+bool ReadFileToVector(const char *file_name,
+                      InternalMmapVectorNoCtor<char> *buff, uptr max_len,
+                      error_t *errno_p) {
+  buff->clear();
+  if (!max_len)
+    return true;
+  uptr PageSize = GetPageSizeCached();
+  fd_t fd = OpenFile(file_name, RdOnly, errno_p);
+  if (fd == kInvalidFd)
+    return false;
+  uptr read_len = 0;
+  while (read_len < max_len) {
+    if (read_len >= buff->size())
+      buff->resize(Min(Max(PageSize, read_len * 2), max_len));
+    CHECK_LT(read_len, buff->size());
+    CHECK_LE(buff->size(), max_len);
+    uptr just_read;
+    if (!ReadFromFile(fd, buff->data() + read_len, buff->size() - read_len,
+                      &just_read, errno_p)) {
+      CloseFile(fd);
+      return false;
+    }
+    read_len += just_read;
+    if (!just_read)
+      break;
+  }
+  CloseFile(fd);
+  buff->resize(read_len);
+  return true;
+}
+
+static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':';
+
+char *FindPathToBinary(const char *name) {
+  if (FileExists(name)) {
+    return internal_strdup(name);
+  }
+
+  const char *path = GetEnv("PATH");
+  if (!path)
+    return nullptr;
+  uptr name_len = internal_strlen(name);
+  InternalMmapVector<char> buffer(kMaxPathLength);
+  const char *beg = path;
+  while (true) {
+    const char *end = internal_strchrnul(beg, kPathSeparator);
+    uptr prefix_len = end - beg;
+    if (prefix_len + name_len + 2 <= kMaxPathLength) {
+      internal_memcpy(buffer.data(), beg, prefix_len);
+      buffer[prefix_len] = '/';
+      internal_memcpy(&buffer[prefix_len + 1], name, name_len);
+      buffer[prefix_len + 1 + name_len] = '\0';
+      if (FileExists(buffer.data()))
+        return internal_strdup(buffer.data());
+    }
+    if (*end == '\0') break;
+    beg = end + 1;
+  }
+  return nullptr;
+}
+
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+extern "C" {
+void __sanitizer_set_report_path(const char *path) {
+  report_file.SetReportPath(path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+  report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
+  report_file.fd_pid = internal_getpid();
+}
+} // extern "C"
+
+#endif  // !SANITIZER_FUCHSIA
diff --git a/lib/tsan/sanitizer_common/sanitizer_file.h b/lib/tsan/sanitizer_common/sanitizer_file.h
new file mode 100644
index 0000000000..26681f0493
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_file.h
@@ -0,0 +1,106 @@
+//===-- sanitizer_file.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between run-time libraries of sanitizers.
+// It declares filesystem-related interfaces.  This is separate from
+// sanitizer_common.h so that it's simpler to disable all the filesystem
+// support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FILE_H
+#define SANITIZER_FILE_H
+
+#include "sanitizer_interface_internal.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+struct ReportFile {
+  void Write(const char *buffer, uptr length);
+  bool SupportsColors();
+  void SetReportPath(const char *path);
+
+  // Don't use fields directly. They are only declared public to allow
+  // aggregate initialization.
+
+  // Protects fields below.
+  StaticSpinMutex *mu;
+  // Opened file descriptor. Defaults to stderr. It may be equal to
+  // kInvalidFd, in which case new file will be opened when necessary.
+  fd_t fd;
+  // Path prefix of report file, set via __sanitizer_set_report_path.
+  char path_prefix[kMaxPathLength];
+  // Full path to report, obtained as <path_prefix>.PID
+  char full_path[kMaxPathLength];
+  // PID of the process that opened fd. If a fork() occurs,
+  // the PID of child will be different from fd_pid.
+  uptr fd_pid;
+
+ private:
+  void ReopenIfNecessary();
+};
+extern ReportFile report_file;
+
+enum FileAccessMode {
+  RdOnly,
+  WrOnly,
+  RdWr
+};
+
+// Returns kInvalidFd on error.
+fd_t OpenFile(const char *filename, FileAccessMode mode,
+              error_t *errno_p = nullptr);
+void CloseFile(fd_t);
+
+// Return true on success, false on error.
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size,
+                  uptr *bytes_read = nullptr, error_t *error_p = nullptr);
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size,
+                 uptr *bytes_written = nullptr, error_t *error_p = nullptr);
+
+// Scoped file handle closer.
+struct FileCloser {
+  explicit FileCloser(fd_t fd) : fd(fd) {}
+  ~FileCloser() { CloseFile(fd); }
+  fd_t fd;
+};
+
+bool SupportsColoredOutput(fd_t fd);
+
+// OS
+const char *GetPwd();
+bool FileExists(const char *filename);
+char *FindPathToBinary(const char *name);
+bool IsPathSeparator(const char c);
+bool IsAbsolutePath(const char *path);
+// Starts a subprocess and returs its pid.
+// If *_fd parameters are not kInvalidFd their corresponding input/output
+// streams will be redirect to the file. The files will always be closed
+// in parent process even in case of an error.
+// The child process will close all fds after STDERR_FILENO
+// before passing control to a program.
+pid_t StartSubprocess(const char *filename, const char *const argv[],
+                      const char *const envp[], fd_t stdin_fd = kInvalidFd,
+                      fd_t stdout_fd = kInvalidFd, fd_t stderr_fd = kInvalidFd);
+// Checks if specified process is still running
+bool IsProcessRunning(pid_t pid);
+// Waits for the process to finish and returns its exit code.
+// Returns -1 in case of an error.
+int WaitForProcess(pid_t pid);
+
+// Maps given file to virtual memory, and returns pointer to it
+// (or NULL if mapping fails). Stores the size of mmaped region
+// in '*buff_size'.
+void *MapFileToMemory(const char *file_name, uptr *buff_size);
+void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FILE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp b/lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp
new file mode 100644
index 0000000000..9e274268bf
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp
@@ -0,0 +1,191 @@
+//===-- sanitizer_flag_parser.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_flag_parser.h"
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_flag_parser.h"
+
+namespace __sanitizer {
+
+LowLevelAllocator FlagParser::Alloc;
+
+class UnknownFlags {
+  static const int kMaxUnknownFlags = 20;
+  const char *unknown_flags_[kMaxUnknownFlags];
+  int n_unknown_flags_;
+
+ public:
+  void Add(const char *name) {
+    CHECK_LT(n_unknown_flags_, kMaxUnknownFlags);
+    unknown_flags_[n_unknown_flags_++] = name;
+  }
+
+  void Report() {
+    if (!n_unknown_flags_) return;
+    Printf("WARNING: found %d unrecognized flag(s):\n", n_unknown_flags_);
+    for (int i = 0; i < n_unknown_flags_; ++i)
+      Printf("    %s\n", unknown_flags_[i]);
+    n_unknown_flags_ = 0;
+  }
+};
+
+UnknownFlags unknown_flags;
+
+void ReportUnrecognizedFlags() {
+  unknown_flags.Report();
+}
+
+char *FlagParser::ll_strndup(const char *s, uptr n) {
+  uptr len = internal_strnlen(s, n);
+  char *s2 = (char*)Alloc.Allocate(len + 1);
+  internal_memcpy(s2, s, len);
+  s2[len] = 0;
+  return s2;
+}
+
+void FlagParser::PrintFlagDescriptions() {
+  char buffer[128];
+  buffer[sizeof(buffer) - 1] = '\0';
+  Printf("Available flags for %s:\n", SanitizerToolName);
+  for (int i = 0; i < n_flags_; ++i) {
+    bool truncated = !(flags_[i].handler->Format(buffer, sizeof(buffer)));
+    CHECK_EQ(buffer[sizeof(buffer) - 1], '\0');
+    const char *truncation_str = truncated ? " Truncated" : "";
+    Printf("\t%s\n\t\t- %s (Current Value%s: %s)\n", flags_[i].name,
+           flags_[i].desc, truncation_str, buffer);
+  }
+}
+
+void FlagParser::fatal_error(const char *err) {
+  Printf("%s: ERROR: %s\n", SanitizerToolName, err);
+  Die();
+}
+
+bool FlagParser::is_space(char c) {
+  return c == ' ' || c == ',' || c == ':' || c == '\n' || c == '\t' ||
+         c == '\r';
+}
+
+void FlagParser::skip_whitespace() {
+  while (is_space(buf_[pos_])) ++pos_;
+}
+
+void FlagParser::parse_flag(const char *env_option_name) {
+  uptr name_start = pos_;
+  while (buf_[pos_] != 0 && buf_[pos_] != '=' && !is_space(buf_[pos_])) ++pos_;
+  if (buf_[pos_] != '=') {
+    if (env_option_name) {
+      Printf("%s: ERROR: expected '=' in %s\n", SanitizerToolName,
+             env_option_name);
+      Die();
+    } else {
+      fatal_error("expected '='");
+    }
+  }
+  char *name = ll_strndup(buf_ + name_start, pos_ - name_start);
+
+  uptr value_start = ++pos_;
+  char *value;
+  if (buf_[pos_] == '\'' || buf_[pos_] == '"') {
+    char quote = buf_[pos_++];
+    while (buf_[pos_] != 0 && buf_[pos_] != quote) ++pos_;
+    if (buf_[pos_] == 0) fatal_error("unterminated string");
+    value = ll_strndup(buf_ + value_start + 1, pos_ - value_start - 1);
+    ++pos_; // consume the closing quote
+  } else {
+    while (buf_[pos_] != 0 && !is_space(buf_[pos_])) ++pos_;
+    if (buf_[pos_] != 0 && !is_space(buf_[pos_]))
+      fatal_error("expected separator or eol");
+    value = ll_strndup(buf_ + value_start, pos_ - value_start);
+  }
+
+  bool res = run_handler(name, value);
+  if (!res) fatal_error("Flag parsing failed.");
+}
+
+void FlagParser::parse_flags(const char *env_option_name) {
+  while (true) {
+    skip_whitespace();
+    if (buf_[pos_] == 0) break;
+    parse_flag(env_option_name);
+  }
+
+  // Do a sanity check for certain flags.
+  if (common_flags_dont_use.malloc_context_size < 1)
+    common_flags_dont_use.malloc_context_size = 1;
+}
+
+void FlagParser::ParseStringFromEnv(const char *env_name) {
+  const char *env = GetEnv(env_name);
+  VPrintf(1, "%s: %s\n", env_name, env ? env : "<empty>");
+  ParseString(env, env_name);
+}
+
+void FlagParser::ParseString(const char *s, const char *env_option_name) {
+  if (!s) return;
+  // Backup current parser state to allow nested ParseString() calls.
+  const char *old_buf_ = buf_;
+  uptr old_pos_ = pos_;
+  buf_ = s;
+  pos_ = 0;
+
+  parse_flags(env_option_name);
+
+  buf_ = old_buf_;
+  pos_ = old_pos_;
+}
+
+bool FlagParser::ParseFile(const char *path, bool ignore_missing) {
+  static const uptr kMaxIncludeSize = 1 << 15;
+  char *data;
+  uptr data_mapped_size;
+  error_t err;
+  uptr len;
+  if (!ReadFileToBuffer(path, &data, &data_mapped_size, &len,
+                        Max(kMaxIncludeSize, GetPageSizeCached()), &err)) {
+    if (ignore_missing)
+      return true;
+    Printf("Failed to read options from '%s': error %d\n", path, err);
+    return false;
+  }
+  ParseString(data, path);
+  UnmapOrDie(data, data_mapped_size);
+  return true;
+}
+
+bool FlagParser::run_handler(const char *name, const char *value) {
+  for (int i = 0; i < n_flags_; ++i) {
+    if (internal_strcmp(name, flags_[i].name) == 0)
+      return flags_[i].handler->Parse(value);
+  }
+  // Unrecognized flag. This is not a fatal error, we may print a warning later.
+  unknown_flags.Add(name);
+  return true;
+}
+
+void FlagParser::RegisterHandler(const char *name, FlagHandlerBase *handler,
+                                 const char *desc) {
+  CHECK_LT(n_flags_, kMaxFlags);
+  flags_[n_flags_].name = name;
+  flags_[n_flags_].desc = desc;
+  flags_[n_flags_].handler = handler;
+  ++n_flags_;
+}
+
+FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) {
+  flags_ = (Flag *)Alloc.Allocate(sizeof(Flag) * kMaxFlags);
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_flag_parser.h b/lib/tsan/sanitizer_common/sanitizer_flag_parser.h
new file mode 100644
index 0000000000..fac5dff346
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_flag_parser.h
@@ -0,0 +1,204 @@
+//===-- sanitizer_flag_parser.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_FLAG_REGISTRY_H
+#define SANITIZER_FLAG_REGISTRY_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+class FlagHandlerBase {
+ public:
+  virtual bool Parse(const char *value) { return false; }
+  // Write the C string representation of the current value (truncated to fit)
+  // into the buffer of size `size`. Returns false if truncation occurred and
+  // returns true otherwise.
+  virtual bool Format(char *buffer, uptr size) {
+    if (size > 0)
+      buffer[0] = '\0';
+    return false;
+  }
+
+ protected:
+  ~FlagHandlerBase() {}
+
+  inline bool FormatString(char *buffer, uptr size, const char *str_to_use) {
+    uptr num_symbols_should_write =
+        internal_snprintf(buffer, size, "%s", str_to_use);
+    return num_symbols_should_write < size;
+  }
+};
+
+template <typename T>
+class FlagHandler : public FlagHandlerBase {
+  T *t_;
+
+ public:
+  explicit FlagHandler(T *t) : t_(t) {}
+  bool Parse(const char *value) final;
+  bool Format(char *buffer, uptr size) final;
+};
+
+inline bool ParseBool(const char *value, bool *b) {
+  if (internal_strcmp(value, "0") == 0 ||
+      internal_strcmp(value, "no") == 0 ||
+      internal_strcmp(value, "false") == 0) {
+    *b = false;
+    return true;
+  }
+  if (internal_strcmp(value, "1") == 0 ||
+      internal_strcmp(value, "yes") == 0 ||
+      internal_strcmp(value, "true") == 0) {
+    *b = true;
+    return true;
+  }
+  return false;
+}
+
+template <>
+inline bool FlagHandler<bool>::Parse(const char *value) {
+  if (ParseBool(value, t_)) return true;
+  Printf("ERROR: Invalid value for bool option: '%s'\n", value);
+  return false;
+}
+
+template <>
+inline bool FlagHandler<bool>::Format(char *buffer, uptr size) {
+  return FormatString(buffer, size, *t_ ? "true" : "false");
+}
+
+template <>
+inline bool FlagHandler<HandleSignalMode>::Parse(const char *value) {
+  bool b;
+  if (ParseBool(value, &b)) {
+    *t_ = b ? kHandleSignalYes : kHandleSignalNo;
+    return true;
+  }
+  if (internal_strcmp(value, "2") == 0 ||
+      internal_strcmp(value, "exclusive") == 0) {
+    *t_ = kHandleSignalExclusive;
+    return true;
+  }
+  Printf("ERROR: Invalid value for signal handler option: '%s'\n", value);
+  return false;
+}
+
+template <>
+inline bool FlagHandler<HandleSignalMode>::Format(char *buffer, uptr size) {
+  uptr num_symbols_should_write = internal_snprintf(buffer, size, "%d", *t_);
+  return num_symbols_should_write < size;
+}
+
+template <>
+inline bool FlagHandler<const char *>::Parse(const char *value) {
+  *t_ = value;
+  return true;
+}
+
+template <>
+inline bool FlagHandler<const char *>::Format(char *buffer, uptr size) {
+  return FormatString(buffer, size, *t_);
+}
+
+template <>
+inline bool FlagHandler<int>::Parse(const char *value) {
+  const char *value_end;
+  *t_ = internal_simple_strtoll(value, &value_end, 10);
+  bool ok = *value_end == 0;
+  if (!ok) Printf("ERROR: Invalid value for int option: '%s'\n", value);
+  return ok;
+}
+
+template <>
+inline bool FlagHandler<int>::Format(char *buffer, uptr size) {
+  uptr num_symbols_should_write = internal_snprintf(buffer, size, "%d", *t_);
+  return num_symbols_should_write < size;
+}
+
+template <>
+inline bool FlagHandler<uptr>::Parse(const char *value) {
+  const char *value_end;
+  *t_ = internal_simple_strtoll(value, &value_end, 10);
+  bool ok = *value_end == 0;
+  if (!ok) Printf("ERROR: Invalid value for uptr option: '%s'\n", value);
+  return ok;
+}
+
+template <>
+inline bool FlagHandler<uptr>::Format(char *buffer, uptr size) {
+  uptr num_symbols_should_write = internal_snprintf(buffer, size, "%p", *t_);
+  return num_symbols_should_write < size;
+}
+
+template <>
+inline bool FlagHandler<s64>::Parse(const char *value) {
+  const char *value_end;
+  *t_ = internal_simple_strtoll(value, &value_end, 10);
+  bool ok = *value_end == 0;
+  if (!ok) Printf("ERROR: Invalid value for s64 option: '%s'\n", value);
+  return ok;
+}
+
+template <>
+inline bool FlagHandler<s64>::Format(char *buffer, uptr size) {
+  uptr num_symbols_should_write = internal_snprintf(buffer, size, "%lld", *t_);
+  return num_symbols_should_write < size;
+}
+
+class FlagParser {
+  static const int kMaxFlags = 200;
+  struct Flag {
+    const char *name;
+    const char *desc;
+    FlagHandlerBase *handler;
+  } *flags_;
+  int n_flags_;
+
+  const char *buf_;
+  uptr pos_;
+
+ public:
+  FlagParser();
+  void RegisterHandler(const char *name, FlagHandlerBase *handler,
+                       const char *desc);
+  void ParseString(const char *s, const char *env_name = 0);
+  void ParseStringFromEnv(const char *env_name);
+  bool ParseFile(const char *path, bool ignore_missing);
+  void PrintFlagDescriptions();
+
+  static LowLevelAllocator Alloc;
+
+ private:
+  void fatal_error(const char *err);
+  bool is_space(char c);
+  void skip_whitespace();
+  void parse_flags(const char *env_option_name);
+  void parse_flag(const char *env_option_name);
+  bool run_handler(const char *name, const char *value);
+  char *ll_strndup(const char *s, uptr n);
+};
+
+template <typename T>
+static void RegisterFlag(FlagParser *parser, const char *name, const char *desc,
+                         T *var) {
+  FlagHandler<T> *fh = new (FlagParser::Alloc) FlagHandler<T>(var);
+  parser->RegisterHandler(name, fh, desc);
+}
+
+void ReportUnrecognizedFlags();
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FLAG_REGISTRY_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_flags.cpp b/lib/tsan/sanitizer_common/sanitizer_flags.cpp
new file mode 100644
index 0000000000..684ee1e0b9
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_flags.cpp
@@ -0,0 +1,129 @@
+//===-- sanitizer_flags.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_flags.h"
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_list.h"
+#include "sanitizer_flag_parser.h"
+
+namespace __sanitizer {
+
+CommonFlags common_flags_dont_use;
+
+void CommonFlags::SetDefaults() {
+#define COMMON_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "sanitizer_flags.inc"
+#undef COMMON_FLAG
+}
+
+void CommonFlags::CopyFrom(const CommonFlags &other) {
+  internal_memcpy(this, &other, sizeof(*this));
+}
+
+// Copy the string from "s" to "out", making the following substitutions:
+// %b = binary basename
+// %p = pid
+void SubstituteForFlagValue(const char *s, char *out, uptr out_size) {
+  char *out_end = out + out_size;
+  while (*s && out < out_end - 1) {
+    if (s[0] != '%') {
+      *out++ = *s++;
+      continue;
+    }
+    switch (s[1]) {
+      case 'b': {
+        const char *base = GetProcessName();
+        CHECK(base);
+        while (*base && out < out_end - 1)
+          *out++ = *base++;
+        s += 2; // skip "%b"
+        break;
+      }
+      case 'p': {
+        int pid = internal_getpid();
+        char buf[32];
+        char *buf_pos = buf + 32;
+        do {
+          *--buf_pos = (pid % 10) + '0';
+          pid /= 10;
+        } while (pid);
+        while (buf_pos < buf + 32 && out < out_end - 1)
+          *out++ = *buf_pos++;
+        s += 2; // skip "%p"
+        break;
+      }
+      default:
+        *out++ = *s++;
+        break;
+    }
+  }
+  CHECK(out < out_end - 1);
+  *out = '\0';
+}
+
+class FlagHandlerInclude : public FlagHandlerBase {
+  FlagParser *parser_;
+  bool ignore_missing_;
+  const char *original_path_;
+
+ public:
+  explicit FlagHandlerInclude(FlagParser *parser, bool ignore_missing)
+      : parser_(parser), ignore_missing_(ignore_missing), original_path_("") {}
+  bool Parse(const char *value) final {
+    original_path_ = value;
+    if (internal_strchr(value, '%')) {
+      char *buf = (char *)MmapOrDie(kMaxPathLength, "FlagHandlerInclude");
+      SubstituteForFlagValue(value, buf, kMaxPathLength);
+      bool res = parser_->ParseFile(buf, ignore_missing_);
+      UnmapOrDie(buf, kMaxPathLength);
+      return res;
+    }
+    return parser_->ParseFile(value, ignore_missing_);
+  }
+  bool Format(char *buffer, uptr size) {
+    // Note `original_path_` isn't actually what's parsed due to `%`
+    // substitutions. Printing the substituted path would require holding onto
+    // mmap'ed memory.
+    return FormatString(buffer, size, original_path_);
+  }
+};
+
+void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf) {
+  FlagHandlerInclude *fh_include = new (FlagParser::Alloc)
+      FlagHandlerInclude(parser, /*ignore_missing*/ false);
+  parser->RegisterHandler("include", fh_include,
+                          "read more options from the given file");
+  FlagHandlerInclude *fh_include_if_exists = new (FlagParser::Alloc)
+      FlagHandlerInclude(parser, /*ignore_missing*/ true);
+  parser->RegisterHandler(
+      "include_if_exists", fh_include_if_exists,
+      "read more options from the given file (if it exists)");
+}
+
+void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf) {
+#define COMMON_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &cf->Name);
+#include "sanitizer_flags.inc"
+#undef COMMON_FLAG
+
+  RegisterIncludeFlags(parser, cf);
+}
+
+void InitializeCommonFlags(CommonFlags *cf) {
+  // need to record coverage to generate coverage report.
+  cf->coverage |= cf->html_cov_report;
+  SetVerbosity(cf->verbosity);
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_flags.h b/lib/tsan/sanitizer_common/sanitizer_flags.h
new file mode 100644
index 0000000000..8f5e987da3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_flags.h
@@ -0,0 +1,67 @@
+//===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_FLAGS_H
+#define SANITIZER_FLAGS_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+enum HandleSignalMode {
+  kHandleSignalNo,
+  kHandleSignalYes,
+  kHandleSignalExclusive,
+};
+
+struct CommonFlags {
+#define COMMON_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "sanitizer_flags.inc"
+#undef COMMON_FLAG
+
+  void SetDefaults();
+  void CopyFrom(const CommonFlags &other);
+};
+
+// Functions to get/set global CommonFlags shared by all sanitizer runtimes:
+extern CommonFlags common_flags_dont_use;
+inline const CommonFlags *common_flags() {
+  return &common_flags_dont_use;
+}
+
+inline void SetCommonFlagsDefaults() {
+  common_flags_dont_use.SetDefaults();
+}
+
+// This function can only be used to setup tool-specific overrides for
+// CommonFlags defaults. Generally, it should only be used right after
+// SetCommonFlagsDefaults(), but before ParseCommonFlagsFromString(), and
+// only during the flags initialization (i.e. before they are used for
+// the first time).
+inline void OverrideCommonFlags(const CommonFlags &cf) {
+  common_flags_dont_use.CopyFrom(cf);
+}
+
+void SubstituteForFlagValue(const char *s, char *out, uptr out_size);
+
+class FlagParser;
+void RegisterCommonFlags(FlagParser *parser,
+                         CommonFlags *cf = &common_flags_dont_use);
+void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf);
+
+// Should be called after parsing all flags. Sets up common flag values
+// and perform initializations common to all sanitizers (e.g. setting
+// verbosity).
+void InitializeCommonFlags(CommonFlags *cf = &common_flags_dont_use);
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FLAGS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_flags.inc b/lib/tsan/sanitizer_common/sanitizer_flags.inc
new file mode 100644
index 0000000000..065258a5a6
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_flags.inc
@@ -0,0 +1,250 @@
+//===-- sanitizer_flags.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes common flags available in all sanitizers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef COMMON_FLAG
+#error "Define COMMON_FLAG prior to including this file!"
+#endif
+
+// COMMON_FLAG(Type, Name, DefaultValue, Description)
+// Supported types: bool, const char *, int, uptr.
+// Default value must be a compile-time constant.
+// Description must be a string literal.
+
+COMMON_FLAG(
+    bool, symbolize, true,
+    "If set, use the online symbolizer from common sanitizer runtime to turn "
+    "virtual addresses to file/line locations.")
+COMMON_FLAG(
+    const char *, external_symbolizer_path, nullptr,
+    "Path to external symbolizer. If empty, the tool will search $PATH for "
+    "the symbolizer.")
+COMMON_FLAG(
+    bool, allow_addr2line, false,
+    "If set, allows online symbolizer to run addr2line binary to symbolize "
+    "stack traces (addr2line will only be used if llvm-symbolizer binary is "
+    "unavailable.")
+COMMON_FLAG(const char *, strip_path_prefix, "",
+            "Strips this prefix from file paths in error reports.")
+COMMON_FLAG(bool, fast_unwind_on_check, false,
+            "If available, use the fast frame-pointer-based unwinder on "
+            "internal CHECK failures.")
+COMMON_FLAG(bool, fast_unwind_on_fatal, false,
+            "If available, use the fast frame-pointer-based unwinder on fatal "
+            "errors.")
+COMMON_FLAG(bool, fast_unwind_on_malloc, true,
+            "If available, use the fast frame-pointer-based unwinder on "
+            "malloc/free.")
+COMMON_FLAG(bool, handle_ioctl, false, "Intercept and handle ioctl requests.")
+COMMON_FLAG(int, malloc_context_size, 1,
+            "Max number of stack frames kept for each allocation/deallocation.")
+COMMON_FLAG(
+    const char *, log_path, "stderr",
+    "Write logs to \"log_path.pid\". The special values are \"stdout\" and "
+    "\"stderr\". The default is \"stderr\".")
+COMMON_FLAG(
+    bool, log_exe_name, false,
+    "Mention name of executable when reporting error and "
+    "append executable name to logs (as in \"log_path.exe_name.pid\").")
+COMMON_FLAG(
+    bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+    "Write all sanitizer output to syslog in addition to other means of "
+    "logging.")
+COMMON_FLAG(
+    int, verbosity, 0,
+    "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
+COMMON_FLAG(bool, strip_env, 1,
+            "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
+            "avoid passing it to children. Default is true.")
+COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
+COMMON_FLAG(
+    bool, leak_check_at_exit, true,
+    "Invoke leak checking in an atexit handler. Has no effect if "
+    "detect_leaks=false, or if __lsan_do_leak_check() is called before the "
+    "handler has a chance to run.")
+COMMON_FLAG(bool, allocator_may_return_null, false,
+            "If false, the allocator will crash instead of returning 0 on "
+            "out-of-memory.")
+COMMON_FLAG(bool, print_summary, true,
+            "If false, disable printing error summaries in addition to error "
+            "reports.")
+COMMON_FLAG(int, print_module_map, 0,
+            "OS X only (0 - don't print, 1 - print only once before process "
+            "exits, 2 - print after each report).")
+COMMON_FLAG(bool, check_printf, true, "Check printf arguments.")
+#define COMMON_FLAG_HANDLE_SIGNAL_HELP(signal) \
+    "Controls custom tool's " #signal " handler (0 - do not registers the " \
+    "handler, 1 - register the handler and allow user to set own, " \
+    "2 - registers the handler and block user from changing it). "
+COMMON_FLAG(HandleSignalMode, handle_segv, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGSEGV))
+COMMON_FLAG(HandleSignalMode, handle_sigbus, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGBUS))
+COMMON_FLAG(HandleSignalMode, handle_abort, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGABRT))
+COMMON_FLAG(HandleSignalMode, handle_sigill, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGILL))
+COMMON_FLAG(HandleSignalMode, handle_sigtrap, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGTRAP))
+COMMON_FLAG(HandleSignalMode, handle_sigfpe, kHandleSignalYes,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGFPE))
+#undef COMMON_FLAG_HANDLE_SIGNAL_HELP
+COMMON_FLAG(bool, allow_user_segv_handler, true,
+            "Deprecated. True has no effect, use handle_sigbus=1. If false, "
+            "handle_*=1 will be upgraded to handle_*=2.")
+COMMON_FLAG(bool, use_sigaltstack, true,
+            "If set, uses alternate stack for signal handling.")
+COMMON_FLAG(bool, detect_deadlocks, true,
+            "If set, deadlock detection is enabled.")
+COMMON_FLAG(
+    uptr, clear_shadow_mmap_threshold, 64 * 1024,
+    "Large shadow regions are zero-filled using mmap(NORESERVE) instead of "
+    "memset(). This is the threshold size in bytes.")
+COMMON_FLAG(const char *, color, "auto",
+            "Colorize reports: (always|never|auto).")
+COMMON_FLAG(
+    bool, legacy_pthread_cond, false,
+    "Enables support for dynamic libraries linked with libpthread 2.2.5.")
+COMMON_FLAG(bool, intercept_tls_get_addr, false, "Intercept __tls_get_addr.")
+COMMON_FLAG(bool, help, false, "Print the flag descriptions.")
+COMMON_FLAG(uptr, mmap_limit_mb, 0,
+            "Limit the amount of mmap-ed memory (excluding shadow) in Mb; "
+            "not a user-facing flag, used mosly for testing the tools")
+COMMON_FLAG(uptr, hard_rss_limit_mb, 0,
+            "Hard RSS limit in Mb."
+            " If non-zero, a background thread is spawned at startup"
+            " which periodically reads RSS and aborts the process if the"
+            " limit is reached")
+COMMON_FLAG(uptr, soft_rss_limit_mb, 0,
+            "Soft RSS limit in Mb."
+            " If non-zero, a background thread is spawned at startup"
+            " which periodically reads RSS. If the limit is reached"
+            " all subsequent malloc/new calls will fail or return NULL"
+            " (depending on the value of allocator_may_return_null)"
+            " until the RSS goes below the soft limit."
+            " This limit does not affect memory allocations other than"
+            " malloc/new.")
+COMMON_FLAG(uptr, max_allocation_size_mb, 0,
+            "If non-zero, malloc/new calls larger than this size will return "
+            "nullptr (or crash if allocator_may_return_null=false).")
+COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only")
+COMMON_FLAG(s32, allocator_release_to_os_interval_ms,
+            ((bool)SANITIZER_FUCHSIA || (bool)SANITIZER_WINDOWS) ? -1 : 5000,
+            "Only affects a 64-bit allocator. If set, tries to release unused "
+            "memory to the OS, but not more often than this interval (in "
+            "milliseconds). Negative values mean do not attempt to release "
+            "memory to the OS.\n")
+COMMON_FLAG(bool, can_use_proc_maps_statm, true,
+            "If false, do not attempt to read /proc/maps/statm."
+            " Mostly useful for testing sanitizers.")
+COMMON_FLAG(
+    bool, coverage, false,
+    "If set, coverage information will be dumped at program shutdown (if the "
+    "coverage instrumentation was enabled at compile time).")
+COMMON_FLAG(const char *, coverage_dir, ".",
+            "Target directory for coverage dumps. Defaults to the current "
+            "directory.")
+COMMON_FLAG(bool, full_address_space, false,
+            "Sanitize complete address space; "
+            "by default kernel area on 32-bit platforms will not be sanitized")
+COMMON_FLAG(bool, print_suppressions, true,
+            "Print matched suppressions at exit.")
+COMMON_FLAG(
+    bool, disable_coredump, (SANITIZER_WORDSIZE == 64) && !SANITIZER_GO,
+    "Disable core dumping. By default, disable_coredump=1 on 64-bit to avoid"
+    " dumping a 16T+ core file. Ignored on OSes that don't dump core by"
+    " default and for sanitizers that don't reserve lots of virtual memory.")
+COMMON_FLAG(bool, use_madv_dontdump, true,
+          "If set, instructs kernel to not store the (huge) shadow "
+          "in core file.")
+COMMON_FLAG(bool, symbolize_inline_frames, true,
+            "Print inlined frames in stacktraces. Defaults to true.")
+COMMON_FLAG(bool, symbolize_vs_style, false,
+            "Print file locations in Visual Studio style (e.g: "
+            " file(10,42): ...")
+COMMON_FLAG(int, dedup_token_length, 0,
+            "If positive, after printing a stack trace also print a short "
+            "string token based on this number of frames that will simplify "
+            "deduplication of the reports. "
+            "Example: 'DEDUP_TOKEN: foo-bar-main'. Default is 0.")
+COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",
+            "Format string used to render stack frames. "
+            "See sanitizer_stacktrace_printer.h for the format description. "
+            "Use DEFAULT to get default format.")
+COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
+            "If true, the shadow is not allowed to use huge pages. ")
+COMMON_FLAG(bool, strict_string_checks, false,
+            "If set check that string arguments are properly null-terminated")
+COMMON_FLAG(bool, intercept_strstr, true,
+            "If set, uses custom wrappers for strstr and strcasestr functions "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strspn, true,
+            "If set, uses custom wrappers for strspn and strcspn function "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strtok, true,
+            "If set, uses a custom wrapper for the strtok function "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strpbrk, true,
+            "If set, uses custom wrappers for strpbrk function "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strlen, true,
+            "If set, uses custom wrappers for strlen and strnlen functions "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strndup, true,
+            "If set, uses custom wrappers for strndup functions "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_strchr, true,
+            "If set, uses custom wrappers for strchr, strchrnul, and strrchr "
+            "functions to find more errors.")
+COMMON_FLAG(bool, intercept_memcmp, true,
+            "If set, uses custom wrappers for memcmp function "
+            "to find more errors.")
+COMMON_FLAG(bool, strict_memcmp, true,
+          "If true, assume that memcmp(p1, p2, n) always reads n bytes before "
+          "comparing p1 and p2.")
+COMMON_FLAG(bool, intercept_memmem, true,
+            "If set, uses a wrapper for memmem() to find more errors.")
+COMMON_FLAG(bool, intercept_intrin, true,
+            "If set, uses custom wrappers for memset/memcpy/memmove "
+            "intrinsics to find more errors.")
+COMMON_FLAG(bool, intercept_stat, true,
+            "If set, uses custom wrappers for *stat functions "
+            "to find more errors.")
+COMMON_FLAG(bool, intercept_send, true,
+            "If set, uses custom wrappers for send* functions "
+            "to find more errors.")
+COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID,
+            "If set, decorate sanitizer mappings in /proc/self/maps with "
+            "user-readable names")
+COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool "
+                              "found an error")
+COMMON_FLAG(
+    bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+    "If set, the tool calls abort() instead of _exit() after printing the "
+    "error report.")
+COMMON_FLAG(bool, suppress_equal_pcs, true,
+            "Deduplicate multiple reports for single source location in "
+            "halt_on_error=false mode (asan only).")
+COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash "
+            "(asan only).")
+COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.")
+COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.")
+COMMON_FLAG(bool, dump_instruction_bytes, false,
+          "If true, dump 16 bytes starting at the instruction that caused SEGV")
+COMMON_FLAG(bool, dump_registers, true,
+          "If true, dump values of CPU registers when SEGV happens. Only "
+          "available on OS X for now.")
+COMMON_FLAG(bool, detect_write_exec, false,
+          "If true, triggers warning when writable-executable pages requests "
+          "are being made")
+COMMON_FLAG(bool, test_only_emulate_no_memorymap, false,
+            "TEST ONLY fail to read memory mappings to emulate sanitized "
+            "\"init\"")
diff --git a/lib/tsan/sanitizer_common/sanitizer_freebsd.h b/lib/tsan/sanitizer_common/sanitizer_freebsd.h
new file mode 100644
index 0000000000..82b227eab6
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_freebsd.h
@@ -0,0 +1,137 @@
+//===-- sanitizer_freebsd.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime. It contains FreeBSD-specific
+// definitions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_FREEBSD_H
+#define SANITIZER_FREEBSD_H
+
+#include "sanitizer_internal_defs.h"
+
+// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
+// 32-bit mode.
+#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
+#include <osreldate.h>
+#if __FreeBSD_version <= 902001  // v9.2
+#include <link.h>
+#include <sys/param.h>
+#include <ucontext.h>
+
+namespace __sanitizer {
+
+typedef unsigned long long __xuint64_t;
+
+typedef __int32_t __xregister_t;
+
+typedef struct __xmcontext {
+  __xregister_t mc_onstack;
+  __xregister_t mc_gs;
+  __xregister_t mc_fs;
+  __xregister_t mc_es;
+  __xregister_t mc_ds;
+  __xregister_t mc_edi;
+  __xregister_t mc_esi;
+  __xregister_t mc_ebp;
+  __xregister_t mc_isp;
+  __xregister_t mc_ebx;
+  __xregister_t mc_edx;
+  __xregister_t mc_ecx;
+  __xregister_t mc_eax;
+  __xregister_t mc_trapno;
+  __xregister_t mc_err;
+  __xregister_t mc_eip;
+  __xregister_t mc_cs;
+  __xregister_t mc_eflags;
+  __xregister_t mc_esp;
+  __xregister_t mc_ss;
+
+  int mc_len;
+  int mc_fpformat;
+  int mc_ownedfp;
+  __xregister_t mc_flags;
+
+  int mc_fpstate[128] __aligned(16);
+  __xregister_t mc_fsbase;
+  __xregister_t mc_gsbase;
+  __xregister_t mc_xfpustate;
+  __xregister_t mc_xfpustate_len;
+
+  int mc_spare2[4];
+} xmcontext_t;
+
+typedef struct __xucontext {
+  sigset_t uc_sigmask;
+  xmcontext_t uc_mcontext;
+
+  struct __ucontext *uc_link;
+  stack_t uc_stack;
+  int uc_flags;
+  int __spare__[4];
+} xucontext_t;
+
+struct xkinfo_vmentry {
+  int kve_structsize;
+  int kve_type;
+  __xuint64_t kve_start;
+  __xuint64_t kve_end;
+  __xuint64_t kve_offset;
+  __xuint64_t kve_vn_fileid;
+  __uint32_t kve_vn_fsid;
+  int kve_flags;
+  int kve_resident;
+  int kve_private_resident;
+  int kve_protection;
+  int kve_ref_count;
+  int kve_shadow_count;
+  int kve_vn_type;
+  __xuint64_t kve_vn_size;
+  __uint32_t kve_vn_rdev;
+  __uint16_t kve_vn_mode;
+  __uint16_t kve_status;
+  int _kve_ispare[12];
+  char kve_path[PATH_MAX];
+};
+
+typedef struct {
+  __uint32_t p_type;
+  __uint32_t p_offset;
+  __uint32_t p_vaddr;
+  __uint32_t p_paddr;
+  __uint32_t p_filesz;
+  __uint32_t p_memsz;
+  __uint32_t p_flags;
+  __uint32_t p_align;
+} XElf32_Phdr;
+
+struct xdl_phdr_info {
+  Elf_Addr dlpi_addr;
+  const char *dlpi_name;
+  const XElf32_Phdr *dlpi_phdr;
+  Elf_Half dlpi_phnum;
+  unsigned long long int dlpi_adds;
+  unsigned long long int dlpi_subs;
+  size_t dlpi_tls_modid;
+  void *dlpi_tls_data;
+};
+
+typedef int (*__xdl_iterate_hdr_callback)(struct xdl_phdr_info *, size_t,
+                                          void *);
+typedef int xdl_iterate_phdr_t(__xdl_iterate_hdr_callback, void *);
+
+#define xdl_iterate_phdr(callback, param) \
+  (((xdl_iterate_phdr_t *)dl_iterate_phdr)((callback), (param)))
+
+}  // namespace __sanitizer
+
+#endif  // __FreeBSD_version <= 902001
+#endif  // SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
+
+#endif  // SANITIZER_FREEBSD_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp b/lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp
new file mode 100644
index 0000000000..6d1ad79467
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp
@@ -0,0 +1,531 @@
+//===-- sanitizer_fuchsia.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and other sanitizer
+// run-time libraries and implements Fuchsia-specific functions from
+// sanitizer_common.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+
+#include <limits.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <zircon/errors.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+namespace __sanitizer {
+
+void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
+
+uptr internal_sched_yield() {
+  zx_status_t status = _zx_nanosleep(0);
+  CHECK_EQ(status, ZX_OK);
+  return 0;  // Why doesn't this return void?
+}
+
+static void internal_nanosleep(zx_time_t ns) {
+  zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns));
+  CHECK_EQ(status, ZX_OK);
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+  internal_nanosleep(ZX_SEC(seconds));
+  return 0;
+}
+
+u64 NanoTime() {
+  zx_time_t time;
+  zx_status_t status = _zx_clock_get(ZX_CLOCK_UTC, &time);
+  CHECK_EQ(status, ZX_OK);
+  return time;
+}
+
+u64 MonotonicNanoTime() { return _zx_clock_get_monotonic(); }
+
+uptr internal_getpid() {
+  zx_info_handle_basic_t info;
+  zx_status_t status =
+      _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &info,
+                          sizeof(info), NULL, NULL);
+  CHECK_EQ(status, ZX_OK);
+  uptr pid = static_cast<uptr>(info.koid);
+  CHECK_EQ(pid, info.koid);
+  return pid;
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+  UNIMPLEMENTED();
+}
+
+uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); }
+
+tid_t GetTid() { return GetThreadSelf(); }
+
+void Abort() { abort(); }
+
+int Atexit(void (*function)(void)) { return atexit(function); }
+
+void SleepForSeconds(int seconds) { internal_sleep(seconds); }
+
+void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); }
+
+void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
+  pthread_attr_t attr;
+  CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
+  void *base;
+  size_t size;
+  CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0);
+  CHECK_EQ(pthread_attr_destroy(&attr), 0);
+
+  *stack_bottom = reinterpret_cast<uptr>(base);
+  *stack_top = *stack_bottom + size;
+}
+
+void InitializePlatformEarly() {}
+void MaybeReexec() {}
+void CheckASLR() {}
+void CheckMPROTECT() {}
+void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void DisableCoreDumperIfNecessary() {}
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
+void SetAlternateSignalStack() {}
+void UnsetAlternateSignalStack() {}
+void InitTlsSize() {}
+
+void PrintModuleMap() {}
+
+bool SignalContext::IsStackOverflow() const { return false; }
+void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
+const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
+
+enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
+
+BlockingMutex::BlockingMutex() {
+  // NOTE!  It's important that this use internal_memset, because plain
+  // memset might be intercepted (e.g., actually be __asan_memset).
+  // Defining this so the compiler initializes each field, e.g.:
+  //   BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
+  // might result in the compiler generating a call to memset, which would
+  // have the same problem.
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  CHECK_EQ(owner_, 0);
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
+    return;
+  while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
+    zx_status_t status =
+        _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), MtxSleeping,
+                       ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
+    if (status != ZX_ERR_BAD_STATE)  // Normal race.
+      CHECK_EQ(status, ZX_OK);
+  }
+}
+
+void BlockingMutex::Unlock() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
+  CHECK_NE(v, MtxUnlocked);
+  if (v == MtxSleeping) {
+    zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
+    CHECK_EQ(status, ZX_OK);
+  }
+}
+
+void BlockingMutex::CheckLocked() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
+}
+
+uptr GetPageSize() { return PAGE_SIZE; }
+
+uptr GetMmapGranularity() { return PAGE_SIZE; }
+
+sanitizer_shadow_bounds_t ShadowBounds;
+
+uptr GetMaxUserVirtualAddress() {
+  ShadowBounds = __sanitizer_shadow_bounds();
+  return ShadowBounds.memory_limit - 1;
+}
+
+uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
+
+static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
+                                  bool raw_report, bool die_for_nomem) {
+  size = RoundUpTo(size, PAGE_SIZE);
+
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status,
+                              raw_report);
+    return nullptr;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+                          internal_strlen(mem_type));
+
+  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+  uintptr_t addr;
+  status =
+      _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
+                   vmo, 0, size, &addr);
+  _zx_handle_close(vmo);
+
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status,
+                              raw_report);
+    return nullptr;
+  }
+
+  IncreaseTotalMmap(size);
+
+  return reinterpret_cast<void *>(addr);
+}
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+  return DoAnonymousMmapOrDie(size, mem_type, raw_report, true);
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+  return MmapOrDie(size, mem_type);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  return DoAnonymousMmapOrDie(size, mem_type, false, false);
+}
+
+uptr ReservedAddressRange::Init(uptr init_size, const char *name,
+                                uptr fixed_addr) {
+  init_size = RoundUpTo(init_size, PAGE_SIZE);
+  DCHECK_EQ(os_handle_, ZX_HANDLE_INVALID);
+  uintptr_t base;
+  zx_handle_t vmar;
+  zx_status_t status =
+      _zx_vmar_allocate(
+          _zx_vmar_root_self(),
+          ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC,
+          0, init_size, &vmar, &base);
+  if (status != ZX_OK)
+    ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status);
+  base_ = reinterpret_cast<void *>(base);
+  size_ = init_size;
+  name_ = name;
+  os_handle_ = vmar;
+
+  return reinterpret_cast<uptr>(base_);
+}
+
+static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size,
+                             void *base, const char *name, bool die_for_nomem) {
+  uptr offset = fixed_addr - reinterpret_cast<uptr>(base);
+  map_size = RoundUpTo(map_size, PAGE_SIZE);
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(map_size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+      ReportMmapFailureAndDie(map_size, name, "zx_vmo_create", status);
+    return 0;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, name, internal_strlen(name));
+  DCHECK_GE(base + size_, map_size + offset);
+  uintptr_t addr;
+
+  status =
+      _zx_vmar_map(vmar, ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC,
+                   offset, vmo, 0, map_size, &addr);
+  _zx_handle_close(vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY || die_for_nomem) {
+      ReportMmapFailureAndDie(map_size, name, "zx_vmar_map", status);
+    }
+    return 0;
+  }
+  IncreaseTotalMmap(map_size);
+  return addr;
+}
+
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size,
+                               const char *name) {
+  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
+                          name_, false);
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size,
+                                    const char *name) {
+  return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
+                          name_, true);
+}
+
+void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) {
+  if (!addr || !size) return;
+  size = RoundUpTo(size, PAGE_SIZE);
+
+  zx_status_t status =
+      _zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size);
+  if (status != ZX_OK) {
+    Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+           SanitizerToolName, size, size, addr);
+    CHECK("unable to unmap" && 0);
+  }
+
+  DecreaseTotalMmap(size);
+}
+
+void ReservedAddressRange::Unmap(uptr addr, uptr size) {
+  CHECK_LE(size, size_);
+  const zx_handle_t vmar = static_cast<zx_handle_t>(os_handle_);
+  if (addr == reinterpret_cast<uptr>(base_)) {
+    if (size == size_) {
+      // Destroying the vmar effectively unmaps the whole mapping.
+      _zx_vmar_destroy(vmar);
+      _zx_handle_close(vmar);
+      os_handle_ = static_cast<uptr>(ZX_HANDLE_INVALID);
+      DecreaseTotalMmap(size);
+      return;
+    }
+  } else {
+    CHECK_EQ(addr + size, reinterpret_cast<uptr>(base_) + size_);
+  }
+  // Partial unmapping does not affect the fact that the initial range is still
+  // reserved, and the resulting unmapped memory can't be reused.
+  UnmapOrDieVmar(reinterpret_cast<void *>(addr), size, vmar);
+}
+
+// This should never be called.
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  UNIMPLEMENTED();
+}
+
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
+  CHECK_GE(size, PAGE_SIZE);
+  CHECK(IsPowerOfTwo(size));
+  CHECK(IsPowerOfTwo(alignment));
+
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, false);
+    return nullptr;
+  }
+  _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+                          internal_strlen(mem_type));
+
+  // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+
+  // Map a larger size to get a chunk of address space big enough that
+  // it surely contains an aligned region of the requested size.  Then
+  // overwrite the aligned middle portion with a mapping from the
+  // beginning of the VMO, and unmap the excess before and after.
+  size_t map_size = size + alignment;
+  uintptr_t addr;
+  status =
+      _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
+                   vmo, 0, map_size, &addr);
+  if (status == ZX_OK) {
+    uintptr_t map_addr = addr;
+    uintptr_t map_end = map_addr + map_size;
+    addr = RoundUpTo(map_addr, alignment);
+    uintptr_t end = addr + size;
+    if (addr != map_addr) {
+      zx_info_vmar_t info;
+      status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info,
+                                   sizeof(info), NULL, NULL);
+      if (status == ZX_OK) {
+        uintptr_t new_addr;
+        status = _zx_vmar_map(
+            _zx_vmar_root_self(),
+            ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE,
+            addr - info.base, vmo, 0, size, &new_addr);
+        if (status == ZX_OK) CHECK_EQ(new_addr, addr);
+      }
+    }
+    if (status == ZX_OK && addr != map_addr)
+      status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr);
+    if (status == ZX_OK && end != map_end)
+      status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end);
+  }
+  _zx_handle_close(vmo);
+
+  if (status != ZX_OK) {
+    if (status != ZX_ERR_NO_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, false);
+    return nullptr;
+  }
+
+  IncreaseTotalMmap(size);
+
+  return reinterpret_cast<void *>(addr);
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+  UnmapOrDieVmar(addr, size, _zx_vmar_root_self());
+}
+
+// This is used on the shadow mapping, which cannot be changed.
+// Zircon doesn't have anything like MADV_DONTNEED.
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
+
+void DumpProcessMap() {
+  // TODO(mcgrathr): write it
+  return;
+}
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+  // TODO(mcgrathr): Figure out a better way.
+  zx_handle_t vmo;
+  zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+  if (status == ZX_OK) {
+    status = _zx_vmo_write(vmo, reinterpret_cast<const void *>(beg), 0, size);
+    _zx_handle_close(vmo);
+  }
+  return status == ZX_OK;
+}
+
+// FIXME implement on this platform.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+                      uptr *read_len, uptr max_len, error_t *errno_p) {
+  zx_handle_t vmo;
+  zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
+  if (status == ZX_OK) {
+    uint64_t vmo_size;
+    status = _zx_vmo_get_size(vmo, &vmo_size);
+    if (status == ZX_OK) {
+      if (vmo_size < max_len) max_len = vmo_size;
+      size_t map_size = RoundUpTo(max_len, PAGE_SIZE);
+      uintptr_t addr;
+      status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0,
+                            map_size, &addr);
+      if (status == ZX_OK) {
+        *buff = reinterpret_cast<char *>(addr);
+        *buff_size = map_size;
+        *read_len = max_len;
+      }
+    }
+    _zx_handle_close(vmo);
+  }
+  if (status != ZX_OK && errno_p) *errno_p = status;
+  return status == ZX_OK;
+}
+
+void RawWrite(const char *buffer) {
+  constexpr size_t size = 128;
+  static _Thread_local char line[size];
+  static _Thread_local size_t lastLineEnd = 0;
+  static _Thread_local size_t cur = 0;
+
+  while (*buffer) {
+    if (cur >= size) {
+      if (lastLineEnd == 0)
+        lastLineEnd = size;
+      __sanitizer_log_write(line, lastLineEnd);
+      internal_memmove(line, line + lastLineEnd, cur - lastLineEnd);
+      cur = cur - lastLineEnd;
+      lastLineEnd = 0;
+    }
+    if (*buffer == '\n')
+      lastLineEnd = cur + 1;
+    line[cur++] = *buffer++;
+  }
+  // Flush all complete lines before returning.
+  if (lastLineEnd != 0) {
+    __sanitizer_log_write(line, lastLineEnd);
+    internal_memmove(line, line + lastLineEnd, cur - lastLineEnd);
+    cur = cur - lastLineEnd;
+    lastLineEnd = 0;
+  }
+}
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+  __sanitizer_log_write(buffer, length);
+}
+
+char **StoredArgv;
+char **StoredEnviron;
+
+char **GetArgv() { return StoredArgv; }
+char **GetEnviron() { return StoredEnviron; }
+
+const char *GetEnv(const char *name) {
+  if (StoredEnviron) {
+    uptr NameLen = internal_strlen(name);
+    for (char **Env = StoredEnviron; *Env != 0; Env++) {
+      if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=')
+        return (*Env) + NameLen + 1;
+    }
+  }
+  return nullptr;
+}
+
+uptr ReadBinaryName(/*out*/ char *buf, uptr buf_len) {
+  const char *argv0 = "<UNKNOWN>";
+  if (StoredArgv && StoredArgv[0]) {
+    argv0 = StoredArgv[0];
+  }
+  internal_strncpy(buf, argv0, buf_len);
+  return internal_strlen(buf);
+}
+
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
+  return ReadBinaryName(buf, buf_len);
+}
+
+uptr MainThreadStackBase, MainThreadStackSize;
+
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN);
+  _zx_cprng_draw(buffer, length);
+  return true;
+}
+
+u32 GetNumberOfCPUs() {
+  return zx_system_get_num_cpus();
+}
+
+uptr GetRSS() { UNIMPLEMENTED(); }
+
+}  // namespace __sanitizer
+
+using namespace __sanitizer;
+
+extern "C" {
+void __sanitizer_startup_hook(int argc, char **argv, char **envp,
+                              void *stack_base, size_t stack_size) {
+  __sanitizer::StoredArgv = argv;
+  __sanitizer::StoredEnviron = envp;
+  __sanitizer::MainThreadStackBase = reinterpret_cast<uintptr_t>(stack_base);
+  __sanitizer::MainThreadStackSize = stack_size;
+}
+
+void __sanitizer_set_report_path(const char *path) {
+  // Handle the initialization code in each sanitizer, but no other calls.
+  // This setting is never consulted on Fuchsia.
+  DCHECK_EQ(path, common_flags()->log_path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+  UNREACHABLE("not available on Fuchsia");
+}
+}  // extern "C"
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/lib/tsan/sanitizer_common/sanitizer_fuchsia.h b/lib/tsan/sanitizer_common/sanitizer_fuchsia.h
new file mode 100644
index 0000000000..96f9cde7ef
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_fuchsia.h
@@ -0,0 +1,36 @@
+//===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// Fuchsia-specific sanitizer support.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FUCHSIA_H
+#define SANITIZER_FUCHSIA_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+
+#include <zircon/sanitizer.h>
+#include <zircon/syscalls/object.h>
+
+namespace __sanitizer {
+
+extern uptr MainThreadStackBase, MainThreadStackSize;
+extern sanitizer_shadow_bounds_t ShadowBounds;
+
+struct MemoryMappingLayoutData {
+  InternalMmapVector<zx_info_maps_t> data;
+  size_t current;  // Current index into the vector.
+};
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FUCHSIA
+#endif  // SANITIZER_FUCHSIA_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_getauxval.h b/lib/tsan/sanitizer_common/sanitizer_getauxval.h
new file mode 100644
index 0000000000..86ad3a5e2c
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_getauxval.h
@@ -0,0 +1,59 @@
+//===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common getauxval() guards and definitions.
+// getauxval() is not defined until glibc version 2.16, or until API level 21
+// for Android.
+// Implement the getauxval() compat function for NetBSD.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_GETAUXVAL_H
+#define SANITIZER_GETAUXVAL_H
+
+#include "sanitizer_platform.h"
+#include "sanitizer_glibc_version.h"
+
+#if SANITIZER_LINUX || SANITIZER_FUCHSIA
+
+# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
+     SANITIZER_FUCHSIA
+#  define SANITIZER_USE_GETAUXVAL 1
+# else
+#  define SANITIZER_USE_GETAUXVAL 0
+# endif
+
+# if SANITIZER_USE_GETAUXVAL
+#  include <sys/auxv.h>
+# else
+// The weak getauxval definition allows to check for the function at runtime.
+// This is useful for Android, when compiled at a lower API level yet running
+// on a more recent platform that offers the function.
+extern "C" SANITIZER_WEAK_ATTRIBUTE unsigned long getauxval(unsigned long type);
+# endif
+
+#elif SANITIZER_NETBSD
+
+#define SANITIZER_USE_GETAUXVAL 1
+
+#include <dlfcn.h>
+#include <elf.h>
+
+static inline decltype(AuxInfo::a_v) getauxval(decltype(AuxInfo::a_type) type) {
+  for (const AuxInfo *aux = (const AuxInfo *)_dlauxinfo();
+       aux->a_type != AT_NULL; ++aux) {
+    if (type == aux->a_type)
+      return aux->a_v;
+  }
+
+  return 0;
+}
+
+#endif
+
+#endif // SANITIZER_GETAUXVAL_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_glibc_version.h b/lib/tsan/sanitizer_common/sanitizer_glibc_version.h
new file mode 100644
index 0000000000..47175f20aa
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_glibc_version.h
@@ -0,0 +1,26 @@
+//===-- sanitizer_glibc_version.h -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_GLIBC_VERSION_H
+#define SANITIZER_GLIBC_VERSION_H
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_FUCHSIA
+#include <features.h>
+#endif
+
+#ifndef __GLIBC_PREREQ
+#define __GLIBC_PREREQ(x, y) 0
+#endif
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_hash.h b/lib/tsan/sanitizer_common/sanitizer_hash.h
new file mode 100644
index 0000000000..3d97dcc5d2
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_hash.h
@@ -0,0 +1,43 @@
+//===-- sanitizer_common.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple hash function.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_HASH_H
+#define SANITIZER_HASH_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+class MurMur2HashBuilder {
+  static const u32 m = 0x5bd1e995;
+  static const u32 seed = 0x9747b28c;
+  static const u32 r = 24;
+  u32 h;
+
+ public:
+  explicit MurMur2HashBuilder(u32 init = 0) { h = seed ^ init; }
+  void add(u32 k) {
+    k *= m;
+    k ^= k >> r;
+    k *= m;
+    h *= m;
+    h ^= k;
+  }
+  u32 get() {
+    u32 x = h;
+    x ^= x >> 13;
+    x *= m;
+    x ^= x >> 15;
+    return x;
+  }
+};
+}  //namespace __sanitizer
+
+#endif  // SANITIZER_HASH_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
new file mode 100644
index 0000000000..576807ea3a
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
@@ -0,0 +1,1535 @@
+//===-- sanitizer_interceptors_ioctl_netbsd.inc -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Ioctl handling in common sanitizer interceptors.
+//===----------------------------------------------------------------------===//
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_flags.h"
+
+struct ioctl_desc {
+  unsigned req;
+  // FIXME: support read+write arguments. Currently READWRITE and WRITE do the
+  // same thing.
+  // XXX: The declarations below may use WRITE instead of READWRITE, unless
+  // explicitly noted.
+  enum { NONE, READ, WRITE, READWRITE, CUSTOM } type : 3;
+  unsigned size : 29;
+  const char *name;
+};
+
+const unsigned ioctl_table_max = 1238;
+static ioctl_desc ioctl_table[ioctl_table_max];
+static unsigned ioctl_table_size = 0;
+
+// This can not be declared as a global, because references to struct_*_sz
+// require a global initializer. And this table must be available before global
+// initializers are run.
+static void ioctl_table_fill() {
+#define _(rq, tp, sz)                                                          \
+  if (IOCTL_##rq != IOCTL_NOT_PRESENT) {                                       \
+    CHECK(ioctl_table_size < ioctl_table_max);                                 \
+    ioctl_table[ioctl_table_size].req = IOCTL_##rq;                            \
+    ioctl_table[ioctl_table_size].type = ioctl_desc::tp;                       \
+    ioctl_table[ioctl_table_size].size = sz;                                   \
+    ioctl_table[ioctl_table_size].name = #rq;                                  \
+    ++ioctl_table_size;                                                        \
+  }
+
+  /* Entries from file: altq/altq_afmap.h */
+  _(AFM_ADDFMAP, READWRITE, struct_atm_flowmap_sz);
+  _(AFM_DELFMAP, READWRITE, struct_atm_flowmap_sz);
+  _(AFM_CLEANFMAP, READWRITE, struct_atm_flowmap_sz);
+  _(AFM_GETFMAP, READWRITE, struct_atm_flowmap_sz);
+  /* Entries from file: altq/altq.h */
+  _(ALTQGTYPE, READWRITE, struct_altqreq_sz);
+  _(ALTQTBRSET, READ, struct_tbrreq_sz);
+  _(ALTQTBRGET, READWRITE, struct_tbrreq_sz);
+  /* Entries from file: altq/altq_blue.h */
+  _(BLUE_IF_ATTACH, READ, struct_blue_interface_sz);
+  _(BLUE_DISABLE, READ, struct_blue_interface_sz);
+  _(BLUE_CONFIG, READWRITE, struct_blue_conf_sz);
+  _(BLUE_GETSTATS, READWRITE, struct_blue_stats_sz);
+  /* Entries from file: altq/altq_cbq.h */
+  _(CBQ_ENABLE, READ, struct_cbq_interface_sz);
+  _(CBQ_ADD_CLASS, READWRITE, struct_cbq_add_class_sz);
+  _(CBQ_DEL_CLASS, READ, struct_cbq_delete_class_sz);
+  _(CBQ_MODIFY_CLASS, READWRITE, struct_cbq_modify_class_sz);
+  _(CBQ_DEL_FILTER, READ, struct_cbq_delete_filter_sz);
+  _(CBQ_GETSTATS, READWRITE, struct_cbq_getstats_sz);
+  /* Entries from file: altq/altq_cdnr.h */
+  _(CDNR_IF_DETACH, READ, struct_cdnr_interface_sz);
+  _(CDNR_ADD_FILTER, READWRITE, struct_cdnr_add_filter_sz);
+  _(CDNR_GETSTATS, READWRITE, struct_cdnr_get_stats_sz);
+  _(CDNR_ADD_ELEM, READWRITE, struct_cdnr_add_element_sz);
+  _(CDNR_DEL_ELEM, READ, struct_cdnr_delete_element_sz);
+  _(CDNR_ADD_TBM, READWRITE, struct_cdnr_add_tbmeter_sz);
+  _(CDNR_MOD_TBM, READ, struct_cdnr_modify_tbmeter_sz);
+  _(CDNR_TBM_STATS, READWRITE, struct_cdnr_tbmeter_stats_sz);
+  _(CDNR_ADD_TCM, READWRITE, struct_cdnr_add_trtcm_sz);
+  _(CDNR_MOD_TCM, READWRITE, struct_cdnr_modify_trtcm_sz);
+  _(CDNR_TCM_STATS, READWRITE, struct_cdnr_tcm_stats_sz);
+  _(CDNR_ADD_TSW, READWRITE, struct_cdnr_add_tswtcm_sz);
+  _(CDNR_MOD_TSW, READWRITE, struct_cdnr_modify_tswtcm_sz);
+  /* Entries from file: altq/altq_fifoq.h */
+  _(FIFOQ_CONFIG, READWRITE, struct_fifoq_conf_sz);
+  _(FIFOQ_GETSTATS, READWRITE, struct_fifoq_getstats_sz);
+  /* Entries from file: altq/altq_hfsc.h */
+  _(HFSC_CLEAR_HIERARCHY, READ, struct_hfsc_interface_sz);
+  _(HFSC_ADD_CLASS, READWRITE, struct_hfsc_add_class_sz);
+  _(HFSC_GETSTATS, READWRITE, struct_hfsc_class_stats_sz);
+  /* Entries from file: altq/altq_jobs.h */
+  _(JOBS_IF_ATTACH, READ, struct_jobs_attach_sz);
+  _(JOBS_IF_DETACH, READ, struct_jobs_interface_sz);
+  _(JOBS_ENABLE, READ, struct_jobs_interface_sz);
+  _(JOBS_DISABLE, READ, struct_jobs_interface_sz);
+  _(JOBS_CLEAR, READ, struct_jobs_interface_sz);
+  _(JOBS_ADD_CLASS, READWRITE, struct_jobs_add_class_sz);
+  _(JOBS_MOD_CLASS, READ, struct_jobs_modify_class_sz);
+  /* Entries from file: altq/altq_priq.h */
+  _(PRIQ_IF_ATTACH, READ, struct_priq_interface_sz);
+  _(PRIQ_CLEAR, READ, struct_priq_interface_sz);
+  _(PRIQ_ADD_CLASS, READWRITE, struct_priq_add_class_sz);
+  _(PRIQ_DEL_CLASS, READ, struct_priq_delete_class_sz);
+  _(PRIQ_MOD_CLASS, READ, struct_priq_modify_class_sz);
+  _(PRIQ_ADD_FILTER, READWRITE, struct_priq_add_filter_sz);
+  _(PRIQ_DEL_FILTER, READ, struct_priq_delete_filter_sz);
+  _(PRIQ_GETSTATS, READWRITE, struct_priq_class_stats_sz);
+  /* Entries from file: altq/altq_red.h */
+  _(RED_CONFIG, READWRITE, struct_red_conf_sz);
+  _(RED_GETSTATS, READWRITE, struct_red_stats_sz);
+  _(RED_SETDEFAULTS, READ, struct_redparams_sz);
+  /* Entries from file: altq/altq_rio.h */
+  _(RIO_CONFIG, READWRITE, struct_rio_conf_sz);
+  _(RIO_GETSTATS, READWRITE, struct_rio_stats_sz);
+  _(RIO_SETDEFAULTS, READ, struct_redparams_sz);
+  /* Entries from file: altq/altq_wfq.h */
+  _(WFQ_CONFIG, READWRITE, struct_wfq_conf_sz);
+  _(WFQ_GET_QID, READWRITE, struct_wfq_getqid_sz);
+  _(WFQ_SET_WEIGHT, READWRITE, struct_wfq_setweight_sz);
+  /* Entries from file: crypto/cryptodev.h */
+  _(CRIOGET, READWRITE, sizeof(u32));
+  _(CIOCFSESSION, READ, sizeof(u32));
+  _(CIOCKEY, READWRITE, struct_crypt_kop_sz);
+  _(CIOCNFKEYM, READWRITE, struct_crypt_mkop_sz);
+  _(CIOCNFSESSION, READ, struct_crypt_sfop_sz);
+  _(CIOCNCRYPTRETM, READWRITE, struct_cryptret_sz);
+  _(CIOCNCRYPTRET, READWRITE, struct_crypt_result_sz);
+  _(CIOCGSESSION, READWRITE, struct_session_op_sz);
+  _(CIOCNGSESSION, READWRITE, struct_crypt_sgop_sz);
+  _(CIOCCRYPT, READWRITE, struct_crypt_op_sz);
+  _(CIOCNCRYPTM, READWRITE, struct_crypt_mop_sz);
+  _(CIOCASYMFEAT, WRITE, sizeof(u32));
+  /* Entries from file: dev/apm/apmio.h */
+  _(APM_IOC_REJECT, READ, struct_apm_event_info_sz);
+  _(OAPM_IOC_GETPOWER, WRITE, struct_apm_power_info_sz);
+  _(APM_IOC_GETPOWER, READWRITE, struct_apm_power_info_sz);
+  _(APM_IOC_NEXTEVENT, WRITE, struct_apm_event_info_sz);
+  _(APM_IOC_DEV_CTL, READ, struct_apm_ctl_sz);
+  /* Entries from file: dev/dm/netbsd-dm.h */
+  _(NETBSD_DM_IOCTL, READWRITE, struct_plistref_sz);
+  /* Entries from file: dev/dmover/dmover_io.h */
+  _(DMIO_SETFUNC, READ, struct_dmio_setfunc_sz);
+  /* Entries from file: dev/dtv/dtvio_demux.h */
+  _(DMX_START, NONE, 0);
+  _(DMX_STOP, NONE, 0);
+  _(DMX_SET_FILTER, READ, struct_dmx_sct_filter_params_sz);
+  _(DMX_SET_PES_FILTER, READ, struct_dmx_pes_filter_params_sz);
+  _(DMX_SET_BUFFER_SIZE, NONE, 0);
+  _(DMX_GET_STC, READWRITE, struct_dmx_stc_sz);
+  _(DMX_ADD_PID, READ, sizeof(u16));
+  _(DMX_REMOVE_PID, READ, sizeof(u16));
+  _(DMX_GET_CAPS, WRITE, struct_dmx_caps_sz);
+  _(DMX_SET_SOURCE, READ, enum_dmx_source_sz);
+  /* Entries from file: dev/dtv/dtvio_frontend.h */
+  _(FE_READ_STATUS, WRITE, enum_fe_status_sz);
+  _(FE_READ_BER, WRITE, sizeof(u32));
+  _(FE_READ_SNR, WRITE, sizeof(u16));
+  _(FE_READ_SIGNAL_STRENGTH, WRITE, sizeof(u16));
+  _(FE_READ_UNCORRECTED_BLOCKS, WRITE, sizeof(u32));
+  _(FE_SET_FRONTEND, READWRITE, struct_dvb_frontend_parameters_sz);
+  _(FE_GET_FRONTEND, WRITE, struct_dvb_frontend_parameters_sz);
+  _(FE_GET_EVENT, WRITE, struct_dvb_frontend_event_sz);
+  _(FE_GET_INFO, WRITE, struct_dvb_frontend_info_sz);
+  _(FE_DISEQC_RESET_OVERLOAD, NONE, 0);
+  _(FE_DISEQC_SEND_MASTER_CMD, READ, struct_dvb_diseqc_master_cmd_sz);
+  _(FE_DISEQC_RECV_SLAVE_REPLY, WRITE, struct_dvb_diseqc_slave_reply_sz);
+  _(FE_DISEQC_SEND_BURST, READ, enum_fe_sec_mini_cmd_sz);
+  _(FE_SET_TONE, READ, enum_fe_sec_tone_mode_sz);
+  _(FE_SET_VOLTAGE, READ, enum_fe_sec_voltage_sz);
+  _(FE_ENABLE_HIGH_LNB_VOLTAGE, READ, sizeof(int));
+  _(FE_SET_FRONTEND_TUNE_MODE, READ, sizeof(unsigned int));
+  _(FE_DISHNETWORK_SEND_LEGACY_CMD, READ, sizeof(unsigned long));
+  /* Entries from file: dev/hdaudio/hdaudioio.h */
+  _(HDAUDIO_FGRP_INFO, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_FGRP_GETCONFIG, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_FGRP_SETCONFIG, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_FGRP_WIDGET_INFO, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_FGRP_CODEC_INFO, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_AFG_WIDGET_INFO, READWRITE, struct_plistref_sz);
+  _(HDAUDIO_AFG_CODEC_INFO, READWRITE, struct_plistref_sz);
+  /* Entries from file: dev/hdmicec/hdmicecio.h */
+  _(CEC_GET_PHYS_ADDR, WRITE, sizeof(u16));
+  _(CEC_GET_LOG_ADDRS, WRITE, sizeof(u16));
+  _(CEC_SET_LOG_ADDRS, READ, sizeof(u16));
+  _(CEC_GET_VENDOR_ID, WRITE, sizeof(u32));
+  /* Entries from file: dev/hpc/hpcfbio.h */
+  _(HPCFBIO_GCONF, READWRITE, struct_hpcfb_fbconf_sz);
+  _(HPCFBIO_SCONF, READ, struct_hpcfb_fbconf_sz);
+  _(HPCFBIO_GDSPCONF, READWRITE, struct_hpcfb_dspconf_sz);
+  _(HPCFBIO_SDSPCONF, READ, struct_hpcfb_dspconf_sz);
+  _(HPCFBIO_GOP, WRITE, struct_hpcfb_dsp_op_sz);
+  _(HPCFBIO_SOP, READWRITE, struct_hpcfb_dsp_op_sz);
+  /* Entries from file: dev/i2o/iopio.h */
+  _(IOPIOCPT, READWRITE, struct_ioppt_sz);
+  _(IOPIOCGLCT, READWRITE, struct_iovec_sz);
+  _(IOPIOCGSTATUS, READWRITE, struct_iovec_sz);
+  _(IOPIOCRECONFIG, NONE, 0);
+  _(IOPIOCGTIDMAP, READWRITE, struct_iovec_sz);
+  /* Entries from file: dev/ic/athioctl.h */
+  _(SIOCGATHSTATS, READWRITE, struct_ifreq_sz);
+  _(SIOCGATHDIAG, READWRITE, struct_ath_diag_sz);
+  /* Entries from file: dev/ic/bt8xx.h */
+  _(METEORCAPTUR, READ, sizeof(int));
+  _(METEORCAPFRM, READ, struct_meteor_capframe_sz);
+  _(METEORSETGEO, READ, struct_meteor_geomet_sz);
+  _(METEORGETGEO, WRITE, struct_meteor_geomet_sz);
+  _(METEORSTATUS, WRITE, sizeof(unsigned short));
+  _(METEORSHUE, READ, sizeof(signed char));
+  _(METEORGHUE, WRITE, sizeof(signed char));
+  _(METEORSFMT, READ, sizeof(unsigned int));
+  _(METEORGFMT, WRITE, sizeof(unsigned int));
+  _(METEORSINPUT, READ, sizeof(unsigned int));
+  _(METEORGINPUT, WRITE, sizeof(unsigned int));
+  _(METEORSCHCV, READ, sizeof(unsigned char));
+  _(METEORGCHCV, WRITE, sizeof(unsigned char));
+  _(METEORSCOUNT, READ, struct_meteor_counts_sz);
+  _(METEORGCOUNT, WRITE, struct_meteor_counts_sz);
+  _(METEORSFPS, READ, sizeof(unsigned short));
+  _(METEORGFPS, WRITE, sizeof(unsigned short));
+  _(METEORSSIGNAL, READ, sizeof(unsigned int));
+  _(METEORGSIGNAL, WRITE, sizeof(unsigned int));
+  _(METEORSVIDEO, READ, struct_meteor_video_sz);
+  _(METEORGVIDEO, WRITE, struct_meteor_video_sz);
+  _(METEORSBRIG, READ, sizeof(unsigned char));
+  _(METEORGBRIG, WRITE, sizeof(unsigned char));
+  _(METEORSCSAT, READ, sizeof(unsigned char));
+  _(METEORGCSAT, WRITE, sizeof(unsigned char));
+  _(METEORSCONT, READ, sizeof(unsigned char));
+  _(METEORGCONT, WRITE, sizeof(unsigned char));
+  _(METEORSHWS, READ, sizeof(unsigned char));
+  _(METEORGHWS, WRITE, sizeof(unsigned char));
+  _(METEORSVWS, READ, sizeof(unsigned char));
+  _(METEORGVWS, WRITE, sizeof(unsigned char));
+  _(METEORSTS, READ, sizeof(unsigned char));
+  _(METEORGTS, WRITE, sizeof(unsigned char));
+  _(TVTUNER_SETCHNL, READ, sizeof(unsigned int));
+  _(TVTUNER_GETCHNL, WRITE, sizeof(unsigned int));
+  _(TVTUNER_SETTYPE, READ, sizeof(unsigned int));
+  _(TVTUNER_GETTYPE, WRITE, sizeof(unsigned int));
+  _(TVTUNER_GETSTATUS, WRITE, sizeof(unsigned int));
+  _(TVTUNER_SETFREQ, READ, sizeof(unsigned int));
+  _(TVTUNER_GETFREQ, WRITE, sizeof(unsigned int));
+  _(TVTUNER_SETAFC, READ, sizeof(int));
+  _(TVTUNER_GETAFC, WRITE, sizeof(int));
+  _(RADIO_SETMODE, READ, sizeof(unsigned int));
+  _(RADIO_GETMODE, WRITE, sizeof(unsigned char));
+  _(RADIO_SETFREQ, READ, sizeof(unsigned int));
+  _(RADIO_GETFREQ, WRITE, sizeof(unsigned int));
+  _(METEORSACTPIXFMT, READ, sizeof(int));
+  _(METEORGACTPIXFMT, WRITE, sizeof(int));
+  _(METEORGSUPPIXFMT, READWRITE, struct_meteor_pixfmt_sz);
+  _(TVTUNER_GETCHNLSET, READWRITE, struct_bktr_chnlset_sz);
+  _(REMOTE_GETKEY, WRITE, struct_bktr_remote_sz);
+  /* Entries from file: dev/ic/icp_ioctl.h */
+  _(GDT_IOCTL_GENERAL, READWRITE, struct_gdt_ucmd_sz);
+  _(GDT_IOCTL_DRVERS, WRITE, sizeof(int));
+  _(GDT_IOCTL_CTRTYPE, READWRITE, struct_gdt_ctrt_sz);
+  _(GDT_IOCTL_OSVERS, WRITE, struct_gdt_osv_sz);
+  _(GDT_IOCTL_CTRCNT, WRITE, sizeof(int));
+  _(GDT_IOCTL_EVENT, READWRITE, struct_gdt_event_sz);
+  _(GDT_IOCTL_STATIST, WRITE, struct_gdt_statist_sz);
+  _(GDT_IOCTL_RESCAN, READWRITE, struct_gdt_rescan_sz);
+  /* Entries from file: dev/ic/isp_ioctl.h */
+  _(ISP_SDBLEV, READWRITE, sizeof(int));
+  _(ISP_RESETHBA, NONE, 0);
+  _(ISP_RESCAN, NONE, 0);
+  _(ISP_SETROLE, READWRITE, sizeof(int));
+  _(ISP_GETROLE, WRITE, sizeof(int));
+  _(ISP_GET_STATS, WRITE, struct_isp_stats_sz);
+  _(ISP_CLR_STATS, NONE, 0);
+  _(ISP_FC_LIP, NONE, 0);
+  _(ISP_FC_GETDINFO, READWRITE, struct_isp_fc_device_sz);
+  _(ISP_GET_FW_CRASH_DUMP, NONE, 0);
+  _(ISP_FORCE_CRASH_DUMP, NONE, 0);
+  _(ISP_FC_GETHINFO, READWRITE, struct_isp_hba_device_sz);
+  _(ISP_TSK_MGMT, READWRITE, struct_isp_fc_tsk_mgmt_sz);
+  _(ISP_FC_GETDLIST, NONE, 0);
+  /* Entries from file: dev/ic/mlxio.h */
+  _(MLXD_STATUS, WRITE, sizeof(int));
+  _(MLXD_CHECKASYNC, WRITE, sizeof(int));
+  _(MLXD_DETACH, READ, sizeof(int));
+  _(MLX_RESCAN_DRIVES, NONE, 0);
+  _(MLX_PAUSE_CHANNEL, READ, struct_mlx_pause_sz);
+  _(MLX_COMMAND, READWRITE, struct_mlx_usercommand_sz);
+  _(MLX_REBUILDASYNC, READWRITE, struct_mlx_rebuild_request_sz);
+  _(MLX_REBUILDSTAT, WRITE, struct_mlx_rebuild_status_sz);
+  _(MLX_GET_SYSDRIVE, READWRITE, sizeof(int));
+  _(MLX_GET_CINFO, WRITE, struct_mlx_cinfo_sz);
+  /* Entries from file: dev/ic/nvmeio.h */
+  _(NVME_PASSTHROUGH_CMD, READWRITE, struct_nvme_pt_command_sz);
+  /* Entries from file: dev/ic/qemufwcfgio.h */
+  _(FWCFGIO_SET_INDEX, READ, sizeof(u16));
+  /* Entries from file: dev/ir/irdaio.h */
+  _(IRDA_RESET_PARAMS, NONE, 0);
+  _(IRDA_SET_PARAMS, READ, struct_irda_params_sz);
+  _(IRDA_GET_SPEEDMASK, WRITE, sizeof(unsigned int));
+  _(IRDA_GET_TURNAROUNDMASK, WRITE, sizeof(unsigned int));
+  _(IRFRAMETTY_GET_DEVICE, WRITE, sizeof(unsigned int));
+  _(IRFRAMETTY_GET_DONGLE, WRITE, sizeof(unsigned int));
+  _(IRFRAMETTY_SET_DONGLE, READ, sizeof(unsigned int));
+  /* Entries from file: dev/isa/isvio.h */
+  _(ISV_CMD, READWRITE, struct_isv_cmd_sz);
+  /* Entries from file: dev/isa/wtreg.h */
+  _(WTQICMD, NONE, 0);
+  /* Entries from file: dev/iscsi/iscsi_ioctl.h */
+  _(ISCSI_GET_VERSION, READWRITE, struct_iscsi_get_version_parameters_sz);
+  _(ISCSI_LOGIN, READWRITE, struct_iscsi_login_parameters_sz);
+  _(ISCSI_LOGOUT, READWRITE, struct_iscsi_logout_parameters_sz);
+  _(ISCSI_ADD_CONNECTION, READWRITE, struct_iscsi_login_parameters_sz);
+  _(ISCSI_RESTORE_CONNECTION, READWRITE, struct_iscsi_login_parameters_sz);
+  _(ISCSI_REMOVE_CONNECTION, READWRITE, struct_iscsi_remove_parameters_sz);
+  _(ISCSI_CONNECTION_STATUS, READWRITE, struct_iscsi_conn_status_parameters_sz);
+  _(ISCSI_SEND_TARGETS, READWRITE, struct_iscsi_send_targets_parameters_sz);
+  _(ISCSI_SET_NODE_NAME, READWRITE, struct_iscsi_set_node_name_parameters_sz);
+  _(ISCSI_IO_COMMAND, READWRITE, struct_iscsi_iocommand_parameters_sz);
+  _(ISCSI_REGISTER_EVENT, READWRITE, struct_iscsi_register_event_parameters_sz);
+  _(ISCSI_DEREGISTER_EVENT, READWRITE,
+    struct_iscsi_register_event_parameters_sz);
+  _(ISCSI_WAIT_EVENT, READWRITE, struct_iscsi_wait_event_parameters_sz);
+  _(ISCSI_POLL_EVENT, READWRITE, struct_iscsi_wait_event_parameters_sz);
+  /* Entries from file: dev/ofw/openfirmio.h */
+  _(OFIOCGET, READWRITE, struct_ofiocdesc_sz);
+  _(OFIOCSET, READ, struct_ofiocdesc_sz);
+  _(OFIOCNEXTPROP, READWRITE, struct_ofiocdesc_sz);
+  _(OFIOCGETOPTNODE, WRITE, sizeof(int));
+  _(OFIOCGETNEXT, READWRITE, sizeof(int));
+  _(OFIOCGETCHILD, READWRITE, sizeof(int));
+  _(OFIOCFINDDEVICE, READWRITE, struct_ofiocdesc_sz);
+  /* Entries from file: dev/pci/amrio.h */
+  _(AMR_IO_VERSION, WRITE, sizeof(int));
+  _(AMR_IO_COMMAND, READWRITE, struct_amr_user_ioctl_sz);
+  /* Entries from file: dev/pci/mlyio.h */
+  _(MLYIO_COMMAND, READWRITE, struct_mly_user_command_sz);
+  _(MLYIO_HEALTH, READ, struct_mly_user_health_sz);
+  /* Entries from file: dev/pci/pciio.h */
+  _(PCI_IOC_CFGREAD, READWRITE, struct_pciio_cfgreg_sz);
+  _(PCI_IOC_CFGWRITE, READ, struct_pciio_cfgreg_sz);
+  _(PCI_IOC_BDF_CFGREAD, READWRITE, struct_pciio_bdf_cfgreg_sz);
+  _(PCI_IOC_BDF_CFGWRITE, READ, struct_pciio_bdf_cfgreg_sz);
+  _(PCI_IOC_BUSINFO, WRITE, struct_pciio_businfo_sz);
+  _(PCI_IOC_DRVNAME, READWRITE, struct_pciio_drvname_sz);
+  _(PCI_IOC_DRVNAMEONBUS, READWRITE, struct_pciio_drvnameonbus_sz);
+  /* Entries from file: dev/pci/tweio.h */
+  _(TWEIO_COMMAND, READWRITE, struct_twe_usercommand_sz);
+  _(TWEIO_STATS, READWRITE, union_twe_statrequest_sz);
+  _(TWEIO_AEN_POLL, WRITE, sizeof(int));
+  _(TWEIO_AEN_WAIT, WRITE, sizeof(int));
+  _(TWEIO_SET_PARAM, READ, struct_twe_paramcommand_sz);
+  _(TWEIO_GET_PARAM, READ, struct_twe_paramcommand_sz);
+  _(TWEIO_RESET, NONE, 0);
+  _(TWEIO_ADD_UNIT, READ, struct_twe_drivecommand_sz);
+  _(TWEIO_DEL_UNIT, READ, struct_twe_drivecommand_sz);
+  /* Entries from file: dev/pcmcia/if_cnwioctl.h */
+  _(SIOCSCNWDOMAIN, READ, struct_ifreq_sz);
+  _(SIOCGCNWDOMAIN, READWRITE, struct_ifreq_sz);
+  _(SIOCSCNWKEY, READWRITE, struct_ifreq_sz);
+  _(SIOCGCNWSTATUS, READWRITE, struct_cnwstatus_sz);
+  _(SIOCGCNWSTATS, READWRITE, struct_cnwistats_sz);
+  _(SIOCGCNWTRAIL, READWRITE, struct_cnwitrail_sz);
+  /* Entries from file: dev/pcmcia/if_rayreg.h */
+  _(SIOCGRAYSIGLEV, READWRITE, struct_ifreq_sz);
+  /* Entries from file: dev/raidframe/raidframeio.h */
+  _(RAIDFRAME_SHUTDOWN, NONE, 0);
+  _(RAIDFRAME_TUR, READ, sizeof(u64));
+  _(RAIDFRAME_FAIL_DISK, READ, struct_rf_recon_req_sz);
+  _(RAIDFRAME_CHECK_RECON_STATUS, READWRITE, sizeof(int));
+  _(RAIDFRAME_REWRITEPARITY, NONE, 0);
+  _(RAIDFRAME_COPYBACK, NONE, 0);
+  _(RAIDFRAME_SPARET_WAIT, WRITE, struct_RF_SparetWait_sz);
+  _(RAIDFRAME_SEND_SPARET, READ, sizeof(uptr));
+  _(RAIDFRAME_ABORT_SPARET_WAIT, NONE, 0);
+  _(RAIDFRAME_START_ATRACE, NONE, 0);
+  _(RAIDFRAME_STOP_ATRACE, NONE, 0);
+  _(RAIDFRAME_GET_SIZE, WRITE, sizeof(int));
+  _(RAIDFRAME_RESET_ACCTOTALS, NONE, 0);
+  _(RAIDFRAME_KEEP_ACCTOTALS, READ, sizeof(int));
+  _(RAIDFRAME_GET_COMPONENT_LABEL, READWRITE, struct_RF_ComponentLabel_sz);
+  _(RAIDFRAME_SET_COMPONENT_LABEL, READ, struct_RF_ComponentLabel_sz);
+  _(RAIDFRAME_INIT_LABELS, READ, struct_RF_ComponentLabel_sz);
+  _(RAIDFRAME_ADD_HOT_SPARE, READ, struct_RF_SingleComponent_sz);
+  _(RAIDFRAME_REMOVE_HOT_SPARE, READ, struct_RF_SingleComponent_sz);
+  _(RAIDFRAME_REBUILD_IN_PLACE, READ, struct_RF_SingleComponent_sz);
+  _(RAIDFRAME_CHECK_PARITY, READWRITE, sizeof(int));
+  _(RAIDFRAME_CHECK_PARITYREWRITE_STATUS, READWRITE, sizeof(int));
+  _(RAIDFRAME_CHECK_COPYBACK_STATUS, READWRITE, sizeof(int));
+  _(RAIDFRAME_SET_AUTOCONFIG, READWRITE, sizeof(int));
+  _(RAIDFRAME_SET_ROOT, READWRITE, sizeof(int));
+  _(RAIDFRAME_DELETE_COMPONENT, READ, struct_RF_SingleComponent_sz);
+  _(RAIDFRAME_INCORPORATE_HOT_SPARE, READ, struct_RF_SingleComponent_sz);
+  _(RAIDFRAME_CHECK_RECON_STATUS_EXT, READWRITE, struct_RF_ProgressInfo_sz);
+  _(RAIDFRAME_CHECK_PARITYREWRITE_STATUS_EXT, READWRITE,
+    struct_RF_ProgressInfo_sz);
+  _(RAIDFRAME_CHECK_COPYBACK_STATUS_EXT, READWRITE, struct_RF_ProgressInfo_sz);
+  _(RAIDFRAME_PARITYMAP_STATUS, WRITE, struct_rf_pmstat_sz);
+  _(RAIDFRAME_PARITYMAP_GET_DISABLE, WRITE, sizeof(int));
+  _(RAIDFRAME_PARITYMAP_SET_DISABLE, READ, sizeof(int));
+  _(RAIDFRAME_PARITYMAP_SET_PARAMS, READ, struct_rf_pmparams_sz);
+  _(RAIDFRAME_SET_LAST_UNIT, READ, sizeof(int));
+  _(RAIDFRAME_GET_INFO, READWRITE, sizeof(uptr));
+  _(RAIDFRAME_CONFIGURE, READ, sizeof(uptr));
+  /* Entries from file: dev/sbus/mbppio.h */
+  _(MBPPIOCSPARAM, READ, struct_mbpp_param_sz);
+  _(MBPPIOCGPARAM, WRITE, struct_mbpp_param_sz);
+  _(MBPPIOCGSTAT, WRITE, sizeof(int));
+  /* Entries from file: dev/scsipi/ses.h */
+  _(SESIOC_GETNOBJ, NONE, 0);
+  _(SESIOC_GETOBJMAP, NONE, 0);
+  _(SESIOC_GETENCSTAT, NONE, 0);
+  _(SESIOC_SETENCSTAT, NONE, 0);
+  _(SESIOC_GETOBJSTAT, NONE, 0);
+  _(SESIOC_SETOBJSTAT, NONE, 0);
+  _(SESIOC_GETTEXT, NONE, 0);
+  _(SESIOC_INIT, NONE, 0);
+  /* Entries from file: dev/sun/disklabel.h */
+  _(SUN_DKIOCGGEOM, WRITE, struct_sun_dkgeom_sz);
+  _(SUN_DKIOCINFO, WRITE, struct_sun_dkctlr_sz);
+  _(SUN_DKIOCGPART, WRITE, struct_sun_dkpart_sz);
+  /* Entries from file: dev/sun/fbio.h */
+  _(FBIOGTYPE, WRITE, struct_fbtype_sz);
+  _(FBIOPUTCMAP, READ, struct_fbcmap_sz);
+  _(FBIOGETCMAP, READ, struct_fbcmap_sz);
+  _(FBIOGATTR, WRITE, struct_fbgattr_sz);
+  _(FBIOSVIDEO, READ, sizeof(int));
+  _(FBIOGVIDEO, WRITE, sizeof(int));
+  _(FBIOSCURSOR, READ, struct_fbcursor_sz);
+  _(FBIOGCURSOR, READWRITE, struct_fbcursor_sz);
+  _(FBIOSCURPOS, READ, struct_fbcurpos_sz);
+  _(FBIOGCURPOS, READ, struct_fbcurpos_sz);
+  _(FBIOGCURMAX, WRITE, struct_fbcurpos_sz);
+  /* Entries from file: dev/sun/kbio.h */
+  _(KIOCTRANS, READ, sizeof(int));
+  _(KIOCSETKEY, READWRITE, struct_okiockey_sz);
+  _(KIOCGETKEY, READWRITE, struct_okiockey_sz);
+  _(KIOCGTRANS, WRITE, sizeof(int));
+  _(KIOCCMD, READ, sizeof(int));
+  _(KIOCTYPE, WRITE, sizeof(int));
+  _(KIOCSDIRECT, READ, sizeof(int));
+  _(KIOCSKEY, READ, struct_kiockeymap_sz);
+  _(KIOCGKEY, READWRITE, struct_kiockeymap_sz);
+  _(KIOCSLED, READ, sizeof(char));
+  _(KIOCGLED, WRITE, sizeof(char));
+  _(KIOCLAYOUT, WRITE, sizeof(int));
+  /* Entries from file: dev/sun/vuid_event.h */
+  _(VUIDSFORMAT, READ, sizeof(int));
+  _(VUIDGFORMAT, WRITE, sizeof(int));
+  /* Entries from file: dev/tc/sticio.h */
+  _(STICIO_GXINFO, WRITE, struct_stic_xinfo_sz);
+  _(STICIO_RESET, NONE, 0);
+  _(STICIO_STARTQ, NONE, 0);
+  _(STICIO_STOPQ, NONE, 0);
+  /* Entries from file: dev/usb/ukyopon.h */
+  _(UKYOPON_IDENTIFY, WRITE, struct_ukyopon_identify_sz);
+  /* Entries from file: dev/usb/usb.h */
+  _(USB_REQUEST, READWRITE, struct_usb_ctl_request_sz);
+  _(USB_SETDEBUG, READ, sizeof(int));
+  _(USB_DISCOVER, NONE, 0);
+  _(USB_DEVICEINFO, READWRITE, struct_usb_device_info_sz);
+  _(USB_DEVICEINFO_OLD, READWRITE, struct_usb_device_info_old_sz);
+  _(USB_DEVICESTATS, WRITE, struct_usb_device_stats_sz);
+  _(USB_GET_REPORT_DESC, WRITE, struct_usb_ctl_report_desc_sz);
+  _(USB_SET_IMMED, READ, sizeof(int));
+  _(USB_GET_REPORT, READWRITE, struct_usb_ctl_report_sz);
+  _(USB_SET_REPORT, READ, struct_usb_ctl_report_sz);
+  _(USB_GET_REPORT_ID, WRITE, sizeof(int));
+  _(USB_GET_CONFIG, WRITE, sizeof(int));
+  _(USB_SET_CONFIG, READ, sizeof(int));
+  _(USB_GET_ALTINTERFACE, READWRITE, struct_usb_alt_interface_sz);
+  _(USB_SET_ALTINTERFACE, READWRITE, struct_usb_alt_interface_sz);
+  _(USB_GET_NO_ALT, READWRITE, struct_usb_alt_interface_sz);
+  _(USB_GET_DEVICE_DESC, WRITE, struct_usb_device_descriptor_sz);
+  _(USB_GET_CONFIG_DESC, READWRITE, struct_usb_config_desc_sz);
+  _(USB_GET_INTERFACE_DESC, READWRITE, struct_usb_interface_desc_sz);
+  _(USB_GET_ENDPOINT_DESC, READWRITE, struct_usb_endpoint_desc_sz);
+  _(USB_GET_FULL_DESC, READWRITE, struct_usb_full_desc_sz);
+  _(USB_GET_STRING_DESC, READWRITE, struct_usb_string_desc_sz);
+  _(USB_DO_REQUEST, READWRITE, struct_usb_ctl_request_sz);
+  _(USB_GET_DEVICEINFO, WRITE, struct_usb_device_info_sz);
+  _(USB_GET_DEVICEINFO_OLD, WRITE, struct_usb_device_info_old_sz);
+  _(USB_SET_SHORT_XFER, READ, sizeof(int));
+  _(USB_SET_TIMEOUT, READ, sizeof(int));
+  _(USB_SET_BULK_RA, READ, sizeof(int));
+  _(USB_SET_BULK_WB, READ, sizeof(int));
+  _(USB_SET_BULK_RA_OPT, READ, struct_usb_bulk_ra_wb_opt_sz);
+  _(USB_SET_BULK_WB_OPT, READ, struct_usb_bulk_ra_wb_opt_sz);
+  _(USB_GET_CM_OVER_DATA, WRITE, sizeof(int));
+  _(USB_SET_CM_OVER_DATA, READ, sizeof(int));
+  /* Entries from file: dev/usb/utoppy.h */
+  _(UTOPPYIOTURBO, READ, sizeof(int));
+  _(UTOPPYIOREBOOT, NONE, 0);
+  _(UTOPPYIOSTATS, WRITE, struct_utoppy_stats_sz);
+  _(UTOPPYIORENAME, READ, struct_utoppy_rename_sz);
+  _(UTOPPYIOMKDIR, READ, sizeof(uptr));
+  _(UTOPPYIODELETE, READ, sizeof(uptr));
+  _(UTOPPYIOREADDIR, READ, sizeof(uptr));
+  _(UTOPPYIOREADFILE, READ, struct_utoppy_readfile_sz);
+  _(UTOPPYIOWRITEFILE, READ, struct_utoppy_writefile_sz);
+  /* Entries from file: dev/vme/xio.h */
+  _(DIOSXDCMD, READWRITE, struct_xd_iocmd_sz);
+  /* Entries from file: dev/wscons/wsdisplay_usl_io.h */
+  _(VT_OPENQRY, WRITE, sizeof(int));
+  _(VT_SETMODE, READ, struct_vt_mode_sz);
+  _(VT_GETMODE, WRITE, struct_vt_mode_sz);
+  _(VT_RELDISP, NONE, 0);
+  _(VT_ACTIVATE, NONE, 0);
+  _(VT_WAITACTIVE, NONE, 0);
+  _(VT_GETACTIVE, WRITE, sizeof(int));
+  _(VT_GETSTATE, WRITE, struct_vt_stat_sz);
+  _(KDGETKBENT, READWRITE, struct_kbentry_sz);
+  _(KDGKBMODE, WRITE, sizeof(int));
+  _(KDSKBMODE, NONE, 0);
+  _(KDMKTONE, NONE, 0);
+  _(KDSETMODE, NONE, 0);
+  _(KDENABIO, NONE, 0);
+  _(KDDISABIO, NONE, 0);
+  _(KDGKBTYPE, WRITE, sizeof(char));
+  _(KDGETLED, WRITE, sizeof(int));
+  _(KDSETLED, NONE, 0);
+  _(KDSETRAD, NONE, 0);
+  _(VGAPCVTID, READWRITE, struct_pcvtid_sz);
+  _(CONS_GETVERS, WRITE, sizeof(int));
+  /* Entries from file: dev/wscons/wsconsio.h */
+  _(WSKBDIO_GTYPE, WRITE, sizeof(unsigned int));
+  _(WSKBDIO_BELL, NONE, 0);
+  _(WSKBDIO_COMPLEXBELL, READ, struct_wskbd_bell_data_sz);
+  _(WSKBDIO_SETBELL, READ, struct_wskbd_bell_data_sz);
+  _(WSKBDIO_GETBELL, WRITE, struct_wskbd_bell_data_sz);
+  _(WSKBDIO_SETDEFAULTBELL, READ, struct_wskbd_bell_data_sz);
+  _(WSKBDIO_GETDEFAULTBELL, WRITE, struct_wskbd_bell_data_sz);
+  _(WSKBDIO_SETKEYREPEAT, READ, struct_wskbd_keyrepeat_data_sz);
+  _(WSKBDIO_GETKEYREPEAT, WRITE, struct_wskbd_keyrepeat_data_sz);
+  _(WSKBDIO_SETDEFAULTKEYREPEAT, READ, struct_wskbd_keyrepeat_data_sz);
+  _(WSKBDIO_GETDEFAULTKEYREPEAT, WRITE, struct_wskbd_keyrepeat_data_sz);
+  _(WSKBDIO_SETLEDS, READ, sizeof(int));
+  _(WSKBDIO_GETLEDS, WRITE, sizeof(int));
+  _(WSKBDIO_GETMAP, READWRITE, struct_wskbd_map_data_sz);
+  _(WSKBDIO_SETMAP, READ, struct_wskbd_map_data_sz);
+  _(WSKBDIO_GETENCODING, WRITE, sizeof(int));
+  _(WSKBDIO_SETENCODING, READ, sizeof(int));
+  _(WSKBDIO_SETMODE, READ, sizeof(int));
+  _(WSKBDIO_GETMODE, WRITE, sizeof(int));
+  _(WSKBDIO_SETKEYCLICK, READ, sizeof(int));
+  _(WSKBDIO_GETKEYCLICK, WRITE, sizeof(int));
+  _(WSKBDIO_GETSCROLL, WRITE, struct_wskbd_scroll_data_sz);
+  _(WSKBDIO_SETSCROLL, READ, struct_wskbd_scroll_data_sz);
+  _(WSKBDIO_SETVERSION, READ, sizeof(int));
+  _(WSMOUSEIO_GTYPE, WRITE, sizeof(unsigned int));
+  _(WSMOUSEIO_SRES, READ, sizeof(unsigned int));
+  _(WSMOUSEIO_SSCALE, READ, sizeof(unsigned int));
+  _(WSMOUSEIO_SRATE, READ, sizeof(unsigned int));
+  _(WSMOUSEIO_SCALIBCOORDS, READ, struct_wsmouse_calibcoords_sz);
+  _(WSMOUSEIO_GCALIBCOORDS, WRITE, struct_wsmouse_calibcoords_sz);
+  _(WSMOUSEIO_GETID, READWRITE, struct_wsmouse_id_sz);
+  _(WSMOUSEIO_GETREPEAT, WRITE, struct_wsmouse_repeat_sz);
+  _(WSMOUSEIO_SETREPEAT, READ, struct_wsmouse_repeat_sz);
+  _(WSMOUSEIO_SETVERSION, READ, sizeof(int));
+  _(WSDISPLAYIO_GTYPE, WRITE, sizeof(unsigned int));
+  _(WSDISPLAYIO_GINFO, WRITE, struct_wsdisplay_fbinfo_sz);
+  _(WSDISPLAYIO_GETCMAP, READ, struct_wsdisplay_cmap_sz);
+  _(WSDISPLAYIO_PUTCMAP, READ, struct_wsdisplay_cmap_sz);
+  _(WSDISPLAYIO_GVIDEO, WRITE, sizeof(unsigned int));
+  _(WSDISPLAYIO_SVIDEO, READ, sizeof(unsigned int));
+  _(WSDISPLAYIO_GCURPOS, WRITE, struct_wsdisplay_curpos_sz);
+  _(WSDISPLAYIO_SCURPOS, READ, struct_wsdisplay_curpos_sz);
+  _(WSDISPLAYIO_GCURMAX, WRITE, struct_wsdisplay_curpos_sz);
+  _(WSDISPLAYIO_GCURSOR, READWRITE, struct_wsdisplay_cursor_sz);
+  _(WSDISPLAYIO_SCURSOR, READ, struct_wsdisplay_cursor_sz);
+  _(WSDISPLAYIO_GMODE, WRITE, sizeof(unsigned int));
+  _(WSDISPLAYIO_SMODE, READ, sizeof(unsigned int));
+  _(WSDISPLAYIO_LDFONT, READ, struct_wsdisplay_font_sz);
+  _(WSDISPLAYIO_ADDSCREEN, READ, struct_wsdisplay_addscreendata_sz);
+  _(WSDISPLAYIO_DELSCREEN, READ, struct_wsdisplay_delscreendata_sz);
+  _(WSDISPLAYIO_SFONT, READ, struct_wsdisplay_usefontdata_sz);
+  _(_O_WSDISPLAYIO_SETKEYBOARD, READWRITE, struct_wsdisplay_kbddata_sz);
+  _(WSDISPLAYIO_GETPARAM, READWRITE, struct_wsdisplay_param_sz);
+  _(WSDISPLAYIO_SETPARAM, READWRITE, struct_wsdisplay_param_sz);
+  _(WSDISPLAYIO_GETACTIVESCREEN, WRITE, sizeof(int));
+  _(WSDISPLAYIO_GETWSCHAR, READWRITE, struct_wsdisplay_char_sz);
+  _(WSDISPLAYIO_PUTWSCHAR, READWRITE, struct_wsdisplay_char_sz);
+  _(WSDISPLAYIO_DGSCROLL, WRITE, struct_wsdisplay_scroll_data_sz);
+  _(WSDISPLAYIO_DSSCROLL, READ, struct_wsdisplay_scroll_data_sz);
+  _(WSDISPLAYIO_GMSGATTRS, WRITE, struct_wsdisplay_msgattrs_sz);
+  _(WSDISPLAYIO_SMSGATTRS, READ, struct_wsdisplay_msgattrs_sz);
+  _(WSDISPLAYIO_GBORDER, WRITE, sizeof(int));
+  _(WSDISPLAYIO_SBORDER, READ, sizeof(int));
+  _(WSDISPLAYIO_SSPLASH, READ, sizeof(int));
+  _(WSDISPLAYIO_SPROGRESS, READ, sizeof(int));
+  _(WSDISPLAYIO_LINEBYTES, WRITE, sizeof(unsigned int));
+  _(WSDISPLAYIO_SETVERSION, READ, sizeof(int));
+  _(WSMUXIO_ADD_DEVICE, READ, struct_wsmux_device_sz);
+  _(WSMUXIO_REMOVE_DEVICE, READ, struct_wsmux_device_sz);
+  _(WSMUXIO_LIST_DEVICES, READWRITE, struct_wsmux_device_list_sz);
+  _(WSMUXIO_INJECTEVENT, READ, struct_wscons_event_sz);
+  _(WSDISPLAYIO_GET_BUSID, WRITE, struct_wsdisplayio_bus_id_sz);
+  _(WSDISPLAYIO_GET_EDID, READWRITE, struct_wsdisplayio_edid_info_sz);
+  _(WSDISPLAYIO_SET_POLLING, READ, sizeof(int));
+  _(WSDISPLAYIO_GET_FBINFO, READWRITE, struct_wsdisplayio_fbinfo_sz);
+  _(WSDISPLAYIO_DOBLIT, READWRITE, struct_wsdisplayio_blit_sz);
+  _(WSDISPLAYIO_WAITBLIT, READWRITE, struct_wsdisplayio_blit_sz);
+  /* Entries from file: dev/biovar.h */
+  _(BIOCLOCATE, READWRITE, struct_bio_locate_sz);
+  _(BIOCINQ, READWRITE, struct_bioc_inq_sz);
+  _(BIOCDISK_NOVOL, READWRITE, struct_bioc_disk_sz);
+  _(BIOCDISK, READWRITE, struct_bioc_disk_sz);
+  _(BIOCVOL, READWRITE, struct_bioc_vol_sz);
+  _(BIOCALARM, READWRITE, struct_bioc_alarm_sz);
+  _(BIOCBLINK, READWRITE, struct_bioc_blink_sz);
+  _(BIOCSETSTATE, READWRITE, struct_bioc_setstate_sz);
+  _(BIOCVOLOPS, READWRITE, struct_bioc_volops_sz);
+  /* Entries from file: dev/md.h */
+  _(MD_GETCONF, WRITE, struct_md_conf_sz);
+  _(MD_SETCONF, READ, struct_md_conf_sz);
+  /* Entries from file: dev/ccdvar.h */
+  _(CCDIOCSET, READWRITE, struct_ccd_ioctl_sz);
+  _(CCDIOCCLR, READ, struct_ccd_ioctl_sz);
+  /* Entries from file: dev/cgdvar.h */
+  _(CGDIOCSET, READWRITE, struct_cgd_ioctl_sz);
+  _(CGDIOCCLR, READ, struct_cgd_ioctl_sz);
+  _(CGDIOCGET, READWRITE, struct_cgd_user_sz);
+  /* Entries from file: dev/fssvar.h */
+  _(FSSIOCSET, READ, struct_fss_set_sz);
+  _(FSSIOCGET, WRITE, struct_fss_get_sz);
+  _(FSSIOCCLR, NONE, 0);
+  _(FSSIOFSET, READ, sizeof(int));
+  _(FSSIOFGET, WRITE, sizeof(int));
+  /* Entries from file: dev/bluetooth/btdev.h */
+  _(BTDEV_ATTACH, READ, struct_plistref_sz);
+  _(BTDEV_DETACH, READ, struct_plistref_sz);
+  /* Entries from file: dev/bluetooth/btsco.h */
+  _(BTSCO_GETINFO, WRITE, struct_btsco_info_sz);
+  /* Entries from file: dev/kttcpio.h */
+  _(KTTCP_IO_SEND, READWRITE, struct_kttcp_io_args_sz);
+  _(KTTCP_IO_RECV, READWRITE, struct_kttcp_io_args_sz);
+  /* Entries from file: dev/lockstat.h */
+  _(IOC_LOCKSTAT_GVERSION, WRITE, sizeof(int));
+  _(IOC_LOCKSTAT_ENABLE, READ, struct_lsenable_sz);
+  _(IOC_LOCKSTAT_DISABLE, WRITE, struct_lsdisable_sz);
+  /* Entries from file: dev/vndvar.h */
+  _(VNDIOCSET, READWRITE, struct_vnd_ioctl_sz);
+  _(VNDIOCCLR, READ, struct_vnd_ioctl_sz);
+  _(VNDIOCGET, READWRITE, struct_vnd_user_sz);
+  /* Entries from file: dev/spkrio.h */
+  _(SPKRTONE, READ, struct_tone_sz);
+  _(SPKRTUNE, NONE, 0);
+  _(SPKRGETVOL, WRITE, sizeof(unsigned int));
+  _(SPKRSETVOL, READ, sizeof(unsigned int));
+#if defined(__x86_64__)
+  /* Entries from file: dev/nvmm/nvmm_ioctl.h */
+  _(NVMM_IOC_CAPABILITY, WRITE, struct_nvmm_ioc_capability_sz);
+  _(NVMM_IOC_MACHINE_CREATE, READWRITE, struct_nvmm_ioc_machine_create_sz);
+  _(NVMM_IOC_MACHINE_DESTROY, READ, struct_nvmm_ioc_machine_destroy_sz);
+  _(NVMM_IOC_MACHINE_CONFIGURE, READ, struct_nvmm_ioc_machine_configure_sz);
+  _(NVMM_IOC_VCPU_CREATE, READ, struct_nvmm_ioc_vcpu_create_sz);
+  _(NVMM_IOC_VCPU_DESTROY, READ, struct_nvmm_ioc_vcpu_destroy_sz);
+  _(NVMM_IOC_VCPU_CONFIGURE, READ, struct_nvmm_ioc_vcpu_configure_sz);
+  _(NVMM_IOC_VCPU_SETSTATE, READ, struct_nvmm_ioc_vcpu_setstate_sz);
+  _(NVMM_IOC_VCPU_GETSTATE, READ, struct_nvmm_ioc_vcpu_getstate_sz);
+  _(NVMM_IOC_VCPU_INJECT, READ, struct_nvmm_ioc_vcpu_inject_sz);
+  _(NVMM_IOC_VCPU_RUN, READWRITE, struct_nvmm_ioc_vcpu_run_sz);
+  _(NVMM_IOC_GPA_MAP, READ, struct_nvmm_ioc_gpa_map_sz);
+  _(NVMM_IOC_GPA_UNMAP, READ, struct_nvmm_ioc_gpa_unmap_sz);
+  _(NVMM_IOC_HVA_MAP, READ, struct_nvmm_ioc_hva_map_sz);
+  _(NVMM_IOC_HVA_UNMAP, READ, struct_nvmm_ioc_hva_unmap_sz);
+  _(NVMM_IOC_CTL, READ, struct_nvmm_ioc_ctl_sz);
+#endif
+  /* Entries from file: dev/spi/spi_io.h */
+  _(SPI_IOCTL_CONFIGURE, READ, struct_spi_ioctl_configure_sz);
+  _(SPI_IOCTL_TRANSFER, READ, struct_spi_ioctl_transfer_sz);
+  /* Entries from file: fs/autofs/autofs_ioctl.h */
+  _(AUTOFSREQUEST, WRITE, struct_autofs_daemon_request_sz);
+  _(AUTOFSDONE, READ, struct_autofs_daemon_done_sz);
+  /* Entries from file: net/bpf.h */
+  _(BIOCGBLEN, WRITE, sizeof(unsigned int));
+  _(BIOCSBLEN, READWRITE, sizeof(unsigned int));
+  _(BIOCSETF, READ, struct_bpf_program_sz);
+  _(BIOCFLUSH, NONE, 0);
+  _(BIOCPROMISC, NONE, 0);
+  _(BIOCGDLT, WRITE, sizeof(unsigned int));
+  _(BIOCGETIF, WRITE, struct_ifreq_sz);
+  _(BIOCSETIF, READ, struct_ifreq_sz);
+  _(BIOCGSTATS, WRITE, struct_bpf_stat_sz);
+  _(BIOCGSTATSOLD, WRITE, struct_bpf_stat_old_sz);
+  _(BIOCIMMEDIATE, READ, sizeof(unsigned int));
+  _(BIOCVERSION, WRITE, struct_bpf_version_sz);
+  _(BIOCSTCPF, READ, struct_bpf_program_sz);
+  _(BIOCSUDPF, READ, struct_bpf_program_sz);
+  _(BIOCGHDRCMPLT, WRITE, sizeof(unsigned int));
+  _(BIOCSHDRCMPLT, READ, sizeof(unsigned int));
+  _(BIOCSDLT, READ, sizeof(unsigned int));
+  _(BIOCGDLTLIST, READWRITE, struct_bpf_dltlist_sz);
+  _(BIOCGDIRECTION, WRITE, sizeof(unsigned int));
+  _(BIOCSDIRECTION, READ, sizeof(unsigned int));
+  _(BIOCSRTIMEOUT, READ, struct_timeval_sz);
+  _(BIOCGRTIMEOUT, WRITE, struct_timeval_sz);
+  _(BIOCGFEEDBACK, WRITE, sizeof(unsigned int));
+  _(BIOCSFEEDBACK, READ, sizeof(unsigned int));
+  /* Entries from file: net/if_gre.h */
+  _(GRESADDRS, READ, struct_ifreq_sz);
+  _(GRESADDRD, READ, struct_ifreq_sz);
+  _(GREGADDRS, READWRITE, struct_ifreq_sz);
+  _(GREGADDRD, READWRITE, struct_ifreq_sz);
+  _(GRESPROTO, READ, struct_ifreq_sz);
+  _(GREGPROTO, READWRITE, struct_ifreq_sz);
+  _(GRESSOCK, READ, struct_ifreq_sz);
+  _(GREDSOCK, READ, struct_ifreq_sz);
+  /* Entries from file: net/if_ppp.h */
+  _(PPPIOCGRAWIN, WRITE, struct_ppp_rawin_sz);
+  _(PPPIOCGFLAGS, WRITE, sizeof(int));
+  _(PPPIOCSFLAGS, READ, sizeof(int));
+  _(PPPIOCGASYNCMAP, WRITE, sizeof(int));
+  _(PPPIOCSASYNCMAP, READ, sizeof(int));
+  _(PPPIOCGUNIT, WRITE, sizeof(int));
+  _(PPPIOCGRASYNCMAP, WRITE, sizeof(int));
+  _(PPPIOCSRASYNCMAP, READ, sizeof(int));
+  _(PPPIOCGMRU, WRITE, sizeof(int));
+  _(PPPIOCSMRU, READ, sizeof(int));
+  _(PPPIOCSMAXCID, READ, sizeof(int));
+  _(PPPIOCGXASYNCMAP, WRITE, (8 * sizeof(u32)));
+  _(PPPIOCSXASYNCMAP, READ, (8 * sizeof(u32)));
+  _(PPPIOCXFERUNIT, NONE, 0);
+  _(PPPIOCSCOMPRESS, READ, struct_ppp_option_data_sz);
+  _(PPPIOCGNPMODE, READWRITE, struct_npioctl_sz);
+  _(PPPIOCSNPMODE, READ, struct_npioctl_sz);
+  _(PPPIOCGIDLE, WRITE, struct_ppp_idle_sz);
+  _(PPPIOCGMTU, WRITE, sizeof(int));
+  _(PPPIOCSMTU, READ, sizeof(int));
+  _(SIOCGPPPSTATS, READWRITE, struct_ifpppstatsreq_sz);
+  _(SIOCGPPPCSTATS, READWRITE, struct_ifpppcstatsreq_sz);
+  /* Entries from file: net/npf.h */
+  _(IOC_NPF_VERSION, WRITE, sizeof(int));
+  _(IOC_NPF_SWITCH, READ, sizeof(int));
+  _(IOC_NPF_LOAD, READWRITE, struct_nvlist_ref_sz);
+  _(IOC_NPF_TABLE, READ, struct_npf_ioctl_table_sz);
+  _(IOC_NPF_STATS, READ, sizeof(uptr));
+  _(IOC_NPF_SAVE, WRITE, struct_nvlist_ref_sz);
+  _(IOC_NPF_RULE, READWRITE, struct_nvlist_ref_sz);
+  _(IOC_NPF_CONN_LOOKUP, READWRITE, struct_nvlist_ref_sz);
+  _(IOC_NPF_TABLE_REPLACE, READWRITE, struct_nvlist_ref_sz);
+  /* Entries from file: net/if_pppoe.h */
+  _(PPPOESETPARMS, READ, struct_pppoediscparms_sz);
+  _(PPPOEGETPARMS, READWRITE, struct_pppoediscparms_sz);
+  _(PPPOEGETSESSION, READWRITE, struct_pppoeconnectionstate_sz);
+  /* Entries from file: net/if_sppp.h */
+  _(SPPPGETAUTHCFG, READWRITE, struct_spppauthcfg_sz);
+  _(SPPPSETAUTHCFG, READ, struct_spppauthcfg_sz);
+  _(SPPPGETLCPCFG, READWRITE, struct_sppplcpcfg_sz);
+  _(SPPPSETLCPCFG, READ, struct_sppplcpcfg_sz);
+  _(SPPPGETSTATUS, READWRITE, struct_spppstatus_sz);
+  _(SPPPGETSTATUSNCP, READWRITE, struct_spppstatusncp_sz);
+  _(SPPPGETIDLETO, READWRITE, struct_spppidletimeout_sz);
+  _(SPPPSETIDLETO, READ, struct_spppidletimeout_sz);
+  _(SPPPGETAUTHFAILURES, READWRITE, struct_spppauthfailurestats_sz);
+  _(SPPPSETAUTHFAILURE, READ, struct_spppauthfailuresettings_sz);
+  _(SPPPSETDNSOPTS, READ, struct_spppdnssettings_sz);
+  _(SPPPGETDNSOPTS, READWRITE, struct_spppdnssettings_sz);
+  _(SPPPGETDNSADDRS, READWRITE, struct_spppdnsaddrs_sz);
+  _(SPPPSETKEEPALIVE, READ, struct_spppkeepalivesettings_sz);
+  _(SPPPGETKEEPALIVE, READWRITE, struct_spppkeepalivesettings_sz);
+  /* Entries from file: net/if_srt.h */
+  _(SRT_GETNRT, WRITE, sizeof(unsigned int));
+  _(SRT_GETRT, READWRITE, struct_srt_rt_sz);
+  _(SRT_SETRT, READ, struct_srt_rt_sz);
+  _(SRT_DELRT, READ, sizeof(unsigned int));
+  _(SRT_SFLAGS, READ, sizeof(unsigned int));
+  _(SRT_GFLAGS, WRITE, sizeof(unsigned int));
+  _(SRT_SGFLAGS, READWRITE, sizeof(unsigned int));
+  _(SRT_DEBUG, READ, sizeof(uptr));
+  /* Entries from file: net/if_tap.h */
+  _(TAPGIFNAME, WRITE, struct_ifreq_sz);
+  /* Entries from file: net/if_tun.h */
+  _(TUNSDEBUG, READ, sizeof(int));
+  _(TUNGDEBUG, WRITE, sizeof(int));
+  _(TUNSIFMODE, READ, sizeof(int));
+  _(TUNSIFHEAD, READ, sizeof(int));
+  _(TUNGIFHEAD, WRITE, sizeof(int));
+  /* Entries from file: net/pfvar.h */
+  _(DIOCSTART, NONE, 0);
+  _(DIOCSTOP, NONE, 0);
+  _(DIOCADDRULE, READWRITE, struct_pfioc_rule_sz);
+  _(DIOCGETRULES, READWRITE, struct_pfioc_rule_sz);
+  _(DIOCGETRULE, READWRITE, struct_pfioc_rule_sz);
+  _(DIOCSETLCK, READWRITE, sizeof(u32));
+  _(DIOCCLRSTATES, READWRITE, struct_pfioc_state_kill_sz);
+  _(DIOCGETSTATE, READWRITE, struct_pfioc_state_sz);
+  _(DIOCSETSTATUSIF, READWRITE, struct_pfioc_if_sz);
+  _(DIOCGETSTATUS, READWRITE, struct_pf_status_sz);
+  _(DIOCCLRSTATUS, NONE, 0);
+  _(DIOCNATLOOK, READWRITE, struct_pfioc_natlook_sz);
+  _(DIOCSETDEBUG, READWRITE, sizeof(u32));
+  _(DIOCGETSTATES, READWRITE, struct_pfioc_states_sz);
+  _(DIOCCHANGERULE, READWRITE, struct_pfioc_rule_sz);
+  _(DIOCSETTIMEOUT, READWRITE, struct_pfioc_tm_sz);
+  _(DIOCGETTIMEOUT, READWRITE, struct_pfioc_tm_sz);
+  _(DIOCADDSTATE, READWRITE, struct_pfioc_state_sz);
+  _(DIOCCLRRULECTRS, NONE, 0);
+  _(DIOCGETLIMIT, READWRITE, struct_pfioc_limit_sz);
+  _(DIOCSETLIMIT, READWRITE, struct_pfioc_limit_sz);
+  _(DIOCKILLSTATES, READWRITE, struct_pfioc_state_kill_sz);
+  _(DIOCSTARTALTQ, NONE, 0);
+  _(DIOCSTOPALTQ, NONE, 0);
+  _(DIOCADDALTQ, READWRITE, struct_pfioc_altq_sz);
+  _(DIOCGETALTQS, READWRITE, struct_pfioc_altq_sz);
+  _(DIOCGETALTQ, READWRITE, struct_pfioc_altq_sz);
+  _(DIOCCHANGEALTQ, READWRITE, struct_pfioc_altq_sz);
+  _(DIOCGETQSTATS, READWRITE, struct_pfioc_qstats_sz);
+  _(DIOCBEGINADDRS, READWRITE, struct_pfioc_pooladdr_sz);
+  _(DIOCADDADDR, READWRITE, struct_pfioc_pooladdr_sz);
+  _(DIOCGETADDRS, READWRITE, struct_pfioc_pooladdr_sz);
+  _(DIOCGETADDR, READWRITE, struct_pfioc_pooladdr_sz);
+  _(DIOCCHANGEADDR, READWRITE, struct_pfioc_pooladdr_sz);
+  _(DIOCADDSTATES, READWRITE, struct_pfioc_states_sz);
+  _(DIOCGETRULESETS, READWRITE, struct_pfioc_ruleset_sz);
+  _(DIOCGETRULESET, READWRITE, struct_pfioc_ruleset_sz);
+  _(DIOCRCLRTABLES, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRADDTABLES, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRDELTABLES, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRGETTABLES, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRGETTSTATS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRCLRTSTATS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRCLRADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRADDADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRDELADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRSETADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRGETADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRGETASTATS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRCLRASTATS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRTSTADDRS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRSETTFLAGS, READWRITE, struct_pfioc_table_sz);
+  _(DIOCRINADEFINE, READWRITE, struct_pfioc_table_sz);
+  _(DIOCOSFPFLUSH, NONE, 0);
+  _(DIOCOSFPADD, READWRITE, struct_pf_osfp_ioctl_sz);
+  _(DIOCOSFPGET, READWRITE, struct_pf_osfp_ioctl_sz);
+  _(DIOCXBEGIN, READWRITE, struct_pfioc_trans_sz);
+  _(DIOCXCOMMIT, READWRITE, struct_pfioc_trans_sz);
+  _(DIOCXROLLBACK, READWRITE, struct_pfioc_trans_sz);
+  _(DIOCGETSRCNODES, READWRITE, struct_pfioc_src_nodes_sz);
+  _(DIOCCLRSRCNODES, NONE, 0);
+  _(DIOCSETHOSTID, READWRITE, sizeof(u32));
+  _(DIOCIGETIFACES, READWRITE, struct_pfioc_iface_sz);
+  _(DIOCSETIFFLAG, READWRITE, struct_pfioc_iface_sz);
+  _(DIOCCLRIFFLAG, READWRITE, struct_pfioc_iface_sz);
+  _(DIOCKILLSRCNODES, READWRITE, struct_pfioc_src_node_kill_sz);
+  /* Entries from file: netbt/hci.h */
+  _(SIOCGBTINFO, READWRITE, struct_btreq_sz);
+  _(SIOCGBTINFOA, READWRITE, struct_btreq_sz);
+  _(SIOCNBTINFO, READWRITE, struct_btreq_sz);
+  _(SIOCSBTFLAGS, READWRITE, struct_btreq_sz);
+  _(SIOCSBTPOLICY, READWRITE, struct_btreq_sz);
+  _(SIOCSBTPTYPE, READWRITE, struct_btreq_sz);
+  _(SIOCGBTSTATS, READWRITE, struct_btreq_sz);
+  _(SIOCZBTSTATS, READWRITE, struct_btreq_sz);
+  _(SIOCBTDUMP, READ, struct_btreq_sz);
+  _(SIOCSBTSCOMTU, READWRITE, struct_btreq_sz);
+  _(SIOCGBTFEAT, READWRITE, struct_btreq_sz);
+  /* Entries from file: netinet/ip_nat.h */
+  _(SIOCADNAT, READ, struct_ipfobj_sz);
+  _(SIOCRMNAT, READ, struct_ipfobj_sz);
+  _(SIOCGNATS, READWRITE, struct_ipfobj_sz);
+  _(SIOCGNATL, READWRITE, struct_ipfobj_sz);
+  _(SIOCPURGENAT, READWRITE, struct_ipfobj_sz);
+  /* Entries from file: netinet/sctp_uio.h */
+  _(SIOCCONNECTX, READWRITE, struct_sctp_connectx_addrs_sz);
+  _(SIOCCONNECTXDEL, READWRITE, struct_sctp_connectx_addrs_sz);
+  /* Entries from file: netinet6/in6_var.h */
+  _(SIOCSIFINFO_FLAGS, READWRITE, struct_in6_ndireq_sz);
+  _(SIOCAADDRCTL_POLICY, READ, struct_in6_addrpolicy_sz);
+  _(SIOCDADDRCTL_POLICY, READ, struct_in6_addrpolicy_sz);
+  /* Entries from file: netsmb/smb_dev.h */
+  _(SMBIOC_OPENSESSION, READ, struct_smbioc_ossn_sz);
+  _(SMBIOC_OPENSHARE, READ, struct_smbioc_oshare_sz);
+  _(SMBIOC_REQUEST, READWRITE, struct_smbioc_rq_sz);
+  _(SMBIOC_SETFLAGS, READ, struct_smbioc_flags_sz);
+  _(SMBIOC_LOOKUP, READ, struct_smbioc_lookup_sz);
+  _(SMBIOC_READ, READWRITE, struct_smbioc_rw_sz);
+  _(SMBIOC_WRITE, READWRITE, struct_smbioc_rw_sz);
+  /* Entries from file: sys/agpio.h */
+  _(AGPIOC_INFO, WRITE, struct__agp_info_sz);
+  _(AGPIOC_ACQUIRE, NONE, 0);
+  _(AGPIOC_RELEASE, NONE, 0);
+  _(AGPIOC_SETUP, READ, struct__agp_setup_sz);
+  _(AGPIOC_ALLOCATE, READWRITE, struct__agp_allocate_sz);
+  _(AGPIOC_DEALLOCATE, READ, sizeof(int));
+  _(AGPIOC_BIND, READ, struct__agp_bind_sz);
+  _(AGPIOC_UNBIND, READ, struct__agp_unbind_sz);
+  /* Entries from file: sys/audioio.h */
+  _(AUDIO_GETINFO, WRITE, struct_audio_info_sz);
+  _(AUDIO_SETINFO, READWRITE, struct_audio_info_sz);
+  _(AUDIO_DRAIN, NONE, 0);
+  _(AUDIO_FLUSH, NONE, 0);
+  _(AUDIO_WSEEK, WRITE, sizeof(unsigned long));
+  _(AUDIO_RERROR, WRITE, sizeof(int));
+  _(AUDIO_GETDEV, WRITE, struct_audio_device_sz);
+  _(AUDIO_GETENC, READWRITE, struct_audio_encoding_sz);
+  _(AUDIO_GETFD, WRITE, sizeof(int));
+  _(AUDIO_SETFD, READWRITE, sizeof(int));
+  _(AUDIO_PERROR, WRITE, sizeof(int));
+  _(AUDIO_GETIOFFS, WRITE, struct_audio_offset_sz);
+  _(AUDIO_GETOOFFS, WRITE, struct_audio_offset_sz);
+  _(AUDIO_GETPROPS, WRITE, sizeof(int));
+  _(AUDIO_GETBUFINFO, WRITE, struct_audio_info_sz);
+  _(AUDIO_SETCHAN, READ, sizeof(int));
+  _(AUDIO_GETCHAN, WRITE, sizeof(int));
+  _(AUDIO_QUERYFORMAT, READWRITE, struct_audio_format_query_sz);
+  _(AUDIO_GETFORMAT, WRITE, struct_audio_info_sz);
+  _(AUDIO_SETFORMAT, READ, struct_audio_info_sz);
+  _(AUDIO_MIXER_READ, READWRITE, struct_mixer_ctrl_sz);
+  _(AUDIO_MIXER_WRITE, READWRITE, struct_mixer_ctrl_sz);
+  _(AUDIO_MIXER_DEVINFO, READWRITE, struct_mixer_devinfo_sz);
+  /* Entries from file: sys/ataio.h */
+  _(ATAIOCCOMMAND, READWRITE, struct_atareq_sz);
+  _(ATABUSIOSCAN, READ, struct_atabusioscan_args_sz);
+  _(ATABUSIORESET, NONE, 0);
+  _(ATABUSIODETACH, READ, struct_atabusiodetach_args_sz);
+  /* Entries from file: sys/cdio.h */
+  _(CDIOCPLAYTRACKS, READ, struct_ioc_play_track_sz);
+  _(CDIOCPLAYBLOCKS, READ, struct_ioc_play_blocks_sz);
+  _(CDIOCREADSUBCHANNEL, READWRITE, struct_ioc_read_subchannel_sz);
+  _(CDIOREADTOCHEADER, WRITE, struct_ioc_toc_header_sz);
+  _(CDIOREADTOCENTRIES, READWRITE, struct_ioc_read_toc_entry_sz);
+  _(CDIOREADMSADDR, READWRITE, sizeof(int));
+  _(CDIOCSETPATCH, READ, struct_ioc_patch_sz);
+  _(CDIOCGETVOL, WRITE, struct_ioc_vol_sz);
+  _(CDIOCSETVOL, READ, struct_ioc_vol_sz);
+  _(CDIOCSETMONO, NONE, 0);
+  _(CDIOCSETSTEREO, NONE, 0);
+  _(CDIOCSETMUTE, NONE, 0);
+  _(CDIOCSETLEFT, NONE, 0);
+  _(CDIOCSETRIGHT, NONE, 0);
+  _(CDIOCSETDEBUG, NONE, 0);
+  _(CDIOCCLRDEBUG, NONE, 0);
+  _(CDIOCPAUSE, NONE, 0);
+  _(CDIOCRESUME, NONE, 0);
+  _(CDIOCRESET, NONE, 0);
+  _(CDIOCSTART, NONE, 0);
+  _(CDIOCSTOP, NONE, 0);
+  _(CDIOCEJECT, NONE, 0);
+  _(CDIOCALLOW, NONE, 0);
+  _(CDIOCPREVENT, NONE, 0);
+  _(CDIOCCLOSE, NONE, 0);
+  _(CDIOCPLAYMSF, READ, struct_ioc_play_msf_sz);
+  _(CDIOCLOADUNLOAD, READ, struct_ioc_load_unload_sz);
+  /* Entries from file: sys/chio.h */
+  _(CHIOMOVE, READ, struct_changer_move_request_sz);
+  _(CHIOEXCHANGE, READ, struct_changer_exchange_request_sz);
+  _(CHIOPOSITION, READ, struct_changer_position_request_sz);
+  _(CHIOSPICKER, READ, sizeof(int));
+  _(CHIOGPARAMS, WRITE, struct_changer_params_sz);
+  _(CHIOIELEM, NONE, 0);
+  _(OCHIOGSTATUS, READ, struct_ochanger_element_status_request_sz);
+  _(CHIOGSTATUS, READ, struct_changer_element_status_request_sz);
+  _(CHIOSVOLTAG, READ, struct_changer_set_voltag_request_sz);
+  /* Entries from file: sys/clockctl.h */
+  _(CLOCKCTL_SETTIMEOFDAY, READ, struct_clockctl_settimeofday_sz);
+  _(CLOCKCTL_ADJTIME, READWRITE, struct_clockctl_adjtime_sz);
+  _(CLOCKCTL_CLOCK_SETTIME, READ, struct_clockctl_clock_settime_sz);
+  _(CLOCKCTL_NTP_ADJTIME, READWRITE, struct_clockctl_ntp_adjtime_sz);
+  /* Entries from file: sys/cpuio.h */
+  _(IOC_CPU_SETSTATE, READ, struct_cpustate_sz);
+  _(IOC_CPU_GETSTATE, READWRITE, struct_cpustate_sz);
+  _(IOC_CPU_GETCOUNT, WRITE, sizeof(int));
+  _(IOC_CPU_MAPID, READWRITE, sizeof(int));
+  _(IOC_CPU_UCODE_GET_VERSION, READWRITE, struct_cpu_ucode_version_sz);
+  _(IOC_CPU_UCODE_APPLY, READ, struct_cpu_ucode_sz);
+  /* Entries from file: sys/dkio.h */
+  _(DIOCGDINFO, WRITE, struct_disklabel_sz);
+  _(DIOCSDINFO, READ, struct_disklabel_sz);
+  _(DIOCWDINFO, READ, 0);
+  _(DIOCRFORMAT, READWRITE, struct_format_op_sz);
+  _(DIOCWFORMAT, READWRITE, struct_format_op_sz);
+  _(DIOCSSTEP, READ, sizeof(int));
+  _(DIOCSRETRIES, READ, sizeof(int));
+  _(DIOCKLABEL, READ, sizeof(int));
+  _(DIOCWLABEL, READ, sizeof(int));
+  _(DIOCSBAD, READ, struct_dkbad_sz);
+  _(DIOCEJECT, READ, sizeof(int));
+  _(ODIOCEJECT, NONE, 0);
+  _(DIOCLOCK, READ, sizeof(int));
+  _(DIOCGDEFLABEL, WRITE, struct_disklabel_sz);
+  _(DIOCCLRLABEL, NONE, 0);
+  _(DIOCGCACHE, WRITE, sizeof(int));
+  _(DIOCSCACHE, READ, sizeof(int));
+  _(DIOCCACHESYNC, READ, sizeof(int));
+  _(DIOCBSLIST, READWRITE, struct_disk_badsecinfo_sz);
+  _(DIOCBSFLUSH, NONE, 0);
+  _(DIOCAWEDGE, READWRITE, struct_dkwedge_info_sz);
+  _(DIOCGWEDGEINFO, WRITE, struct_dkwedge_info_sz);
+  _(DIOCDWEDGE, READ, struct_dkwedge_info_sz);
+  _(DIOCLWEDGES, READWRITE, struct_dkwedge_list_sz);
+  _(DIOCGSTRATEGY, WRITE, struct_disk_strategy_sz);
+  _(DIOCSSTRATEGY, READ, struct_disk_strategy_sz);
+  _(DIOCGDISKINFO, WRITE, struct_plistref_sz);
+  _(DIOCTUR, WRITE, sizeof(int));
+  _(DIOCMWEDGES, WRITE, sizeof(int));
+  _(DIOCGSECTORSIZE, WRITE, sizeof(unsigned int));
+  _(DIOCGMEDIASIZE, WRITE, sizeof(uptr));
+  _(DIOCRMWEDGES, WRITE, sizeof(int));
+  /* Entries from file: sys/drvctlio.h */
+  _(DRVDETACHDEV, READ, struct_devdetachargs_sz);
+  _(DRVRESCANBUS, READ, struct_devrescanargs_sz);
+  _(DRVCTLCOMMAND, READWRITE, struct_plistref_sz);
+  _(DRVRESUMEDEV, READ, struct_devpmargs_sz);
+  _(DRVLISTDEV, READWRITE, struct_devlistargs_sz);
+  _(DRVGETEVENT, WRITE, struct_plistref_sz);
+  _(DRVSUSPENDDEV, READ, struct_devpmargs_sz);
+  /* Entries from file: sys/dvdio.h */
+  _(DVD_READ_STRUCT, READWRITE, union_dvd_struct_sz);
+  _(DVD_WRITE_STRUCT, READWRITE, union_dvd_struct_sz);
+  _(DVD_AUTH, READWRITE, union_dvd_authinfo_sz);
+  /* Entries from file: sys/envsys.h */
+  _(ENVSYS_GETDICTIONARY, READWRITE, struct_plistref_sz);
+  _(ENVSYS_SETDICTIONARY, READWRITE, struct_plistref_sz);
+  _(ENVSYS_REMOVEPROPS, READWRITE, struct_plistref_sz);
+  _(ENVSYS_GTREDATA, READWRITE, struct_envsys_tre_data_sz);
+  _(ENVSYS_GTREINFO, READWRITE, struct_envsys_basic_info_sz);
+  /* Entries from file: sys/event.h */
+  _(KFILTER_BYFILTER, READWRITE, struct_kfilter_mapping_sz);
+  _(KFILTER_BYNAME, READWRITE, struct_kfilter_mapping_sz);
+  /* Entries from file: sys/fdio.h */
+  _(FDIOCGETOPTS, WRITE, 0);
+  _(FDIOCSETOPTS, READ, sizeof(int));
+  _(FDIOCSETFORMAT, READ, struct_fdformat_parms_sz);
+  _(FDIOCGETFORMAT, WRITE, struct_fdformat_parms_sz);
+  _(FDIOCFORMAT_TRACK, READ, struct_fdformat_cmd_sz);
+  /* Entries from file: sys/filio.h */
+  _(FIOCLEX, NONE, 0);
+  _(FIONCLEX, NONE, 0);
+  _(FIOSEEKDATA, READWRITE, sizeof(uptr));
+  _(FIOSEEKHOLE, READWRITE, sizeof(uptr));
+  _(FIONREAD, WRITE, sizeof(int));
+  _(FIONBIO, READ, sizeof(int));
+  _(FIOASYNC, READ, sizeof(int));
+  _(FIOSETOWN, READ, sizeof(int));
+  _(FIOGETOWN, WRITE, sizeof(int));
+  _(OFIOGETBMAP, READWRITE, sizeof(u32));
+  _(FIOGETBMAP, READWRITE, sizeof(u64));
+  _(FIONWRITE, WRITE, sizeof(int));
+  _(FIONSPACE, WRITE, sizeof(int));
+  /* Entries from file: sys/gpio.h */
+  _(GPIOINFO, WRITE, struct_gpio_info_sz);
+  _(GPIOSET, READWRITE, struct_gpio_set_sz);
+  _(GPIOUNSET, READWRITE, struct_gpio_set_sz);
+  _(GPIOREAD, READWRITE, struct_gpio_req_sz);
+  _(GPIOWRITE, READWRITE, struct_gpio_req_sz);
+  _(GPIOTOGGLE, READWRITE, struct_gpio_req_sz);
+  _(GPIOATTACH, READWRITE, struct_gpio_attach_sz);
+  /* Entries from file: sys/ioctl.h */
+  _(PTIOCNETBSD, READ, struct_ioctl_pt_sz);
+  _(PTIOCSUNOS, READ, struct_ioctl_pt_sz);
+  _(PTIOCLINUX, READ, struct_ioctl_pt_sz);
+  _(PTIOCFREEBSD, READ, struct_ioctl_pt_sz);
+  _(PTIOCULTRIX, READ, struct_ioctl_pt_sz);
+  /* Entries from file: sys/ioctl_compat.h */
+  _(TIOCHPCL, NONE, 0);
+  _(TIOCGETP, WRITE, struct_sgttyb_sz);
+  _(TIOCSETP, READ, struct_sgttyb_sz);
+  _(TIOCSETN, READ, 0);
+  _(TIOCSETC, READ, struct_tchars_sz);
+  _(TIOCGETC, WRITE, struct_tchars_sz);
+  _(TIOCLBIS, READ, sizeof(int));
+  _(TIOCLBIC, READ, sizeof(int));
+  _(TIOCLSET, READ, sizeof(int));
+  _(TIOCLGET, WRITE, sizeof(int));
+  _(TIOCSLTC, READ, struct_ltchars_sz);
+  _(TIOCGLTC, WRITE, struct_ltchars_sz);
+  _(OTIOCCONS, NONE, 0);
+  /* Entries from file: sys/joystick.h */
+  _(JOY_SETTIMEOUT, READ, sizeof(int));
+  _(JOY_GETTIMEOUT, WRITE, sizeof(int));
+  _(JOY_SET_X_OFFSET, READ, sizeof(int));
+  _(JOY_SET_Y_OFFSET, READ, sizeof(int));
+  _(JOY_GET_Y_OFFSET, WRITE, sizeof(int));
+  /* Entries from file: sys/ksyms.h */
+  _(OKIOCGSYMBOL, READ, struct_ksyms_ogsymbol_sz);
+  _(OKIOCGVALUE, READ, struct_ksyms_ogsymbol_sz);
+  _(KIOCGSIZE, WRITE, sizeof(int));
+  _(KIOCGVALUE, READWRITE, struct_ksyms_gvalue_sz);
+  _(KIOCGSYMBOL, READWRITE, struct_ksyms_gsymbol_sz);
+  /* Entries from file: sys/lua.h */
+  _(LUAINFO, READWRITE, struct_lua_info_sz);
+  _(LUACREATE, READWRITE, struct_lua_create_sz);
+  _(LUADESTROY, READWRITE, struct_lua_create_sz);
+  _(LUAREQUIRE, READWRITE, struct_lua_require_sz);
+  _(LUALOAD, READWRITE, struct_lua_load_sz);
+  /* Entries from file: sys/midiio.h */
+  _(MIDI_PRETIME, READWRITE, sizeof(int));
+  _(MIDI_MPUMODE, READWRITE, sizeof(int));
+  _(MIDI_MPUCMD, READWRITE, struct_mpu_command_rec_sz);
+  _(SEQUENCER_RESET, NONE, 0);
+  _(SEQUENCER_SYNC, NONE, 0);
+  _(SEQUENCER_INFO, READWRITE, struct_synth_info_sz);
+  _(SEQUENCER_CTRLRATE, READWRITE, sizeof(int));
+  _(SEQUENCER_GETOUTCOUNT, WRITE, sizeof(int));
+  _(SEQUENCER_GETINCOUNT, WRITE, sizeof(int));
+  _(SEQUENCER_RESETSAMPLES, READ, sizeof(int));
+  _(SEQUENCER_NRSYNTHS, WRITE, sizeof(int));
+  _(SEQUENCER_NRMIDIS, WRITE, sizeof(int));
+  _(SEQUENCER_THRESHOLD, READ, sizeof(int));
+  _(SEQUENCER_MEMAVL, READWRITE, sizeof(int));
+  _(SEQUENCER_PANIC, NONE, 0);
+  _(SEQUENCER_OUTOFBAND, READ, struct_seq_event_rec_sz);
+  _(SEQUENCER_GETTIME, WRITE, sizeof(int));
+  _(SEQUENCER_TMR_TIMEBASE, READWRITE, sizeof(int));
+  _(SEQUENCER_TMR_START, NONE, 0);
+  _(SEQUENCER_TMR_STOP, NONE, 0);
+  _(SEQUENCER_TMR_CONTINUE, NONE, 0);
+  _(SEQUENCER_TMR_TEMPO, READWRITE, sizeof(int));
+  _(SEQUENCER_TMR_SOURCE, READWRITE, sizeof(int));
+  _(SEQUENCER_TMR_METRONOME, READ, sizeof(int));
+  _(SEQUENCER_TMR_SELECT, READ, sizeof(int));
+  /* Entries from file: sys/mtio.h */
+  _(MTIOCTOP, READ, struct_mtop_sz);
+  _(MTIOCGET, WRITE, struct_mtget_sz);
+  _(MTIOCIEOT, NONE, 0);
+  _(MTIOCEEOT, NONE, 0);
+  _(MTIOCRDSPOS, WRITE, sizeof(u32));
+  _(MTIOCRDHPOS, WRITE, sizeof(u32));
+  _(MTIOCSLOCATE, READ, sizeof(u32));
+  _(MTIOCHLOCATE, READ, sizeof(u32));
+  /* Entries from file: sys/power.h */
+  _(POWER_EVENT_RECVDICT, READWRITE, struct_plistref_sz);
+  _(POWER_IOC_GET_TYPE, WRITE, struct_power_type_sz);
+  /* Entries from file: sys/radioio.h */
+  _(RIOCGINFO, WRITE, struct_radio_info_sz);
+  _(RIOCSINFO, READWRITE, struct_radio_info_sz);
+  _(RIOCSSRCH, READ, sizeof(int));
+  /* Entries from file: sys/rndio.h */
+  _(RNDGETENTCNT, WRITE, sizeof(u32));
+  _(RNDGETSRCNUM, READWRITE, struct_rndstat_sz);
+  _(RNDGETSRCNAME, READWRITE, struct_rndstat_name_sz);
+  _(RNDCTL, READ, struct_rndctl_sz);
+  _(RNDADDDATA, READ, struct_rnddata_sz);
+  _(RNDGETPOOLSTAT, WRITE, struct_rndpoolstat_sz);
+  _(RNDGETESTNUM, READWRITE, struct_rndstat_est_sz);
+  _(RNDGETESTNAME, READWRITE, struct_rndstat_est_name_sz);
+  /* Entries from file: sys/scanio.h */
+  _(SCIOCGET, WRITE, struct_scan_io_sz);
+  _(SCIOCSET, READ, struct_scan_io_sz);
+  _(SCIOCRESTART, NONE, 0);
+  /* Entries from file: sys/scsiio.h */
+  _(SCIOCCOMMAND, READWRITE, struct_scsireq_sz);
+  _(SCIOCDEBUG, READ, sizeof(int));
+  _(SCIOCIDENTIFY, WRITE, struct_scsi_addr_sz);
+  _(OSCIOCIDENTIFY, WRITE, struct_oscsi_addr_sz);
+  _(SCIOCDECONFIG, NONE, 0);
+  _(SCIOCRECONFIG, NONE, 0);
+  _(SCIOCRESET, NONE, 0);
+  _(SCBUSIOSCAN, READ, struct_scbusioscan_args_sz);
+  _(SCBUSIORESET, NONE, 0);
+  _(SCBUSIODETACH, READ, struct_scbusiodetach_args_sz);
+  _(SCBUSACCEL, READ, struct_scbusaccel_args_sz);
+  /* Entries from file: sys/sockio.h */
+  _(SIOCSHIWAT, READ, sizeof(int));
+  _(SIOCGHIWAT, WRITE, sizeof(int));
+  _(SIOCSLOWAT, READ, sizeof(int));
+  _(SIOCGLOWAT, WRITE, sizeof(int));
+  _(SIOCATMARK, WRITE, sizeof(int));
+  _(SIOCSPGRP, READ, sizeof(int));
+  _(SIOCGPGRP, WRITE, sizeof(int));
+  _(SIOCPEELOFF, READWRITE, sizeof(int));
+  _(SIOCADDRT, READ, struct_ortentry_sz);
+  _(SIOCDELRT, READ, struct_ortentry_sz);
+  _(SIOCSIFADDR, READ, struct_ifreq_sz);
+  _(SIOCGIFADDR, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFDSTADDR, READ, struct_ifreq_sz);
+  _(SIOCGIFDSTADDR, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFFLAGS, READ, struct_ifreq_sz);
+  _(SIOCGIFFLAGS, READWRITE, struct_ifreq_sz);
+  _(SIOCGIFBRDADDR, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFBRDADDR, READ, struct_ifreq_sz);
+  _(SIOCGIFCONF, READWRITE, struct_ifconf_sz);
+  _(SIOCGIFNETMASK, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFNETMASK, READ, struct_ifreq_sz);
+  _(SIOCGIFMETRIC, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFMETRIC, READ, struct_ifreq_sz);
+  _(SIOCDIFADDR, READ, struct_ifreq_sz);
+  _(SIOCAIFADDR, READ, struct_ifaliasreq_sz);
+  _(SIOCGIFALIAS, READWRITE, struct_ifaliasreq_sz);
+  _(SIOCGIFAFLAG_IN, READWRITE, struct_ifreq_sz);
+  _(SIOCALIFADDR, READ, struct_if_laddrreq_sz);
+  _(SIOCGLIFADDR, READWRITE, struct_if_laddrreq_sz);
+  _(SIOCDLIFADDR, READ, struct_if_laddrreq_sz);
+  _(SIOCSIFADDRPREF, READ, struct_if_addrprefreq_sz);
+  _(SIOCGIFADDRPREF, READWRITE, struct_if_addrprefreq_sz);
+  _(SIOCADDMULTI, READ, struct_ifreq_sz);
+  _(SIOCDELMULTI, READ, struct_ifreq_sz);
+  _(SIOCGETVIFCNT, READWRITE, struct_sioc_vif_req_sz);
+  _(SIOCGETSGCNT, READWRITE, struct_sioc_sg_req_sz);
+  _(SIOCSIFMEDIA, READWRITE, struct_ifreq_sz);
+  _(SIOCGIFMEDIA, READWRITE, struct_ifmediareq_sz);
+  _(SIOCSIFGENERIC, READ, struct_ifreq_sz);
+  _(SIOCGIFGENERIC, READWRITE, struct_ifreq_sz);
+  _(SIOCSIFPHYADDR, READ, struct_ifaliasreq_sz);
+  _(SIOCGIFPSRCADDR, READWRITE, struct_ifreq_sz);
+  _(SIOCGIFPDSTADDR, READWRITE, struct_ifreq_sz);
+  _(SIOCDIFPHYADDR, READ, struct_ifreq_sz);
+  _(SIOCSLIFPHYADDR, READ, struct_if_laddrreq_sz);
+  _(SIOCGLIFPHYADDR, READWRITE, struct_if_laddrreq_sz);
+  _(SIOCSIFMTU, READ, struct_ifreq_sz);
+  _(SIOCGIFMTU, READWRITE, struct_ifreq_sz);
+  _(SIOCSDRVSPEC, READ, struct_ifdrv_sz);
+  _(SIOCGDRVSPEC, READWRITE, struct_ifdrv_sz);
+  _(SIOCIFCREATE, READ, struct_ifreq_sz);
+  _(SIOCIFDESTROY, READ, struct_ifreq_sz);
+  _(SIOCIFGCLONERS, READWRITE, struct_if_clonereq_sz);
+  _(SIOCGIFDLT, READWRITE, struct_ifreq_sz);
+  _(SIOCGIFCAP, READWRITE, struct_ifcapreq_sz);
+  _(SIOCSIFCAP, READ, struct_ifcapreq_sz);
+  _(SIOCSVH, READWRITE, struct_ifreq_sz);
+  _(SIOCGVH, READWRITE, struct_ifreq_sz);
+  _(SIOCINITIFADDR, READWRITE, struct_ifaddr_sz);
+  _(SIOCGIFDATA, READWRITE, struct_ifdatareq_sz);
+  _(SIOCZIFDATA, READWRITE, struct_ifdatareq_sz);
+  _(SIOCGLINKSTR, READWRITE, struct_ifdrv_sz);
+  _(SIOCSLINKSTR, READ, struct_ifdrv_sz);
+  _(SIOCGETHERCAP, READWRITE, struct_eccapreq_sz);
+  _(SIOCGIFINDEX, READWRITE, struct_ifreq_sz);
+  _(SIOCSETHERCAP, READ, struct_eccapreq_sz);
+  _(SIOCSIFDESCR, READ, struct_ifreq_sz);
+  _(SIOCGIFDESCR, READWRITE, struct_ifreq_sz);
+  _(SIOCGUMBINFO, READWRITE, struct_ifreq_sz);
+  _(SIOCSUMBPARAM, READ, struct_ifreq_sz);
+  _(SIOCGUMBPARAM, READWRITE, struct_ifreq_sz);
+  _(SIOCSETPFSYNC, READ, struct_ifreq_sz);
+  _(SIOCGETPFSYNC, READWRITE, struct_ifreq_sz);
+  /* Entries from file: sys/timepps.h */
+  _(PPS_IOC_CREATE, NONE, 0);
+  _(PPS_IOC_DESTROY, NONE, 0);
+  _(PPS_IOC_SETPARAMS, READ, struct_pps_params_sz);
+  _(PPS_IOC_GETPARAMS, WRITE, struct_pps_params_sz);
+  _(PPS_IOC_GETCAP, WRITE, sizeof(int));
+  _(PPS_IOC_FETCH, READWRITE, struct_pps_info_sz);
+  _(PPS_IOC_KCBIND, READ, sizeof(int));
+  /* Entries from file: sys/ttycom.h */
+  _(TIOCEXCL, NONE, 0);
+  _(TIOCNXCL, NONE, 0);
+  _(TIOCFLUSH, READ, sizeof(int));
+  _(TIOCGETA, WRITE, struct_termios_sz);
+  _(TIOCSETA, READ, struct_termios_sz);
+  _(TIOCSETAW, READ, 0);
+  _(TIOCSETAF, READ, 0);
+  _(TIOCGETD, WRITE, sizeof(int));
+  _(TIOCSETD, READ, sizeof(int));
+  _(TIOCGLINED, WRITE, (32 * sizeof(char)));
+  _(TIOCSLINED, READ, (32 * sizeof(char)));
+  _(TIOCSBRK, NONE, 0);
+  _(TIOCCBRK, NONE, 0);
+  _(TIOCSDTR, NONE, 0);
+  _(TIOCCDTR, NONE, 0);
+  _(TIOCGPGRP, WRITE, sizeof(int));
+  _(TIOCSPGRP, READ, sizeof(int));
+  _(TIOCOUTQ, WRITE, sizeof(int));
+  _(TIOCSTI, READ, sizeof(char));
+  _(TIOCNOTTY, NONE, 0);
+  _(TIOCPKT, READ, sizeof(int));
+  _(TIOCSTOP, NONE, 0);
+  _(TIOCSTART, NONE, 0);
+  _(TIOCMSET, READ, sizeof(int));
+  _(TIOCMBIS, READ, sizeof(int));
+  _(TIOCMBIC, READ, sizeof(int));
+  _(TIOCMGET, WRITE, sizeof(int));
+  _(TIOCREMOTE, READ, sizeof(int));
+  _(TIOCGWINSZ, WRITE, struct_winsize_sz);
+  _(TIOCSWINSZ, READ, struct_winsize_sz);
+  _(TIOCUCNTL, READ, sizeof(int));
+  _(TIOCSTAT, READ, sizeof(int));
+  _(TIOCGSID, WRITE, sizeof(int));
+  _(TIOCCONS, READ, sizeof(int));
+  _(TIOCSCTTY, NONE, 0);
+  _(TIOCEXT, READ, sizeof(int));
+  _(TIOCSIG, NONE, 0);
+  _(TIOCDRAIN, NONE, 0);
+  _(TIOCGFLAGS, WRITE, sizeof(int));
+  _(TIOCSFLAGS, READ, sizeof(int));
+  _(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz);
+  _(TIOCRCVFRAME, READ, sizeof(uptr));
+  _(TIOCXMTFRAME, READ, sizeof(uptr));
+  _(TIOCPTMGET, WRITE, struct_ptmget_sz);
+  _(TIOCGRANTPT, NONE, 0);
+  _(TIOCPTSNAME, WRITE, struct_ptmget_sz);
+  _(TIOCSQSIZE, READ, sizeof(int));
+  _(TIOCGQSIZE, WRITE, sizeof(int));
+  /* Entries from file: sys/verified_exec.h */
+  _(VERIEXEC_LOAD, READ, struct_plistref_sz);
+  _(VERIEXEC_TABLESIZE, READ, struct_plistref_sz);
+  _(VERIEXEC_DELETE, READ, struct_plistref_sz);
+  _(VERIEXEC_QUERY, READWRITE, struct_plistref_sz);
+  _(VERIEXEC_DUMP, WRITE, struct_plistref_sz);
+  _(VERIEXEC_FLUSH, NONE, 0);
+  /* Entries from file: sys/videoio.h */
+  _(VIDIOC_QUERYCAP, WRITE, struct_v4l2_capability_sz);
+  _(VIDIOC_RESERVED, NONE, 0);
+  _(VIDIOC_ENUM_FMT, READWRITE, struct_v4l2_fmtdesc_sz);
+  _(VIDIOC_G_FMT, READWRITE, struct_v4l2_format_sz);
+  _(VIDIOC_S_FMT, READWRITE, struct_v4l2_format_sz);
+  _(VIDIOC_REQBUFS, READWRITE, struct_v4l2_requestbuffers_sz);
+  _(VIDIOC_QUERYBUF, READWRITE, struct_v4l2_buffer_sz);
+  _(VIDIOC_G_FBUF, WRITE, struct_v4l2_framebuffer_sz);
+  _(VIDIOC_S_FBUF, READ, struct_v4l2_framebuffer_sz);
+  _(VIDIOC_OVERLAY, READ, sizeof(int));
+  _(VIDIOC_QBUF, READWRITE, struct_v4l2_buffer_sz);
+  _(VIDIOC_DQBUF, READWRITE, struct_v4l2_buffer_sz);
+  _(VIDIOC_STREAMON, READ, sizeof(int));
+  _(VIDIOC_STREAMOFF, READ, sizeof(int));
+  _(VIDIOC_G_PARM, READWRITE, struct_v4l2_streamparm_sz);
+  _(VIDIOC_S_PARM, READWRITE, struct_v4l2_streamparm_sz);
+  _(VIDIOC_G_STD, WRITE, sizeof(u64));
+  _(VIDIOC_S_STD, READ, sizeof(u64));
+  _(VIDIOC_ENUMSTD, READWRITE, struct_v4l2_standard_sz);
+  _(VIDIOC_ENUMINPUT, READWRITE, struct_v4l2_input_sz);
+  _(VIDIOC_G_CTRL, READWRITE, struct_v4l2_control_sz);
+  _(VIDIOC_S_CTRL, READWRITE, struct_v4l2_control_sz);
+  _(VIDIOC_G_TUNER, READWRITE, struct_v4l2_tuner_sz);
+  _(VIDIOC_S_TUNER, READ, struct_v4l2_tuner_sz);
+  _(VIDIOC_G_AUDIO, WRITE, struct_v4l2_audio_sz);
+  _(VIDIOC_S_AUDIO, READ, struct_v4l2_audio_sz);
+  _(VIDIOC_QUERYCTRL, READWRITE, struct_v4l2_queryctrl_sz);
+  _(VIDIOC_QUERYMENU, READWRITE, struct_v4l2_querymenu_sz);
+  _(VIDIOC_G_INPUT, WRITE, sizeof(int));
+  _(VIDIOC_S_INPUT, READWRITE, sizeof(int));
+  _(VIDIOC_G_OUTPUT, WRITE, sizeof(int));
+  _(VIDIOC_S_OUTPUT, READWRITE, sizeof(int));
+  _(VIDIOC_ENUMOUTPUT, READWRITE, struct_v4l2_output_sz);
+  _(VIDIOC_G_AUDOUT, WRITE, struct_v4l2_audioout_sz);
+  _(VIDIOC_S_AUDOUT, READ, struct_v4l2_audioout_sz);
+  _(VIDIOC_G_MODULATOR, READWRITE, struct_v4l2_modulator_sz);
+  _(VIDIOC_S_MODULATOR, READ, struct_v4l2_modulator_sz);
+  _(VIDIOC_G_FREQUENCY, READWRITE, struct_v4l2_frequency_sz);
+  _(VIDIOC_S_FREQUENCY, READ, struct_v4l2_frequency_sz);
+  _(VIDIOC_CROPCAP, READWRITE, struct_v4l2_cropcap_sz);
+  _(VIDIOC_G_CROP, READWRITE, struct_v4l2_crop_sz);
+  _(VIDIOC_S_CROP, READ, struct_v4l2_crop_sz);
+  _(VIDIOC_G_JPEGCOMP, WRITE, struct_v4l2_jpegcompression_sz);
+  _(VIDIOC_S_JPEGCOMP, READ, struct_v4l2_jpegcompression_sz);
+  _(VIDIOC_QUERYSTD, WRITE, sizeof(u64));
+  _(VIDIOC_TRY_FMT, READWRITE, struct_v4l2_format_sz);
+  _(VIDIOC_ENUMAUDIO, READWRITE, struct_v4l2_audio_sz);
+  _(VIDIOC_ENUMAUDOUT, READWRITE, struct_v4l2_audioout_sz);
+  _(VIDIOC_G_PRIORITY, WRITE, enum_v4l2_priority_sz);
+  _(VIDIOC_S_PRIORITY, READ, enum_v4l2_priority_sz);
+  _(VIDIOC_ENUM_FRAMESIZES, READWRITE, struct_v4l2_frmsizeenum_sz);
+  _(VIDIOC_ENUM_FRAMEINTERVALS, READWRITE, struct_v4l2_frmivalenum_sz);
+  /* Entries from file: sys/wdog.h */
+  _(WDOGIOC_GMODE, READWRITE, struct_wdog_mode_sz);
+  _(WDOGIOC_SMODE, READ, struct_wdog_mode_sz);
+  _(WDOGIOC_WHICH, WRITE, struct_wdog_mode_sz);
+  _(WDOGIOC_TICKLE, NONE, 0);
+  _(WDOGIOC_GTICKLER, WRITE, sizeof(int));
+  _(WDOGIOC_GWDOGS, READWRITE, struct_wdog_conf_sz);
+  /* Entries from file: sys/kcov.h */
+  _(KCOV_IOC_SETBUFSIZE, READ, sizeof(u64));
+  _(KCOV_IOC_ENABLE, READ, sizeof(int));
+  _(KCOV_IOC_DISABLE, NONE, 0);
+  /* Entries from file: sys/ipmi.h */
+  _(IPMICTL_RECEIVE_MSG_TRUNC, READWRITE, struct_ipmi_recv_sz);
+  _(IPMICTL_RECEIVE_MSG, READWRITE, struct_ipmi_recv_sz);
+  _(IPMICTL_SEND_COMMAND, READ, struct_ipmi_req_sz);
+  _(IPMICTL_REGISTER_FOR_CMD, READ, struct_ipmi_cmdspec_sz);
+  _(IPMICTL_UNREGISTER_FOR_CMD, READ, struct_ipmi_cmdspec_sz);
+  _(IPMICTL_SET_GETS_EVENTS_CMD, READ, sizeof(int));
+  _(IPMICTL_SET_MY_ADDRESS_CMD, READ, sizeof(unsigned int));
+  _(IPMICTL_GET_MY_ADDRESS_CMD, WRITE, sizeof(unsigned int));
+  _(IPMICTL_SET_MY_LUN_CMD, READ, sizeof(unsigned int));
+  _(IPMICTL_GET_MY_LUN_CMD, WRITE, sizeof(unsigned int));
+  /* Entries from file: soundcard.h */
+  _(SNDCTL_DSP_RESET, NONE, 0);
+  _(SNDCTL_DSP_SYNC, NONE, 0);
+  _(SNDCTL_DSP_SPEED, READWRITE, sizeof(int));
+  _(SOUND_PCM_READ_RATE, WRITE, sizeof(int));
+  _(SNDCTL_DSP_STEREO, READWRITE, sizeof(int));
+  _(SNDCTL_DSP_GETBLKSIZE, READWRITE, sizeof(int));
+  _(SNDCTL_DSP_SETFMT, READWRITE, sizeof(int));
+  _(SOUND_PCM_READ_BITS, WRITE, sizeof(int));
+  _(SNDCTL_DSP_CHANNELS, READWRITE, sizeof(int));
+  _(SOUND_PCM_READ_CHANNELS, WRITE, sizeof(int));
+  _(SOUND_PCM_WRITE_FILTER, READWRITE, sizeof(int));
+  _(SOUND_PCM_READ_FILTER, WRITE, sizeof(int));
+  _(SNDCTL_DSP_POST, NONE, 0);
+  _(SNDCTL_DSP_SUBDIVIDE, READWRITE, sizeof(int));
+  _(SNDCTL_DSP_SETFRAGMENT, READWRITE, sizeof(int));
+  _(SNDCTL_DSP_GETFMTS, WRITE, sizeof(int));
+  _(SNDCTL_DSP_GETOSPACE, WRITE, struct_audio_buf_info_sz);
+  _(SNDCTL_DSP_GETISPACE, WRITE, struct_audio_buf_info_sz);
+  _(SNDCTL_DSP_NONBLOCK, NONE, 0);
+  _(SNDCTL_DSP_GETCAPS, WRITE, sizeof(int));
+  _(SNDCTL_DSP_GETTRIGGER, WRITE, sizeof(int));
+  _(SNDCTL_DSP_SETTRIGGER, READ, sizeof(int));
+  _(SNDCTL_DSP_GETIPTR, WRITE, struct_count_info_sz);
+  _(SNDCTL_DSP_GETOPTR, WRITE, struct_count_info_sz);
+  _(SNDCTL_DSP_MAPINBUF, WRITE, struct_buffmem_desc_sz);
+  _(SNDCTL_DSP_MAPOUTBUF, WRITE, struct_buffmem_desc_sz);
+  _(SNDCTL_DSP_SETSYNCRO, NONE, 0);
+  _(SNDCTL_DSP_SETDUPLEX, NONE, 0);
+  _(SNDCTL_DSP_PROFILE, READ, sizeof(int));
+  _(SNDCTL_DSP_GETODELAY, WRITE, sizeof(int));
+  _(SOUND_MIXER_INFO, WRITE, struct_mixer_info_sz);
+  _(SOUND_OLD_MIXER_INFO, WRITE, struct__old_mixer_info_sz);
+  _(OSS_GETVERSION, WRITE, sizeof(int));
+  _(SNDCTL_SYSINFO, WRITE, struct_oss_sysinfo_sz);
+  _(SNDCTL_AUDIOINFO, READWRITE, struct_oss_audioinfo_sz);
+  _(SNDCTL_ENGINEINFO, READWRITE, struct_oss_audioinfo_sz);
+  _(SNDCTL_DSP_GETPLAYVOL, WRITE, sizeof(unsigned int));
+  _(SNDCTL_DSP_SETPLAYVOL, READ, sizeof(unsigned int));
+  _(SNDCTL_DSP_GETRECVOL, WRITE, sizeof(unsigned int));
+  _(SNDCTL_DSP_SETRECVOL, READ, sizeof(unsigned int));
+  _(SNDCTL_DSP_SKIP, NONE, 0);
+  _(SNDCTL_DSP_SILENCE, NONE, 0);
+  /* Entries from file: dev/filemon/filemon.h (compat <= 9.99.26) */
+  _(FILEMON_SET_FD, READWRITE, sizeof(int));
+  _(FILEMON_SET_PID, READWRITE, sizeof(int));
+  /* Entries from file: dev/usb/urio.h (compat <= 9.99.43) */
+  _(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
+  _(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
+#undef _
+} // NOLINT
+
+static bool ioctl_initialized = false;
+
+struct ioctl_desc_compare {
+  bool operator()(const ioctl_desc &left, const ioctl_desc &right) const {
+    return left.req < right.req;
+  }
+};
+
+static void ioctl_init() {
+  ioctl_table_fill();
+  Sort(ioctl_table, ioctl_table_size, ioctl_desc_compare());
+
+  bool bad = false;
+  for (unsigned i = 0; i < ioctl_table_size - 1; ++i) {
+    if (ioctl_table[i].req >= ioctl_table[i + 1].req) {
+      Printf("Duplicate or unsorted ioctl request id %x >= %x (%s vs %s)\n",
+             ioctl_table[i].req, ioctl_table[i + 1].req, ioctl_table[i].name,
+             ioctl_table[i + 1].name);
+      bad = true;
+    }
+  }
+
+  if (bad)
+    Die();
+
+  ioctl_initialized = true;
+}
+
+static const ioctl_desc *ioctl_table_lookup(unsigned req) {
+  int left = 0;
+  int right = ioctl_table_size;
+  while (left < right) {
+    int mid = (left + right) / 2;
+    if (ioctl_table[mid].req < req)
+      left = mid + 1;
+    else
+      right = mid;
+  }
+  if (left == right && ioctl_table[left].req == req)
+    return ioctl_table + left;
+  else
+    return nullptr;
+}
+
+static bool ioctl_decode(unsigned req, ioctl_desc *desc) {
+  CHECK(desc);
+  desc->req = req;
+  desc->name = "<DECODED_IOCTL>";
+  desc->size = IOC_SIZE(req);
+  // Sanity check.
+  if (desc->size > 0xFFFF)
+    return false;
+  unsigned dir = IOC_DIR(req);
+  switch (dir) {
+  case IOC_NONE:
+    desc->type = ioctl_desc::NONE;
+    break;
+  case IOC_READ | IOC_WRITE:
+    desc->type = ioctl_desc::READWRITE;
+    break;
+  case IOC_READ:
+    desc->type = ioctl_desc::WRITE;
+    break;
+  case IOC_WRITE:
+    desc->type = ioctl_desc::READ;
+    break;
+  default:
+    return false;
+  }
+  // Size can be 0 iff type is NONE.
+  if ((desc->type == IOC_NONE) != (desc->size == 0))
+    return false;
+  // Sanity check.
+  if (IOC_TYPE(req) == 0)
+    return false;
+  return true;
+}
+
+static const ioctl_desc *ioctl_lookup(unsigned req) {
+  const ioctl_desc *desc = ioctl_table_lookup(req);
+  if (desc)
+    return desc;
+
+  // Try stripping access size from the request id.
+  desc = ioctl_table_lookup(req & ~(IOC_SIZEMASK << IOC_SIZESHIFT));
+  // Sanity check: requests that encode access size are either read or write and
+  // have size of 0 in the table.
+  if (desc && desc->size == 0 &&
+      (desc->type == ioctl_desc::READWRITE || desc->type == ioctl_desc::WRITE ||
+       desc->type == ioctl_desc::READ))
+    return desc;
+  return nullptr;
+}
+
+static void ioctl_common_pre(void *ctx, const ioctl_desc *desc, int d,
+                             unsigned request, void *arg) {
+  if (desc->type == ioctl_desc::READ || desc->type == ioctl_desc::READWRITE) {
+    unsigned size = desc->size ? desc->size : IOC_SIZE(request);
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, arg, size);
+  }
+  if (desc->type != ioctl_desc::CUSTOM)
+    return;
+  if (request == IOCTL_SIOCGIFCONF) {
+    struct __sanitizer_ifconf *ifc = (__sanitizer_ifconf *)arg;
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, (char *)&ifc->ifc_len,
+                                  sizeof(ifc->ifc_len));
+  }
+}
+
+static void ioctl_common_post(void *ctx, const ioctl_desc *desc, int res, int d,
+                              unsigned request, void *arg) {
+  if (desc->type == ioctl_desc::WRITE || desc->type == ioctl_desc::READWRITE) {
+    // FIXME: add verbose output
+    unsigned size = desc->size ? desc->size : IOC_SIZE(request);
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg, size);
+  }
+  if (desc->type != ioctl_desc::CUSTOM)
+    return;
+  if (request == IOCTL_SIOCGIFCONF) {
+    struct __sanitizer_ifconf *ifc = (__sanitizer_ifconf *)arg;
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifc->ifc_ifcu.ifcu_req, ifc->ifc_len);
+  }
+}
+
+#endif // SANITIZER_NETBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_interface_internal.h b/lib/tsan/sanitizer_common/sanitizer_interface_internal.h
new file mode 100644
index 0000000000..be8023e9e1
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_interface_internal.h
@@ -0,0 +1,118 @@
+//===-- sanitizer_interface_internal.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between run-time libraries of sanitizers.
+//
+// This header declares the sanitizer runtime interface functions.
+// The runtime library has to define these functions so the instrumented program
+// could call them.
+//
+// See also include/sanitizer/common_interface_defs.h
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_INTERFACE_INTERNAL_H
+#define SANITIZER_INTERFACE_INTERNAL_H
+
+#include "sanitizer_internal_defs.h"
+
+extern "C" {
+  // Tell the tools to write their reports to "path.<pid>" instead of stderr.
+  // The special values are "stdout" and "stderr".
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __sanitizer_set_report_path(const char *path);
+  // Tell the tools to write their reports to the provided file descriptor
+  // (casted to void *).
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __sanitizer_set_report_fd(void *fd);
+
+  typedef struct {
+      int coverage_sandboxed;
+      __sanitizer::sptr coverage_fd;
+      unsigned int coverage_max_block_size;
+  } __sanitizer_sandbox_arguments;
+
+  // Notify the tools that the sandbox is going to be turned on.
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+      __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
+
+  // This function is called by the tool when it has just finished reporting
+  // an error. 'error_summary' is a one-line string that summarizes
+  // the error message. This function can be overridden by the client.
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_report_error_summary(const char *error_summary);
+
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
+      const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
+
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
+
+  // Returns 1 on the first call, then returns 0 thereafter.  Called by the tool
+  // to ensure only one report is printed when multiple errors occur
+  // simultaneously.
+  SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
+
+  SANITIZER_INTERFACE_ATTRIBUTE
+  void __sanitizer_annotate_contiguous_container(const void *beg,
+                                                 const void *end,
+                                                 const void *old_mid,
+                                                 const void *new_mid);
+  SANITIZER_INTERFACE_ATTRIBUTE
+  int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
+                                              const void *end);
+  SANITIZER_INTERFACE_ATTRIBUTE
+  const void *__sanitizer_contiguous_container_find_bad_address(
+      const void *beg, const void *mid, const void *end);
+
+  SANITIZER_INTERFACE_ATTRIBUTE
+  int __sanitizer_get_module_and_offset_for_pc(
+      __sanitizer::uptr pc, char *module_path,
+      __sanitizer::uptr module_path_len, __sanitizer::uptr *pc_offset);
+
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp1();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp2();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_cmp8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp1();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp2();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_const_cmp8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_switch();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_div4();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_div8();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_gep();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_indir();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_guard(__sanitizer::u32*);
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,
+                                           __sanitizer::u32*);
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+  void __sanitizer_cov_8bit_counters_init();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+  __sanitizer_cov_bool_flag_init();
+  SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+  __sanitizer_cov_pcs_init();
+} // extern "C"
+
+#endif  // SANITIZER_INTERFACE_INTERNAL_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_internal_defs.h b/lib/tsan/sanitizer_common/sanitizer_internal_defs.h
new file mode 100644
index 0000000000..d0ffc79b06
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_internal_defs.h
@@ -0,0 +1,459 @@
+//===-- sanitizer_internal_defs.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer.
+// It contains macro used in run-time libraries code.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_DEFS_H
+#define SANITIZER_DEFS_H
+
+#include "sanitizer_platform.h"
+
+#ifndef SANITIZER_DEBUG
+# define SANITIZER_DEBUG 0
+#endif
+
+#define SANITIZER_STRINGIFY_(S) #S
+#define SANITIZER_STRINGIFY(S) SANITIZER_STRINGIFY_(S)
+
+// Only use SANITIZER_*ATTRIBUTE* before the function return type!
+#if SANITIZER_WINDOWS
+#if SANITIZER_IMPORT_INTERFACE
+# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport)
+#else
+# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
+#endif
+# define SANITIZER_WEAK_ATTRIBUTE
+#elif SANITIZER_GO
+# define SANITIZER_INTERFACE_ATTRIBUTE
+# define SANITIZER_WEAK_ATTRIBUTE
+#else
+# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
+# define SANITIZER_WEAK_ATTRIBUTE  __attribute__((weak))
+#endif
+
+// TLS is handled differently on different platforms
+#if SANITIZER_LINUX || SANITIZER_NETBSD || \
+  SANITIZER_FREEBSD || SANITIZER_OPENBSD
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
+    __attribute__((tls_model("initial-exec"))) thread_local
+#else
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
+#endif
+
+//--------------------------- WEAK FUNCTIONS ---------------------------------//
+// When working with weak functions, to simplify the code and make it more
+// portable, when possible define a default implementation using this macro:
+//
+// SANITIZER_INTERFACE_WEAK_DEF(<return_type>, <name>, <parameter list>)
+//
+// For example:
+//   SANITIZER_INTERFACE_WEAK_DEF(bool, compare, int a, int b) { return a > b; }
+//
+#if SANITIZER_WINDOWS
+#include "sanitizer_win_defs.h"
+# define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...)                   \
+  WIN_WEAK_EXPORT_DEF(ReturnType, Name, __VA_ARGS__)
+#else
+# define SANITIZER_INTERFACE_WEAK_DEF(ReturnType, Name, ...)                   \
+  extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE            \
+  ReturnType Name(__VA_ARGS__)
+#endif
+
+// SANITIZER_SUPPORTS_WEAK_HOOKS means that we support real weak functions that
+// will evaluate to a null pointer when not defined.
+#ifndef SANITIZER_SUPPORTS_WEAK_HOOKS
+#if (SANITIZER_LINUX || SANITIZER_SOLARIS) && !SANITIZER_GO
+# define SANITIZER_SUPPORTS_WEAK_HOOKS 1
+// Before Xcode 4.5, the Darwin linker doesn't reliably support undefined
+// weak symbols.  Mac OS X 10.9/Darwin 13 is the first release only supported
+// by Xcode >= 4.5.
+#elif SANITIZER_MAC && \
+    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1090 && !SANITIZER_GO
+# define SANITIZER_SUPPORTS_WEAK_HOOKS 1
+#else
+# define SANITIZER_SUPPORTS_WEAK_HOOKS 0
+#endif
+#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
+// For some weak hooks that will be called very often and we want to avoid the
+// overhead of executing the default implementation when it is not necessary,
+// we can use the flag SANITIZER_SUPPORTS_WEAK_HOOKS to only define the default
+// implementation for platforms that doesn't support weak symbols. For example:
+//
+//   #if !SANITIZER_SUPPORT_WEAK_HOOKS
+//     SANITIZER_INTERFACE_WEAK_DEF(bool, compare_hook, int a, int b) {
+//       return a > b;
+//     }
+//   #endif
+//
+// And then use it as: if (compare_hook) compare_hook(a, b);
+//----------------------------------------------------------------------------//
+
+
+// We can use .preinit_array section on Linux to call sanitizer initialization
+// functions very early in the process startup (unless PIC macro is defined).
+//
+// On FreeBSD, .preinit_array functions are called with rtld_bind_lock writer
+// lock held. It will lead to dead lock if unresolved PLT functions (which helds
+// rtld_bind_lock reader lock) are called inside .preinit_array functions.
+//
+// FIXME: do we have anything like this on Mac?
+#ifndef SANITIZER_CAN_USE_PREINIT_ARRAY
+#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD || \
+     SANITIZER_FUCHSIA || SANITIZER_NETBSD) && !defined(PIC)
+#define SANITIZER_CAN_USE_PREINIT_ARRAY 1
+// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld.
+// FIXME: Check for those conditions.
+#elif SANITIZER_SOLARIS && !defined(PIC)
+# define SANITIZER_CAN_USE_PREINIT_ARRAY 1
+#else
+# define SANITIZER_CAN_USE_PREINIT_ARRAY 0
+#endif
+#endif  // SANITIZER_CAN_USE_PREINIT_ARRAY
+
+// GCC does not understand __has_feature
+#if !defined(__has_feature)
+# define __has_feature(x) 0
+#endif
+
+// Older GCCs do not understand __has_attribute.
+#if !defined(__has_attribute)
+# define __has_attribute(x) 0
+#endif
+
+// For portability reasons we do not include stddef.h, stdint.h or any other
+// system header, but we do need some basic types that are not defined
+// in a portable way by the language itself.
+namespace __sanitizer {
+
+#if defined(_WIN64)
+// 64-bit Windows uses LLP64 data model.
+typedef unsigned long long uptr;
+typedef signed long long sptr;
+#else
+typedef unsigned long uptr;
+typedef signed long sptr;
+#endif  // defined(_WIN64)
+#if defined(__x86_64__)
+// Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
+// 64-bit pointer to unwind stack frame.
+typedef unsigned long long uhwptr;
+#else
+typedef uptr uhwptr;
+#endif
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long long u64;
+typedef signed char s8;
+typedef signed short s16;
+typedef signed int s32;
+typedef signed long long s64;
+#if SANITIZER_WINDOWS
+// On Windows, files are HANDLE, which is a synonim of void*.
+// Use void* to avoid including <windows.h> everywhere.
+typedef void* fd_t;
+typedef unsigned error_t;
+#else
+typedef int fd_t;
+typedef int error_t;
+#endif
+#if SANITIZER_SOLARIS && !defined(_LP64)
+typedef long pid_t;
+#else
+typedef int pid_t;
+#endif
+
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_MAC || \
+    (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
+    (SANITIZER_LINUX && defined(__x86_64__))
+typedef u64 OFF_T;
+#else
+typedef uptr OFF_T;
+#endif
+typedef u64  OFF64_T;
+
+#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC
+typedef uptr operator_new_size_type;
+#else
+# if SANITIZER_OPENBSD || defined(__s390__) && !defined(__s390x__)
+// Special case: 31-bit s390 has unsigned long as size_t.
+typedef unsigned long operator_new_size_type;
+# else
+typedef u32 operator_new_size_type;
+# endif
+#endif
+
+typedef u64 tid_t;
+
+// ----------- ATTENTION -------------
+// This header should NOT include any other headers to avoid portability issues.
+
+// Common defs.
+#ifndef INLINE
+#define INLINE inline
+#endif
+#define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE
+#define SANITIZER_WEAK_DEFAULT_IMPL \
+  extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE
+#define SANITIZER_WEAK_CXX_DEFAULT_IMPL \
+  extern "C++" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE
+
+// Platform-specific defs.
+#if defined(_MSC_VER)
+# define ALWAYS_INLINE __forceinline
+// FIXME(timurrrr): do we need this on Windows?
+# define ALIAS(x)
+# define ALIGNED(x) __declspec(align(x))
+# define FORMAT(f, a)
+# define NOINLINE __declspec(noinline)
+# define NORETURN __declspec(noreturn)
+# define THREADLOCAL   __declspec(thread)
+# define LIKELY(x) (x)
+# define UNLIKELY(x) (x)
+# define PREFETCH(x) /* _mm_prefetch(x, _MM_HINT_NTA) */ (void)0
+# define WARN_UNUSED_RESULT
+#else  // _MSC_VER
+# define ALWAYS_INLINE inline __attribute__((always_inline))
+# define ALIAS(x) __attribute__((alias(x)))
+// Please only use the ALIGNED macro before the type.
+// Using ALIGNED after the variable declaration is not portable!
+# define ALIGNED(x) __attribute__((aligned(x)))
+# define FORMAT(f, a)  __attribute__((format(printf, f, a)))
+# define NOINLINE __attribute__((noinline))
+# define NORETURN  __attribute__((noreturn))
+# define THREADLOCAL   __thread
+# define LIKELY(x)     __builtin_expect(!!(x), 1)
+# define UNLIKELY(x)   __builtin_expect(!!(x), 0)
+# if defined(__i386__) || defined(__x86_64__)
+// __builtin_prefetch(x) generates prefetchnt0 on x86
+#  define PREFETCH(x) __asm__("prefetchnta (%0)" : : "r" (x))
+# else
+#  define PREFETCH(x) __builtin_prefetch(x)
+# endif
+# define WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#endif  // _MSC_VER
+
+#if !defined(_MSC_VER) || defined(__clang__)
+# define UNUSED __attribute__((unused))
+# define USED __attribute__((used))
+#else
+# define UNUSED
+# define USED
+#endif
+
+#if !defined(_MSC_VER) || defined(__clang__) || MSC_PREREQ(1900)
+# define NOEXCEPT noexcept
+#else
+# define NOEXCEPT throw()
+#endif
+
+// Unaligned versions of basic types.
+typedef ALIGNED(1) u16 uu16;
+typedef ALIGNED(1) u32 uu32;
+typedef ALIGNED(1) u64 uu64;
+typedef ALIGNED(1) s16 us16;
+typedef ALIGNED(1) s32 us32;
+typedef ALIGNED(1) s64 us64;
+
+#if SANITIZER_WINDOWS
+}  // namespace __sanitizer
+typedef unsigned long DWORD;
+namespace __sanitizer {
+typedef DWORD thread_return_t;
+# define THREAD_CALLING_CONV __stdcall
+#else  // _WIN32
+typedef void* thread_return_t;
+# define THREAD_CALLING_CONV
+#endif  // _WIN32
+typedef thread_return_t (THREAD_CALLING_CONV *thread_callback_t)(void* arg);
+
+// NOTE: Functions below must be defined in each run-time.
+void NORETURN Die();
+
+void NORETURN CheckFailed(const char *file, int line, const char *cond,
+                          u64 v1, u64 v2);
+
+// Check macro
+#define RAW_CHECK_MSG(expr, msg) do { \
+  if (UNLIKELY(!(expr))) { \
+    RawWrite(msg); \
+    Die(); \
+  } \
+} while (0)
+
+#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr)
+
+#define CHECK_IMPL(c1, op, c2) \
+  do { \
+    __sanitizer::u64 v1 = (__sanitizer::u64)(c1); \
+    __sanitizer::u64 v2 = (__sanitizer::u64)(c2); \
+    if (UNLIKELY(!(v1 op v2))) \
+      __sanitizer::CheckFailed(__FILE__, __LINE__, \
+        "(" #c1 ") " #op " (" #c2 ")", v1, v2); \
+  } while (false) \
+/**/
+
+#define CHECK(a)       CHECK_IMPL((a), !=, 0)
+#define CHECK_EQ(a, b) CHECK_IMPL((a), ==, (b))
+#define CHECK_NE(a, b) CHECK_IMPL((a), !=, (b))
+#define CHECK_LT(a, b) CHECK_IMPL((a), <,  (b))
+#define CHECK_LE(a, b) CHECK_IMPL((a), <=, (b))
+#define CHECK_GT(a, b) CHECK_IMPL((a), >,  (b))
+#define CHECK_GE(a, b) CHECK_IMPL((a), >=, (b))
+
+#if SANITIZER_DEBUG
+#define DCHECK(a)       CHECK(a)
+#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
+#define DCHECK_NE(a, b) CHECK_NE(a, b)
+#define DCHECK_LT(a, b) CHECK_LT(a, b)
+#define DCHECK_LE(a, b) CHECK_LE(a, b)
+#define DCHECK_GT(a, b) CHECK_GT(a, b)
+#define DCHECK_GE(a, b) CHECK_GE(a, b)
+#else
+#define DCHECK(a)
+#define DCHECK_EQ(a, b)
+#define DCHECK_NE(a, b)
+#define DCHECK_LT(a, b)
+#define DCHECK_LE(a, b)
+#define DCHECK_GT(a, b)
+#define DCHECK_GE(a, b)
+#endif
+
+#define UNREACHABLE(msg) do { \
+  CHECK(0 && msg); \
+  Die(); \
+} while (0)
+
+#define UNIMPLEMENTED() UNREACHABLE("unimplemented")
+
+#define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__)
+
+#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
+
+#define IMPL_PASTE(a, b) a##b
+#define IMPL_COMPILER_ASSERT(pred, line) \
+    typedef char IMPL_PASTE(assertion_failed_##_, line)[2*(int)(pred)-1]
+
+// Limits for integral types. We have to redefine it in case we don't
+// have stdint.h (like in Visual Studio 9).
+#undef __INT64_C
+#undef __UINT64_C
+#if SANITIZER_WORDSIZE == 64
+# define __INT64_C(c)  c ## L
+# define __UINT64_C(c) c ## UL
+#else
+# define __INT64_C(c)  c ## LL
+# define __UINT64_C(c) c ## ULL
+#endif  // SANITIZER_WORDSIZE == 64
+#undef INT32_MIN
+#define INT32_MIN              (-2147483647-1)
+#undef INT32_MAX
+#define INT32_MAX              (2147483647)
+#undef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#undef INT64_MIN
+#define INT64_MIN              (-__INT64_C(9223372036854775807)-1)
+#undef INT64_MAX
+#define INT64_MAX              (__INT64_C(9223372036854775807))
+#undef UINT64_MAX
+#define UINT64_MAX             (__UINT64_C(18446744073709551615))
+#undef UINTPTR_MAX
+#if SANITIZER_WORDSIZE == 64
+# define UINTPTR_MAX           (18446744073709551615UL)
+#else
+# define UINTPTR_MAX           (4294967295U)
+#endif  // SANITIZER_WORDSIZE == 64
+
+enum LinkerInitialized { LINKER_INITIALIZED = 0 };
+
+#if !defined(_MSC_VER) || defined(__clang__)
+#if SANITIZER_S390_31
+#define GET_CALLER_PC() \
+  (__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0))
+#else
+#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0)
+#endif
+#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0)
+inline void Trap() {
+  __builtin_trap();
+}
+#else
+extern "C" void* _ReturnAddress(void);
+extern "C" void* _AddressOfReturnAddress(void);
+# pragma intrinsic(_ReturnAddress)
+# pragma intrinsic(_AddressOfReturnAddress)
+#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress()
+// CaptureStackBackTrace doesn't need to know BP on Windows.
+#define GET_CURRENT_FRAME() \
+  (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
+
+extern "C" void __ud2(void);
+# pragma intrinsic(__ud2)
+inline void Trap() {
+  __ud2();
+}
+#endif
+
+#define HANDLE_EINTR(res, f)                                       \
+  {                                                                \
+    int rverrno;                                                   \
+    do {                                                           \
+      res = (f);                                                   \
+    } while (internal_iserror(res, &rverrno) && rverrno == EINTR); \
+  }
+
+// Forces the compiler to generate a frame pointer in the function.
+#define ENABLE_FRAME_POINTER              \
+  do {                                    \
+    volatile __sanitizer::uptr enable_fp; \
+    enable_fp = GET_CURRENT_FRAME();      \
+    (void)enable_fp;                      \
+  } while (0)
+
+}  // namespace __sanitizer
+
+namespace __asan {
+using namespace __sanitizer;
+}
+namespace __dsan {
+using namespace __sanitizer;
+}
+namespace __dfsan {
+using namespace __sanitizer;
+}
+namespace __lsan {
+using namespace __sanitizer;
+}
+namespace __msan {
+using namespace __sanitizer;
+}
+namespace __hwasan {
+using namespace __sanitizer;
+}
+namespace __tsan {
+using namespace __sanitizer;
+}
+namespace __scudo {
+using namespace __sanitizer;
+}
+namespace __ubsan {
+using namespace __sanitizer;
+}
+namespace __xray {
+using namespace __sanitizer;
+}
+namespace __interception {
+using namespace __sanitizer;
+}
+namespace __hwasan {
+using namespace __sanitizer;
+}
+
+#endif  // SANITIZER_DEFS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_lfstack.h b/lib/tsan/sanitizer_common/sanitizer_lfstack.h
new file mode 100644
index 0000000000..af2ca55ec3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_lfstack.h
@@ -0,0 +1,72 @@
+//===-- sanitizer_lfstack.h -=-----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lock-free stack.
+// Uses 32/17 bits as ABA-counter on 32/64-bit platforms.
+// The memory passed to Push() must not be ever munmap'ed.
+// The type T must contain T *next field.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LFSTACK_H
+#define SANITIZER_LFSTACK_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+#include "sanitizer_atomic.h"
+
+namespace __sanitizer {
+
+template<typename T>
+struct LFStack {
+  void Clear() {
+    atomic_store(&head_, 0, memory_order_relaxed);
+  }
+
+  bool Empty() const {
+    return (atomic_load(&head_, memory_order_relaxed) & kPtrMask) == 0;
+  }
+
+  void Push(T *p) {
+    u64 cmp = atomic_load(&head_, memory_order_relaxed);
+    for (;;) {
+      u64 cnt = (cmp & kCounterMask) + kCounterInc;
+      u64 xch = (u64)(uptr)p | cnt;
+      p->next = (T*)(uptr)(cmp & kPtrMask);
+      if (atomic_compare_exchange_weak(&head_, &cmp, xch,
+                                       memory_order_release))
+        break;
+    }
+  }
+
+  T *Pop() {
+    u64 cmp = atomic_load(&head_, memory_order_acquire);
+    for (;;) {
+      T *cur = (T*)(uptr)(cmp & kPtrMask);
+      if (!cur)
+        return nullptr;
+      T *nxt = cur->next;
+      u64 cnt = (cmp & kCounterMask);
+      u64 xch = (u64)(uptr)nxt | cnt;
+      if (atomic_compare_exchange_weak(&head_, &cmp, xch,
+                                       memory_order_acquire))
+        return cur;
+    }
+  }
+
+  // private:
+  static const int kCounterBits = FIRST_32_SECOND_64(32, 17);
+  static const u64 kPtrMask = ((u64)-1) >> kCounterBits;
+  static const u64 kCounterMask = ~kPtrMask;
+  static const u64 kCounterInc = kPtrMask + 1;
+
+  atomic_uint64_t head_;
+};
+} // namespace __sanitizer
+
+#endif // SANITIZER_LFSTACK_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_libc.cpp b/lib/tsan/sanitizer_common/sanitizer_libc.cpp
new file mode 100644
index 0000000000..4bc04b4868
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_libc.cpp
@@ -0,0 +1,280 @@
+//===-- sanitizer_libc.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries. See sanitizer_libc.h for details.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+
+namespace __sanitizer {
+
+s64 internal_atoll(const char *nptr) {
+  return internal_simple_strtoll(nptr, nullptr, 10);
+}
+
+void *internal_memchr(const void *s, int c, uptr n) {
+  const char *t = (const char *)s;
+  for (uptr i = 0; i < n; ++i, ++t)
+    if (*t == c)
+      return reinterpret_cast<void *>(const_cast<char *>(t));
+  return nullptr;
+}
+
+void *internal_memrchr(const void *s, int c, uptr n) {
+  const char *t = (const char *)s;
+  void *res = nullptr;
+  for (uptr i = 0; i < n; ++i, ++t) {
+    if (*t == c) res = reinterpret_cast<void *>(const_cast<char *>(t));
+  }
+  return res;
+}
+
+int internal_memcmp(const void* s1, const void* s2, uptr n) {
+  const char *t1 = (const char *)s1;
+  const char *t2 = (const char *)s2;
+  for (uptr i = 0; i < n; ++i, ++t1, ++t2)
+    if (*t1 != *t2)
+      return *t1 < *t2 ? -1 : 1;
+  return 0;
+}
+
+void *internal_memcpy(void *dest, const void *src, uptr n) {
+  char *d = (char*)dest;
+  const char *s = (const char *)src;
+  for (uptr i = 0; i < n; ++i)
+    d[i] = s[i];
+  return dest;
+}
+
+void *internal_memmove(void *dest, const void *src, uptr n) {
+  char *d = (char*)dest;
+  const char *s = (const char *)src;
+  sptr i, signed_n = (sptr)n;
+  CHECK_GE(signed_n, 0);
+  if (d < s) {
+    for (i = 0; i < signed_n; ++i)
+      d[i] = s[i];
+  } else {
+    if (d > s && signed_n > 0) {
+      for (i = signed_n - 1; i >= 0; --i) {
+        d[i] = s[i];
+      }
+    }
+  }
+  return dest;
+}
+
+void *internal_memset(void* s, int c, uptr n) {
+  // Optimize for the most performance-critical case:
+  if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) {
+    u64 *p = reinterpret_cast<u64*>(s);
+    u64 *e = p + n / 8;
+    u64 v = c;
+    v |= v << 8;
+    v |= v << 16;
+    v |= v << 32;
+    for (; p < e; p += 2)
+      p[0] = p[1] = v;
+    return s;
+  }
+  // The next line prevents Clang from making a call to memset() instead of the
+  // loop below.
+  // FIXME: building the runtime with -ffreestanding is a better idea. However
+  // there currently are linktime problems due to PR12396.
+  char volatile *t = (char*)s;
+  for (uptr i = 0; i < n; ++i, ++t) {
+    *t = c;
+  }
+  return s;
+}
+
+uptr internal_strcspn(const char *s, const char *reject) {
+  uptr i;
+  for (i = 0; s[i]; i++) {
+    if (internal_strchr(reject, s[i]))
+      return i;
+  }
+  return i;
+}
+
+char* internal_strdup(const char *s) {
+  uptr len = internal_strlen(s);
+  char *s2 = (char*)InternalAlloc(len + 1);
+  internal_memcpy(s2, s, len);
+  s2[len] = 0;
+  return s2;
+}
+
+int internal_strcmp(const char *s1, const char *s2) {
+  while (true) {
+    unsigned c1 = *s1;
+    unsigned c2 = *s2;
+    if (c1 != c2) return (c1 < c2) ? -1 : 1;
+    if (c1 == 0) break;
+    s1++;
+    s2++;
+  }
+  return 0;
+}
+
+int internal_strncmp(const char *s1, const char *s2, uptr n) {
+  for (uptr i = 0; i < n; i++) {
+    unsigned c1 = *s1;
+    unsigned c2 = *s2;
+    if (c1 != c2) return (c1 < c2) ? -1 : 1;
+    if (c1 == 0) break;
+    s1++;
+    s2++;
+  }
+  return 0;
+}
+
+char* internal_strchr(const char *s, int c) {
+  while (true) {
+    if (*s == (char)c)
+      return const_cast<char *>(s);
+    if (*s == 0)
+      return nullptr;
+    s++;
+  }
+}
+
+char *internal_strchrnul(const char *s, int c) {
+  char *res = internal_strchr(s, c);
+  if (!res)
+    res = const_cast<char *>(s) + internal_strlen(s);
+  return res;
+}
+
+char *internal_strrchr(const char *s, int c) {
+  const char *res = nullptr;
+  for (uptr i = 0; s[i]; i++) {
+    if (s[i] == c) res = s + i;
+  }
+  return const_cast<char *>(res);
+}
+
+uptr internal_strlen(const char *s) {
+  uptr i = 0;
+  while (s[i]) i++;
+  return i;
+}
+
+uptr internal_strlcat(char *dst, const char *src, uptr maxlen) {
+  const uptr srclen = internal_strlen(src);
+  const uptr dstlen = internal_strnlen(dst, maxlen);
+  if (dstlen == maxlen) return maxlen + srclen;
+  if (srclen < maxlen - dstlen) {
+    internal_memmove(dst + dstlen, src, srclen + 1);
+  } else {
+    internal_memmove(dst + dstlen, src, maxlen - dstlen - 1);
+    dst[maxlen - 1] = '\0';
+  }
+  return dstlen + srclen;
+}
+
+char *internal_strncat(char *dst, const char *src, uptr n) {
+  uptr len = internal_strlen(dst);
+  uptr i;
+  for (i = 0; i < n && src[i]; i++)
+    dst[len + i] = src[i];
+  dst[len + i] = 0;
+  return dst;
+}
+
+uptr internal_strlcpy(char *dst, const char *src, uptr maxlen) {
+  const uptr srclen = internal_strlen(src);
+  if (srclen < maxlen) {
+    internal_memmove(dst, src, srclen + 1);
+  } else if (maxlen != 0) {
+    internal_memmove(dst, src, maxlen - 1);
+    dst[maxlen - 1] = '\0';
+  }
+  return srclen;
+}
+
+char *internal_strncpy(char *dst, const char *src, uptr n) {
+  uptr i;
+  for (i = 0; i < n && src[i]; i++)
+    dst[i] = src[i];
+  internal_memset(dst + i, '\0', n - i);
+  return dst;
+}
+
+uptr internal_strnlen(const char *s, uptr maxlen) {
+  uptr i = 0;
+  while (i < maxlen && s[i]) i++;
+  return i;
+}
+
+char *internal_strstr(const char *haystack, const char *needle) {
+  // This is O(N^2), but we are not using it in hot places.
+  uptr len1 = internal_strlen(haystack);
+  uptr len2 = internal_strlen(needle);
+  if (len1 < len2) return nullptr;
+  for (uptr pos = 0; pos <= len1 - len2; pos++) {
+    if (internal_memcmp(haystack + pos, needle, len2) == 0)
+      return const_cast<char *>(haystack) + pos;
+  }
+  return nullptr;
+}
+
+s64 internal_simple_strtoll(const char *nptr, const char **endptr, int base) {
+  CHECK_EQ(base, 10);
+  while (IsSpace(*nptr)) nptr++;
+  int sgn = 1;
+  u64 res = 0;
+  bool have_digits = false;
+  char *old_nptr = const_cast<char *>(nptr);
+  if (*nptr == '+') {
+    sgn = 1;
+    nptr++;
+  } else if (*nptr == '-') {
+    sgn = -1;
+    nptr++;
+  }
+  while (IsDigit(*nptr)) {
+    res = (res <= UINT64_MAX / 10) ? res * 10 : UINT64_MAX;
+    int digit = ((*nptr) - '0');
+    res = (res <= UINT64_MAX - digit) ? res + digit : UINT64_MAX;
+    have_digits = true;
+    nptr++;
+  }
+  if (endptr) {
+    *endptr = (have_digits) ? const_cast<char *>(nptr) : old_nptr;
+  }
+  if (sgn > 0) {
+    return (s64)(Min((u64)INT64_MAX, res));
+  } else {
+    return (res > INT64_MAX) ? INT64_MIN : ((s64)res * -1);
+  }
+}
+
+bool mem_is_zero(const char *beg, uptr size) {
+  CHECK_LE(size, 1ULL << FIRST_32_SECOND_64(30, 40));  // Sanity check.
+  const char *end = beg + size;
+  uptr *aligned_beg = (uptr *)RoundUpTo((uptr)beg, sizeof(uptr));
+  uptr *aligned_end = (uptr *)RoundDownTo((uptr)end, sizeof(uptr));
+  uptr all = 0;
+  // Prologue.
+  for (const char *mem = beg; mem < (char*)aligned_beg && mem < end; mem++)
+    all |= *mem;
+  // Aligned loop.
+  for (; aligned_beg < aligned_end; aligned_beg++)
+    all |= *aligned_beg;
+  // Epilogue.
+  if ((char *)aligned_end >= beg) {
+    for (const char *mem = (char *)aligned_end; mem < end; mem++) all |= *mem;
+  }
+  return all == 0;
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_libc.h b/lib/tsan/sanitizer_common/sanitizer_libc.h
new file mode 100644
index 0000000000..ec0a6ded00
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_libc.h
@@ -0,0 +1,85 @@
+//===-- sanitizer_libc.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// These tools can not use some of the libc functions directly because those
+// functions are intercepted. Instead, we implement a tiny subset of libc here.
+// FIXME: Some of functions declared in this file are in fact POSIX, not libc.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LIBC_H
+#define SANITIZER_LIBC_H
+
+// ----------- ATTENTION -------------
+// This header should NOT include any other headers from sanitizer runtime.
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+// internal_X() is a custom implementation of X() for use in RTL.
+
+// String functions
+s64 internal_atoll(const char *nptr);
+void *internal_memchr(const void *s, int c, uptr n);
+void *internal_memrchr(const void *s, int c, uptr n);
+int internal_memcmp(const void* s1, const void* s2, uptr n);
+void *internal_memcpy(void *dest, const void *src, uptr n);
+void *internal_memmove(void *dest, const void *src, uptr n);
+// Should not be used in performance-critical places.
+void *internal_memset(void *s, int c, uptr n);
+char* internal_strchr(const char *s, int c);
+char *internal_strchrnul(const char *s, int c);
+int internal_strcmp(const char *s1, const char *s2);
+uptr internal_strcspn(const char *s, const char *reject);
+char *internal_strdup(const char *s);
+uptr internal_strlen(const char *s);
+uptr internal_strlcat(char *dst, const char *src, uptr maxlen);
+char *internal_strncat(char *dst, const char *src, uptr n);
+int internal_strncmp(const char *s1, const char *s2, uptr n);
+uptr internal_strlcpy(char *dst, const char *src, uptr maxlen);
+char *internal_strncpy(char *dst, const char *src, uptr n);
+uptr internal_strnlen(const char *s, uptr maxlen);
+char *internal_strrchr(const char *s, int c);
+char *internal_strstr(const char *haystack, const char *needle);
+// Works only for base=10 and doesn't set errno.
+s64 internal_simple_strtoll(const char *nptr, const char **endptr, int base);
+int internal_snprintf(char *buffer, uptr length, const char *format, ...);
+
+// Return true if all bytes in [mem, mem+size) are zero.
+// Optimized for the case when the result is true.
+bool mem_is_zero(const char *mem, uptr size);
+
+// I/O
+// Define these as macros so we can use them in linker initialized global
+// structs without dynamic initialization.
+#define kInvalidFd ((fd_t)-1)
+#define kStdinFd ((fd_t)0)
+#define kStdoutFd ((fd_t)1)
+#define kStderrFd ((fd_t)2)
+
+uptr internal_ftruncate(fd_t fd, uptr size);
+
+// OS
+void NORETURN internal__exit(int exitcode);
+unsigned int internal_sleep(unsigned int seconds);
+
+uptr internal_getpid();
+uptr internal_getppid();
+
+int internal_dlinfo(void *handle, int request, void *p);
+
+// Threading
+uptr internal_sched_yield();
+
+// Error handling
+bool internal_iserror(uptr retval, int *rverrno = nullptr);
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_LIBC_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_libignore.cpp b/lib/tsan/sanitizer_common/sanitizer_libignore.cpp
new file mode 100644
index 0000000000..eb9bb76501
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_libignore.cpp
@@ -0,0 +1,129 @@
+//===-- sanitizer_libignore.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
+    SANITIZER_NETBSD || SANITIZER_OPENBSD
+
+#include "sanitizer_libignore.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_posix.h"
+#include "sanitizer_procmaps.h"
+
+namespace __sanitizer {
+
+LibIgnore::LibIgnore(LinkerInitialized) {
+}
+
+void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
+  BlockingMutexLock lock(&mutex_);
+  if (count_ >= kMaxLibs) {
+    Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName,
+           kMaxLibs);
+    Die();
+  }
+  Lib *lib = &libs_[count_++];
+  lib->templ = internal_strdup(name_templ);
+  lib->name = nullptr;
+  lib->real_name = nullptr;
+  lib->loaded = false;
+}
+
+void LibIgnore::OnLibraryLoaded(const char *name) {
+  BlockingMutexLock lock(&mutex_);
+  // Try to match suppressions with symlink target.
+  InternalScopedString buf(kMaxPathLength);
+  if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
+      buf[0]) {
+    for (uptr i = 0; i < count_; i++) {
+      Lib *lib = &libs_[i];
+      if (!lib->loaded && (!lib->real_name) &&
+          TemplateMatch(lib->templ, name))
+        lib->real_name = internal_strdup(buf.data());
+    }
+  }
+
+  // Scan suppressions list and find newly loaded and unloaded libraries.
+  ListOfModules modules;
+  modules.init();
+  for (uptr i = 0; i < count_; i++) {
+    Lib *lib = &libs_[i];
+    bool loaded = false;
+    for (const auto &mod : modules) {
+      for (const auto &range : mod.ranges()) {
+        if (!range.executable)
+          continue;
+        if (!TemplateMatch(lib->templ, mod.full_name()) &&
+            !(lib->real_name &&
+            internal_strcmp(lib->real_name, mod.full_name()) == 0))
+          continue;
+        if (loaded) {
+          Report("%s: called_from_lib suppression '%s' is matched against"
+                 " 2 libraries: '%s' and '%s'\n",
+                 SanitizerToolName, lib->templ, lib->name, mod.full_name());
+          Die();
+        }
+        loaded = true;
+        if (lib->loaded)
+          continue;
+        VReport(1,
+                "Matched called_from_lib suppression '%s' against library"
+                " '%s'\n",
+                lib->templ, mod.full_name());
+        lib->loaded = true;
+        lib->name = internal_strdup(mod.full_name());
+        const uptr idx =
+            atomic_load(&ignored_ranges_count_, memory_order_relaxed);
+        CHECK_LT(idx, ARRAY_SIZE(ignored_code_ranges_));
+        ignored_code_ranges_[idx].begin = range.beg;
+        ignored_code_ranges_[idx].end = range.end;
+        atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release);
+        break;
+      }
+    }
+    if (lib->loaded && !loaded) {
+      Report("%s: library '%s' that was matched against called_from_lib"
+             " suppression '%s' is unloaded\n",
+             SanitizerToolName, lib->name, lib->templ);
+      Die();
+    }
+  }
+
+  // Track instrumented ranges.
+  if (track_instrumented_libs_) {
+    for (const auto &mod : modules) {
+      if (!mod.instrumented())
+        continue;
+      for (const auto &range : mod.ranges()) {
+        if (!range.executable)
+          continue;
+        if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
+          continue;
+        VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
+                range.beg, range.end, mod.full_name());
+        const uptr idx =
+            atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
+        CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));
+        instrumented_code_ranges_[idx].begin = range.beg;
+        instrumented_code_ranges_[idx].end = range.end;
+        atomic_store(&instrumented_ranges_count_, idx + 1,
+                     memory_order_release);
+      }
+    }
+  }
+}
+
+void LibIgnore::OnLibraryUnloaded() {
+  OnLibraryLoaded(nullptr);
+}
+
+} // namespace __sanitizer
+
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC ||
+        // SANITIZER_NETBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_libignore.h b/lib/tsan/sanitizer_common/sanitizer_libignore.h
new file mode 100644
index 0000000000..256f685979
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_libignore.h
@@ -0,0 +1,115 @@
+//===-- sanitizer_libignore.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// LibIgnore allows to ignore all interceptors called from a particular set
+// of dynamic libraries. LibIgnore can be initialized with several templates
+// of names of libraries to be ignored. It finds code ranges for the libraries;
+// and checks whether the provided PC value belongs to the code ranges.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LIBIGNORE_H
+#define SANITIZER_LIBIGNORE_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+class LibIgnore {
+ public:
+  explicit LibIgnore(LinkerInitialized);
+
+  // Must be called during initialization.
+  void AddIgnoredLibrary(const char *name_templ);
+  void IgnoreNoninstrumentedModules(bool enable) {
+    track_instrumented_libs_ = enable;
+  }
+
+  // Must be called after a new dynamic library is loaded.
+  void OnLibraryLoaded(const char *name);
+
+  // Must be called after a dynamic library is unloaded.
+  void OnLibraryUnloaded();
+
+  // Checks whether the provided PC belongs to one of the ignored libraries or
+  // the PC should be ignored because it belongs to an non-instrumented module
+  // (when ignore_noninstrumented_modules=1). Also returns true via
+  // "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise.
+  bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const;
+
+  // Checks whether the provided PC belongs to an instrumented module.
+  bool IsPcInstrumented(uptr pc) const;
+
+ private:
+  struct Lib {
+    char *templ;
+    char *name;
+    char *real_name;  // target of symlink
+    bool loaded;
+  };
+
+  struct LibCodeRange {
+    uptr begin;
+    uptr end;
+  };
+
+  inline bool IsInRange(uptr pc, const LibCodeRange &range) const {
+    return (pc >= range.begin && pc < range.end);
+  }
+
+  static const uptr kMaxIgnoredRanges = 128;
+  static const uptr kMaxInstrumentedRanges = 1024;
+  static const uptr kMaxLibs = 1024;
+
+  // Hot part:
+  atomic_uintptr_t ignored_ranges_count_;
+  LibCodeRange ignored_code_ranges_[kMaxIgnoredRanges];
+
+  atomic_uintptr_t instrumented_ranges_count_;
+  LibCodeRange instrumented_code_ranges_[kMaxInstrumentedRanges];
+
+  // Cold part:
+  BlockingMutex mutex_;
+  uptr count_;
+  Lib libs_[kMaxLibs];
+  bool track_instrumented_libs_;
+
+  // Disallow copying of LibIgnore objects.
+  LibIgnore(const LibIgnore&);  // not implemented
+  void operator = (const LibIgnore&);  // not implemented
+};
+
+inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
+  const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
+  for (uptr i = 0; i < n; i++) {
+    if (IsInRange(pc, ignored_code_ranges_[i])) {
+      *pc_in_ignored_lib = true;
+      return true;
+    }
+  }
+  *pc_in_ignored_lib = false;
+  if (track_instrumented_libs_ && !IsPcInstrumented(pc))
+    return true;
+  return false;
+}
+
+inline bool LibIgnore::IsPcInstrumented(uptr pc) const {
+  const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
+  for (uptr i = 0; i < n; i++) {
+    if (IsInRange(pc, instrumented_code_ranges_[i]))
+      return true;
+  }
+  return false;
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_LIBIGNORE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_linux.cpp b/lib/tsan/sanitizer_common/sanitizer_linux.cpp
new file mode 100644
index 0000000000..470f4b70f0
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_linux.cpp
@@ -0,0 +1,2269 @@
+//===-- sanitizer_linux.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements linux-specific functions from
+// sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_getauxval.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_procmaps.h"
+
+#if SANITIZER_LINUX && !SANITIZER_GO
+#include <asm/param.h>
+#endif
+
+// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'
+// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
+// access stat from asm/stat.h, without conflicting with definition in
+// sys/stat.h, we use this trick.
+#if defined(__mips64)
+#include <asm/unistd.h>
+#include <sys/types.h>
+#define stat kernel_stat
+#include <asm/stat.h>
+#undef stat
+#endif
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <link.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#if !SANITIZER_SOLARIS
+#include <sys/ptrace.h>
+#endif
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#if !SANITIZER_OPENBSD
+#include <ucontext.h>
+#endif
+#if SANITIZER_OPENBSD
+#include <sys/futex.h>
+#include <sys/sysctl.h>
+#endif
+#include <unistd.h>
+
+#if SANITIZER_LINUX
+#include <sys/utsname.h>
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#include <sys/personality.h>
+#endif
+
+#if SANITIZER_FREEBSD
+#include <sys/exec.h>
+#include <sys/sysctl.h>
+#include <machine/atomic.h>
+extern "C" {
+// <sys/umtx.h> must be included after <errno.h> and <sys/types.h> on
+// FreeBSD 9.2 and 10.0.
+#include <sys/umtx.h>
+}
+#include <sys/thr.h>
+#endif  // SANITIZER_FREEBSD
+
+#if SANITIZER_NETBSD
+#include <limits.h>  // For NAME_MAX
+#include <sys/sysctl.h>
+#include <sys/exec.h>
+extern struct ps_strings *__ps_strings;
+#endif  // SANITIZER_NETBSD
+
+#if SANITIZER_SOLARIS
+#include <stdlib.h>
+#include <thread.h>
+#define environ _environ
+#endif
+
+extern char **environ;
+
+#if SANITIZER_LINUX
+// <linux/time.h>
+struct kernel_timeval {
+  long tv_sec;
+  long tv_usec;
+};
+
+// <linux/futex.h> is broken on some linux distributions.
+const int FUTEX_WAIT = 0;
+const int FUTEX_WAKE = 1;
+const int FUTEX_PRIVATE_FLAG = 128;
+const int FUTEX_WAIT_PRIVATE = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
+const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
+#endif  // SANITIZER_LINUX
+
+// Are we using 32-bit or 64-bit Linux syscalls?
+// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
+// but it still needs to use 64-bit syscalls.
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__powerpc64__) ||       \
+                        SANITIZER_WORDSIZE == 64)
+# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
+#else
+# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
+#endif
+
+// Note : FreeBSD had implemented both
+// Linux and OpenBSD apis, available from
+// future 12.x version most likely
+#if SANITIZER_LINUX && defined(__NR_getrandom)
+# if !defined(GRND_NONBLOCK)
+#  define GRND_NONBLOCK 1
+# endif
+# define SANITIZER_USE_GETRANDOM 1
+#else
+# define SANITIZER_USE_GETRANDOM 0
+#endif  // SANITIZER_LINUX && defined(__NR_getrandom)
+
+#if SANITIZER_OPENBSD
+# define SANITIZER_USE_GETENTROPY 1
+#else
+# if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
+#   define SANITIZER_USE_GETENTROPY 1
+# else
+#   define SANITIZER_USE_GETENTROPY 0
+# endif
+#endif // SANITIZER_USE_GETENTROPY
+
+namespace __sanitizer {
+
+#if SANITIZER_LINUX && defined(__x86_64__)
+#include "sanitizer_syscall_linux_x86_64.inc"
+#elif SANITIZER_LINUX && defined(__aarch64__)
+#include "sanitizer_syscall_linux_aarch64.inc"
+#elif SANITIZER_LINUX && defined(__arm__)
+#include "sanitizer_syscall_linux_arm.inc"
+#else
+#include "sanitizer_syscall_generic.inc"
+#endif
+
+// --------------- sanitizer_libc.h
+#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+#if !SANITIZER_S390 && !SANITIZER_OPENBSD
+uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
+                   u64 offset) {
+#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+  return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
+                          offset);
+#else
+  // mmap2 specifies file offset in 4096-byte units.
+  CHECK(IsAligned(offset, 4096));
+  return internal_syscall(SYSCALL(mmap2), addr, length, prot, flags, fd,
+                          offset / 4096);
+#endif
+}
+#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD
+
+#if !SANITIZER_OPENBSD
+uptr internal_munmap(void *addr, uptr length) {
+  return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
+}
+
+int internal_mprotect(void *addr, uptr length, int prot) {
+  return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
+}
+#endif
+
+uptr internal_close(fd_t fd) {
+  return internal_syscall(SYSCALL(close), fd);
+}
+
+uptr internal_open(const char *filename, int flags) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
+#else
+  return internal_syscall(SYSCALL(open), (uptr)filename, flags);
+#endif
+}
+
+uptr internal_open(const char *filename, int flags, u32 mode) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
+                          mode);
+#else
+  return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode);
+#endif
+}
+
+uptr internal_read(fd_t fd, void *buf, uptr count) {
+  sptr res;
+  HANDLE_EINTR(res,
+               (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf, count));
+  return res;
+}
+
+uptr internal_write(fd_t fd, const void *buf, uptr count) {
+  sptr res;
+  HANDLE_EINTR(res,
+               (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf, count));
+  return res;
+}
+
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  sptr res;
+  HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd,
+               (OFF_T)size));
+  return res;
+}
+
+#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
+static void stat64_to_stat(struct stat64 *in, struct stat *out) {
+  internal_memset(out, 0, sizeof(*out));
+  out->st_dev = in->st_dev;
+  out->st_ino = in->st_ino;
+  out->st_mode = in->st_mode;
+  out->st_nlink = in->st_nlink;
+  out->st_uid = in->st_uid;
+  out->st_gid = in->st_gid;
+  out->st_rdev = in->st_rdev;
+  out->st_size = in->st_size;
+  out->st_blksize = in->st_blksize;
+  out->st_blocks = in->st_blocks;
+  out->st_atime = in->st_atime;
+  out->st_mtime = in->st_mtime;
+  out->st_ctime = in->st_ctime;
+}
+#endif
+
+#if defined(__mips64)
+// Undefine compatibility macros from <sys/stat.h>
+// so that they would not clash with the kernel_stat
+// st_[a|m|c]time fields
+#undef st_atime
+#undef st_mtime
+#undef st_ctime
+#if defined(SANITIZER_ANDROID)
+// Bionic sys/stat.h defines additional macros
+// for compatibility with the old NDKs and
+// they clash with the kernel_stat structure
+// st_[a|m|c]time_nsec fields.
+#undef st_atime_nsec
+#undef st_mtime_nsec
+#undef st_ctime_nsec
+#endif
+static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
+  internal_memset(out, 0, sizeof(*out));
+  out->st_dev = in->st_dev;
+  out->st_ino = in->st_ino;
+  out->st_mode = in->st_mode;
+  out->st_nlink = in->st_nlink;
+  out->st_uid = in->st_uid;
+  out->st_gid = in->st_gid;
+  out->st_rdev = in->st_rdev;
+  out->st_size = in->st_size;
+  out->st_blksize = in->st_blksize;
+  out->st_blocks = in->st_blocks;
+#if defined(__USE_MISC)     || \
+    defined(__USE_XOPEN2K8) || \
+    defined(SANITIZER_ANDROID)
+  out->st_atim.tv_sec = in->st_atime;
+  out->st_atim.tv_nsec = in->st_atime_nsec;
+  out->st_mtim.tv_sec = in->st_mtime;
+  out->st_mtim.tv_nsec = in->st_mtime_nsec;
+  out->st_ctim.tv_sec = in->st_ctime;
+  out->st_ctim.tv_nsec = in->st_ctime_nsec;
+#else
+  out->st_atime = in->st_atime;
+  out->st_atimensec = in->st_atime_nsec;
+  out->st_mtime = in->st_mtime;
+  out->st_mtimensec = in->st_mtime_nsec;
+  out->st_ctime = in->st_ctime;
+  out->st_atimensec = in->st_ctime_nsec;
+#endif
+}
+#endif
+
+uptr internal_stat(const char *path, void *buf) {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
+                          0);
+#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
+# if defined(__mips64)
+  // For mips64, stat syscall fills buffer in the format of kernel_stat
+  struct kernel_stat kbuf;
+  int res = internal_syscall(SYSCALL(stat), path, &kbuf);
+  kernel_stat_to_stat(&kbuf, (struct stat *)buf);
+  return res;
+# else
+  return internal_syscall(SYSCALL(stat), (uptr)path, (uptr)buf);
+# endif
+#else
+  struct stat64 buf64;
+  int res = internal_syscall(SYSCALL(stat64), path, &buf64);
+  stat64_to_stat(&buf64, (struct stat *)buf);
+  return res;
+#endif
+}
+
+uptr internal_lstat(const char *path, void *buf) {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
+                          AT_SYMLINK_NOFOLLOW);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
+                          AT_SYMLINK_NOFOLLOW);
+#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
+# if SANITIZER_MIPS64
+  // For mips64, lstat syscall fills buffer in the format of kernel_stat
+  struct kernel_stat kbuf;
+  int res = internal_syscall(SYSCALL(lstat), path, &kbuf);
+  kernel_stat_to_stat(&kbuf, (struct stat *)buf);
+  return res;
+# else
+  return internal_syscall(SYSCALL(lstat), (uptr)path, (uptr)buf);
+# endif
+#else
+  struct stat64 buf64;
+  int res = internal_syscall(SYSCALL(lstat64), path, &buf64);
+  stat64_to_stat(&buf64, (struct stat *)buf);
+  return res;
+#endif
+}
+
+uptr internal_fstat(fd_t fd, void *buf) {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD || \
+    SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_MIPS64 && !SANITIZER_OPENBSD
+  // For mips64, fstat syscall fills buffer in the format of kernel_stat
+  struct kernel_stat kbuf;
+  int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
+  kernel_stat_to_stat(&kbuf, (struct stat *)buf);
+  return res;
+# else
+  return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
+# endif
+#else
+  struct stat64 buf64;
+  int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
+  stat64_to_stat(&buf64, (struct stat *)buf);
+  return res;
+#endif
+}
+
+uptr internal_filesize(fd_t fd) {
+  struct stat st;
+  if (internal_fstat(fd, &st))
+    return -1;
+  return (uptr)st.st_size;
+}
+
+uptr internal_dup(int oldfd) {
+  return internal_syscall(SYSCALL(dup), oldfd);
+}
+
+uptr internal_dup2(int oldfd, int newfd) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);
+#else
+  return internal_syscall(SYSCALL(dup2), oldfd, newfd);
+#endif
+}
+
+uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
+                          bufsize);
+#elif SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
+                          bufsize);
+#else
+  return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
+#endif
+}
+
+uptr internal_unlink(const char *path) {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
+#else
+  return internal_syscall(SYSCALL(unlink), (uptr)path);
+#endif
+}
+
+uptr internal_rename(const char *oldpath, const char *newpath) {
+#if defined(__riscv)
+  return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
+                          (uptr)newpath, 0);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
+                          (uptr)newpath);
+#else
+  return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
+#endif
+}
+
+uptr internal_sched_yield() {
+  return internal_syscall(SYSCALL(sched_yield));
+}
+
+void internal__exit(int exitcode) {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+  internal_syscall(SYSCALL(exit), exitcode);
+#else
+  internal_syscall(SYSCALL(exit_group), exitcode);
+#endif
+  Die();  // Unreachable.
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+  struct timespec ts;
+  ts.tv_sec = seconds;
+  ts.tv_nsec = 0;
+  int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts);
+  if (res) return ts.tv_sec;
+  return 0;
+}
+
+uptr internal_execve(const char *filename, char *const argv[],
+                     char *const envp[]) {
+  return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv,
+                          (uptr)envp);
+}
+#endif  // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+
+// ----------------- sanitizer_common.h
+bool FileExists(const char *filename) {
+  if (ShouldMockFailureToOpen(filename))
+    return false;
+  struct stat st;
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  if (internal_syscall(SYSCALL(newfstatat), AT_FDCWD, filename, &st, 0))
+#else
+  if (internal_stat(filename, &st))
+#endif
+    return false;
+  // Sanity check: filename is a regular file.
+  return S_ISREG(st.st_mode);
+}
+
+#if !SANITIZER_NETBSD
+tid_t GetTid() {
+#if SANITIZER_FREEBSD
+  long Tid;
+  thr_self(&Tid);
+  return Tid;
+#elif SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(getthrid));
+#elif SANITIZER_SOLARIS
+  return thr_self();
+#else
+  return internal_syscall(SYSCALL(gettid));
+#endif
+}
+
+int TgKill(pid_t pid, tid_t tid, int sig) {
+#if SANITIZER_LINUX
+  return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
+#elif SANITIZER_FREEBSD
+  return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);
+#elif SANITIZER_OPENBSD
+  (void)pid;
+  return internal_syscall(SYSCALL(thrkill), tid, sig, nullptr);
+#elif SANITIZER_SOLARIS
+  (void)pid;
+  return thr_kill(tid, sig);
+#endif
+}
+#endif
+
+#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+u64 NanoTime() {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+  timeval tv;
+#else
+  kernel_timeval tv;
+#endif
+  internal_memset(&tv, 0, sizeof(tv));
+  internal_syscall(SYSCALL(gettimeofday), &tv, 0);
+  return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
+}
+
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+  return internal_syscall(SYSCALL(clock_gettime), clk_id, tp);
+}
+#endif  // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+
+// Like getenv, but reads env directly from /proc (on Linux) or parses the
+// 'environ' array (on some others) and does not use libc. This function
+// should be called first inside __asan_init.
+const char *GetEnv(const char *name) {
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
+    SANITIZER_SOLARIS
+  if (::environ != 0) {
+    uptr NameLen = internal_strlen(name);
+    for (char **Env = ::environ; *Env != 0; Env++) {
+      if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=')
+        return (*Env) + NameLen + 1;
+    }
+  }
+  return 0;  // Not found.
+#elif SANITIZER_LINUX
+  static char *environ;
+  static uptr len;
+  static bool inited;
+  if (!inited) {
+    inited = true;
+    uptr environ_size;
+    if (!ReadFileToBuffer("/proc/self/environ", &environ, &environ_size, &len))
+      environ = nullptr;
+  }
+  if (!environ || len == 0) return nullptr;
+  uptr namelen = internal_strlen(name);
+  const char *p = environ;
+  while (*p != '\0') {  // will happen at the \0\0 that terminates the buffer
+    // proc file has the format NAME=value\0NAME=value\0NAME=value\0...
+    const char* endp =
+        (char*)internal_memchr(p, '\0', len - (p - environ));
+    if (!endp)  // this entry isn't NUL terminated
+      return nullptr;
+    else if (!internal_memcmp(p, name, namelen) && p[namelen] == '=')  // Match.
+      return p + namelen + 1;  // point after =
+    p = endp + 1;
+  }
+  return nullptr;  // Not found.
+#else
+#error "Unsupported platform"
+#endif
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_OPENBSD && \
+    !SANITIZER_GO
+extern "C" {
+SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
+}
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD &&                \
+    !SANITIZER_OPENBSD
+static void ReadNullSepFileToArray(const char *path, char ***arr,
+                                   int arr_size) {
+  char *buff;
+  uptr buff_size;
+  uptr buff_len;
+  *arr = (char **)MmapOrDie(arr_size * sizeof(char *), "NullSepFileArray");
+  if (!ReadFileToBuffer(path, &buff, &buff_size, &buff_len, 1024 * 1024)) {
+    (*arr)[0] = nullptr;
+    return;
+  }
+  (*arr)[0] = buff;
+  int count, i;
+  for (count = 1, i = 1; ; i++) {
+    if (buff[i] == 0) {
+      if (buff[i+1] == 0) break;
+      (*arr)[count] = &buff[i+1];
+      CHECK_LE(count, arr_size - 1);  // FIXME: make this more flexible.
+      count++;
+    }
+  }
+  (*arr)[count] = nullptr;
+}
+#endif
+
+#if !SANITIZER_OPENBSD
+static void GetArgsAndEnv(char ***argv, char ***envp) {
+#if SANITIZER_FREEBSD
+  // On FreeBSD, retrieving the argument and environment arrays is done via the
+  // kern.ps_strings sysctl, which returns a pointer to a structure containing
+  // this information. See also <sys/exec.h>.
+  ps_strings *pss;
+  uptr sz = sizeof(pss);
+  if (internal_sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) {
+    Printf("sysctl kern.ps_strings failed\n");
+    Die();
+  }
+  *argv = pss->ps_argvstr;
+  *envp = pss->ps_envstr;
+#elif SANITIZER_NETBSD
+  *argv = __ps_strings->ps_argvstr;
+  *envp = __ps_strings->ps_envstr;
+#else // SANITIZER_FREEBSD
+#if !SANITIZER_GO
+  if (&__libc_stack_end) {
+    uptr* stack_end = (uptr*)__libc_stack_end;
+    // Normally argc can be obtained from *stack_end, however, on ARM glibc's
+    // _start clobbers it:
+    // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
+    // Do not special-case ARM and infer argc from argv everywhere.
+    int argc = 0;
+    while (stack_end[argc + 1]) argc++;
+    *argv = (char**)(stack_end + 1);
+    *envp = (char**)(stack_end + argc + 2);
+  } else {
+#endif // !SANITIZER_GO
+    static const int kMaxArgv = 2000, kMaxEnvp = 2000;
+    ReadNullSepFileToArray("/proc/self/cmdline", argv, kMaxArgv);
+    ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);
+#if !SANITIZER_GO
+  }
+#endif // !SANITIZER_GO
+#endif // SANITIZER_FREEBSD
+}
+
+char **GetArgv() {
+  char **argv, **envp;
+  GetArgsAndEnv(&argv, &envp);
+  return argv;
+}
+
+char **GetEnviron() {
+  char **argv, **envp;
+  GetArgsAndEnv(&argv, &envp);
+  return envp;
+}
+
+#endif  // !SANITIZER_OPENBSD
+
+#if !SANITIZER_SOLARIS
+enum MutexState {
+  MtxUnlocked = 0,
+  MtxLocked = 1,
+  MtxSleeping = 2
+};
+
+BlockingMutex::BlockingMutex() {
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  CHECK_EQ(owner_, 0);
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
+    return;
+  while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
+#if SANITIZER_FREEBSD
+    _umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0);
+#elif SANITIZER_NETBSD
+    sched_yield(); /* No userspace futex-like synchronization */
+#else
+    internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT_PRIVATE, MtxSleeping,
+                     0, 0, 0);
+#endif
+  }
+}
+
+void BlockingMutex::Unlock() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
+  CHECK_NE(v, MtxUnlocked);
+  if (v == MtxSleeping) {
+#if SANITIZER_FREEBSD
+    _umtx_op(m, UMTX_OP_WAKE, 1, 0, 0);
+#elif SANITIZER_NETBSD
+                   /* No userspace futex-like synchronization */
+#else
+    internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0);
+#endif
+  }
+}
+
+void BlockingMutex::CheckLocked() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
+}
+#endif // !SANITIZER_SOLARIS
+
+// ----------------- sanitizer_linux.h
+// The actual size of this structure is specified by d_reclen.
+// Note that getdents64 uses a different structure format. We only provide the
+// 32-bit syscall here.
+#if SANITIZER_NETBSD
+// Not used
+#elif SANITIZER_OPENBSD
+// struct dirent is different for Linux and us. At this moment, we use only
+// d_fileno (Linux call this d_ino), d_reclen, and d_name.
+struct linux_dirent {
+  u64 d_ino;  // d_fileno
+  u16 d_reclen;
+  u16 d_namlen;  // not used
+  u8 d_type;     // not used
+  char d_name[NAME_MAX + 1];
+};
+#else
+struct linux_dirent {
+#if SANITIZER_X32 || defined(__aarch64__)
+  u64 d_ino;
+  u64 d_off;
+#else
+  unsigned long      d_ino;
+  unsigned long      d_off;
+#endif
+  unsigned short     d_reclen;
+#ifdef __aarch64__
+  unsigned char      d_type;
+#endif
+  char               d_name[256];
+};
+#endif
+
+#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+// Syscall wrappers.
+uptr internal_ptrace(int request, int pid, void *addr, void *data) {
+  return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,
+                          (uptr)data);
+}
+
+uptr internal_waitpid(int pid, int *status, int options) {
+  return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options,
+                          0 /* rusage */);
+}
+
+uptr internal_getpid() {
+  return internal_syscall(SYSCALL(getpid));
+}
+
+uptr internal_getppid() {
+  return internal_syscall(SYSCALL(getppid));
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+#if SANITIZER_FREEBSD
+  return dlinfo(handle, request, p);
+#else
+  UNIMPLEMENTED();
+#endif
+}
+
+uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
+#if SANITIZER_FREEBSD
+  return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
+#else
+  return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);
+#endif
+}
+
+uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
+  return internal_syscall(SYSCALL(lseek), fd, offset, whence);
+}
+
+#if SANITIZER_LINUX
+uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
+  return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);
+}
+#endif
+
+uptr internal_sigaltstack(const void *ss, void *oss) {
+  return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
+}
+
+int internal_fork() {
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+  return internal_syscall(SYSCALL(clone), SIGCHLD, 0);
+#else
+  return internal_syscall(SYSCALL(fork));
+#endif
+}
+
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
+                    uptr *oldlenp, const void *newp, uptr newlen) {
+#if SANITIZER_OPENBSD
+  return sysctl(name, namelen, oldp, (size_t *)oldlenp, (void *)newp,
+                (size_t)newlen);
+#else
+  return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,
+                          (size_t *)oldlenp, newp, (size_t)newlen);
+#endif
+}
+
+#if SANITIZER_FREEBSD
+int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
+                          const void *newp, uptr newlen) {
+  static decltype(sysctlbyname) *real = nullptr;
+  if (!real)
+    real = (decltype(sysctlbyname) *)dlsym(RTLD_NEXT, "sysctlbyname");
+  CHECK(real);
+  return real(sname, oldp, (size_t *)oldlenp, newp, (size_t)newlen);
+}
+#endif
+#endif
+
+#if SANITIZER_LINUX
+#define SA_RESTORER 0x04000000
+// Doesn't set sa_restorer if the caller did not set it, so use with caution
+//(see below).
+int internal_sigaction_norestorer(int signum, const void *act, void *oldact) {
+  __sanitizer_kernel_sigaction_t k_act, k_oldact;
+  internal_memset(&k_act, 0, sizeof(__sanitizer_kernel_sigaction_t));
+  internal_memset(&k_oldact, 0, sizeof(__sanitizer_kernel_sigaction_t));
+  const __sanitizer_sigaction *u_act = (const __sanitizer_sigaction *)act;
+  __sanitizer_sigaction *u_oldact = (__sanitizer_sigaction *)oldact;
+  if (u_act) {
+    k_act.handler = u_act->handler;
+    k_act.sigaction = u_act->sigaction;
+    internal_memcpy(&k_act.sa_mask, &u_act->sa_mask,
+                    sizeof(__sanitizer_kernel_sigset_t));
+    // Without SA_RESTORER kernel ignores the calls (probably returns EINVAL).
+    k_act.sa_flags = u_act->sa_flags | SA_RESTORER;
+    // FIXME: most often sa_restorer is unset, however the kernel requires it
+    // to point to a valid signal restorer that calls the rt_sigreturn syscall.
+    // If sa_restorer passed to the kernel is NULL, the program may crash upon
+    // signal delivery or fail to unwind the stack in the signal handler.
+    // libc implementation of sigaction() passes its own restorer to
+    // rt_sigaction, so we need to do the same (we'll need to reimplement the
+    // restorers; for x86_64 the restorer address can be obtained from
+    // oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact).
+#if !SANITIZER_ANDROID || !SANITIZER_MIPS32
+    k_act.sa_restorer = u_act->sa_restorer;
+#endif
+  }
+
+  uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum,
+      (uptr)(u_act ? &k_act : nullptr),
+      (uptr)(u_oldact ? &k_oldact : nullptr),
+      (uptr)sizeof(__sanitizer_kernel_sigset_t));
+
+  if ((result == 0) && u_oldact) {
+    u_oldact->handler = k_oldact.handler;
+    u_oldact->sigaction = k_oldact.sigaction;
+    internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask,
+                    sizeof(__sanitizer_kernel_sigset_t));
+    u_oldact->sa_flags = k_oldact.sa_flags;
+#if !SANITIZER_ANDROID || !SANITIZER_MIPS32
+    u_oldact->sa_restorer = k_oldact.sa_restorer;
+#endif
+  }
+  return result;
+}
+#endif  // SANITIZER_LINUX
+
+uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
+                          __sanitizer_sigset_t *oldset) {
+#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+  return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
+#else
+  __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
+  __sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset;
+  return internal_syscall(SYSCALL(rt_sigprocmask), (uptr)how, (uptr)k_set,
+                          (uptr)k_oldset, sizeof(__sanitizer_kernel_sigset_t));
+#endif
+}
+
+void internal_sigfillset(__sanitizer_sigset_t *set) {
+  internal_memset(set, 0xff, sizeof(*set));
+}
+
+void internal_sigemptyset(__sanitizer_sigset_t *set) {
+  internal_memset(set, 0, sizeof(*set));
+}
+
+#if SANITIZER_LINUX
+void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
+  signum -= 1;
+  CHECK_GE(signum, 0);
+  CHECK_LT(signum, sizeof(*set) * 8);
+  __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
+  const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
+  const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
+  k_set->sig[idx] &= ~(1 << bit);
+}
+
+bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
+  signum -= 1;
+  CHECK_GE(signum, 0);
+  CHECK_LT(signum, sizeof(*set) * 8);
+  __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
+  const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
+  const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
+  return k_set->sig[idx] & (1 << bit);
+}
+#elif SANITIZER_FREEBSD
+void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
+  sigset_t *rset = reinterpret_cast<sigset_t *>(set);
+  sigdelset(rset, signum);
+}
+
+bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
+  sigset_t *rset = reinterpret_cast<sigset_t *>(set);
+  return sigismember(rset, signum);
+}
+#endif
+#endif // !SANITIZER_SOLARIS
+
+#if !SANITIZER_NETBSD
+// ThreadLister implementation.
+ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {
+  char task_directory_path[80];
+  internal_snprintf(task_directory_path, sizeof(task_directory_path),
+                    "/proc/%d/task/", pid);
+  descriptor_ = internal_open(task_directory_path, O_RDONLY | O_DIRECTORY);
+  if (internal_iserror(descriptor_)) {
+    Report("Can't open /proc/%d/task for reading.\n", pid);
+  }
+}
+
+ThreadLister::Result ThreadLister::ListThreads(
+    InternalMmapVector<tid_t> *threads) {
+  if (internal_iserror(descriptor_))
+    return Error;
+  internal_lseek(descriptor_, 0, SEEK_SET);
+  threads->clear();
+
+  Result result = Ok;
+  for (bool first_read = true;; first_read = false) {
+    // Resize to max capacity if it was downsized by IsAlive.
+    buffer_.resize(buffer_.capacity());
+    CHECK_GE(buffer_.size(), 4096);
+    uptr read = internal_getdents(
+        descriptor_, (struct linux_dirent *)buffer_.data(), buffer_.size());
+    if (!read)
+      return result;
+    if (internal_iserror(read)) {
+      Report("Can't read directory entries from /proc/%d/task.\n", pid_);
+      return Error;
+    }
+
+    for (uptr begin = (uptr)buffer_.data(), end = begin + read; begin < end;) {
+      struct linux_dirent *entry = (struct linux_dirent *)begin;
+      begin += entry->d_reclen;
+      if (entry->d_ino == 1) {
+        // Inode 1 is for bad blocks and also can be a reason for early return.
+        // Should be emitted if kernel tried to output terminating thread.
+        // See proc_task_readdir implementation in Linux.
+        result = Incomplete;
+      }
+      if (entry->d_ino && *entry->d_name >= '0' && *entry->d_name <= '9')
+        threads->push_back(internal_atoll(entry->d_name));
+    }
+
+    // Now we are going to detect short-read or early EOF. In such cases Linux
+    // can return inconsistent list with missing alive threads.
+    // Code will just remember that the list can be incomplete but it will
+    // continue reads to return as much as possible.
+    if (!first_read) {
+      // The first one was a short-read by definition.
+      result = Incomplete;
+    } else if (read > buffer_.size() - 1024) {
+      // Read was close to the buffer size. So double the size and assume the
+      // worst.
+      buffer_.resize(buffer_.size() * 2);
+      result = Incomplete;
+    } else if (!threads->empty() && !IsAlive(threads->back())) {
+      // Maybe Linux early returned from read on terminated thread (!pid_alive)
+      // and failed to restore read position.
+      // See next_tid and proc_task_instantiate in Linux.
+      result = Incomplete;
+    }
+  }
+}
+
+bool ThreadLister::IsAlive(int tid) {
+  // /proc/%d/task/%d/status uses same call to detect alive threads as
+  // proc_task_readdir. See task_state implementation in Linux.
+  char path[80];
+  internal_snprintf(path, sizeof(path), "/proc/%d/task/%d/status", pid_, tid);
+  if (!ReadFileToVector(path, &buffer_) || buffer_.empty())
+    return false;
+  buffer_.push_back(0);
+  static const char kPrefix[] = "\nPPid:";
+  const char *field = internal_strstr(buffer_.data(), kPrefix);
+  if (!field)
+    return false;
+  field += internal_strlen(kPrefix);
+  return (int)internal_atoll(field) != 0;
+}
+
+ThreadLister::~ThreadLister() {
+  if (!internal_iserror(descriptor_))
+    internal_close(descriptor_);
+}
+#endif
+
+#if SANITIZER_WORDSIZE == 32
+// Take care of unusable kernel area in top gigabyte.
+static uptr GetKernelAreaSize() {
+#if SANITIZER_LINUX && !SANITIZER_X32
+  const uptr gbyte = 1UL << 30;
+
+  // Firstly check if there are writable segments
+  // mapped to top gigabyte (e.g. stack).
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  if (proc_maps.Error())
+    return 0;
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;
+  }
+
+#if !SANITIZER_ANDROID
+  // Even if nothing is mapped, top Gb may still be accessible
+  // if we are running on 64-bit kernel.
+  // Uname may report misleading results if personality type
+  // is modified (e.g. under schroot) so check this as well.
+  struct utsname uname_info;
+  int pers = personality(0xffffffffUL);
+  if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&
+      internal_strstr(uname_info.machine, "64"))
+    return 0;
+#endif  // SANITIZER_ANDROID
+
+  // Top gigabyte is reserved for kernel.
+  return gbyte;
+#else
+  return 0;
+#endif  // SANITIZER_LINUX && !SANITIZER_X32
+}
+#endif  // SANITIZER_WORDSIZE == 32
+
+uptr GetMaxVirtualAddress() {
+#if (SANITIZER_NETBSD || SANITIZER_OPENBSD) && defined(__x86_64__)
+  return 0x7f7ffffff000ULL;  // (0x00007f8000000000 - PAGE_SIZE)
+#elif SANITIZER_WORDSIZE == 64
+# if defined(__powerpc64__) || defined(__aarch64__)
+  // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
+  // We somehow need to figure out which one we are using now and choose
+  // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
+  // Note that with 'ulimit -s unlimited' the stack is moved away from the top
+  // of the address space, so simply checking the stack address is not enough.
+  // This should (does) work for both PowerPC64 Endian modes.
+  // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
+  return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+# elif defined(__mips64)
+  return (1ULL << 40) - 1;  // 0x000000ffffffffffUL;
+# elif defined(__s390x__)
+  return (1ULL << 53) - 1;  // 0x001fffffffffffffUL;
+#elif defined(__sparc__)
+  return ~(uptr)0;
+# else
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# endif
+#else  // SANITIZER_WORDSIZE == 32
+# if defined(__s390__)
+  return (1ULL << 31) - 1;  // 0x7fffffff;
+# else
+  return (1ULL << 32) - 1;  // 0xffffffff;
+# endif
+#endif  // SANITIZER_WORDSIZE
+}
+
+uptr GetMaxUserVirtualAddress() {
+  uptr addr = GetMaxVirtualAddress();
+#if SANITIZER_WORDSIZE == 32 && !defined(__s390__)
+  if (!common_flags()->full_address_space)
+    addr -= GetKernelAreaSize();
+  CHECK_LT(reinterpret_cast<uptr>(&addr), addr);
+#endif
+  return addr;
+}
+
+#if !SANITIZER_ANDROID
+uptr GetPageSize() {
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) && \
+    defined(EXEC_PAGESIZE)
+  return EXEC_PAGESIZE;
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
+// Use sysctl as sysconf can trigger interceptors internally.
+  int pz = 0;
+  uptr pzl = sizeof(pz);
+  int mib[2] = {CTL_HW, HW_PAGESIZE};
+  int rv = internal_sysctl(mib, 2, &pz, &pzl, nullptr, 0);
+  CHECK_EQ(rv, 0);
+  return (uptr)pz;
+#elif SANITIZER_USE_GETAUXVAL
+  return getauxval(AT_PAGESZ);
+#else
+  return sysconf(_SC_PAGESIZE);  // EXEC_PAGESIZE may not be trustworthy.
+#endif
+}
+#endif // !SANITIZER_ANDROID
+
+#if !SANITIZER_OPENBSD
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+#if SANITIZER_SOLARIS
+  const char *default_module_name = getexecname();
+  CHECK_NE(default_module_name, NULL);
+  return internal_snprintf(buf, buf_len, "%s", default_module_name);
+#else
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+#if SANITIZER_FREEBSD
+  const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
+#else
+  const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};
+#endif
+  const char *default_module_name = "kern.proc.pathname";
+  uptr Size = buf_len;
+  bool IsErr =
+      (internal_sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);
+  int readlink_error = IsErr ? errno : 0;
+  uptr module_name_len = Size;
+#else
+  const char *default_module_name = "/proc/self/exe";
+  uptr module_name_len = internal_readlink(
+      default_module_name, buf, buf_len);
+  int readlink_error;
+  bool IsErr = internal_iserror(module_name_len, &readlink_error);
+#endif  // SANITIZER_SOLARIS
+  if (IsErr) {
+    // We can't read binary name for some reason, assume it's unknown.
+    Report("WARNING: reading executable name failed with errno %d, "
+           "some stack frames may not be symbolized\n", readlink_error);
+    module_name_len = internal_snprintf(buf, buf_len, "%s",
+                                        default_module_name);
+    CHECK_LT(module_name_len, buf_len);
+  }
+  return module_name_len;
+#endif
+}
+#endif // !SANITIZER_OPENBSD
+
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
+#if SANITIZER_LINUX
+  char *tmpbuf;
+  uptr tmpsize;
+  uptr tmplen;
+  if (ReadFileToBuffer("/proc/self/cmdline", &tmpbuf, &tmpsize, &tmplen,
+                       1024 * 1024)) {
+    internal_strncpy(buf, tmpbuf, buf_len);
+    UnmapOrDie(tmpbuf, tmpsize);
+    return internal_strlen(buf);
+  }
+#endif
+  return ReadBinaryName(buf, buf_len);
+}
+
+// Match full names of the form /path/to/base_name{-,.}*
+bool LibraryNameIs(const char *full_name, const char *base_name) {
+  const char *name = full_name;
+  // Strip path.
+  while (*name != '\0') name++;
+  while (name > full_name && *name != '/') name--;
+  if (*name == '/') name++;
+  uptr base_name_length = internal_strlen(base_name);
+  if (internal_strncmp(name, base_name, base_name_length)) return false;
+  return (name[base_name_length] == '-' || name[base_name_length] == '.');
+}
+
+#if !SANITIZER_ANDROID
+// Call cb for each region mapped by map.
+void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)) {
+  CHECK_NE(map, nullptr);
+#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+  typedef ElfW(Phdr) Elf_Phdr;
+  typedef ElfW(Ehdr) Elf_Ehdr;
+#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+  char *base = (char *)map->l_addr;
+  Elf_Ehdr *ehdr = (Elf_Ehdr *)base;
+  char *phdrs = base + ehdr->e_phoff;
+  char *phdrs_end = phdrs + ehdr->e_phnum * ehdr->e_phentsize;
+
+  // Find the segment with the minimum base so we can "relocate" the p_vaddr
+  // fields.  Typically ET_DYN objects (DSOs) have base of zero and ET_EXEC
+  // objects have a non-zero base.
+  uptr preferred_base = (uptr)-1;
+  for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) {
+    Elf_Phdr *phdr = (Elf_Phdr *)iter;
+    if (phdr->p_type == PT_LOAD && preferred_base > (uptr)phdr->p_vaddr)
+      preferred_base = (uptr)phdr->p_vaddr;
+  }
+
+  // Compute the delta from the real base to get a relocation delta.
+  sptr delta = (uptr)base - preferred_base;
+  // Now we can figure out what the loader really mapped.
+  for (char *iter = phdrs; iter != phdrs_end; iter += ehdr->e_phentsize) {
+    Elf_Phdr *phdr = (Elf_Phdr *)iter;
+    if (phdr->p_type == PT_LOAD) {
+      uptr seg_start = phdr->p_vaddr + delta;
+      uptr seg_end = seg_start + phdr->p_memsz;
+      // None of these values are aligned.  We consider the ragged edges of the
+      // load command as defined, since they are mapped from the file.
+      seg_start = RoundDownTo(seg_start, GetPageSizeCached());
+      seg_end = RoundUpTo(seg_end, GetPageSizeCached());
+      cb((void *)seg_start, seg_end - seg_start);
+    }
+  }
+}
+#endif
+
+#if defined(__x86_64__) && SANITIZER_LINUX
+// We cannot use glibc's clone wrapper, because it messes with the child
+// task's TLS. It writes the PID and TID of the child task to its thread
+// descriptor, but in our case the child task shares the thread descriptor with
+// the parent (because we don't know how to allocate a new thread
+// descriptor to keep glibc happy). So the stock version of clone(), when
+// used with CLONE_VM, would end up corrupting the parent's thread descriptor.
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  long long res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  child_stack = (char *)child_stack - 2 * sizeof(unsigned long long);
+  ((unsigned long long *)child_stack)[0] = (uptr)fn;
+  ((unsigned long long *)child_stack)[1] = (uptr)arg;
+  register void *r8 __asm__("r8") = newtls;
+  register int *r10 __asm__("r10") = child_tidptr;
+  __asm__ __volatile__(
+                       /* %rax = syscall(%rax = SYSCALL(clone),
+                        *                %rdi = flags,
+                        *                %rsi = child_stack,
+                        *                %rdx = parent_tidptr,
+                        *                %r8  = new_tls,
+                        *                %r10 = child_tidptr)
+                        */
+                       "syscall\n"
+
+                       /* if (%rax != 0)
+                        *   return;
+                        */
+                       "testq  %%rax,%%rax\n"
+                       "jnz    1f\n"
+
+                       /* In the child. Terminate unwind chain. */
+                       // XXX: We should also terminate the CFI unwind chain
+                       // here. Unfortunately clang 3.2 doesn't support the
+                       // necessary CFI directives, so we skip that part.
+                       "xorq   %%rbp,%%rbp\n"
+
+                       /* Call "fn(arg)". */
+                       "popq   %%rax\n"
+                       "popq   %%rdi\n"
+                       "call   *%%rax\n"
+
+                       /* Call _exit(%rax). */
+                       "movq   %%rax,%%rdi\n"
+                       "movq   %2,%%rax\n"
+                       "syscall\n"
+
+                       /* Return to parent. */
+                     "1:\n"
+                       : "=a" (res)
+                       : "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),
+                         "S"(child_stack),
+                         "D"(flags),
+                         "d"(parent_tidptr),
+                         "r"(r8),
+                         "r"(r10)
+                       : "memory", "r11", "rcx");
+  return res;
+}
+#elif defined(__mips__)
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  long long res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  child_stack = (char *)child_stack - 2 * sizeof(unsigned long long);
+  ((unsigned long long *)child_stack)[0] = (uptr)fn;
+  ((unsigned long long *)child_stack)[1] = (uptr)arg;
+  register void *a3 __asm__("$7") = newtls;
+  register int *a4 __asm__("$8") = child_tidptr;
+  // We don't have proper CFI directives here because it requires alot of code
+  // for very marginal benefits.
+  __asm__ __volatile__(
+                       /* $v0 = syscall($v0 = __NR_clone,
+                        * $a0 = flags,
+                        * $a1 = child_stack,
+                        * $a2 = parent_tidptr,
+                        * $a3 = new_tls,
+                        * $a4 = child_tidptr)
+                        */
+                       ".cprestore 16;\n"
+                       "move $4,%1;\n"
+                       "move $5,%2;\n"
+                       "move $6,%3;\n"
+                       "move $7,%4;\n"
+                       /* Store the fifth argument on stack
+                        * if we are using 32-bit abi.
+                        */
+#if SANITIZER_WORDSIZE == 32
+                       "lw %5,16($29);\n"
+#else
+                       "move $8,%5;\n"
+#endif
+                       "li $2,%6;\n"
+                       "syscall;\n"
+
+                       /* if ($v0 != 0)
+                        * return;
+                        */
+                       "bnez $2,1f;\n"
+
+                       /* Call "fn(arg)". */
+#if SANITIZER_WORDSIZE == 32
+#ifdef __BIG_ENDIAN__
+                       "lw $25,4($29);\n"
+                       "lw $4,12($29);\n"
+#else
+                       "lw $25,0($29);\n"
+                       "lw $4,8($29);\n"
+#endif
+#else
+                       "ld $25,0($29);\n"
+                       "ld $4,8($29);\n"
+#endif
+                       "jal $25;\n"
+
+                       /* Call _exit($v0). */
+                       "move $4,$2;\n"
+                       "li $2,%7;\n"
+                       "syscall;\n"
+
+                       /* Return to parent. */
+                     "1:\n"
+                       : "=r" (res)
+                       : "r"(flags),
+                         "r"(child_stack),
+                         "r"(parent_tidptr),
+                         "r"(a3),
+                         "r"(a4),
+                         "i"(__NR_clone),
+                         "i"(__NR_exit)
+                       : "memory", "$29" );
+  return res;
+}
+#elif defined(__aarch64__)
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  long long res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  child_stack = (char *)child_stack - 2 * sizeof(unsigned long long);
+  ((unsigned long long *)child_stack)[0] = (uptr)fn;
+  ((unsigned long long *)child_stack)[1] = (uptr)arg;
+
+  register int (*__fn)(void *)  __asm__("x0") = fn;
+  register void *__stack __asm__("x1") = child_stack;
+  register int   __flags __asm__("x2") = flags;
+  register void *__arg   __asm__("x3") = arg;
+  register int  *__ptid  __asm__("x4") = parent_tidptr;
+  register void *__tls   __asm__("x5") = newtls;
+  register int  *__ctid  __asm__("x6") = child_tidptr;
+
+  __asm__ __volatile__(
+                       "mov x0,x2\n" /* flags  */
+                       "mov x2,x4\n" /* ptid  */
+                       "mov x3,x5\n" /* tls  */
+                       "mov x4,x6\n" /* ctid  */
+                       "mov x8,%9\n" /* clone  */
+
+                       "svc 0x0\n"
+
+                       /* if (%r0 != 0)
+                        *   return %r0;
+                        */
+                       "cmp x0, #0\n"
+                       "bne 1f\n"
+
+                       /* In the child, now. Call "fn(arg)". */
+                       "ldp x1, x0, [sp], #16\n"
+                       "blr x1\n"
+
+                       /* Call _exit(%r0).  */
+                       "mov x8, %10\n"
+                       "svc 0x0\n"
+                     "1:\n"
+
+                       : "=r" (res)
+                       : "i"(-EINVAL),
+                         "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg),
+                         "r"(__ptid), "r"(__tls), "r"(__ctid),
+                         "i"(__NR_clone), "i"(__NR_exit)
+                       : "x30", "memory");
+  return res;
+}
+#elif defined(__powerpc64__)
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                   int *parent_tidptr, void *newtls, int *child_tidptr) {
+  long long res;
+// Stack frame structure.
+#if SANITIZER_PPC64V1
+//   Back chain == 0        (SP + 112)
+// Frame (112 bytes):
+//   Parameter save area    (SP + 48), 8 doublewords
+//   TOC save area          (SP + 40)
+//   Link editor doubleword (SP + 32)
+//   Compiler doubleword    (SP + 24)
+//   LR save area           (SP + 16)
+//   CR save area           (SP + 8)
+//   Back chain             (SP + 0)
+# define FRAME_SIZE 112
+# define FRAME_TOC_SAVE_OFFSET 40
+#elif SANITIZER_PPC64V2
+//   Back chain == 0        (SP + 32)
+// Frame (32 bytes):
+//   TOC save area          (SP + 24)
+//   LR save area           (SP + 16)
+//   CR save area           (SP + 8)
+//   Back chain             (SP + 0)
+# define FRAME_SIZE 32
+# define FRAME_TOC_SAVE_OFFSET 24
+#else
+# error "Unsupported PPC64 ABI"
+#endif
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+
+  register int (*__fn)(void *) __asm__("r3") = fn;
+  register void *__cstack      __asm__("r4") = child_stack;
+  register int __flags         __asm__("r5") = flags;
+  register void *__arg         __asm__("r6") = arg;
+  register int *__ptidptr      __asm__("r7") = parent_tidptr;
+  register void *__newtls      __asm__("r8") = newtls;
+  register int *__ctidptr      __asm__("r9") = child_tidptr;
+
+ __asm__ __volatile__(
+           /* fn and arg are saved across the syscall */
+           "mr 28, %5\n\t"
+           "mr 27, %8\n\t"
+
+           /* syscall
+             r0 == __NR_clone
+             r3 == flags
+             r4 == child_stack
+             r5 == parent_tidptr
+             r6 == newtls
+             r7 == child_tidptr */
+           "mr 3, %7\n\t"
+           "mr 5, %9\n\t"
+           "mr 6, %10\n\t"
+           "mr 7, %11\n\t"
+           "li 0, %3\n\t"
+           "sc\n\t"
+
+           /* Test if syscall was successful */
+           "cmpdi  cr1, 3, 0\n\t"
+           "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t"
+           "bne-   cr1, 1f\n\t"
+
+           /* Set up stack frame */
+           "li    29, 0\n\t"
+           "stdu  29, -8(1)\n\t"
+           "stdu  1, -%12(1)\n\t"
+           /* Do the function call */
+           "std   2, %13(1)\n\t"
+#if SANITIZER_PPC64V1
+           "ld    0, 0(28)\n\t"
+           "ld    2, 8(28)\n\t"
+           "mtctr 0\n\t"
+#elif SANITIZER_PPC64V2
+           "mr    12, 28\n\t"
+           "mtctr 12\n\t"
+#else
+# error "Unsupported PPC64 ABI"
+#endif
+           "mr    3, 27\n\t"
+           "bctrl\n\t"
+           "ld    2, %13(1)\n\t"
+
+           /* Call _exit(r3) */
+           "li 0, %4\n\t"
+           "sc\n\t"
+
+           /* Return to parent */
+           "1:\n\t"
+           "mr %0, 3\n\t"
+             : "=r" (res)
+             : "0" (-1),
+               "i" (EINVAL),
+               "i" (__NR_clone),
+               "i" (__NR_exit),
+               "r" (__fn),
+               "r" (__cstack),
+               "r" (__flags),
+               "r" (__arg),
+               "r" (__ptidptr),
+               "r" (__newtls),
+               "r" (__ctidptr),
+               "i" (FRAME_SIZE),
+               "i" (FRAME_TOC_SAVE_OFFSET)
+             : "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
+  return res;
+}
+#elif defined(__i386__) && SANITIZER_LINUX
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  int res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  child_stack = (char *)child_stack - 7 * sizeof(unsigned int);
+  ((unsigned int *)child_stack)[0] = (uptr)flags;
+  ((unsigned int *)child_stack)[1] = (uptr)0;
+  ((unsigned int *)child_stack)[2] = (uptr)fn;
+  ((unsigned int *)child_stack)[3] = (uptr)arg;
+  __asm__ __volatile__(
+                       /* %eax = syscall(%eax = SYSCALL(clone),
+                        *                %ebx = flags,
+                        *                %ecx = child_stack,
+                        *                %edx = parent_tidptr,
+                        *                %esi  = new_tls,
+                        *                %edi = child_tidptr)
+                        */
+
+                        /* Obtain flags */
+                        "movl    (%%ecx), %%ebx\n"
+                        /* Do the system call */
+                        "pushl   %%ebx\n"
+                        "pushl   %%esi\n"
+                        "pushl   %%edi\n"
+                        /* Remember the flag value.  */
+                        "movl    %%ebx, (%%ecx)\n"
+                        "int     $0x80\n"
+                        "popl    %%edi\n"
+                        "popl    %%esi\n"
+                        "popl    %%ebx\n"
+
+                        /* if (%eax != 0)
+                         *   return;
+                         */
+
+                        "test    %%eax,%%eax\n"
+                        "jnz    1f\n"
+
+                        /* terminate the stack frame */
+                        "xorl   %%ebp,%%ebp\n"
+                        /* Call FN. */
+                        "call    *%%ebx\n"
+#ifdef PIC
+                        "call    here\n"
+                        "here:\n"
+                        "popl    %%ebx\n"
+                        "addl    $_GLOBAL_OFFSET_TABLE_+[.-here], %%ebx\n"
+#endif
+                        /* Call exit */
+                        "movl    %%eax, %%ebx\n"
+                        "movl    %2, %%eax\n"
+                        "int     $0x80\n"
+                        "1:\n"
+                       : "=a" (res)
+                       : "a"(SYSCALL(clone)), "i"(SYSCALL(exit)),
+                         "c"(child_stack),
+                         "d"(parent_tidptr),
+                         "S"(newtls),
+                         "D"(child_tidptr)
+                       : "memory");
+  return res;
+}
+#elif defined(__arm__) && SANITIZER_LINUX
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  unsigned int res;
+  if (!fn || !child_stack)
+    return -EINVAL;
+  child_stack = (char *)child_stack - 2 * sizeof(unsigned int);
+  ((unsigned int *)child_stack)[0] = (uptr)fn;
+  ((unsigned int *)child_stack)[1] = (uptr)arg;
+  register int r0 __asm__("r0") = flags;
+  register void *r1 __asm__("r1") = child_stack;
+  register int *r2 __asm__("r2") = parent_tidptr;
+  register void *r3 __asm__("r3") = newtls;
+  register int *r4 __asm__("r4") = child_tidptr;
+  register int r7 __asm__("r7") = __NR_clone;
+
+#if __ARM_ARCH > 4 || defined (__ARM_ARCH_4T__)
+# define ARCH_HAS_BX
+#endif
+#if __ARM_ARCH > 4
+# define ARCH_HAS_BLX
+#endif
+
+#ifdef ARCH_HAS_BX
+# ifdef ARCH_HAS_BLX
+#  define BLX(R) "blx "  #R "\n"
+# else
+#  define BLX(R) "mov lr, pc; bx " #R "\n"
+# endif
+#else
+# define BLX(R)  "mov lr, pc; mov pc," #R "\n"
+#endif
+
+  __asm__ __volatile__(
+                       /* %r0 = syscall(%r7 = SYSCALL(clone),
+                        *               %r0 = flags,
+                        *               %r1 = child_stack,
+                        *               %r2 = parent_tidptr,
+                        *               %r3  = new_tls,
+                        *               %r4 = child_tidptr)
+                        */
+
+                       /* Do the system call */
+                       "swi 0x0\n"
+
+                       /* if (%r0 != 0)
+                        *   return %r0;
+                        */
+                       "cmp r0, #0\n"
+                       "bne 1f\n"
+
+                       /* In the child, now. Call "fn(arg)". */
+                       "ldr r0, [sp, #4]\n"
+                       "ldr ip, [sp], #8\n"
+                       BLX(ip)
+                       /* Call _exit(%r0). */
+                       "mov r7, %7\n"
+                       "swi 0x0\n"
+                       "1:\n"
+                       "mov %0, r0\n"
+                       : "=r"(res)
+                       : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7),
+                         "i"(__NR_exit)
+                       : "memory");
+  return res;
+}
+#endif  // defined(__x86_64__) && SANITIZER_LINUX
+
+#if SANITIZER_LINUX
+int internal_uname(struct utsname *buf) {
+  return internal_syscall(SYSCALL(uname), buf);
+}
+#endif
+
+#if SANITIZER_ANDROID
+#if __ANDROID_API__ < 21
+extern "C" __attribute__((weak)) int dl_iterate_phdr(
+    int (*)(struct dl_phdr_info *, size_t, void *), void *);
+#endif
+
+static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
+                                   void *data) {
+  // Any name starting with "lib" indicates a bug in L where library base names
+  // are returned instead of paths.
+  if (info->dlpi_name && info->dlpi_name[0] == 'l' &&
+      info->dlpi_name[1] == 'i' && info->dlpi_name[2] == 'b') {
+    *(bool *)data = true;
+    return 1;
+  }
+  return 0;
+}
+
+static atomic_uint32_t android_api_level;
+
+static AndroidApiLevel AndroidDetectApiLevelStatic() {
+#if __ANDROID_API__ <= 19
+  return ANDROID_KITKAT;
+#elif __ANDROID_API__ <= 22
+  return ANDROID_LOLLIPOP_MR1;
+#else
+  return ANDROID_POST_LOLLIPOP;
+#endif
+}
+
+static AndroidApiLevel AndroidDetectApiLevel() {
+  if (!&dl_iterate_phdr)
+    return ANDROID_KITKAT; // K or lower
+  bool base_name_seen = false;
+  dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
+  if (base_name_seen)
+    return ANDROID_LOLLIPOP_MR1; // L MR1
+  return ANDROID_POST_LOLLIPOP;   // post-L
+  // Plain L (API level 21) is completely broken wrt ASan and not very
+  // interesting to detect.
+}
+
+extern "C" __attribute__((weak)) void* _DYNAMIC;
+
+AndroidApiLevel AndroidGetApiLevel() {
+  AndroidApiLevel level =
+      (AndroidApiLevel)atomic_load(&android_api_level, memory_order_relaxed);
+  if (level) return level;
+  level = &_DYNAMIC == nullptr ? AndroidDetectApiLevelStatic()
+                               : AndroidDetectApiLevel();
+  atomic_store(&android_api_level, level, memory_order_relaxed);
+  return level;
+}
+
+#endif
+
+static HandleSignalMode GetHandleSignalModeImpl(int signum) {
+  switch (signum) {
+    case SIGABRT:
+      return common_flags()->handle_abort;
+    case SIGILL:
+      return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
+    case SIGFPE:
+      return common_flags()->handle_sigfpe;
+    case SIGSEGV:
+      return common_flags()->handle_segv;
+    case SIGBUS:
+      return common_flags()->handle_sigbus;
+  }
+  return kHandleSignalNo;
+}
+
+HandleSignalMode GetHandleSignalMode(int signum) {
+  HandleSignalMode result = GetHandleSignalModeImpl(signum);
+  if (result == kHandleSignalYes && !common_flags()->allow_user_segv_handler)
+    return kHandleSignalExclusive;
+  return result;
+}
+
+#if !SANITIZER_GO
+void *internal_start_thread(void *(*func)(void *arg), void *arg) {
+  // Start the thread with signals blocked, otherwise it can steal user signals.
+  __sanitizer_sigset_t set, old;
+  internal_sigfillset(&set);
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  // Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
+  // on any thread, setuid call hangs (see test/tsan/setuid.c).
+  internal_sigdelset(&set, 33);
+#endif
+  internal_sigprocmask(SIG_SETMASK, &set, &old);
+  void *th;
+  real_pthread_create(&th, nullptr, func, arg);
+  internal_sigprocmask(SIG_SETMASK, &old, nullptr);
+  return th;
+}
+
+void internal_join_thread(void *th) {
+  real_pthread_join(th, nullptr);
+}
+#else
+void *internal_start_thread(void *(*func)(void *), void *arg) { return 0; }
+
+void internal_join_thread(void *th) {}
+#endif
+
+#if defined(__aarch64__)
+// Android headers in the older NDK releases miss this definition.
+struct __sanitizer_esr_context {
+  struct _aarch64_ctx head;
+  uint64_t esr;
+};
+
+static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) {
+  static const u32 kEsrMagic = 0x45535201;
+  u8 *aux = ucontext->uc_mcontext.__reserved;
+  while (true) {
+    _aarch64_ctx *ctx = (_aarch64_ctx *)aux;
+    if (ctx->size == 0) break;
+    if (ctx->magic == kEsrMagic) {
+      *esr = ((__sanitizer_esr_context *)ctx)->esr;
+      return true;
+    }
+    aux += ctx->size;
+  }
+  return false;
+}
+#endif
+
+#if SANITIZER_OPENBSD
+using Context = sigcontext;
+#else
+using Context = ucontext_t;
+#endif
+
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
+  Context *ucontext = (Context *)context;
+#if defined(__x86_64__) || defined(__i386__)
+  static const uptr PF_WRITE = 1U << 1;
+#if SANITIZER_FREEBSD
+  uptr err = ucontext->uc_mcontext.mc_err;
+#elif SANITIZER_NETBSD
+  uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
+#elif SANITIZER_OPENBSD
+  uptr err = ucontext->sc_err;
+#elif SANITIZER_SOLARIS && defined(__i386__)
+  const int Err = 13;
+  uptr err = ucontext->uc_mcontext.gregs[Err];
+#else
+  uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
+#endif // SANITIZER_FREEBSD
+  return err & PF_WRITE ? WRITE : READ;
+#elif defined(__mips__)
+  uint32_t *exception_source;
+  uint32_t faulty_instruction;
+  uint32_t op_code;
+
+  exception_source = (uint32_t *)ucontext->uc_mcontext.pc;
+  faulty_instruction = (uint32_t)(*exception_source);
+
+  op_code = (faulty_instruction >> 26) & 0x3f;
+
+  // FIXME: Add support for FPU, microMIPS, DSP, MSA memory instructions.
+  switch (op_code) {
+    case 0x28:  // sb
+    case 0x29:  // sh
+    case 0x2b:  // sw
+    case 0x3f:  // sd
+#if __mips_isa_rev < 6
+    case 0x2c:  // sdl
+    case 0x2d:  // sdr
+    case 0x2a:  // swl
+    case 0x2e:  // swr
+#endif
+      return SignalContext::WRITE;
+
+    case 0x20:  // lb
+    case 0x24:  // lbu
+    case 0x21:  // lh
+    case 0x25:  // lhu
+    case 0x23:  // lw
+    case 0x27:  // lwu
+    case 0x37:  // ld
+#if __mips_isa_rev < 6
+    case 0x1a:  // ldl
+    case 0x1b:  // ldr
+    case 0x22:  // lwl
+    case 0x26:  // lwr
+#endif
+      return SignalContext::READ;
+#if __mips_isa_rev == 6
+    case 0x3b:  // pcrel
+      op_code = (faulty_instruction >> 19) & 0x3;
+      switch (op_code) {
+        case 0x1:  // lwpc
+        case 0x2:  // lwupc
+          return SignalContext::READ;
+      }
+#endif
+  }
+  return SignalContext::UNKNOWN;
+#elif defined(__arm__)
+  static const uptr FSR_WRITE = 1U << 11;
+  uptr fsr = ucontext->uc_mcontext.error_code;
+  return fsr & FSR_WRITE ? WRITE : READ;
+#elif defined(__aarch64__)
+  static const u64 ESR_ELx_WNR = 1U << 6;
+  u64 esr;
+  if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN;
+  return esr & ESR_ELx_WNR ? WRITE : READ;
+#elif defined(__sparc__)
+  // Decode the instruction to determine the access type.
+  // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
+#if SANITIZER_SOLARIS
+  uptr pc = ucontext->uc_mcontext.gregs[REG_PC];
+#else
+  // Historical BSDism here.
+  struct sigcontext *scontext = (struct sigcontext *)context;
+#if defined(__arch64__)
+  uptr pc = scontext->sigc_regs.tpc;
+#else
+  uptr pc = scontext->si_regs.pc;
+#endif
+#endif
+  u32 instr = *(u32 *)pc;
+  return (instr >> 21) & 1 ? WRITE: READ;
+#elif defined(__riscv)
+  unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
+  unsigned faulty_instruction = *(uint16_t *)pc;
+
+#if defined(__riscv_compressed)
+  if ((faulty_instruction & 0x3) != 0x3) {  // it's a compressed instruction
+    // set op_bits to the instruction bits [1, 0, 15, 14, 13]
+    unsigned op_bits =
+        ((faulty_instruction & 0x3) << 3) | (faulty_instruction >> 13);
+    unsigned rd = faulty_instruction & 0xF80;  // bits 7-11, inclusive
+    switch (op_bits) {
+      case 0b10'010:  // c.lwsp (rd != x0)
+#if __riscv_xlen == 64
+      case 0b10'011:  // c.ldsp (rd != x0)
+#endif
+        return rd ? SignalContext::READ : SignalContext::UNKNOWN;
+      case 0b00'010:  // c.lw
+#if __riscv_flen >= 32 && __riscv_xlen == 32
+      case 0b10'011:  // c.flwsp
+#endif
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+      case 0b00'011:  // c.flw / c.ld
+#endif
+#if __riscv_flen == 64
+      case 0b00'001:  // c.fld
+      case 0b10'001:  // c.fldsp
+#endif
+        return SignalContext::READ;
+      case 0b00'110:  // c.sw
+      case 0b10'110:  // c.swsp
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+      case 0b00'111:  // c.fsw / c.sd
+      case 0b10'111:  // c.fswsp / c.sdsp
+#endif
+#if __riscv_flen == 64
+      case 0b00'101:  // c.fsd
+      case 0b10'101:  // c.fsdsp
+#endif
+        return SignalContext::WRITE;
+      default:
+        return SignalContext::UNKNOWN;
+    }
+  }
+#endif
+
+  unsigned opcode = faulty_instruction & 0x7f;         // lower 7 bits
+  unsigned funct3 = (faulty_instruction >> 12) & 0x7;  // bits 12-14, inclusive
+  switch (opcode) {
+    case 0b0000011:  // loads
+      switch (funct3) {
+        case 0b000:  // lb
+        case 0b001:  // lh
+        case 0b010:  // lw
+#if __riscv_xlen == 64
+        case 0b011:  // ld
+#endif
+        case 0b100:  // lbu
+        case 0b101:  // lhu
+          return SignalContext::READ;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+    case 0b0100011:  // stores
+      switch (funct3) {
+        case 0b000:  // sb
+        case 0b001:  // sh
+        case 0b010:  // sw
+#if __riscv_xlen == 64
+        case 0b011:  // sd
+#endif
+          return SignalContext::WRITE;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+#if __riscv_flen >= 32
+    case 0b0000111:  // floating-point loads
+      switch (funct3) {
+        case 0b010:  // flw
+#if __riscv_flen == 64
+        case 0b011:  // fld
+#endif
+          return SignalContext::READ;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+    case 0b0100111:  // floating-point stores
+      switch (funct3) {
+        case 0b010:  // fsw
+#if __riscv_flen == 64
+        case 0b011:  // fsd
+#endif
+          return SignalContext::WRITE;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+#endif
+    default:
+      return SignalContext::UNKNOWN;
+  }
+#else
+  (void)ucontext;
+  return UNKNOWN;  // FIXME: Implement.
+#endif
+}
+
+bool SignalContext::IsTrueFaultingAddress() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  // SIGSEGV signals without a true fault address have si_code set to 128.
+  return si->si_signo == SIGSEGV && si->si_code != 128;
+}
+
+void SignalContext::DumpAllRegisters(void *context) {
+  // FIXME: Implement this.
+}
+
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+#if SANITIZER_NETBSD
+  // This covers all NetBSD architectures
+  ucontext_t *ucontext = (ucontext_t *)context;
+  *pc = _UC_MACHINE_PC(ucontext);
+  *bp = _UC_MACHINE_FP(ucontext);
+  *sp = _UC_MACHINE_SP(ucontext);
+#elif defined(__arm__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.arm_pc;
+  *bp = ucontext->uc_mcontext.arm_fp;
+  *sp = ucontext->uc_mcontext.arm_sp;
+#elif defined(__aarch64__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.pc;
+  *bp = ucontext->uc_mcontext.regs[29];
+  *sp = ucontext->uc_mcontext.sp;
+#elif defined(__hppa__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.sc_iaoq[0];
+  /* GCC uses %r3 whenever a frame pointer is needed.  */
+  *bp = ucontext->uc_mcontext.sc_gr[3];
+  *sp = ucontext->uc_mcontext.sc_gr[30];
+#elif defined(__x86_64__)
+# if SANITIZER_FREEBSD
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.mc_rip;
+  *bp = ucontext->uc_mcontext.mc_rbp;
+  *sp = ucontext->uc_mcontext.mc_rsp;
+#elif SANITIZER_OPENBSD
+  sigcontext *ucontext = (sigcontext *)context;
+  *pc = ucontext->sc_rip;
+  *bp = ucontext->sc_rbp;
+  *sp = ucontext->sc_rsp;
+# else
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.gregs[REG_RIP];
+  *bp = ucontext->uc_mcontext.gregs[REG_RBP];
+  *sp = ucontext->uc_mcontext.gregs[REG_RSP];
+# endif
+#elif defined(__i386__)
+# if SANITIZER_FREEBSD
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.mc_eip;
+  *bp = ucontext->uc_mcontext.mc_ebp;
+  *sp = ucontext->uc_mcontext.mc_esp;
+#elif SANITIZER_OPENBSD
+  sigcontext *ucontext = (sigcontext *)context;
+  *pc = ucontext->sc_eip;
+  *bp = ucontext->sc_ebp;
+  *sp = ucontext->sc_esp;
+# else
+  ucontext_t *ucontext = (ucontext_t*)context;
+# if SANITIZER_SOLARIS
+  /* Use the numeric values: the symbolic ones are undefined by llvm
+     include/llvm/Support/Solaris.h.  */
+# ifndef REG_EIP
+#  define REG_EIP 14 // REG_PC
+# endif
+# ifndef REG_EBP
+#  define REG_EBP  6 // REG_FP
+# endif
+# ifndef REG_UESP
+#  define REG_UESP 17 // REG_SP
+# endif
+# endif
+  *pc = ucontext->uc_mcontext.gregs[REG_EIP];
+  *bp = ucontext->uc_mcontext.gregs[REG_EBP];
+  *sp = ucontext->uc_mcontext.gregs[REG_UESP];
+# endif
+#elif defined(__powerpc__) || defined(__powerpc64__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.regs->nip;
+  *sp = ucontext->uc_mcontext.regs->gpr[PT_R1];
+  // The powerpc{,64}-linux ABIs do not specify r31 as the frame
+  // pointer, but GCC always uses r31 when we need a frame pointer.
+  *bp = ucontext->uc_mcontext.regs->gpr[PT_R31];
+#elif defined(__sparc__)
+#if defined(__arch64__) || defined(__sparcv9)
+#define STACK_BIAS 2047
+#else
+#define STACK_BIAS 0
+# endif
+# if SANITIZER_SOLARIS
+  ucontext_t *ucontext = (ucontext_t *)context;
+  *pc = ucontext->uc_mcontext.gregs[REG_PC];
+  *sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;
+#else
+  // Historical BSDism here.
+  struct sigcontext *scontext = (struct sigcontext *)context;
+#if defined(__arch64__)
+  *pc = scontext->sigc_regs.tpc;
+  *sp = scontext->sigc_regs.u_regs[14] + STACK_BIAS;
+#else
+  *pc = scontext->si_regs.pc;
+  *sp = scontext->si_regs.u_regs[14];
+#endif
+# endif
+  *bp = (uptr)((uhwptr *)*sp)[14] + STACK_BIAS;
+#elif defined(__mips__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.pc;
+  *bp = ucontext->uc_mcontext.gregs[30];
+  *sp = ucontext->uc_mcontext.gregs[29];
+#elif defined(__s390__)
+  ucontext_t *ucontext = (ucontext_t*)context;
+# if defined(__s390x__)
+  *pc = ucontext->uc_mcontext.psw.addr;
+# else
+  *pc = ucontext->uc_mcontext.psw.addr & 0x7fffffff;
+# endif
+  *bp = ucontext->uc_mcontext.gregs[11];
+  *sp = ucontext->uc_mcontext.gregs[15];
+#elif defined(__riscv)
+  ucontext_t *ucontext = (ucontext_t*)context;
+  *pc = ucontext->uc_mcontext.__gregs[REG_PC];
+  *bp = ucontext->uc_mcontext.__gregs[REG_S0];
+  *sp = ucontext->uc_mcontext.__gregs[REG_SP];
+#else
+# error "Unsupported arch"
+#endif
+}
+
+void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+
+void InitializePlatformEarly() {
+  // Do nothing.
+}
+
+void MaybeReexec() {
+  // No need to re-exec on Linux.
+}
+
+void CheckASLR() {
+#if SANITIZER_NETBSD
+  int mib[3];
+  int paxflags;
+  uptr len = sizeof(paxflags);
+
+  mib[0] = CTL_PROC;
+  mib[1] = internal_getpid();
+  mib[2] = PROC_PID_PAXFLAGS;
+
+  if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) {
+    Printf("sysctl failed\n");
+    Die();
+  }
+
+  if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_ASLR)) {
+    Printf("This sanitizer is not compatible with enabled ASLR.\n"
+           "To disable ASLR, please run \"paxctl +a %s\" and try again.\n",
+           GetArgv()[0]);
+    Die();
+  }
+#elif SANITIZER_PPC64V2
+  // Disable ASLR for Linux PPC64LE.
+  int old_personality = personality(0xffffffff);
+  if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
+    VReport(1, "WARNING: Program is being run with address space layout "
+               "randomization (ASLR) enabled which prevents the thread and "
+               "memory sanitizers from working on powerpc64le.\n"
+               "ASLR will be disabled and the program re-executed.\n");
+    CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+    ReExec();
+  }
+#elif SANITIZER_FREEBSD
+  int aslr_pie;
+  uptr len = sizeof(aslr_pie);
+#if SANITIZER_WORDSIZE == 64
+  if (UNLIKELY(internal_sysctlbyname("kern.elf64.aslr.pie_enable",
+      &aslr_pie, &len, NULL, 0) == -1)) {
+    // We're making things less 'dramatic' here since
+    // the OID is not necessarily guaranteed to be here
+    // just yet regarding FreeBSD release
+    return;
+  }
+
+  if (aslr_pie > 0) {
+    Printf("This sanitizer is not compatible with enabled ASLR "
+           "and binaries compiled with PIE\n");
+    Die();
+  }
+#endif
+  // there might be 32 bits compat for 64 bits
+  if (UNLIKELY(internal_sysctlbyname("kern.elf32.aslr.pie_enable",
+      &aslr_pie, &len, NULL, 0) == -1)) {
+    return;
+  }
+
+  if (aslr_pie > 0) {
+    Printf("This sanitizer is not compatible with enabled ASLR "
+           "and binaries compiled with PIE\n");
+    Die();
+  }
+#else
+  // Do nothing
+#endif
+}
+
+void CheckMPROTECT() {
+#if SANITIZER_NETBSD
+  int mib[3];
+  int paxflags;
+  uptr len = sizeof(paxflags);
+
+  mib[0] = CTL_PROC;
+  mib[1] = internal_getpid();
+  mib[2] = PROC_PID_PAXFLAGS;
+
+  if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) {
+    Printf("sysctl failed\n");
+    Die();
+  }
+
+  if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_MPROTECT)) {
+    Printf("This sanitizer is not compatible with enabled MPROTECT\n");
+    Die();
+  }
+#else
+  // Do nothing
+#endif
+}
+
+void PrintModuleMap() { }
+
+void CheckNoDeepBind(const char *filename, int flag) {
+#ifdef RTLD_DEEPBIND
+  if (flag & RTLD_DEEPBIND) {
+    Report(
+        "You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
+        " which is incompatible with sanitizer runtime "
+        "(see https://github.com/google/sanitizers/issues/611 for details"
+        "). If you want to run %s library under sanitizers please remove "
+        "RTLD_DEEPBIND from dlopen flags.\n",
+        filename, filename);
+    Die();
+  }
+#endif
+}
+
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
+  UNREACHABLE("FindAvailableMemoryRange is not available");
+  return 0;
+}
+
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  if (!buffer || !length || length > 256)
+    return false;
+#if SANITIZER_USE_GETENTROPY
+  uptr rnd = getentropy(buffer, length);
+  int rverrno = 0;
+  if (internal_iserror(rnd, &rverrno) && rverrno == EFAULT)
+    return false;
+  else if (rnd == 0)
+    return true;
+#endif // SANITIZER_USE_GETENTROPY
+
+#if SANITIZER_USE_GETRANDOM
+  static atomic_uint8_t skip_getrandom_syscall;
+  if (!atomic_load_relaxed(&skip_getrandom_syscall)) {
+    // Up to 256 bytes, getrandom will not be interrupted.
+    uptr res = internal_syscall(SYSCALL(getrandom), buffer, length,
+                                blocking ? 0 : GRND_NONBLOCK);
+    int rverrno = 0;
+    if (internal_iserror(res, &rverrno) && rverrno == ENOSYS)
+      atomic_store_relaxed(&skip_getrandom_syscall, 1);
+    else if (res == length)
+      return true;
+  }
+#endif // SANITIZER_USE_GETRANDOM
+  // Up to 256 bytes, a read off /dev/urandom will not be interrupted.
+  // blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
+  uptr fd = internal_open("/dev/urandom", O_RDONLY);
+  if (internal_iserror(fd))
+    return false;
+  uptr res = internal_read(fd, buffer, length);
+  if (internal_iserror(res))
+    return false;
+  internal_close(fd);
+  return true;
+}
+
+} // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_linux.h b/lib/tsan/sanitizer_common/sanitizer_linux.h
new file mode 100644
index 0000000000..c162d1ca5d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_linux.h
@@ -0,0 +1,162 @@
+//===-- sanitizer_linux.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Linux-specific syscall wrappers and classes.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_LINUX_H
+#define SANITIZER_LINUX_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||                \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_freebsd.h"
+#include "sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_platform_limits_openbsd.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
+#include "sanitizer_posix.h"
+
+struct link_map;  // Opaque type returned by dlopen().
+struct utsname;
+
+namespace __sanitizer {
+// Dirent structure for getdents(). Note that this structure is different from
+// the one in <dirent.h>, which is used by readdir().
+struct linux_dirent;
+
+struct ProcSelfMapsBuff {
+  char *data;
+  uptr mmaped_size;
+  uptr len;
+};
+
+struct MemoryMappingLayoutData {
+  ProcSelfMapsBuff proc_self_maps;
+  const char *current;
+};
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
+
+// Syscall wrappers.
+uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
+uptr internal_sigaltstack(const void* ss, void* oss);
+uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
+    __sanitizer_sigset_t *oldset);
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
+
+// Linux-only syscalls.
+#if SANITIZER_LINUX
+uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
+// Used only by sanitizer_stoptheworld. Signal handlers that are actually used
+// (like the process-wide error reporting SEGV handler) must use
+// internal_sigaction instead.
+int internal_sigaction_norestorer(int signum, const void *act, void *oldact);
+void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
+#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \
+  || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \
+  || defined(__arm__)
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr);
+#endif
+int internal_uname(struct utsname *buf);
+#elif SANITIZER_FREEBSD
+void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
+#elif SANITIZER_NETBSD
+void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg);
+#endif  // SANITIZER_LINUX
+
+// This class reads thread IDs from /proc/<pid>/task using only syscalls.
+class ThreadLister {
+ public:
+  explicit ThreadLister(pid_t pid);
+  ~ThreadLister();
+  enum Result {
+    Error,
+    Incomplete,
+    Ok,
+  };
+  Result ListThreads(InternalMmapVector<tid_t> *threads);
+
+ private:
+  bool IsAlive(int tid);
+
+  pid_t pid_;
+  int descriptor_ = -1;
+  InternalMmapVector<char> buffer_;
+};
+
+// Exposed for testing.
+uptr ThreadDescriptorSize();
+uptr ThreadSelf();
+uptr ThreadSelfOffset();
+
+// Matches a library's file name against a base name (stripping path and version
+// information).
+bool LibraryNameIs(const char *full_name, const char *base_name);
+
+// Call cb for each region mapped by map.
+void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
+
+// Releases memory pages entirely within the [beg, end] address range.
+// The pages no longer count toward RSS; reads are guaranteed to return 0.
+// Requires (but does not verify!) that pages are MAP_PRIVATE.
+INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+  // man madvise on Linux promises zero-fill for anonymous private pages.
+  // Testing shows the same behaviour for private (but not anonymous) mappings
+  // of shm_open() files, as long as the underlying file is untouched.
+  CHECK(SANITIZER_LINUX);
+  ReleaseMemoryPagesToOS(beg, end);
+}
+
+#if SANITIZER_ANDROID
+
+#if defined(__aarch64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrs %0, tpidr_el0" : "=r"(__v)); __v; })
+#elif defined(__arm__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); __v; })
+#elif defined(__mips__)
+// On mips32r1, this goes via a kernel illegal instruction trap that's
+// optimized for v1.
+# define __get_tls() \
+    ({ register void** __v asm("v1"); \
+       __asm__(".set    push\n" \
+               ".set    mips32r2\n" \
+               "rdhwr   %0,$29\n" \
+               ".set    pop\n" : "=r"(__v)); \
+       __v; })
+#elif defined(__i386__)
+# define __get_tls() \
+    ({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })
+#elif defined(__x86_64__)
+# define __get_tls() \
+    ({ void** __v; __asm__("mov %%fs:0, %0" : "=r"(__v)); __v; })
+#else
+#error "Unsupported architecture."
+#endif
+
+// The Android Bionic team has allocated a TLS slot for sanitizers starting
+// with Q, given that Android currently doesn't support ELF TLS. It is used to
+// store sanitizer thread specific data.
+static const int TLS_SLOT_SANITIZER = 6;
+
+ALWAYS_INLINE uptr *get_android_tls_ptr() {
+  return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]);
+}
+
+#endif  // SANITIZER_ANDROID
+
+}  // namespace __sanitizer
+
+#endif
+#endif  // SANITIZER_LINUX_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp b/lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp
new file mode 100644
index 0000000000..bb2f5b5f9f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp
@@ -0,0 +1,222 @@
+//===-- sanitizer_linux_s390.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements s390-linux-specific functions from
+// sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX && SANITIZER_S390
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <sys/syscall.h>
+#include <sys/utsname.h>
+#include <unistd.h>
+
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+
+namespace __sanitizer {
+
+// --------------- sanitizer_libc.h
+uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
+                   u64 offset) {
+  struct s390_mmap_params {
+    unsigned long addr;
+    unsigned long length;
+    unsigned long prot;
+    unsigned long flags;
+    unsigned long fd;
+    unsigned long offset;
+  } params = {
+    (unsigned long)addr,
+    (unsigned long)length,
+    (unsigned long)prot,
+    (unsigned long)flags,
+    (unsigned long)fd,
+# ifdef __s390x__
+    (unsigned long)offset,
+# else
+    (unsigned long)(offset / 4096),
+# endif
+  };
+# ifdef __s390x__
+  return syscall(__NR_mmap, &params);
+# else
+  return syscall(__NR_mmap2, &params);
+# endif
+}
+
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+                    int *parent_tidptr, void *newtls, int *child_tidptr) {
+  if (!fn || !child_stack)
+    return -EINVAL;
+  CHECK_EQ(0, (uptr)child_stack % 16);
+  // Minimum frame size.
+#ifdef __s390x__
+  child_stack = (char *)child_stack - 160;
+#else
+  child_stack = (char *)child_stack - 96;
+#endif
+  // Terminate unwind chain.
+  ((unsigned long *)child_stack)[0] = 0;
+  // And pass parameters.
+  ((unsigned long *)child_stack)[1] = (uptr)fn;
+  ((unsigned long *)child_stack)[2] = (uptr)arg;
+  register long res __asm__("r2");
+  register void *__cstack      __asm__("r2") = child_stack;
+  register int __flags         __asm__("r3") = flags;
+  register int * __ptidptr     __asm__("r4") = parent_tidptr;
+  register int * __ctidptr     __asm__("r5") = child_tidptr;
+  register void * __newtls     __asm__("r6") = newtls;
+
+  __asm__ __volatile__(
+                       /* Clone. */
+                       "svc    %1\n"
+
+                       /* if (%r2 != 0)
+                        *   return;
+                        */
+#ifdef __s390x__
+                       "cghi   %%r2, 0\n"
+#else
+                       "chi    %%r2, 0\n"
+#endif
+                       "jne    1f\n"
+
+                       /* Call "fn(arg)". */
+#ifdef __s390x__
+                       "lmg    %%r1, %%r2, 8(%%r15)\n"
+#else
+                       "lm     %%r1, %%r2, 4(%%r15)\n"
+#endif
+                       "basr   %%r14, %%r1\n"
+
+                       /* Call _exit(%r2). */
+                       "svc %2\n"
+
+                       /* Return to parent. */
+                     "1:\n"
+                       : "=r" (res)
+                       : "i"(__NR_clone), "i"(__NR_exit),
+                         "r"(__cstack),
+                         "r"(__flags),
+                         "r"(__ptidptr),
+                         "r"(__ctidptr),
+                         "r"(__newtls)
+                       : "memory", "cc");
+  return res;
+}
+
+#if SANITIZER_S390_64
+static bool FixedCVE_2016_2143() {
+  // Try to determine if the running kernel has a fix for CVE-2016-2143,
+  // return false if in doubt (better safe than sorry).  Distros may want to
+  // adjust this for their own kernels.
+  struct utsname buf;
+  unsigned int major, minor, patch = 0;
+  // This should never fail, but just in case...
+  if (internal_uname(&buf))
+    return false;
+  const char *ptr = buf.release;
+  major = internal_simple_strtoll(ptr, &ptr, 10);
+  // At least first 2 should be matched.
+  if (ptr[0] != '.')
+    return false;
+  minor = internal_simple_strtoll(ptr+1, &ptr, 10);
+  // Third is optional.
+  if (ptr[0] == '.')
+    patch = internal_simple_strtoll(ptr+1, &ptr, 10);
+  if (major < 3) {
+    if (major == 2 && minor == 6 && patch == 32 && ptr[0] == '-' &&
+        internal_strstr(ptr, ".el6")) {
+      // Check RHEL6
+      int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
+      if (r1 >= 657) // 2.6.32-657.el6 or later
+        return true;
+      if (r1 == 642 && ptr[0] == '.') {
+        int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);
+        if (r2 >= 9) // 2.6.32-642.9.1.el6 or later
+          return true;
+      }
+    }
+    // <3.0 is bad.
+    return false;
+  } else if (major == 3) {
+    // 3.2.79+ is OK.
+    if (minor == 2 && patch >= 79)
+      return true;
+    // 3.12.58+ is OK.
+    if (minor == 12 && patch >= 58)
+      return true;
+    if (minor == 10 && patch == 0 && ptr[0] == '-' &&
+        internal_strstr(ptr, ".el7")) {
+      // Check RHEL7
+      int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
+      if (r1 >= 426) // 3.10.0-426.el7 or later
+        return true;
+      if (r1 == 327 && ptr[0] == '.') {
+        int r2 = internal_simple_strtoll(ptr+1, &ptr, 10);
+        if (r2 >= 27) // 3.10.0-327.27.1.el7 or later
+          return true;
+      }
+    }
+    // Otherwise, bad.
+    return false;
+  } else if (major == 4) {
+    // 4.1.21+ is OK.
+    if (minor == 1 && patch >= 21)
+      return true;
+    // 4.4.6+ is OK.
+    if (minor == 4 && patch >= 6)
+      return true;
+    if (minor == 4 && patch == 0 && ptr[0] == '-' &&
+        internal_strstr(buf.version, "Ubuntu")) {
+      // Check Ubuntu 16.04
+      int r1 = internal_simple_strtoll(ptr+1, &ptr, 10);
+      if (r1 >= 13) // 4.4.0-13 or later
+        return true;
+    }
+    // Otherwise, OK if 4.5+.
+    return minor >= 5;
+  } else {
+    // Linux 5 and up are fine.
+    return true;
+  }
+}
+
+void AvoidCVE_2016_2143() {
+  // Older kernels are affected by CVE-2016-2143 - they will crash hard
+  // if someone uses 4-level page tables (ie. virtual addresses >= 4TB)
+  // and fork() in the same process.  Unfortunately, sanitizers tend to
+  // require such addresses.  Since this is very likely to crash the whole
+  // machine (sanitizers themselves use fork() for llvm-symbolizer, for one),
+  // abort the process at initialization instead.
+  if (FixedCVE_2016_2143())
+    return;
+  if (GetEnv("SANITIZER_IGNORE_CVE_2016_2143"))
+    return;
+  Report(
+    "ERROR: Your kernel seems to be vulnerable to CVE-2016-2143.  Using ASan,\n"
+    "MSan, TSan, DFSan or LSan with such kernel can and will crash your\n"
+    "machine, or worse.\n"
+    "\n"
+    "If you are certain your kernel is not vulnerable (you have compiled it\n"
+    "yourself, or are using an unrecognized distribution kernel), you can\n"
+    "override this safety check by exporting SANITIZER_IGNORE_CVE_2016_2143\n"
+    "with any value.\n");
+  Die();
+}
+#endif
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_LINUX && SANITIZER_S390
diff --git a/lib/tsan/sanitizer_common/sanitizer_list.h b/lib/tsan/sanitizer_common/sanitizer_list.h
new file mode 100644
index 0000000000..f0b925945e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_list.h
@@ -0,0 +1,166 @@
+//===-- sanitizer_list.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains implementation of a list class to be used by
+// ThreadSanitizer, etc run-times.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_LIST_H
+#define SANITIZER_LIST_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+// Intrusive singly-linked list with size(), push_back(), push_front()
+// pop_front(), append_front() and append_back().
+// This class should be a POD (so that it can be put into TLS)
+// and an object with all zero fields should represent a valid empty list.
+// This class does not have a CTOR, so clear() should be called on all
+// non-zero-initialized objects before using.
+template<class Item>
+struct IntrusiveList {
+  friend class Iterator;
+
+  void clear() {
+    first_ = last_ = nullptr;
+    size_ = 0;
+  }
+
+  bool empty() const { return size_ == 0; }
+  uptr size() const { return size_; }
+
+  void push_back(Item *x) {
+    if (empty()) {
+      x->next = nullptr;
+      first_ = last_ = x;
+      size_ = 1;
+    } else {
+      x->next = nullptr;
+      last_->next = x;
+      last_ = x;
+      size_++;
+    }
+  }
+
+  void push_front(Item *x) {
+    if (empty()) {
+      x->next = nullptr;
+      first_ = last_ = x;
+      size_ = 1;
+    } else {
+      x->next = first_;
+      first_ = x;
+      size_++;
+    }
+  }
+
+  void pop_front() {
+    CHECK(!empty());
+    first_ = first_->next;
+    if (!first_)
+      last_ = nullptr;
+    size_--;
+  }
+
+  void extract(Item *prev, Item *x) {
+    CHECK(!empty());
+    CHECK_NE(prev, nullptr);
+    CHECK_NE(x, nullptr);
+    CHECK_EQ(prev->next, x);
+    prev->next = x->next;
+    if (last_ == x)
+      last_ = prev;
+    size_--;
+  }
+
+  Item *front() { return first_; }
+  const Item *front() const { return first_; }
+  Item *back() { return last_; }
+  const Item *back() const { return last_; }
+
+  void append_front(IntrusiveList<Item> *l) {
+    CHECK_NE(this, l);
+    if (l->empty())
+      return;
+    if (empty()) {
+      *this = *l;
+    } else if (!l->empty()) {
+      l->last_->next = first_;
+      first_ = l->first_;
+      size_ += l->size();
+    }
+    l->clear();
+  }
+
+  void append_back(IntrusiveList<Item> *l) {
+    CHECK_NE(this, l);
+    if (l->empty())
+      return;
+    if (empty()) {
+      *this = *l;
+    } else {
+      last_->next = l->first_;
+      last_ = l->last_;
+      size_ += l->size();
+    }
+    l->clear();
+  }
+
+  void CheckConsistency() {
+    if (size_ == 0) {
+      CHECK_EQ(first_, 0);
+      CHECK_EQ(last_, 0);
+    } else {
+      uptr count = 0;
+      for (Item *i = first_; ; i = i->next) {
+        count++;
+        if (i == last_) break;
+      }
+      CHECK_EQ(size(), count);
+      CHECK_EQ(last_->next, 0);
+    }
+  }
+
+  template<class ItemTy>
+  class IteratorBase {
+   public:
+    explicit IteratorBase(ItemTy *current) : current_(current) {}
+    IteratorBase &operator++() {
+      current_ = current_->next;
+      return *this;
+    }
+    bool operator!=(IteratorBase other) const {
+      return current_ != other.current_;
+    }
+    ItemTy &operator*() {
+      return *current_;
+    }
+   private:
+    ItemTy *current_;
+  };
+
+  typedef IteratorBase<Item> Iterator;
+  typedef IteratorBase<const Item> ConstIterator;
+
+  Iterator begin() { return Iterator(first_); }
+  Iterator end() { return Iterator(0); }
+
+  ConstIterator begin() const { return ConstIterator(first_); }
+  ConstIterator end() const { return ConstIterator(0); }
+
+// private, don't use directly.
+  uptr size_;
+  Item *first_;
+  Item *last_;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_LIST_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h b/lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h
new file mode 100644
index 0000000000..5d1b5264b5
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h
@@ -0,0 +1,76 @@
+//===-- sanitizer_local_address_space_view.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// `LocalAddressSpaceView` provides the local (i.e. target and current address
+// space are the same) implementation of the `AddressSpaveView` interface which
+// provides a simple interface to load memory from another process (i.e.
+// out-of-process)
+//
+// The `AddressSpaceView` interface requires that the type can be used as a
+// template parameter to objects that wish to be able to operate in an
+// out-of-process manner. In normal usage, objects are in-process and are thus
+// instantiated with the `LocalAddressSpaceView` type. This type is used to
+// load any pointers in instance methods. This implementation is effectively
+// a no-op. When an object is to be used in an out-of-process manner it is
+// instansiated with the `RemoteAddressSpaceView` type.
+//
+// By making `AddressSpaceView` a template parameter of an object, it can
+// change its implementation at compile time which has no run time overhead.
+// This also allows unifying in-process and out-of-process code which avoids
+// code duplication.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H
+#define SANITIZER_LOCAL_ADDRES_SPACE_VIEW_H
+
+namespace __sanitizer {
+struct LocalAddressSpaceView {
+  // Load memory `sizeof(T) * num_elements` bytes of memory from the target
+  // process (always local for this implementation) starting at address
+  // `target_address`. The local copy of this memory is returned as a pointer.
+  // The caller should not write to this memory. The behaviour when doing so is
+  // undefined. Callers should use `LoadWritable()` to get access to memory
+  // that is writable.
+  //
+  // The lifetime of loaded memory is implementation defined.
+  template <typename T>
+  static const T *Load(const T *target_address, uptr num_elements = 1) {
+    // The target address space is the local address space so
+    // nothing needs to be copied. Just return the pointer.
+    return target_address;
+  }
+
+  // Load memory `sizeof(T) * num_elements` bytes of memory from the target
+  // process (always local for this implementation) starting at address
+  // `target_address`. The local copy of this memory is returned as a pointer.
+  // The memory returned may be written to.
+  //
+  // Writes made to the returned memory will be visible in the memory returned
+  // by subsequent `Load()` or `LoadWritable()` calls provided the
+  // `target_address` parameter is the same. It is not guaranteed that the
+  // memory returned by previous calls to `Load()` will contain any performed
+  // writes.  If two or more overlapping regions of memory are loaded via
+  // separate calls to `LoadWritable()`, it is implementation defined whether
+  // writes made to the region returned by one call are visible in the regions
+  // returned by other calls.
+  //
+  // Given the above it is recommended to load the largest possible object
+  // that requires modification (e.g. a class) rather than individual fields
+  // from a class to avoid issues with overlapping writable regions.
+  //
+  // The lifetime of loaded memory is implementation defined.
+  template <typename T>
+  static T *LoadWritable(T *target_address, uptr num_elements = 1) {
+    // The target address space is the local address space so
+    // nothing needs to be copied. Just return the pointer.
+    return target_address;
+  }
+};
+}  // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_mac.cpp
new file mode 100644
index 0000000000..7a3dfbcc27
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_mac.cpp
@@ -0,0 +1,1228 @@
+//===-- sanitizer_mac.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements OSX-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_mac.h"
+#include "interception/interception.h"
+
+// Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so
+// the clients will most certainly use 64-bit ones as well.
+#ifndef _DARWIN_USE_64_BIT_INODE
+#define _DARWIN_USE_64_BIT_INODE 1
+#endif
+#include <stdio.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_ptrauth.h"
+
+#if !SANITIZER_IOS
+#include <crt_externs.h>  // for _NSGetEnviron
+#else
+extern char **environ;
+#endif
+
+#if defined(__has_include) && __has_include(<os/trace.h>)
+#define SANITIZER_OS_TRACE 1
+#include <os/trace.h>
+#else
+#define SANITIZER_OS_TRACE 0
+#endif
+
+#if !SANITIZER_IOS
+#include <crt_externs.h>  // for _NSGetArgv and _NSGetEnviron
+#else
+extern "C" {
+  extern char ***_NSGetArgv(void);
+}
+#endif
+
+#include <asl.h>
+#include <dlfcn.h>  // for dladdr()
+#include <errno.h>
+#include <fcntl.h>
+#include <libkern/OSAtomic.h>
+#include <mach-o/dyld.h>
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <mach/vm_statistics.h>
+#include <malloc/malloc.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <spawn.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <util.h>
+
+// From <crt_externs.h>, but we don't have that file on iOS.
+extern "C" {
+  extern char ***_NSGetArgv(void);
+  extern char ***_NSGetEnviron(void);
+}
+
+// From <mach/mach_vm.h>, but we don't have that file on iOS.
+extern "C" {
+  extern kern_return_t mach_vm_region_recurse(
+    vm_map_t target_task,
+    mach_vm_address_t *address,
+    mach_vm_size_t *size,
+    natural_t *nesting_depth,
+    vm_region_recurse_info_t info,
+    mach_msg_type_number_t *infoCnt);
+}
+
+namespace __sanitizer {
+
+#include "sanitizer_syscall_generic.inc"
+
+// Direct syscalls, don't call libmalloc hooks (but not available on 10.6).
+extern "C" void *__mmap(void *addr, size_t len, int prot, int flags, int fildes,
+                        off_t off) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int __munmap(void *, size_t) SANITIZER_WEAK_ATTRIBUTE;
+
+// ---------------------- sanitizer_libc.h
+
+// From <mach/vm_statistics.h>, but not on older OSs.
+#ifndef VM_MEMORY_SANITIZER
+#define VM_MEMORY_SANITIZER 99
+#endif
+
+// XNU on Darwin provides a mmap flag that optimizes allocation/deallocation of
+// giant memory regions (i.e. shadow memory regions).
+#define kXnuFastMmapFd 0x4
+static size_t kXnuFastMmapThreshold = 2 << 30; // 2 GB
+static bool use_xnu_fast_mmap = false;
+
+uptr internal_mmap(void *addr, size_t length, int prot, int flags,
+                   int fd, u64 offset) {
+  if (fd == -1) {
+    fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER);
+    if (length >= kXnuFastMmapThreshold) {
+      if (use_xnu_fast_mmap) fd |= kXnuFastMmapFd;
+    }
+  }
+  if (&__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset);
+  return (uptr)mmap(addr, length, prot, flags, fd, offset);
+}
+
+uptr internal_munmap(void *addr, uptr length) {
+  if (&__munmap) return __munmap(addr, length);
+  return munmap(addr, length);
+}
+
+int internal_mprotect(void *addr, uptr length, int prot) {
+  return mprotect(addr, length, prot);
+}
+
+uptr internal_close(fd_t fd) {
+  return close(fd);
+}
+
+uptr internal_open(const char *filename, int flags) {
+  return open(filename, flags);
+}
+
+uptr internal_open(const char *filename, int flags, u32 mode) {
+  return open(filename, flags, mode);
+}
+
+uptr internal_read(fd_t fd, void *buf, uptr count) {
+  return read(fd, buf, count);
+}
+
+uptr internal_write(fd_t fd, const void *buf, uptr count) {
+  return write(fd, buf, count);
+}
+
+uptr internal_stat(const char *path, void *buf) {
+  return stat(path, (struct stat *)buf);
+}
+
+uptr internal_lstat(const char *path, void *buf) {
+  return lstat(path, (struct stat *)buf);
+}
+
+uptr internal_fstat(fd_t fd, void *buf) {
+  return fstat(fd, (struct stat *)buf);
+}
+
+uptr internal_filesize(fd_t fd) {
+  struct stat st;
+  if (internal_fstat(fd, &st))
+    return -1;
+  return (uptr)st.st_size;
+}
+
+uptr internal_dup(int oldfd) {
+  return dup(oldfd);
+}
+
+uptr internal_dup2(int oldfd, int newfd) {
+  return dup2(oldfd, newfd);
+}
+
+uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
+  return readlink(path, buf, bufsize);
+}
+
+uptr internal_unlink(const char *path) {
+  return unlink(path);
+}
+
+uptr internal_sched_yield() {
+  return sched_yield();
+}
+
+void internal__exit(int exitcode) {
+  _exit(exitcode);
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+  return sleep(seconds);
+}
+
+uptr internal_getpid() {
+  return getpid();
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+  UNIMPLEMENTED();
+}
+
+int internal_sigaction(int signum, const void *act, void *oldact) {
+  return sigaction(signum,
+                   (const struct sigaction *)act, (struct sigaction *)oldact);
+}
+
+void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
+
+uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
+                          __sanitizer_sigset_t *oldset) {
+  // Don't use sigprocmask here, because it affects all threads.
+  return pthread_sigmask(how, set, oldset);
+}
+
+// Doesn't call pthread_atfork() handlers (but not available on 10.6).
+extern "C" pid_t __fork(void) SANITIZER_WEAK_ATTRIBUTE;
+
+int internal_fork() {
+  if (&__fork)
+    return __fork();
+  return fork();
+}
+
+int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
+                    uptr *oldlenp, const void *newp, uptr newlen) {
+  return sysctl(const_cast<int *>(name), namelen, oldp, (size_t *)oldlenp,
+                const_cast<void *>(newp), (size_t)newlen);
+}
+
+int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
+                          const void *newp, uptr newlen) {
+  return sysctlbyname(sname, oldp, (size_t *)oldlenp, const_cast<void *>(newp),
+                      (size_t)newlen);
+}
+
+static fd_t internal_spawn_impl(const char *argv[], const char *envp[],
+                                pid_t *pid) {
+  fd_t master_fd = kInvalidFd;
+  fd_t slave_fd = kInvalidFd;
+
+  auto fd_closer = at_scope_exit([&] {
+    internal_close(master_fd);
+    internal_close(slave_fd);
+  });
+
+  // We need a new pseudoterminal to avoid buffering problems. The 'atos' tool
+  // in particular detects when it's talking to a pipe and forgets to flush the
+  // output stream after sending a response.
+  master_fd = posix_openpt(O_RDWR);
+  if (master_fd == kInvalidFd) return kInvalidFd;
+
+  int res = grantpt(master_fd) || unlockpt(master_fd);
+  if (res != 0) return kInvalidFd;
+
+  // Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.
+  char slave_pty_name[128];
+  res = ioctl(master_fd, TIOCPTYGNAME, slave_pty_name);
+  if (res == -1) return kInvalidFd;
+
+  slave_fd = internal_open(slave_pty_name, O_RDWR);
+  if (slave_fd == kInvalidFd) return kInvalidFd;
+
+  // File descriptor actions
+  posix_spawn_file_actions_t acts;
+  res = posix_spawn_file_actions_init(&acts);
+  if (res != 0) return kInvalidFd;
+
+  auto acts_cleanup = at_scope_exit([&] {
+    posix_spawn_file_actions_destroy(&acts);
+  });
+
+  res = posix_spawn_file_actions_adddup2(&acts, slave_fd, STDIN_FILENO) ||
+        posix_spawn_file_actions_adddup2(&acts, slave_fd, STDOUT_FILENO) ||
+        posix_spawn_file_actions_addclose(&acts, slave_fd);
+  if (res != 0) return kInvalidFd;
+
+  // Spawn attributes
+  posix_spawnattr_t attrs;
+  res = posix_spawnattr_init(&attrs);
+  if (res != 0) return kInvalidFd;
+
+  auto attrs_cleanup  = at_scope_exit([&] {
+    posix_spawnattr_destroy(&attrs);
+  });
+
+  // In the spawned process, close all file descriptors that are not explicitly
+  // described by the file actions object. This is Darwin-specific extension.
+  res = posix_spawnattr_setflags(&attrs, POSIX_SPAWN_CLOEXEC_DEFAULT);
+  if (res != 0) return kInvalidFd;
+
+  // posix_spawn
+  char **argv_casted = const_cast<char **>(argv);
+  char **envp_casted = const_cast<char **>(envp);
+  res = posix_spawn(pid, argv[0], &acts, &attrs, argv_casted, envp_casted);
+  if (res != 0) return kInvalidFd;
+
+  // Disable echo in the new terminal, disable CR.
+  struct termios termflags;
+  tcgetattr(master_fd, &termflags);
+  termflags.c_oflag &= ~ONLCR;
+  termflags.c_lflag &= ~ECHO;
+  tcsetattr(master_fd, TCSANOW, &termflags);
+
+  // On success, do not close master_fd on scope exit.
+  fd_t fd = master_fd;
+  master_fd = kInvalidFd;
+
+  return fd;
+}
+
+fd_t internal_spawn(const char *argv[], const char *envp[], pid_t *pid) {
+  // The client program may close its stdin and/or stdout and/or stderr thus
+  // allowing open/posix_openpt to reuse file descriptors 0, 1 or 2. In this
+  // case the communication is broken if either the parent or the child tries to
+  // close or duplicate these descriptors. We temporarily reserve these
+  // descriptors here to prevent this.
+  fd_t low_fds[3];
+  size_t count = 0;
+
+  for (; count < 3; count++) {
+    low_fds[count] = posix_openpt(O_RDWR);
+    if (low_fds[count] >= STDERR_FILENO)
+      break;
+  }
+
+  fd_t fd = internal_spawn_impl(argv, envp, pid);
+
+  for (; count > 0; count--) {
+    internal_close(low_fds[count]);
+  }
+
+  return fd;
+}
+
+uptr internal_rename(const char *oldpath, const char *newpath) {
+  return rename(oldpath, newpath);
+}
+
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  return ftruncate(fd, size);
+}
+
+uptr internal_execve(const char *filename, char *const argv[],
+                     char *const envp[]) {
+  return execve(filename, argv, envp);
+}
+
+uptr internal_waitpid(int pid, int *status, int options) {
+  return waitpid(pid, status, options);
+}
+
+// ----------------- sanitizer_common.h
+bool FileExists(const char *filename) {
+  if (ShouldMockFailureToOpen(filename))
+    return false;
+  struct stat st;
+  if (stat(filename, &st))
+    return false;
+  // Sanity check: filename is a regular file.
+  return S_ISREG(st.st_mode);
+}
+
+tid_t GetTid() {
+  tid_t tid;
+  pthread_threadid_np(nullptr, &tid);
+  return tid;
+}
+
+void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
+                                uptr *stack_bottom) {
+  CHECK(stack_top);
+  CHECK(stack_bottom);
+  uptr stacksize = pthread_get_stacksize_np(pthread_self());
+  // pthread_get_stacksize_np() returns an incorrect stack size for the main
+  // thread on Mavericks. See
+  // https://github.com/google/sanitizers/issues/261
+  if ((GetMacosAlignedVersion() >= MacosVersion(10, 9)) && at_initialization &&
+      stacksize == (1 << 19))  {
+    struct rlimit rl;
+    CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
+    // Most often rl.rlim_cur will be the desired 8M.
+    if (rl.rlim_cur < kMaxThreadStackSize) {
+      stacksize = rl.rlim_cur;
+    } else {
+      stacksize = kMaxThreadStackSize;
+    }
+  }
+  void *stackaddr = pthread_get_stackaddr_np(pthread_self());
+  *stack_top = (uptr)stackaddr;
+  *stack_bottom = *stack_top - stacksize;
+}
+
+char **GetEnviron() {
+#if !SANITIZER_IOS
+  char ***env_ptr = _NSGetEnviron();
+  if (!env_ptr) {
+    Report("_NSGetEnviron() returned NULL. Please make sure __asan_init() is "
+           "called after libSystem_initializer().\n");
+    CHECK(env_ptr);
+  }
+  char **environ = *env_ptr;
+#endif
+  CHECK(environ);
+  return environ;
+}
+
+const char *GetEnv(const char *name) {
+  char **env = GetEnviron();
+  uptr name_len = internal_strlen(name);
+  while (*env != 0) {
+    uptr len = internal_strlen(*env);
+    if (len > name_len) {
+      const char *p = *env;
+      if (!internal_memcmp(p, name, name_len) &&
+          p[name_len] == '=') {  // Match.
+        return *env + name_len + 1;  // String starting after =.
+      }
+    }
+    env++;
+  }
+  return 0;
+}
+
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+  CHECK_LE(kMaxPathLength, buf_len);
+
+  // On OS X the executable path is saved to the stack by dyld. Reading it
+  // from there is much faster than calling dladdr, especially for large
+  // binaries with symbols.
+  InternalScopedString exe_path(kMaxPathLength);
+  uint32_t size = exe_path.size();
+  if (_NSGetExecutablePath(exe_path.data(), &size) == 0 &&
+      realpath(exe_path.data(), buf) != 0) {
+    return internal_strlen(buf);
+  }
+  return 0;
+}
+
+uptr ReadLongProcessName(/*out*/char *buf, uptr buf_len) {
+  return ReadBinaryName(buf, buf_len);
+}
+
+void ReExec() {
+  UNIMPLEMENTED();
+}
+
+void CheckASLR() {
+  // Do nothing
+}
+
+void CheckMPROTECT() {
+  // Do nothing
+}
+
+uptr GetPageSize() {
+  return sysconf(_SC_PAGESIZE);
+}
+
+extern "C" unsigned malloc_num_zones;
+extern "C" malloc_zone_t **malloc_zones;
+malloc_zone_t sanitizer_zone;
+
+// We need to make sure that sanitizer_zone is registered as malloc_zones[0]. If
+// libmalloc tries to set up a different zone as malloc_zones[0], it will call
+// mprotect(malloc_zones, ..., PROT_READ).  This interceptor will catch that and
+// make sure we are still the first (default) zone.
+void MprotectMallocZones(void *addr, int prot) {
+  if (addr == malloc_zones && prot == PROT_READ) {
+    if (malloc_num_zones > 1 && malloc_zones[0] != &sanitizer_zone) {
+      for (unsigned i = 1; i < malloc_num_zones; i++) {
+        if (malloc_zones[i] == &sanitizer_zone) {
+          // Swap malloc_zones[0] and malloc_zones[i].
+          malloc_zones[i] = malloc_zones[0];
+          malloc_zones[0] = &sanitizer_zone;
+          break;
+        }
+      }
+    }
+  }
+}
+
+BlockingMutex::BlockingMutex() {
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  CHECK(sizeof(OSSpinLock) <= sizeof(opaque_storage_));
+  CHECK_EQ(OS_SPINLOCK_INIT, 0);
+  CHECK_EQ(owner_, 0);
+  OSSpinLockLock((OSSpinLock*)&opaque_storage_);
+}
+
+void BlockingMutex::Unlock() {
+  OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
+}
+
+void BlockingMutex::CheckLocked() {
+  CHECK_NE(*(OSSpinLock*)&opaque_storage_, 0);
+}
+
+u64 NanoTime() {
+  timeval tv;
+  internal_memset(&tv, 0, sizeof(tv));
+  gettimeofday(&tv, 0);
+  return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
+}
+
+// This needs to be called during initialization to avoid being racy.
+u64 MonotonicNanoTime() {
+  static mach_timebase_info_data_t timebase_info;
+  if (timebase_info.denom == 0) mach_timebase_info(&timebase_info);
+  return (mach_absolute_time() * timebase_info.numer) / timebase_info.denom;
+}
+
+uptr GetTlsSize() {
+  return 0;
+}
+
+void InitTlsSize() {
+}
+
+uptr TlsBaseAddr() {
+  uptr segbase = 0;
+#if defined(__x86_64__)
+  asm("movq %%gs:0,%0" : "=r"(segbase));
+#elif defined(__i386__)
+  asm("movl %%gs:0,%0" : "=r"(segbase));
+#endif
+  return segbase;
+}
+
+// The size of the tls on darwin does not appear to be well documented,
+// however the vm memory map suggests that it is 1024 uptrs in size,
+// with a size of 0x2000 bytes on x86_64 and 0x1000 bytes on i386.
+uptr TlsSize() {
+#if defined(__x86_64__) || defined(__i386__)
+  return 1024 * sizeof(uptr);
+#else
+  return 0;
+#endif
+}
+
+void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+                          uptr *tls_addr, uptr *tls_size) {
+#if !SANITIZER_GO
+  uptr stack_top, stack_bottom;
+  GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
+  *stk_addr = stack_bottom;
+  *stk_size = stack_top - stack_bottom;
+  *tls_addr = TlsBaseAddr();
+  *tls_size = TlsSize();
+#else
+  *stk_addr = 0;
+  *stk_size = 0;
+  *tls_addr = 0;
+  *tls_size = 0;
+#endif
+}
+
+void ListOfModules::init() {
+  clearOrInit();
+  MemoryMappingLayout memory_mapping(false);
+  memory_mapping.DumpListOfModules(&modules_);
+}
+
+void ListOfModules::fallbackInit() { clear(); }
+
+static HandleSignalMode GetHandleSignalModeImpl(int signum) {
+  switch (signum) {
+    case SIGABRT:
+      return common_flags()->handle_abort;
+    case SIGILL:
+      return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
+    case SIGFPE:
+      return common_flags()->handle_sigfpe;
+    case SIGSEGV:
+      return common_flags()->handle_segv;
+    case SIGBUS:
+      return common_flags()->handle_sigbus;
+  }
+  return kHandleSignalNo;
+}
+
+HandleSignalMode GetHandleSignalMode(int signum) {
+  // Handling fatal signals on watchOS and tvOS devices is disallowed.
+  if ((SANITIZER_WATCHOS || SANITIZER_TVOS) && !(SANITIZER_IOSSIM))
+    return kHandleSignalNo;
+  HandleSignalMode result = GetHandleSignalModeImpl(signum);
+  if (result == kHandleSignalYes && !common_flags()->allow_user_segv_handler)
+    return kHandleSignalExclusive;
+  return result;
+}
+
+// This corresponds to Triple::getMacOSXVersion() in the Clang driver.
+static MacosVersion GetMacosAlignedVersionInternal() {
+  u16 kernel_major = GetDarwinKernelVersion().major;
+  // Darwin 0-3  -> unsupported
+  // Darwin 4-19 -> macOS 10.x
+  // Darwin 20+  -> macOS 11+
+  CHECK_GE(kernel_major, 4);
+  u16 major, minor;
+  if (kernel_major < 20) {
+    major = 10;
+    minor = kernel_major - 4;
+  } else {
+    major = 11 + kernel_major - 20;
+    minor = 0;
+  }
+  return MacosVersion(major, minor);
+}
+
+static_assert(sizeof(MacosVersion) == sizeof(atomic_uint32_t::Type),
+              "MacosVersion cache size");
+static atomic_uint32_t cached_macos_version;
+
+MacosVersion GetMacosAlignedVersion() {
+  atomic_uint32_t::Type result =
+      atomic_load(&cached_macos_version, memory_order_acquire);
+  if (!result) {
+    MacosVersion version = GetMacosAlignedVersionInternal();
+    result = *reinterpret_cast<atomic_uint32_t::Type *>(&version);
+    atomic_store(&cached_macos_version, result, memory_order_release);
+  }
+  return *reinterpret_cast<MacosVersion *>(&result);
+}
+
+void ParseVersion(const char *vers, u16 *major, u16 *minor) {
+  // Format: <major>.<minor>.<patch>\0
+  CHECK_GE(internal_strlen(vers), 5);
+  const char *p = vers;
+  *major = internal_simple_strtoll(p, &p, /*base=*/10);
+  CHECK_EQ(*p, '.');
+  p += 1;
+  *minor = internal_simple_strtoll(p, &p, /*base=*/10);
+}
+
+DarwinKernelVersion GetDarwinKernelVersion() {
+  char buf[100];
+  size_t len = sizeof(buf);
+  int res = internal_sysctlbyname("kern.osrelease", buf, &len, nullptr, 0);
+  CHECK_EQ(res, 0);
+
+  u16 major, minor;
+  ParseVersion(buf, &major, &minor);
+
+  return DarwinKernelVersion(major, minor);
+}
+
+uptr GetRSS() {
+  struct task_basic_info info;
+  unsigned count = TASK_BASIC_INFO_COUNT;
+  kern_return_t result =
+      task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &count);
+  if (UNLIKELY(result != KERN_SUCCESS)) {
+    Report("Cannot get task info. Error: %d\n", result);
+    Die();
+  }
+  return info.resident_size;
+}
+
+void *internal_start_thread(void *(*func)(void *arg), void *arg) {
+  // Start the thread with signals blocked, otherwise it can steal user signals.
+  __sanitizer_sigset_t set, old;
+  internal_sigfillset(&set);
+  internal_sigprocmask(SIG_SETMASK, &set, &old);
+  pthread_t th;
+  pthread_create(&th, 0, func, arg);
+  internal_sigprocmask(SIG_SETMASK, &old, 0);
+  return th;
+}
+
+void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
+
+#if !SANITIZER_GO
+static BlockingMutex syslog_lock(LINKER_INITIALIZED);
+#endif
+
+void WriteOneLineToSyslog(const char *s) {
+#if !SANITIZER_GO
+  syslog_lock.CheckLocked();
+  asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);
+#endif
+}
+
+void LogMessageOnPrintf(const char *str) {
+  // Log all printf output to CrashLog.
+  if (common_flags()->abort_on_error)
+    CRAppendCrashLogMessage(str);
+}
+
+void LogFullErrorReport(const char *buffer) {
+#if !SANITIZER_GO
+  // Log with os_trace. This will make it into the crash log.
+#if SANITIZER_OS_TRACE
+  if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) {
+    // os_trace requires the message (format parameter) to be a string literal.
+    if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
+                         sizeof("AddressSanitizer") - 1) == 0)
+      os_trace("Address Sanitizer reported a failure.");
+    else if (internal_strncmp(SanitizerToolName, "UndefinedBehaviorSanitizer",
+                              sizeof("UndefinedBehaviorSanitizer") - 1) == 0)
+      os_trace("Undefined Behavior Sanitizer reported a failure.");
+    else if (internal_strncmp(SanitizerToolName, "ThreadSanitizer",
+                              sizeof("ThreadSanitizer") - 1) == 0)
+      os_trace("Thread Sanitizer reported a failure.");
+    else
+      os_trace("Sanitizer tool reported a failure.");
+
+    if (common_flags()->log_to_syslog)
+      os_trace("Consult syslog for more information.");
+  }
+#endif
+
+  // Log to syslog.
+  // The logging on OS X may call pthread_create so we need the threading
+  // environment to be fully initialized. Also, this should never be called when
+  // holding the thread registry lock since that may result in a deadlock. If
+  // the reporting thread holds the thread registry mutex, and asl_log waits
+  // for GCD to dispatch a new thread, the process will deadlock, because the
+  // pthread_create wrapper needs to acquire the lock as well.
+  BlockingMutexLock l(&syslog_lock);
+  if (common_flags()->log_to_syslog)
+    WriteToSyslog(buffer);
+
+  // The report is added to CrashLog as part of logging all of Printf output.
+#endif
+}
+
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
+#if defined(__x86_64__) || defined(__i386__)
+  ucontext_t *ucontext = static_cast<ucontext_t*>(context);
+  return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ;
+#else
+  return UNKNOWN;
+#endif
+}
+
+bool SignalContext::IsTrueFaultingAddress() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  // "Real" SIGSEGV codes (e.g., SEGV_MAPERR, SEGV_MAPERR) are non-zero.
+  return si->si_signo == SIGSEGV && si->si_code != 0;
+}
+
+#if defined(__aarch64__) && defined(arm_thread_state64_get_sp)
+  #define AARCH64_GET_REG(r) \
+    (uptr)ptrauth_strip(     \
+        (void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
+#else
+  #define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r
+#endif
+
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+  ucontext_t *ucontext = (ucontext_t*)context;
+# if defined(__aarch64__)
+  *pc = AARCH64_GET_REG(pc);
+#   if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0
+  *bp = AARCH64_GET_REG(fp);
+#   else
+  *bp = AARCH64_GET_REG(lr);
+#   endif
+  *sp = AARCH64_GET_REG(sp);
+# elif defined(__x86_64__)
+  *pc = ucontext->uc_mcontext->__ss.__rip;
+  *bp = ucontext->uc_mcontext->__ss.__rbp;
+  *sp = ucontext->uc_mcontext->__ss.__rsp;
+# elif defined(__arm__)
+  *pc = ucontext->uc_mcontext->__ss.__pc;
+  *bp = ucontext->uc_mcontext->__ss.__r[7];
+  *sp = ucontext->uc_mcontext->__ss.__sp;
+# elif defined(__i386__)
+  *pc = ucontext->uc_mcontext->__ss.__eip;
+  *bp = ucontext->uc_mcontext->__ss.__ebp;
+  *sp = ucontext->uc_mcontext->__ss.__esp;
+# else
+# error "Unknown architecture"
+# endif
+}
+
+void SignalContext::InitPcSpBp() {
+  addr = (uptr)ptrauth_strip((void *)addr, 0);
+  GetPcSpBp(context, &pc, &sp, &bp);
+}
+
+void InitializePlatformEarly() {
+  // Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
+  use_xnu_fast_mmap =
+#if defined(__x86_64__)
+      GetDarwinKernelVersion() >= DarwinKernelVersion(17, 5);
+#else
+      false;
+#endif
+}
+
+#if !SANITIZER_GO
+static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
+LowLevelAllocator allocator_for_env;
+
+// Change the value of the env var |name|, leaking the original value.
+// If |name_value| is NULL, the variable is deleted from the environment,
+// otherwise the corresponding "NAME=value" string is replaced with
+// |name_value|.
+void LeakyResetEnv(const char *name, const char *name_value) {
+  char **env = GetEnviron();
+  uptr name_len = internal_strlen(name);
+  while (*env != 0) {
+    uptr len = internal_strlen(*env);
+    if (len > name_len) {
+      const char *p = *env;
+      if (!internal_memcmp(p, name, name_len) && p[name_len] == '=') {
+        // Match.
+        if (name_value) {
+          // Replace the old value with the new one.
+          *env = const_cast<char*>(name_value);
+        } else {
+          // Shift the subsequent pointers back.
+          char **del = env;
+          do {
+            del[0] = del[1];
+          } while (*del++);
+        }
+      }
+    }
+    env++;
+  }
+}
+
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+bool ReexecDisabled() {
+  return false;
+}
+
+extern "C" SANITIZER_WEAK_ATTRIBUTE double dyldVersionNumber;
+static const double kMinDyldVersionWithAutoInterposition = 360.0;
+
+bool DyldNeedsEnvVariable() {
+  // Although sanitizer support was added to LLVM on OS X 10.7+, GCC users
+  // still may want use them on older systems. On older Darwin platforms, dyld
+  // doesn't export dyldVersionNumber symbol and we simply return true.
+  if (!&dyldVersionNumber) return true;
+  // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
+  // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via
+  // GetMacosAlignedVersion() doesn't work for the simulator. Let's instead
+  // check `dyldVersionNumber`, which is exported by dyld, against a known
+  // version number from the first OS release where this appeared.
+  return dyldVersionNumber < kMinDyldVersionWithAutoInterposition;
+}
+
+void MaybeReexec() {
+  // FIXME: This should really live in some "InitializePlatform" method.
+  MonotonicNanoTime();
+
+  if (ReexecDisabled()) return;
+
+  // Make sure the dynamic runtime library is preloaded so that the
+  // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec
+  // ourselves.
+  Dl_info info;
+  RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info));
+  char *dyld_insert_libraries =
+      const_cast<char*>(GetEnv(kDyldInsertLibraries));
+  uptr old_env_len = dyld_insert_libraries ?
+      internal_strlen(dyld_insert_libraries) : 0;
+  uptr fname_len = internal_strlen(info.dli_fname);
+  const char *dylib_name = StripModuleName(info.dli_fname);
+  uptr dylib_name_len = internal_strlen(dylib_name);
+
+  bool lib_is_in_env = dyld_insert_libraries &&
+                       internal_strstr(dyld_insert_libraries, dylib_name);
+  if (DyldNeedsEnvVariable() && !lib_is_in_env) {
+    // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
+    // library.
+    InternalScopedString program_name(1024);
+    uint32_t buf_size = program_name.size();
+    _NSGetExecutablePath(program_name.data(), &buf_size);
+    char *new_env = const_cast<char*>(info.dli_fname);
+    if (dyld_insert_libraries) {
+      // Append the runtime dylib name to the existing value of
+      // DYLD_INSERT_LIBRARIES.
+      new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2);
+      internal_strncpy(new_env, dyld_insert_libraries, old_env_len);
+      new_env[old_env_len] = ':';
+      // Copy fname_len and add a trailing zero.
+      internal_strncpy(new_env + old_env_len + 1, info.dli_fname,
+                       fname_len + 1);
+      // Ok to use setenv() since the wrappers don't depend on the value of
+      // asan_inited.
+      setenv(kDyldInsertLibraries, new_env, /*overwrite*/1);
+    } else {
+      // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name.
+      setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0);
+    }
+    VReport(1, "exec()-ing the program with\n");
+    VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env);
+    VReport(1, "to enable wrappers.\n");
+    execv(program_name.data(), *_NSGetArgv());
+
+    // We get here only if execv() failed.
+    Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, "
+           "which is required for the sanitizer to work. We tried to set the "
+           "environment variable and re-execute itself, but execv() failed, "
+           "possibly because of sandbox restrictions. Make sure to launch the "
+           "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env);
+    RAW_CHECK("execv failed" && 0);
+  }
+
+  // Verify that interceptors really work.  We'll use dlsym to locate
+  // "pthread_create", if interceptors are working, it should really point to
+  // "wrap_pthread_create" within our own dylib.
+  Dl_info info_pthread_create;
+  void *dlopen_addr = dlsym(RTLD_DEFAULT, "pthread_create");
+  RAW_CHECK(dladdr(dlopen_addr, &info_pthread_create));
+  if (internal_strcmp(info.dli_fname, info_pthread_create.dli_fname) != 0) {
+    Report(
+        "ERROR: Interceptors are not working. This may be because %s is "
+        "loaded too late (e.g. via dlopen). Please launch the executable "
+        "with:\n%s=%s\n",
+        SanitizerToolName, kDyldInsertLibraries, info.dli_fname);
+    RAW_CHECK("interceptors not installed" && 0);
+  }
+
+  if (!lib_is_in_env)
+    return;
+
+  if (!common_flags()->strip_env)
+    return;
+
+  // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove
+  // the dylib from the environment variable, because interceptors are installed
+  // and we don't want our children to inherit the variable.
+
+  uptr env_name_len = internal_strlen(kDyldInsertLibraries);
+  // Allocate memory to hold the previous env var name, its value, the '='
+  // sign and the '\0' char.
+  char *new_env = (char*)allocator_for_env.Allocate(
+      old_env_len + 2 + env_name_len);
+  RAW_CHECK(new_env);
+  internal_memset(new_env, '\0', old_env_len + 2 + env_name_len);
+  internal_strncpy(new_env, kDyldInsertLibraries, env_name_len);
+  new_env[env_name_len] = '=';
+  char *new_env_pos = new_env + env_name_len + 1;
+
+  // Iterate over colon-separated pieces of |dyld_insert_libraries|.
+  char *piece_start = dyld_insert_libraries;
+  char *piece_end = NULL;
+  char *old_env_end = dyld_insert_libraries + old_env_len;
+  do {
+    if (piece_start[0] == ':') piece_start++;
+    piece_end = internal_strchr(piece_start, ':');
+    if (!piece_end) piece_end = dyld_insert_libraries + old_env_len;
+    if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break;
+    uptr piece_len = piece_end - piece_start;
+
+    char *filename_start =
+        (char *)internal_memrchr(piece_start, '/', piece_len);
+    uptr filename_len = piece_len;
+    if (filename_start) {
+      filename_start += 1;
+      filename_len = piece_len - (filename_start - piece_start);
+    } else {
+      filename_start = piece_start;
+    }
+
+    // If the current piece isn't the runtime library name,
+    // append it to new_env.
+    if ((dylib_name_len != filename_len) ||
+        (internal_memcmp(filename_start, dylib_name, dylib_name_len) != 0)) {
+      if (new_env_pos != new_env + env_name_len + 1) {
+        new_env_pos[0] = ':';
+        new_env_pos++;
+      }
+      internal_strncpy(new_env_pos, piece_start, piece_len);
+      new_env_pos += piece_len;
+    }
+    // Move on to the next piece.
+    piece_start = piece_end;
+  } while (piece_start < old_env_end);
+
+  // Can't use setenv() here, because it requires the allocator to be
+  // initialized.
+  // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in
+  // a separate function called after InitializeAllocator().
+  if (new_env_pos == new_env + env_name_len + 1) new_env = NULL;
+  LeakyResetEnv(kDyldInsertLibraries, new_env);
+}
+#endif  // SANITIZER_GO
+
+char **GetArgv() {
+  return *_NSGetArgv();
+}
+
+#if SANITIZER_IOS
+// The task_vm_info struct is normally provided by the macOS SDK, but we need
+// fields only available in 10.12+. Declare the struct manually to be able to
+// build against older SDKs.
+struct __sanitizer_task_vm_info {
+  mach_vm_size_t virtual_size;
+  integer_t region_count;
+  integer_t page_size;
+  mach_vm_size_t resident_size;
+  mach_vm_size_t resident_size_peak;
+  mach_vm_size_t device;
+  mach_vm_size_t device_peak;
+  mach_vm_size_t internal;
+  mach_vm_size_t internal_peak;
+  mach_vm_size_t external;
+  mach_vm_size_t external_peak;
+  mach_vm_size_t reusable;
+  mach_vm_size_t reusable_peak;
+  mach_vm_size_t purgeable_volatile_pmap;
+  mach_vm_size_t purgeable_volatile_resident;
+  mach_vm_size_t purgeable_volatile_virtual;
+  mach_vm_size_t compressed;
+  mach_vm_size_t compressed_peak;
+  mach_vm_size_t compressed_lifetime;
+  mach_vm_size_t phys_footprint;
+  mach_vm_address_t min_address;
+  mach_vm_address_t max_address;
+};
+#define __SANITIZER_TASK_VM_INFO_COUNT ((mach_msg_type_number_t) \
+    (sizeof(__sanitizer_task_vm_info) / sizeof(natural_t)))
+
+static uptr GetTaskInfoMaxAddress() {
+  __sanitizer_task_vm_info vm_info = {} /* zero initialize */;
+  mach_msg_type_number_t count = __SANITIZER_TASK_VM_INFO_COUNT;
+  int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count);
+  return err ? 0 : vm_info.max_address;
+}
+
+uptr GetMaxUserVirtualAddress() {
+  static uptr max_vm = GetTaskInfoMaxAddress();
+  if (max_vm != 0)
+    return max_vm - 1;
+
+  // xnu cannot provide vm address limit
+# if SANITIZER_WORDSIZE == 32
+  return 0xffe00000 - 1;
+# else
+  return 0x200000000 - 1;
+# endif
+}
+
+#else // !SANITIZER_IOS
+
+uptr GetMaxUserVirtualAddress() {
+# if SANITIZER_WORDSIZE == 64
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# else // SANITIZER_WORDSIZE == 32
+  static_assert(SANITIZER_WORDSIZE == 32, "Wrong wordsize");
+  return (1ULL << 32) - 1;  // 0xffffffff;
+# endif
+}
+#endif
+
+uptr GetMaxVirtualAddress() {
+  return GetMaxUserVirtualAddress();
+}
+
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
+  typedef vm_region_submap_short_info_data_64_t RegionInfo;
+  enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
+  // Start searching for available memory region past PAGEZERO, which is
+  // 4KB on 32-bit and 4GB on 64-bit.
+  mach_vm_address_t start_address =
+    (SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
+
+  mach_vm_address_t address = start_address;
+  mach_vm_address_t free_begin = start_address;
+  kern_return_t kr = KERN_SUCCESS;
+  if (largest_gap_found) *largest_gap_found = 0;
+  if (max_occupied_addr) *max_occupied_addr = 0;
+  while (kr == KERN_SUCCESS) {
+    mach_vm_size_t vmsize = 0;
+    natural_t depth = 0;
+    RegionInfo vminfo;
+    mach_msg_type_number_t count = kRegionInfoSize;
+    kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
+                                (vm_region_info_t)&vminfo, &count);
+    if (kr == KERN_INVALID_ADDRESS) {
+      // No more regions beyond "address", consider the gap at the end of VM.
+      address = GetMaxVirtualAddress() + 1;
+      vmsize = 0;
+    } else {
+      if (max_occupied_addr) *max_occupied_addr = address + vmsize;
+    }
+    if (free_begin != address) {
+      // We found a free region [free_begin..address-1].
+      uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
+      uptr gap_end = RoundDownTo((uptr)address, alignment);
+      uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
+      if (size < gap_size) {
+        return gap_start;
+      }
+
+      if (largest_gap_found && *largest_gap_found < gap_size) {
+        *largest_gap_found = gap_size;
+      }
+    }
+    // Move to the next region.
+    address += vmsize;
+    free_begin = address;
+  }
+
+  // We looked at all free regions and could not find one large enough.
+  return 0;
+}
+
+// FIXME implement on this platform.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+
+void SignalContext::DumpAllRegisters(void *context) {
+  Report("Register values:\n");
+
+  ucontext_t *ucontext = (ucontext_t*)context;
+# define DUMPREG64(r) \
+    Printf("%s = 0x%016llx  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
+# define DUMPREGA64(r) \
+    Printf("   %s = 0x%016llx  ", #r, AARCH64_GET_REG(r));
+# define DUMPREG32(r) \
+    Printf("%s = 0x%08x  ", #r, ucontext->uc_mcontext->__ss.__ ## r);
+# define DUMPREG_(r)   Printf(" "); DUMPREG(r);
+# define DUMPREG__(r)  Printf("  "); DUMPREG(r);
+# define DUMPREG___(r) Printf("   "); DUMPREG(r);
+
+# if defined(__x86_64__)
+#  define DUMPREG(r) DUMPREG64(r)
+  DUMPREG(rax); DUMPREG(rbx); DUMPREG(rcx); DUMPREG(rdx); Printf("\n");
+  DUMPREG(rdi); DUMPREG(rsi); DUMPREG(rbp); DUMPREG(rsp); Printf("\n");
+  DUMPREG_(r8); DUMPREG_(r9); DUMPREG(r10); DUMPREG(r11); Printf("\n");
+  DUMPREG(r12); DUMPREG(r13); DUMPREG(r14); DUMPREG(r15); Printf("\n");
+# elif defined(__i386__)
+#  define DUMPREG(r) DUMPREG32(r)
+  DUMPREG(eax); DUMPREG(ebx); DUMPREG(ecx); DUMPREG(edx); Printf("\n");
+  DUMPREG(edi); DUMPREG(esi); DUMPREG(ebp); DUMPREG(esp); Printf("\n");
+# elif defined(__aarch64__)
+#  define DUMPREG(r) DUMPREG64(r)
+  DUMPREG_(x[0]); DUMPREG_(x[1]); DUMPREG_(x[2]); DUMPREG_(x[3]); Printf("\n");
+  DUMPREG_(x[4]); DUMPREG_(x[5]); DUMPREG_(x[6]); DUMPREG_(x[7]); Printf("\n");
+  DUMPREG_(x[8]); DUMPREG_(x[9]); DUMPREG(x[10]); DUMPREG(x[11]); Printf("\n");
+  DUMPREG(x[12]); DUMPREG(x[13]); DUMPREG(x[14]); DUMPREG(x[15]); Printf("\n");
+  DUMPREG(x[16]); DUMPREG(x[17]); DUMPREG(x[18]); DUMPREG(x[19]); Printf("\n");
+  DUMPREG(x[20]); DUMPREG(x[21]); DUMPREG(x[22]); DUMPREG(x[23]); Printf("\n");
+  DUMPREG(x[24]); DUMPREG(x[25]); DUMPREG(x[26]); DUMPREG(x[27]); Printf("\n");
+  DUMPREG(x[28]); DUMPREGA64(fp); DUMPREGA64(lr); DUMPREGA64(sp); Printf("\n");
+# elif defined(__arm__)
+#  define DUMPREG(r) DUMPREG32(r)
+  DUMPREG_(r[0]); DUMPREG_(r[1]); DUMPREG_(r[2]); DUMPREG_(r[3]); Printf("\n");
+  DUMPREG_(r[4]); DUMPREG_(r[5]); DUMPREG_(r[6]); DUMPREG_(r[7]); Printf("\n");
+  DUMPREG_(r[8]); DUMPREG_(r[9]); DUMPREG(r[10]); DUMPREG(r[11]); Printf("\n");
+  DUMPREG(r[12]); DUMPREG___(sp); DUMPREG___(lr); DUMPREG___(pc); Printf("\n");
+# else
+# error "Unknown architecture"
+# endif
+
+# undef DUMPREG64
+# undef DUMPREG32
+# undef DUMPREG_
+# undef DUMPREG__
+# undef DUMPREG___
+# undef DUMPREG
+}
+
+static inline bool CompareBaseAddress(const LoadedModule &a,
+                                      const LoadedModule &b) {
+  return a.base_address() < b.base_address();
+}
+
+void FormatUUID(char *out, uptr size, const u8 *uuid) {
+  internal_snprintf(out, size,
+                    "<%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-"
+                    "%02X%02X%02X%02X%02X%02X>",
+                    uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5],
+                    uuid[6], uuid[7], uuid[8], uuid[9], uuid[10], uuid[11],
+                    uuid[12], uuid[13], uuid[14], uuid[15]);
+}
+
+void PrintModuleMap() {
+  Printf("Process module map:\n");
+  MemoryMappingLayout memory_mapping(false);
+  InternalMmapVector<LoadedModule> modules;
+  modules.reserve(128);
+  memory_mapping.DumpListOfModules(&modules);
+  Sort(modules.data(), modules.size(), CompareBaseAddress);
+  for (uptr i = 0; i < modules.size(); ++i) {
+    char uuid_str[128];
+    FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
+    Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
+           modules[i].max_executable_address(), modules[i].full_name(),
+           ModuleArchToString(modules[i].arch()), uuid_str);
+  }
+  Printf("End of module map.\n");
+}
+
+void CheckNoDeepBind(const char *filename, int flag) {
+  // Do nothing.
+}
+
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  if (!buffer || !length || length > 256)
+    return false;
+  // arc4random never fails.
+  REAL(arc4random_buf)(buffer, length);
+  return true;
+}
+
+u32 GetNumberOfCPUs() {
+  return (u32)sysconf(_SC_NPROCESSORS_ONLN);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_mac.h b/lib/tsan/sanitizer_common/sanitizer_mac.h
new file mode 100644
index 0000000000..90ecff4815
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_mac.h
@@ -0,0 +1,84 @@
+//===-- sanitizer_mac.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// provides definitions for OSX-specific functions.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_MAC_H
+#define SANITIZER_MAC_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_posix.h"
+
+namespace __sanitizer {
+
+struct MemoryMappingLayoutData {
+  int current_image;
+  u32 current_magic;
+  u32 current_filetype;
+  ModuleArch current_arch;
+  u8 current_uuid[kModuleUUIDSize];
+  int current_load_cmd_count;
+  const char *current_load_cmd_addr;
+  bool current_instrumented;
+};
+
+template <typename VersionType>
+struct VersionBase {
+  u16 major;
+  u16 minor;
+
+  VersionBase(u16 major, u16 minor) : major(major), minor(minor) {}
+
+  bool operator==(const VersionType &other) const {
+    return major == other.major && minor == other.minor;
+  }
+  bool operator>=(const VersionType &other) const {
+    return major > other.major ||
+           (major == other.major && minor >= other.minor);
+  }
+};
+
+struct MacosVersion : VersionBase<MacosVersion> {
+  MacosVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
+};
+
+struct DarwinKernelVersion : VersionBase<DarwinKernelVersion> {
+  DarwinKernelVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
+};
+
+MacosVersion GetMacosAlignedVersion();
+DarwinKernelVersion GetDarwinKernelVersion();
+
+char **GetEnviron();
+
+void RestrictMemoryToMaxAddress(uptr max_address);
+
+}  // namespace __sanitizer
+
+extern "C" {
+static char __crashreporter_info_buff__[__sanitizer::kErrorMessageBufferSize] =
+  {};
+static const char *__crashreporter_info__ __attribute__((__used__)) =
+  &__crashreporter_info_buff__[0];
+asm(".desc ___crashreporter_info__, 0x10");
+} // extern "C"
+
+namespace __sanitizer {
+static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
+
+INLINE void CRAppendCrashLogMessage(const char *msg) {
+  BlockingMutexLock l(&crashreporter_info_mutex);
+  internal_strlcat(__crashreporter_info_buff__, msg,
+                   sizeof(__crashreporter_info_buff__)); }
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
+#endif  // SANITIZER_MAC_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc b/lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc
new file mode 100644
index 0000000000..647bcdfe10
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc
@@ -0,0 +1,420 @@
+//===-- sanitizer_malloc_mac.inc --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains Mac-specific malloc interceptors and a custom zone
+// implementation, which together replace the system allocator.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if !SANITIZER_MAC
+#error "This file should only be compiled on Darwin."
+#endif
+
+#include <AvailabilityMacros.h>
+#include <CoreFoundation/CFBase.h>
+#include <dlfcn.h>
+#include <malloc/malloc.h>
+#include <sys/mman.h>
+
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_mac.h"
+
+// Similar code is used in Google Perftools,
+// https://github.com/gperftools/gperftools.
+
+namespace __sanitizer {
+
+extern malloc_zone_t sanitizer_zone;
+
+struct sanitizer_malloc_introspection_t : public malloc_introspection_t {
+  // IMPORTANT: Do not change the order, alignment, or types of these fields to
+  // maintain binary compatibility. You should only add fields to this struct.
+
+  // Used to track changes to the allocator that will affect
+  // zone enumeration.
+  u64 allocator_enumeration_version;
+  uptr allocator_ptr;
+  uptr allocator_size;
+};
+
+u64 GetMallocZoneAllocatorEnumerationVersion() {
+  // This represents the current allocator ABI version.
+  // This field should be incremented every time the Allocator
+  // ABI changes in a way that breaks allocator enumeration.
+  return 0;
+}
+
+}  // namespace __sanitizer
+
+INTERCEPTOR(malloc_zone_t *, malloc_create_zone,
+                             vm_size_t start_size, unsigned zone_flags) {
+  COMMON_MALLOC_ENTER();
+  uptr page_size = GetPageSizeCached();
+  uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size);
+  COMMON_MALLOC_MEMALIGN(page_size, allocated_size);
+  malloc_zone_t *new_zone = (malloc_zone_t *)p;
+  internal_memcpy(new_zone, &sanitizer_zone, sizeof(sanitizer_zone));
+  new_zone->zone_name = NULL;  // The name will be changed anyway.
+  // Prevent the client app from overwriting the zone contents.
+  // Library functions that need to modify the zone will set PROT_WRITE on it.
+  // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher.
+  mprotect(new_zone, allocated_size, PROT_READ);
+  // We're explicitly *NOT* registering the zone.
+  return new_zone;
+}
+
+INTERCEPTOR(void, malloc_destroy_zone, malloc_zone_t *zone) {
+  COMMON_MALLOC_ENTER();
+  // We don't need to do anything here.  We're not registering new zones, so we
+  // don't to unregister.  Just un-mprotect and free() the zone.
+  uptr page_size = GetPageSizeCached();
+  uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size);
+  mprotect(zone, allocated_size, PROT_READ | PROT_WRITE);
+  if (zone->zone_name) {
+    COMMON_MALLOC_FREE((void *)zone->zone_name);
+  }
+  COMMON_MALLOC_FREE(zone);
+}
+
+INTERCEPTOR(malloc_zone_t *, malloc_default_zone, void) {
+  COMMON_MALLOC_ENTER();
+  return &sanitizer_zone;
+}
+
+INTERCEPTOR(malloc_zone_t *, malloc_zone_from_ptr, const void *ptr) {
+  COMMON_MALLOC_ENTER();
+  size_t size = sanitizer_zone.size(&sanitizer_zone, ptr);
+  if (size) { // Claimed by sanitizer zone?
+    return &sanitizer_zone;
+  }
+  return REAL(malloc_zone_from_ptr)(ptr);
+}
+
+INTERCEPTOR(malloc_zone_t *, malloc_default_purgeable_zone, void) {
+  // FIXME: ASan should support purgeable allocations.
+  // https://github.com/google/sanitizers/issues/139
+  COMMON_MALLOC_ENTER();
+  return &sanitizer_zone;
+}
+
+INTERCEPTOR(void, malloc_make_purgeable, void *ptr) {
+  // FIXME: ASan should support purgeable allocations. Ignoring them is fine
+  // for now.
+  COMMON_MALLOC_ENTER();
+}
+
+INTERCEPTOR(int, malloc_make_nonpurgeable, void *ptr) {
+  // FIXME: ASan should support purgeable allocations. Ignoring them is fine
+  // for now.
+  COMMON_MALLOC_ENTER();
+  // Must return 0 if the contents were not purged since the last call to
+  // malloc_make_purgeable().
+  return 0;
+}
+
+INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) {
+  COMMON_MALLOC_ENTER();
+  // Allocate |sizeof(COMMON_MALLOC_ZONE_NAME "-") + internal_strlen(name)|
+  // bytes.
+  size_t buflen =
+      sizeof(COMMON_MALLOC_ZONE_NAME "-") + (name ? internal_strlen(name) : 0);
+  InternalScopedString new_name(buflen);
+  if (name && zone->introspect == sanitizer_zone.introspect) {
+    new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name);
+    name = new_name.data();
+  }
+
+  // Call the system malloc's implementation for both external and our zones,
+  // since that appropriately changes VM region protections on the zone.
+  REAL(malloc_set_zone_name)(zone, name);
+}
+
+INTERCEPTOR(void *, malloc, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_MALLOC(size);
+  return p;
+}
+
+INTERCEPTOR(void, free, void *ptr) {
+  COMMON_MALLOC_ENTER();
+  if (!ptr) return;
+  COMMON_MALLOC_FREE(ptr);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_REALLOC(ptr, size);
+  return p;
+}
+
+INTERCEPTOR(void *, calloc, size_t nmemb, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_CALLOC(nmemb, size);
+  return p;
+}
+
+INTERCEPTOR(void *, valloc, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_VALLOC(size);
+  return p;
+}
+
+INTERCEPTOR(size_t, malloc_good_size, size_t size) {
+  COMMON_MALLOC_ENTER();
+  return sanitizer_zone.introspect->good_size(&sanitizer_zone, size);
+}
+
+INTERCEPTOR(int, posix_memalign, void **memptr, size_t alignment, size_t size) {
+  COMMON_MALLOC_ENTER();
+  CHECK(memptr);
+  COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size);
+  return res;
+}
+
+namespace {
+
+// TODO(glider): the __sanitizer_mz_* functions should be united with the Linux
+// wrappers, as they are basically copied from there.
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+size_t __sanitizer_mz_size(malloc_zone_t* zone, const void* ptr) {
+  COMMON_MALLOC_SIZE(ptr);
+  return size;
+}
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_MALLOC(size);
+  return p;
+}
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
+  if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
+    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+    const size_t kCallocPoolSize = 1024;
+    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
+    static size_t allocated;
+    size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
+    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
+    allocated += size_in_words;
+    CHECK(allocated < kCallocPoolSize);
+    return mem;
+  }
+  COMMON_MALLOC_CALLOC(nmemb, size);
+  return p;
+}
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__sanitizer_mz_valloc(malloc_zone_t *zone, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_VALLOC(size);
+  return p;
+}
+
+// TODO(glider): the allocation callbacks need to be refactored.
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
+  if (!ptr) return;
+  COMMON_MALLOC_FREE(ptr);
+}
+
+#define GET_ZONE_FOR_PTR(ptr) \
+  malloc_zone_t *zone_ptr = WRAP(malloc_zone_from_ptr)(ptr); \
+  const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__sanitizer_mz_realloc(malloc_zone_t *zone, void *ptr, size_t new_size) {
+  if (!ptr) {
+    COMMON_MALLOC_MALLOC(new_size);
+    return p;
+  } else {
+    COMMON_MALLOC_SIZE(ptr);
+    if (size) {
+      COMMON_MALLOC_REALLOC(ptr, new_size);
+      return p;
+    } else {
+      // We can't recover from reallocating an unknown address, because
+      // this would require reading at most |new_size| bytes from
+      // potentially unaccessible memory.
+      GET_ZONE_FOR_PTR(ptr);
+      COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name);
+      return nullptr;
+    }
+  }
+}
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_mz_destroy(malloc_zone_t* zone) {
+  // A no-op -- we will not be destroyed!
+  Report("__sanitizer_mz_destroy() called -- ignoring\n");
+}
+
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__sanitizer_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_MEMALIGN(align, size);
+  return p;
+}
+
+// This public API exists purely for testing purposes.
+extern "C"
+SANITIZER_INTERFACE_ATTRIBUTE
+malloc_zone_t* __sanitizer_mz_default_zone() {
+  return &sanitizer_zone;
+}
+
+// This function is currently unused, and we build with -Werror.
+#if 0
+void __sanitizer_mz_free_definite_size(
+    malloc_zone_t* zone, void *ptr, size_t size) {
+  // TODO(glider): check that |size| is valid.
+  UNIMPLEMENTED();
+}
+#endif
+
+#ifndef COMMON_MALLOC_HAS_ZONE_ENUMERATOR
+#error "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be defined"
+#endif
+static_assert((COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 0 ||
+                  (COMMON_MALLOC_HAS_ZONE_ENUMERATOR) == 1,
+              "COMMON_MALLOC_HAS_ZONE_ENUMERATOR must be 0 or 1");
+
+#if COMMON_MALLOC_HAS_ZONE_ENUMERATOR
+// Forward declare and expect the implementation to provided by
+// includer.
+kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask,
+                            vm_address_t zone_address, memory_reader_t reader,
+                            vm_range_recorder_t recorder);
+#else
+// Provide stub implementation that fails.
+kern_return_t mi_enumerator(task_t task, void *, unsigned type_mask,
+                            vm_address_t zone_address, memory_reader_t reader,
+                            vm_range_recorder_t recorder) {
+  // Not supported.
+  return KERN_FAILURE;
+}
+#endif
+
+#ifndef COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT
+#error "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be defined"
+#endif
+static_assert((COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 0 ||
+                  (COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT) == 1,
+              "COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT must be 0 or 1");
+#if COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT
+// Forward declare and expect the implementation to provided by
+// includer.
+void mi_extra_init(
+    sanitizer_malloc_introspection_t *mi);
+#else
+void mi_extra_init(
+    sanitizer_malloc_introspection_t *mi) {
+  // Just zero initialize the fields.
+  mi->allocator_ptr = 0;
+  mi->allocator_size = 0;
+}
+#endif
+
+size_t mi_good_size(malloc_zone_t *zone, size_t size) {
+  // I think it's always safe to return size, but we maybe could do better.
+  return size;
+}
+
+boolean_t mi_check(malloc_zone_t *zone) {
+  UNIMPLEMENTED();
+}
+
+void mi_print(malloc_zone_t *zone, boolean_t verbose) {
+  UNIMPLEMENTED();
+}
+
+void mi_log(malloc_zone_t *zone, void *address) {
+  // I don't think we support anything like this
+}
+
+void mi_force_lock(malloc_zone_t *zone) {
+  COMMON_MALLOC_FORCE_LOCK();
+}
+
+void mi_force_unlock(malloc_zone_t *zone) {
+  COMMON_MALLOC_FORCE_UNLOCK();
+}
+
+void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) {
+  COMMON_MALLOC_FILL_STATS(zone, stats);
+}
+
+boolean_t mi_zone_locked(malloc_zone_t *zone) {
+  // UNIMPLEMENTED();
+  return false;
+}
+
+}  // unnamed namespace
+
+namespace COMMON_MALLOC_NAMESPACE {
+
+void InitMallocZoneFields() {
+  static sanitizer_malloc_introspection_t sanitizer_zone_introspection;
+  // Ok to use internal_memset, these places are not performance-critical.
+  internal_memset(&sanitizer_zone_introspection, 0,
+                  sizeof(sanitizer_zone_introspection));
+
+  sanitizer_zone_introspection.enumerator = &mi_enumerator;
+  sanitizer_zone_introspection.good_size = &mi_good_size;
+  sanitizer_zone_introspection.check = &mi_check;
+  sanitizer_zone_introspection.print = &mi_print;
+  sanitizer_zone_introspection.log = &mi_log;
+  sanitizer_zone_introspection.force_lock = &mi_force_lock;
+  sanitizer_zone_introspection.force_unlock = &mi_force_unlock;
+  sanitizer_zone_introspection.statistics = &mi_statistics;
+  sanitizer_zone_introspection.zone_locked = &mi_zone_locked;
+
+  // Set current allocator enumeration version.
+  sanitizer_zone_introspection.allocator_enumeration_version =
+      GetMallocZoneAllocatorEnumerationVersion();
+
+  // Perform any sanitizer specific initialization.
+  mi_extra_init(&sanitizer_zone_introspection);
+
+  internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t));
+
+  // Use version 6 for OSX >= 10.6.
+  sanitizer_zone.version = 6;
+  sanitizer_zone.zone_name = COMMON_MALLOC_ZONE_NAME;
+  sanitizer_zone.size = &__sanitizer_mz_size;
+  sanitizer_zone.malloc = &__sanitizer_mz_malloc;
+  sanitizer_zone.calloc = &__sanitizer_mz_calloc;
+  sanitizer_zone.valloc = &__sanitizer_mz_valloc;
+  sanitizer_zone.free = &__sanitizer_mz_free;
+  sanitizer_zone.realloc = &__sanitizer_mz_realloc;
+  sanitizer_zone.destroy = &__sanitizer_mz_destroy;
+  sanitizer_zone.batch_malloc = 0;
+  sanitizer_zone.batch_free = 0;
+  sanitizer_zone.free_definite_size = 0;
+  sanitizer_zone.memalign = &__sanitizer_mz_memalign;
+  sanitizer_zone.introspect = &sanitizer_zone_introspection;
+}
+
+void ReplaceSystemMalloc() {
+  InitMallocZoneFields();
+
+  // Register the zone.
+  malloc_zone_register(&sanitizer_zone);
+}
+
+}  // namespace COMMON_MALLOC_NAMESPACE
diff --git a/lib/tsan/sanitizer_common/sanitizer_mutex.h b/lib/tsan/sanitizer_common/sanitizer_mutex.h
new file mode 100644
index 0000000000..40a6591429
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_mutex.h
@@ -0,0 +1,223 @@
+//===-- sanitizer_mutex.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_MUTEX_H
+#define SANITIZER_MUTEX_H
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+
+namespace __sanitizer {
+
+class StaticSpinMutex {
+ public:
+  void Init() {
+    atomic_store(&state_, 0, memory_order_relaxed);
+  }
+
+  void Lock() {
+    if (TryLock())
+      return;
+    LockSlow();
+  }
+
+  bool TryLock() {
+    return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
+  }
+
+  void Unlock() {
+    atomic_store(&state_, 0, memory_order_release);
+  }
+
+  void CheckLocked() {
+    CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
+  }
+
+ private:
+  atomic_uint8_t state_;
+
+  void NOINLINE LockSlow() {
+    for (int i = 0;; i++) {
+      if (i < 10)
+        proc_yield(10);
+      else
+        internal_sched_yield();
+      if (atomic_load(&state_, memory_order_relaxed) == 0
+          && atomic_exchange(&state_, 1, memory_order_acquire) == 0)
+        return;
+    }
+  }
+};
+
+class SpinMutex : public StaticSpinMutex {
+ public:
+  SpinMutex() {
+    Init();
+  }
+
+ private:
+  SpinMutex(const SpinMutex&);
+  void operator=(const SpinMutex&);
+};
+
+class BlockingMutex {
+ public:
+  explicit constexpr BlockingMutex(LinkerInitialized)
+      : opaque_storage_ {0, }, owner_ {0} {}
+  BlockingMutex();
+  void Lock();
+  void Unlock();
+
+  // This function does not guarantee an explicit check that the calling thread
+  // is the thread which owns the mutex. This behavior, while more strictly
+  // correct, causes problems in cases like StopTheWorld, where a parent thread
+  // owns the mutex but a child checks that it is locked. Rather than
+  // maintaining complex state to work around those situations, the check only
+  // checks that the mutex is owned, and assumes callers to be generally
+  // well-behaved.
+  void CheckLocked();
+
+ private:
+  // Solaris mutex_t has a member that requires 64-bit alignment.
+  ALIGNED(8) uptr opaque_storage_[10];
+  uptr owner_;  // for debugging
+};
+
+// Reader-writer spin mutex.
+class RWMutex {
+ public:
+  RWMutex() {
+    atomic_store(&state_, kUnlocked, memory_order_relaxed);
+  }
+
+  ~RWMutex() {
+    CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
+  }
+
+  void Lock() {
+    u32 cmp = kUnlocked;
+    if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
+                                       memory_order_acquire))
+      return;
+    LockSlow();
+  }
+
+  void Unlock() {
+    u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
+    DCHECK_NE(prev & kWriteLock, 0);
+    (void)prev;
+  }
+
+  void ReadLock() {
+    u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
+    if ((prev & kWriteLock) == 0)
+      return;
+    ReadLockSlow();
+  }
+
+  void ReadUnlock() {
+    u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
+    DCHECK_EQ(prev & kWriteLock, 0);
+    DCHECK_GT(prev & ~kWriteLock, 0);
+    (void)prev;
+  }
+
+  void CheckLocked() {
+    CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
+  }
+
+ private:
+  atomic_uint32_t state_;
+
+  enum {
+    kUnlocked = 0,
+    kWriteLock = 1,
+    kReadLock = 2
+  };
+
+  void NOINLINE LockSlow() {
+    for (int i = 0;; i++) {
+      if (i < 10)
+        proc_yield(10);
+      else
+        internal_sched_yield();
+      u32 cmp = atomic_load(&state_, memory_order_relaxed);
+      if (cmp == kUnlocked &&
+          atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
+                                       memory_order_acquire))
+          return;
+    }
+  }
+
+  void NOINLINE ReadLockSlow() {
+    for (int i = 0;; i++) {
+      if (i < 10)
+        proc_yield(10);
+      else
+        internal_sched_yield();
+      u32 prev = atomic_load(&state_, memory_order_acquire);
+      if ((prev & kWriteLock) == 0)
+        return;
+    }
+  }
+
+  RWMutex(const RWMutex&);
+  void operator = (const RWMutex&);
+};
+
+template<typename MutexType>
+class GenericScopedLock {
+ public:
+  explicit GenericScopedLock(MutexType *mu)
+      : mu_(mu) {
+    mu_->Lock();
+  }
+
+  ~GenericScopedLock() {
+    mu_->Unlock();
+  }
+
+ private:
+  MutexType *mu_;
+
+  GenericScopedLock(const GenericScopedLock&);
+  void operator=(const GenericScopedLock&);
+};
+
+template<typename MutexType>
+class GenericScopedReadLock {
+ public:
+  explicit GenericScopedReadLock(MutexType *mu)
+      : mu_(mu) {
+    mu_->ReadLock();
+  }
+
+  ~GenericScopedReadLock() {
+    mu_->ReadUnlock();
+  }
+
+ private:
+  MutexType *mu_;
+
+  GenericScopedReadLock(const GenericScopedReadLock&);
+  void operator=(const GenericScopedReadLock&);
+};
+
+typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
+typedef GenericScopedLock<BlockingMutex> BlockingMutexLock;
+typedef GenericScopedLock<RWMutex> RWMutexLock;
+typedef GenericScopedReadLock<RWMutex> RWMutexReadLock;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MUTEX_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_netbsd.cpp b/lib/tsan/sanitizer_common/sanitizer_netbsd.cpp
new file mode 100644
index 0000000000..d9aff51d8a
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_netbsd.cpp
@@ -0,0 +1,343 @@
+//===-- sanitizer_netbsd.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between Sanitizer run-time libraries and implements
+// NetBSD-specific functions from sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_getauxval.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_procmaps.h"
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <sys/exec.h>
+#include <sys/mman.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <link.h>
+#include <lwp.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <ucontext.h>
+#include <unistd.h>
+
+extern "C" void *__mmap(void *, size_t, int, int, int, int,
+                        off_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int __sysctl(const int *, unsigned int, void *, size_t *,
+                        const void *, size_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys_close(int) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys_open(const char *, int, ...) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" ssize_t _sys_read(int, void *, size_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" ssize_t _sys_write(int, const void *,
+                              size_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int __ftruncate(int, int, off_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" ssize_t _sys_readlink(const char *, char *,
+                                 size_t) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys_sched_yield() SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys___nanosleep50(const void *,
+                                  void *) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys_execve(const char *, char *const[],
+                           char *const[]) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" off_t __lseek(int, int, off_t, int) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int __fork() SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys___sigprocmask14(int, const void *,
+                                    void *) SANITIZER_WEAK_ATTRIBUTE;
+extern "C" int _sys___wait450(int wpid, int *, int,
+                              void *) SANITIZER_WEAK_ATTRIBUTE;
+
+namespace __sanitizer {
+
+static void *GetRealLibcAddress(const char *symbol) {
+  void *real = dlsym(RTLD_NEXT, symbol);
+  if (!real)
+    real = dlsym(RTLD_DEFAULT, symbol);
+  if (!real) {
+    Printf("GetRealLibcAddress failed for symbol=%s", symbol);
+    Die();
+  }
+  return real;
+}
+
+#define _REAL(func, ...) real##_##func(__VA_ARGS__)
+#define DEFINE__REAL(ret_type, func, ...)                              \
+  static ret_type (*real_##func)(__VA_ARGS__) = NULL;                  \
+  if (!real_##func) {                                                  \
+    real_##func = (ret_type(*)(__VA_ARGS__))GetRealLibcAddress(#func); \
+  }                                                                    \
+  CHECK(real_##func);
+
+// --------------- sanitizer_libc.h
+uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
+                   u64 offset) {
+  CHECK(&__mmap);
+  return (uptr)__mmap(addr, length, prot, flags, fd, 0, offset);
+}
+
+uptr internal_munmap(void *addr, uptr length) {
+  DEFINE__REAL(int, munmap, void *a, uptr b);
+  return _REAL(munmap, addr, length);
+}
+
+int internal_mprotect(void *addr, uptr length, int prot) {
+  DEFINE__REAL(int, mprotect, void *a, uptr b, int c);
+  return _REAL(mprotect, addr, length, prot);
+}
+
+uptr internal_close(fd_t fd) {
+  CHECK(&_sys_close);
+  return _sys_close(fd);
+}
+
+uptr internal_open(const char *filename, int flags) {
+  CHECK(&_sys_open);
+  return _sys_open(filename, flags);
+}
+
+uptr internal_open(const char *filename, int flags, u32 mode) {
+  CHECK(&_sys_open);
+  return _sys_open(filename, flags, mode);
+}
+
+uptr internal_read(fd_t fd, void *buf, uptr count) {
+  sptr res;
+  CHECK(&_sys_read);
+  HANDLE_EINTR(res, (sptr)_sys_read(fd, buf, (size_t)count));
+  return res;
+}
+
+uptr internal_write(fd_t fd, const void *buf, uptr count) {
+  sptr res;
+  CHECK(&_sys_write);
+  HANDLE_EINTR(res, (sptr)_sys_write(fd, buf, count));
+  return res;
+}
+
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  sptr res;
+  CHECK(&__ftruncate);
+  HANDLE_EINTR(res, __ftruncate(fd, 0, (s64)size));
+  return res;
+}
+
+uptr internal_stat(const char *path, void *buf) {
+  DEFINE__REAL(int, __stat50, const char *a, void *b);
+  return _REAL(__stat50, path, buf);
+}
+
+uptr internal_lstat(const char *path, void *buf) {
+  DEFINE__REAL(int, __lstat50, const char *a, void *b);
+  return _REAL(__lstat50, path, buf);
+}
+
+uptr internal_fstat(fd_t fd, void *buf) {
+  DEFINE__REAL(int, __fstat50, int a, void *b);
+  return _REAL(__fstat50, fd, buf);
+}
+
+uptr internal_filesize(fd_t fd) {
+  struct stat st;
+  if (internal_fstat(fd, &st))
+    return -1;
+  return (uptr)st.st_size;
+}
+
+uptr internal_dup(int oldfd) {
+  DEFINE__REAL(int, dup, int a);
+  return _REAL(dup, oldfd);
+}
+
+uptr internal_dup2(int oldfd, int newfd) {
+  DEFINE__REAL(int, dup2, int a, int b);
+  return _REAL(dup2, oldfd, newfd);
+}
+
+uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
+  CHECK(&_sys_readlink);
+  return (uptr)_sys_readlink(path, buf, bufsize);
+}
+
+uptr internal_unlink(const char *path) {
+  DEFINE__REAL(int, unlink, const char *a);
+  return _REAL(unlink, path);
+}
+
+uptr internal_rename(const char *oldpath, const char *newpath) {
+  DEFINE__REAL(int, rename, const char *a, const char *b);
+  return _REAL(rename, oldpath, newpath);
+}
+
+uptr internal_sched_yield() {
+  CHECK(&_sys_sched_yield);
+  return _sys_sched_yield();
+}
+
+void internal__exit(int exitcode) {
+  DEFINE__REAL(void, _exit, int a);
+  _REAL(_exit, exitcode);
+  Die();  // Unreachable.
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+  struct timespec ts;
+  ts.tv_sec = seconds;
+  ts.tv_nsec = 0;
+  CHECK(&_sys___nanosleep50);
+  int res = _sys___nanosleep50(&ts, &ts);
+  if (res)
+    return ts.tv_sec;
+  return 0;
+}
+
+uptr internal_execve(const char *filename, char *const argv[],
+                     char *const envp[]) {
+  CHECK(&_sys_execve);
+  return _sys_execve(filename, argv, envp);
+}
+
+tid_t GetTid() {
+  DEFINE__REAL(int, _lwp_self);
+  return _REAL(_lwp_self);
+}
+
+int TgKill(pid_t pid, tid_t tid, int sig) {
+  DEFINE__REAL(int, _lwp_kill, int a, int b);
+  (void)pid;
+  return _REAL(_lwp_kill, tid, sig);
+}
+
+u64 NanoTime() {
+  timeval tv;
+  DEFINE__REAL(int, __gettimeofday50, void *a, void *b);
+  internal_memset(&tv, 0, sizeof(tv));
+  _REAL(__gettimeofday50, &tv, 0);
+  return (u64)tv.tv_sec * 1000 * 1000 * 1000 + tv.tv_usec * 1000;
+}
+
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+  DEFINE__REAL(int, __clock_gettime50, __sanitizer_clockid_t a, void *b);
+  return _REAL(__clock_gettime50, clk_id, tp);
+}
+
+uptr internal_ptrace(int request, int pid, void *addr, int data) {
+  DEFINE__REAL(int, ptrace, int a, int b, void *c, int d);
+  return _REAL(ptrace, request, pid, addr, data);
+}
+
+uptr internal_waitpid(int pid, int *status, int options) {
+  CHECK(&_sys___wait450);
+  return _sys___wait450(pid, status, options, 0 /* rusage */);
+}
+
+uptr internal_getpid() {
+  DEFINE__REAL(int, getpid);
+  return _REAL(getpid);
+}
+
+uptr internal_getppid() {
+  DEFINE__REAL(int, getppid);
+  return _REAL(getppid);
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+  DEFINE__REAL(int, dlinfo, void *a, int b, void *c);
+  return _REAL(dlinfo, handle, request, p);
+}
+
+uptr internal_getdents(fd_t fd, void *dirp, unsigned int count) {
+  DEFINE__REAL(int, __getdents30, int a, void *b, size_t c);
+  return _REAL(__getdents30, fd, dirp, count);
+}
+
+uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
+  CHECK(&__lseek);
+  return __lseek(fd, 0, offset, whence);
+}
+
+uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
+  Printf("internal_prctl not implemented for NetBSD");
+  Die();
+  return 0;
+}
+
+uptr internal_sigaltstack(const void *ss, void *oss) {
+  DEFINE__REAL(int, __sigaltstack14, const void *a, void *b);
+  return _REAL(__sigaltstack14, ss, oss);
+}
+
+int internal_fork() {
+  CHECK(&__fork);
+  return __fork();
+}
+
+int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
+                    uptr *oldlenp, const void *newp, uptr newlen) {
+  CHECK(&__sysctl);
+  return __sysctl(name, namelen, oldp, (size_t *)oldlenp, newp, (size_t)newlen);
+}
+
+int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
+                          const void *newp, uptr newlen) {
+  DEFINE__REAL(int, sysctlbyname, const char *a, void *b, size_t *c,
+               const void *d, size_t e);
+  return _REAL(sysctlbyname, sname, oldp, (size_t *)oldlenp, newp,
+               (size_t)newlen);
+}
+
+uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
+                          __sanitizer_sigset_t *oldset) {
+  CHECK(&_sys___sigprocmask14);
+  return _sys___sigprocmask14(how, set, oldset);
+}
+
+void internal_sigfillset(__sanitizer_sigset_t *set) {
+  DEFINE__REAL(int, __sigfillset14, const void *a);
+  (void)_REAL(__sigfillset14, set);
+}
+
+void internal_sigemptyset(__sanitizer_sigset_t *set) {
+  DEFINE__REAL(int, __sigemptyset14, const void *a);
+  (void)_REAL(__sigemptyset14, set);
+}
+
+void internal_sigdelset(__sanitizer_sigset_t *set, int signo) {
+  DEFINE__REAL(int, __sigdelset14, const void *a, int b);
+  (void)_REAL(__sigdelset14, set, signo);
+}
+
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags,
+                    void *arg) {
+  DEFINE__REAL(int, clone, int (*a)(void *b), void *c, int d, void *e);
+
+  return _REAL(clone, fn, child_stack, flags, arg);
+}
+
+}  // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_openbsd.cpp b/lib/tsan/sanitizer_common/sanitizer_openbsd.cpp
new file mode 100644
index 0000000000..ed2d8edeb7
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_openbsd.cpp
@@ -0,0 +1,115 @@
+//===-- sanitizer_openbsd.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements Solaris-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_OPENBSD
+
+#include <stdio.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_procmaps.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/shm.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+extern char **environ;
+
+namespace __sanitizer {
+
+uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd,
+                   u64 offset) {
+  return (uptr)mmap(addr, length, prot, flags, fd, offset);
+}
+
+uptr internal_munmap(void *addr, uptr length) { return munmap(addr, length); }
+
+int internal_mprotect(void *addr, uptr length, int prot) {
+  return mprotect(addr, length, prot);
+}
+
+int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
+                          const void *newp, uptr newlen) {
+  Printf("internal_sysctlbyname not implemented for OpenBSD");
+  Die();
+  return 0;
+}
+
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+  // On OpenBSD we cannot get the full path
+  struct kinfo_proc kp;
+  uptr kl;
+  const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()};
+  if (internal_sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1)
+    return internal_snprintf(buf,
+                             (KI_MAXCOMLEN < buf_len ? KI_MAXCOMLEN : buf_len),
+                             "%s", kp.p_comm);
+  return (uptr)0;
+}
+
+static void GetArgsAndEnv(char ***argv, char ***envp) {
+  uptr nargv;
+  uptr nenv;
+  int argvmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV};
+  int envmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ENV};
+  if (internal_sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) {
+    Printf("sysctl KERN_PROC_NARGV failed\n");
+    Die();
+  }
+  if (internal_sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) {
+    Printf("sysctl KERN_PROC_NENV failed\n");
+    Die();
+  }
+  if (internal_sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) {
+    Printf("sysctl KERN_PROC_ARGV failed\n");
+    Die();
+  }
+  if (internal_sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) {
+    Printf("sysctl KERN_PROC_ENV failed\n");
+    Die();
+  }
+}
+
+char **GetArgv() {
+  char **argv, **envp;
+  GetArgsAndEnv(&argv, &envp);
+  return argv;
+}
+
+char **GetEnviron() {
+  char **argv, **envp;
+  GetArgsAndEnv(&argv, &envp);
+  return envp;
+}
+
+void ReExec() {
+  UNIMPLEMENTED();
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_OPENBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp b/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp
new file mode 100644
index 0000000000..1ca0375b8a
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp
@@ -0,0 +1,18 @@
+//===-- sanitizer_persistent_allocator.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+#include "sanitizer_persistent_allocator.h"
+
+namespace __sanitizer {
+
+PersistentAllocator thePersistentAllocator;
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h b/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h
new file mode 100644
index 0000000000..de4fb6ebc3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h
@@ -0,0 +1,71 @@
+//===-- sanitizer_persistent_allocator.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A fast memory allocator that does not support free() nor realloc().
+// All allocations are forever.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PERSISTENT_ALLOCATOR_H
+#define SANITIZER_PERSISTENT_ALLOCATOR_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+class PersistentAllocator {
+ public:
+  void *alloc(uptr size);
+
+ private:
+  void *tryAlloc(uptr size);
+  StaticSpinMutex mtx;  // Protects alloc of new blocks for region allocator.
+  atomic_uintptr_t region_pos;  // Region allocator for Node's.
+  atomic_uintptr_t region_end;
+};
+
+inline void *PersistentAllocator::tryAlloc(uptr size) {
+  // Optimisic lock-free allocation, essentially try to bump the region ptr.
+  for (;;) {
+    uptr cmp = atomic_load(&region_pos, memory_order_acquire);
+    uptr end = atomic_load(&region_end, memory_order_acquire);
+    if (cmp == 0 || cmp + size > end) return nullptr;
+    if (atomic_compare_exchange_weak(&region_pos, &cmp, cmp + size,
+                                     memory_order_acquire))
+      return (void *)cmp;
+  }
+}
+
+inline void *PersistentAllocator::alloc(uptr size) {
+  // First, try to allocate optimisitically.
+  void *s = tryAlloc(size);
+  if (s) return s;
+  // If failed, lock, retry and alloc new superblock.
+  SpinMutexLock l(&mtx);
+  for (;;) {
+    s = tryAlloc(size);
+    if (s) return s;
+    atomic_store(&region_pos, 0, memory_order_relaxed);
+    uptr allocsz = 64 * 1024;
+    if (allocsz < size) allocsz = size;
+    uptr mem = (uptr)MmapOrDie(allocsz, "stack depot");
+    atomic_store(&region_end, mem + allocsz, memory_order_release);
+    atomic_store(&region_pos, mem, memory_order_release);
+  }
+}
+
+extern PersistentAllocator thePersistentAllocator;
+inline void *PersistentAlloc(uptr sz) {
+  return thePersistentAllocator.alloc(sz);
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_PERSISTENT_ALLOCATOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_placement_new.h b/lib/tsan/sanitizer_common/sanitizer_placement_new.h
new file mode 100644
index 0000000000..1ceb8b9092
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_placement_new.h
@@ -0,0 +1,24 @@
+//===-- sanitizer_placement_new.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//
+// The file provides 'placement new'.
+// Do not include it into header files, only into source files.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_PLACEMENT_NEW_H
+#define SANITIZER_PLACEMENT_NEW_H
+
+#include "sanitizer_internal_defs.h"
+
+inline void *operator new(__sanitizer::operator_new_size_type sz, void *p) {
+  return p;
+}
+
+#endif  // SANITIZER_PLACEMENT_NEW_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform.h b/lib/tsan/sanitizer_common/sanitizer_platform.h
new file mode 100644
index 0000000000..f0b1e04d1d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform.h
@@ -0,0 +1,364 @@
+//===-- sanitizer_platform.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common platform macros.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_H
+#define SANITIZER_PLATFORM_H
+
+#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
+  !defined(__OpenBSD__) && !defined(__APPLE__) && !defined(_WIN32) && \
+  !defined(__Fuchsia__) && !defined(__rtems__) && \
+  !(defined(__sun__) && defined(__svr4__))
+# error "This operating system is not supported"
+#endif
+
+#if defined(__linux__)
+# define SANITIZER_LINUX   1
+#else
+# define SANITIZER_LINUX   0
+#endif
+
+#if defined(__FreeBSD__)
+# define SANITIZER_FREEBSD 1
+#else
+# define SANITIZER_FREEBSD 0
+#endif
+
+#if defined(__NetBSD__)
+# define SANITIZER_NETBSD 1
+#else
+# define SANITIZER_NETBSD 0
+#endif
+
+#if defined(__OpenBSD__)
+# define SANITIZER_OPENBSD 1
+#else
+# define SANITIZER_OPENBSD 0
+#endif
+
+#if defined(__sun__) && defined(__svr4__)
+# define SANITIZER_SOLARIS 1
+#else
+# define SANITIZER_SOLARIS 0
+#endif
+
+#if defined(__APPLE__)
+# define SANITIZER_MAC     1
+# include <TargetConditionals.h>
+# if TARGET_OS_IPHONE
+#  define SANITIZER_IOS    1
+# else
+#  define SANITIZER_IOS    0
+# endif
+# if TARGET_OS_SIMULATOR
+#  define SANITIZER_IOSSIM 1
+# else
+#  define SANITIZER_IOSSIM 0
+# endif
+#else
+# define SANITIZER_MAC     0
+# define SANITIZER_IOS     0
+# define SANITIZER_IOSSIM  0
+#endif
+
+#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH
+# define SANITIZER_WATCHOS 1
+#else
+# define SANITIZER_WATCHOS 0
+#endif
+
+#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV
+# define SANITIZER_TVOS 1
+#else
+# define SANITIZER_TVOS 0
+#endif
+
+#if defined(_WIN32)
+# define SANITIZER_WINDOWS 1
+#else
+# define SANITIZER_WINDOWS 0
+#endif
+
+#if defined(_WIN64)
+# define SANITIZER_WINDOWS64 1
+#else
+# define SANITIZER_WINDOWS64 0
+#endif
+
+#if defined(__ANDROID__)
+# define SANITIZER_ANDROID 1
+#else
+# define SANITIZER_ANDROID 0
+#endif
+
+#if defined(__Fuchsia__)
+# define SANITIZER_FUCHSIA 1
+#else
+# define SANITIZER_FUCHSIA 0
+#endif
+
+#if defined(__rtems__)
+# define SANITIZER_RTEMS 1
+#else
+# define SANITIZER_RTEMS 0
+#endif
+
+#define SANITIZER_POSIX \
+  (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
+    SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_SOLARIS)
+
+#if __LP64__ || defined(_WIN64)
+#  define SANITIZER_WORDSIZE 64
+#else
+#  define SANITIZER_WORDSIZE 32
+#endif
+
+#if SANITIZER_WORDSIZE == 64
+# define FIRST_32_SECOND_64(a, b) (b)
+#else
+# define FIRST_32_SECOND_64(a, b) (a)
+#endif
+
+#if defined(__x86_64__) && !defined(_LP64)
+# define SANITIZER_X32 1
+#else
+# define SANITIZER_X32 0
+#endif
+
+#if defined(__i386__) || defined(_M_IX86)
+# define SANITIZER_I386 1
+#else
+# define SANITIZER_I386 0
+#endif
+
+#if defined(__mips__)
+# define SANITIZER_MIPS 1
+# if defined(__mips64)
+#  define SANITIZER_MIPS32 0
+#  define SANITIZER_MIPS64 1
+# else
+#  define SANITIZER_MIPS32 1
+#  define SANITIZER_MIPS64 0
+# endif
+#else
+# define SANITIZER_MIPS 0
+# define SANITIZER_MIPS32 0
+# define SANITIZER_MIPS64 0
+#endif
+
+#if defined(__s390__)
+# define SANITIZER_S390 1
+# if defined(__s390x__)
+#  define SANITIZER_S390_31 0
+#  define SANITIZER_S390_64 1
+# else
+#  define SANITIZER_S390_31 1
+#  define SANITIZER_S390_64 0
+# endif
+#else
+# define SANITIZER_S390 0
+# define SANITIZER_S390_31 0
+# define SANITIZER_S390_64 0
+#endif
+
+#if defined(__powerpc__)
+# define SANITIZER_PPC 1
+# if defined(__powerpc64__)
+#  define SANITIZER_PPC32 0
+#  define SANITIZER_PPC64 1
+// 64-bit PPC has two ABIs (v1 and v2).  The old powerpc64 target is
+// big-endian, and uses v1 ABI (known for its function descriptors),
+// while the new powerpc64le target is little-endian and uses v2.
+// In theory, you could convince gcc to compile for their evil twins
+// (eg. big-endian v2), but you won't find such combinations in the wild
+// (it'd require bootstrapping a whole system, which would be quite painful
+// - there's no target triple for that).  LLVM doesn't support them either.
+#  if _CALL_ELF == 2
+#   define SANITIZER_PPC64V1 0
+#   define SANITIZER_PPC64V2 1
+#  else
+#   define SANITIZER_PPC64V1 1
+#   define SANITIZER_PPC64V2 0
+#  endif
+# else
+#  define SANITIZER_PPC32 1
+#  define SANITIZER_PPC64 0
+#  define SANITIZER_PPC64V1 0
+#  define SANITIZER_PPC64V2 0
+# endif
+#else
+# define SANITIZER_PPC 0
+# define SANITIZER_PPC32 0
+# define SANITIZER_PPC64 0
+# define SANITIZER_PPC64V1 0
+# define SANITIZER_PPC64V2 0
+#endif
+
+#if defined(__arm__)
+# define SANITIZER_ARM 1
+#else
+# define SANITIZER_ARM 0
+#endif
+
+#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
+# define SANITIZER_SOLARIS32 1
+#else
+# define SANITIZER_SOLARIS32 0
+#endif
+
+#if defined(__myriad2__)
+# define SANITIZER_MYRIAD2 1
+#else
+# define SANITIZER_MYRIAD2 0
+#endif
+
+// By default we allow to use SizeClassAllocator64 on 64-bit platform.
+// But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64
+// does not work well and we need to fallback to SizeClassAllocator32.
+// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
+// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
+#ifndef SANITIZER_CAN_USE_ALLOCATOR64
+# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA
+#  define SANITIZER_CAN_USE_ALLOCATOR64 1
+# elif defined(__mips64) || defined(__aarch64__)
+#  define SANITIZER_CAN_USE_ALLOCATOR64 0
+# else
+#  define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
+# endif
+#endif
+
+// The range of addresses which can be returned my mmap.
+// FIXME: this value should be different on different platforms.  Larger values
+// will still work but will consume more memory for TwoLevelByteMap.
+#if defined(__mips__)
+# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
+#elif defined(__aarch64__)
+# if SANITIZER_MAC
+// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
+#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
+# else
+#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
+# endif
+#elif defined(__sparc__)
+#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
+#else
+# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#endif
+
+// Whether the addresses are sign-extended from the VMA range to the word.
+// The SPARC64 Linux port implements this to split the VMA space into two
+// non-contiguous halves with a huge hole in the middle.
+#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
+#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
+#else
+#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
+#endif
+
+// The AArch64 and RISC-V linux ports use the canonical syscall set as
+// mandated by the upstream linux community for all new ports. Other ports
+// may still use legacy syscalls.
+#ifndef SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
+# if (defined(__aarch64__) || defined(__riscv)) && SANITIZER_LINUX
+# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 1
+# else
+# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 0
+# endif
+#endif
+
+// udi16 syscalls can only be used when the following conditions are
+// met:
+// * target is one of arm32, x86-32, sparc32, sh or m68k
+// * libc version is libc5, glibc-2.0, glibc-2.1 or glibc-2.2 to 2.15
+//   built against > linux-2.2 kernel headers
+// Since we don't want to include libc headers here, we check the
+// target only.
+#if defined(__arm__) || SANITIZER_X32 || defined(__sparc__)
+#define SANITIZER_USES_UID16_SYSCALLS 1
+#else
+#define SANITIZER_USES_UID16_SYSCALLS 0
+#endif
+
+#if defined(__mips__)
+# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
+#else
+# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
+#endif
+
+/// \macro MSC_PREREQ
+/// \brief Is the compiler MSVC of at least the specified version?
+/// The common \param version values to check for are:
+///  * 1800: Microsoft Visual Studio 2013 / 12.0
+///  * 1900: Microsoft Visual Studio 2015 / 14.0
+#ifdef _MSC_VER
+# define MSC_PREREQ(version) (_MSC_VER >= (version))
+#else
+# define MSC_PREREQ(version) 0
+#endif
+
+#if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS)
+# define SANITIZER_NON_UNIQUE_TYPEINFO 0
+#else
+# define SANITIZER_NON_UNIQUE_TYPEINFO 1
+#endif
+
+// On linux, some architectures had an ABI transition from 64-bit long double
+// (ie. same as double) to 128-bit long double.  On those, glibc symbols
+// involving long doubles come in two versions, and we need to pass the
+// correct one to dlvsym when intercepting them.
+#if SANITIZER_LINUX && (SANITIZER_S390 || SANITIZER_PPC32 || SANITIZER_PPC64V1)
+#define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
+#endif
+
+#if SANITIZER_GO == 0
+# define SANITIZER_GO 0
+#endif
+
+// On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.
+// pthread_exit() performs unwinding that leads to dlopen'ing libgcc_s.so.
+// dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize
+// that this allocation happens in dynamic linker and should be ignored.
+#if SANITIZER_PPC || defined(__thumb__)
+# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
+#else
+# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
+#endif
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \
+  SANITIZER_OPENBSD || SANITIZER_SOLARIS
+# define SANITIZER_MADVISE_DONTNEED MADV_FREE
+#else
+# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
+#endif
+
+// Older gcc have issues aligning to a constexpr, and require an integer.
+// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others.
+#if defined(__powerpc__) || defined(__powerpc64__)
+# define SANITIZER_CACHE_LINE_SIZE 128
+#else
+# define SANITIZER_CACHE_LINE_SIZE 64
+#endif
+
+// Enable offline markup symbolizer for Fuchsia and RTEMS.
+#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
+#define SANITIZER_SYMBOLIZER_MARKUP 1
+#else
+#define SANITIZER_SYMBOLIZER_MARKUP 0
+#endif
+
+// Enable ability to support sanitizer initialization that is
+// compatible with the sanitizer library being loaded via
+// `dlopen()`.
+#if SANITIZER_MAC
+#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
+#else
+#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
+#endif
+
+#endif // SANITIZER_PLATFORM_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h b/lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h
new file mode 100644
index 0000000000..e28bb937ae
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h
@@ -0,0 +1,609 @@
+//===-- sanitizer_platform_interceptors.h -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines macro telling whether sanitizer tools can/should intercept
+// given library functions on a given platform.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
+#define SANITIZER_PLATFORM_INTERCEPTORS_H
+
+#include "sanitizer_glibc_version.h"
+#include "sanitizer_internal_defs.h"
+
+#if SANITIZER_POSIX
+# define SI_POSIX 1
+#else
+# define SI_POSIX 0
+#endif
+
+#if !SANITIZER_WINDOWS
+# define SI_WINDOWS 0
+#else
+# define SI_WINDOWS 1
+#endif
+
+#if SI_WINDOWS && SI_POSIX
+# error "Windows is not POSIX!"
+#endif
+
+#if SI_POSIX
+# include "sanitizer_platform_limits_freebsd.h"
+# include "sanitizer_platform_limits_netbsd.h"
+# include "sanitizer_platform_limits_openbsd.h"
+# include "sanitizer_platform_limits_posix.h"
+# include "sanitizer_platform_limits_solaris.h"
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+# define SI_LINUX_NOT_ANDROID 1
+#else
+# define SI_LINUX_NOT_ANDROID 0
+#endif
+
+#if SANITIZER_ANDROID
+# define SI_ANDROID 1
+#else
+# define SI_ANDROID 0
+#endif
+
+#if SANITIZER_FREEBSD
+# define SI_FREEBSD 1
+#else
+# define SI_FREEBSD 0
+#endif
+
+#if SANITIZER_NETBSD
+# define SI_NETBSD 1
+#else
+# define SI_NETBSD 0
+#endif
+
+#if SANITIZER_OPENBSD
+#define SI_OPENBSD 1
+#else
+#define SI_OPENBSD 0
+#endif
+
+#if SANITIZER_LINUX
+# define SI_LINUX 1
+#else
+# define SI_LINUX 0
+#endif
+
+#if SANITIZER_MAC
+# define SI_MAC 1
+# define SI_NOT_MAC 0
+#else
+# define SI_MAC 0
+# define SI_NOT_MAC 1
+#endif
+
+#if SANITIZER_IOS
+# define SI_IOS 1
+#else
+# define SI_IOS 0
+#endif
+
+#if SANITIZER_IOSSIM
+# define SI_IOSSIM 1
+#else
+# define SI_IOSSIM 0
+#endif
+
+#if SANITIZER_WATCHOS
+# define SI_WATCHOS 1
+#else
+# define SI_WATCHOS 0
+#endif
+
+#if SANITIZER_TVOS
+# define SI_TVOS 1
+#else
+# define SI_TVOS 0
+#endif
+
+#if SANITIZER_FUCHSIA
+# define SI_NOT_FUCHSIA 0
+#else
+# define SI_NOT_FUCHSIA 1
+#endif
+
+#if SANITIZER_RTEMS
+# define SI_NOT_RTEMS 0
+#else
+# define SI_NOT_RTEMS 1
+#endif
+
+#if SANITIZER_SOLARIS
+# define SI_SOLARIS 1
+#else
+# define SI_SOLARIS 0
+#endif
+
+#if SANITIZER_SOLARIS32
+# define SI_SOLARIS32 1
+#else
+# define SI_SOLARIS32 0
+#endif
+
+#if SANITIZER_POSIX && !SANITIZER_MAC
+# define SI_POSIX_NOT_MAC 1
+#else
+# define SI_POSIX_NOT_MAC 0
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_FREEBSD
+# define SI_LINUX_NOT_FREEBSD 1
+# else
+# define SI_LINUX_NOT_FREEBSD 0
+#endif
+
+#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX
+#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC
+#define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX
+#define SANITIZER_INTERCEPT_MEMSET 1
+#define SANITIZER_INTERCEPT_MEMMOVE 1
+#define SANITIZER_INTERCEPT_MEMCPY 1
+#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_BCMP \
+  SANITIZER_INTERCEPT_MEMCMP &&  \
+      ((SI_POSIX && _GNU_SOURCE) || SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_STRNDUP SI_POSIX
+#define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD
+#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
+    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070
+# define SI_MAC_DEPLOYMENT_BELOW_10_7 1
+#else
+# define SI_MAC_DEPLOYMENT_BELOW_10_7 0
+#endif
+// memmem on Darwin doesn't exist on 10.6
+// FIXME: enable memmem on Windows.
+#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7)
+#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_MEMRCHR \
+  (SI_FREEBSD || SI_LINUX || SI_NETBSD || SI_OPENBSD)
+
+#define SANITIZER_INTERCEPT_READ SI_POSIX
+#define SANITIZER_INTERCEPT_PREAD SI_POSIX
+#define SANITIZER_INTERCEPT_WRITE SI_POSIX
+#define SANITIZER_INTERCEPT_PWRITE SI_POSIX
+
+#define SANITIZER_INTERCEPT_FREAD SI_POSIX
+#define SANITIZER_INTERCEPT_FWRITE SI_POSIX
+#define SANITIZER_INTERCEPT_FGETS SI_POSIX
+#define SANITIZER_INTERCEPT_FPUTS SI_POSIX
+#define SANITIZER_INTERCEPT_PUTS SI_POSIX
+
+#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+
+#define SANITIZER_INTERCEPT_READV SI_POSIX
+#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
+
+#define SANITIZER_INTERCEPT_PREADV \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PREADV64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PWRITEV64 SI_LINUX_NOT_ANDROID
+
+#define SANITIZER_INTERCEPT_PRCTL   SI_LINUX
+
+#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX
+#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX
+
+#define SANITIZER_INTERCEPT_SCANF SI_POSIX
+#define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID
+
+#ifndef SANITIZER_INTERCEPT_PRINTF
+# define SANITIZER_INTERCEPT_PRINTF SI_POSIX
+# define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
+# define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID
+#endif
+
+#define SANITIZER_INTERCEPT___PRINTF_CHK \
+  (SANITIZER_INTERCEPT_PRINTF && SI_LINUX_NOT_ANDROID)
+
+#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX
+
+#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX
+#define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS                            \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETPWENT                                          \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETGRENT_R                                       \
+  (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_GETPWENT_R \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETPWENT_R \
+  (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SETPWENT \
+  (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CLOCK_GETTIME \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID SI_LINUX
+#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
+#define SANITIZER_INTERCEPT_TIME SI_POSIX
+#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_GLOB64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WAIT SI_POSIX
+#define SANITIZER_INTERCEPT_INET SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME2 SI_POSIX && !SI_SOLARIS
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R \
+  (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R \
+  (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R \
+  (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETHOSTENT_R \
+  (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
+#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
+#define SANITIZER_INTERCEPT_ACCEPT4 \
+  (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
+#define SANITIZER_INTERCEPT_MODF SI_POSIX
+#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
+#define SANITIZER_INTERCEPT_SENDMSG SI_POSIX
+#define SANITIZER_INTERCEPT_RECVMMSG SI_LINUX
+#define SANITIZER_INTERCEPT_SENDMMSG SI_LINUX
+#define SANITIZER_INTERCEPT_SYSMSG SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX
+#define SANITIZER_INTERCEPT_IOCTL SI_POSIX
+#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
+#define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
+#define SANITIZER_INTERCEPT_READDIR SI_POSIX
+#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#if SI_LINUX_NOT_ANDROID && \
+  (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+    defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+    defined(__s390__))
+#define SANITIZER_INTERCEPT_PTRACE 1
+#else
+#define SANITIZER_INTERCEPT_PTRACE 0
+#endif
+#define SANITIZER_INTERCEPT_SETLOCALE SI_POSIX
+#define SANITIZER_INTERCEPT_GETCWD SI_POSIX
+#define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STRTOIMAX SI_POSIX
+#define SANITIZER_INTERCEPT_MBSTOWCS SI_POSIX
+#define SANITIZER_INTERCEPT_MBSNRTOWCS \
+  (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCSTOMBS SI_POSIX
+#define SANITIZER_INTERCEPT_STRXFRM SI_POSIX
+#define SANITIZER_INTERCEPT___STRXFRM_L SI_LINUX
+#define SANITIZER_INTERCEPT_WCSXFRM SI_POSIX
+#define SANITIZER_INTERCEPT___WCSXFRM_L SI_LINUX
+#define SANITIZER_INTERCEPT_WCSNRTOMBS                                        \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCRTOMB                                           \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCTOMB                                           \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_REALPATH SI_POSIX
+#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \
+  (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CONFSTR                                           \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_STRERROR SI_POSIX
+#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
+#define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_SCANDIR \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_POLL SI_POSIX
+#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_WORDEXP \
+  (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \
+    SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
+#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_SIGSETOPS \
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX
+#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_SIGMASK SI_POSIX
+#define SANITIZER_INTERCEPT_BACKTRACE \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX
+#define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STATFS \
+  (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_STATFS64 \
+  (((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_STATVFS \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_ETHER_HOST \
+  (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_SHMCTL                                       \
+  (((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) || \
+   SI_NETBSD || SI_OPENBSD || SI_SOLARIS)  // NOLINT
+#define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
+  (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
+  (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \
+  (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \
+  (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST \
+  (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED \
+  (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED \
+  (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK \
+  (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
+  (SI_LINUX_NOT_ANDROID && !SI_NETBSD && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD
+#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
+#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_TTYNAME SI_POSIX
+#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX
+#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX
+#define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS
+#define SANITIZER_INTERCEPT_REMQUO SI_POSIX
+#define SANITIZER_INTERCEPT_REMQUOL (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX
+#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_RAND_R                                            \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_ICONV \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_TIMES SI_POSIX
+
+// FIXME: getline seems to be available on OSX 10.7
+#define SANITIZER_INTERCEPT_GETLINE \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+
+#define SANITIZER_INTERCEPT__EXIT \
+  (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_SOLARIS)
+
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEX SI_POSIX
+#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___LIBC_MUTEX SI_NETBSD
+#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP \
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+
+#define SANITIZER_INTERCEPT_TLS_GET_ADDR \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+
+#define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
+#define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
+#define SANITIZER_INTERCEPT_GETRESID SI_LINUX
+#define SANITIZER_INTERCEPT_GETIFADDRS                                        \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_IF_INDEXTONAME                                    \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
+  SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID
+#if SI_LINUX && defined(__arm__)
+#define SANITIZER_INTERCEPT_AEABI_MEM 1
+#else
+#define SANITIZER_INTERCEPT_AEABI_MEM 0
+#endif
+#define SANITIZER_INTERCEPT___BZERO SI_MAC || SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_BZERO SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_FTIME \
+  (!SI_FREEBSD && !SI_NETBSD && !SI_OPENBSD && SI_POSIX)
+#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_TSEARCH \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_FOPEN SI_POSIX
+#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM \
+  (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX
+#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX
+
+#ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
+#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE                                    \
+  (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
+  SI_SOLARIS)
+#endif
+
+#define SANITIZER_INTERCEPT_GETPASS \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_OPENBSD)
+#define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID
+
+#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX
+#define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_SEM \
+  (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
+#define SANITIZER_INTERCEPT_MINCORE \
+  (SI_LINUX || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
+#define SANITIZER_INTERCEPT_CTERMID \
+  (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD || SI_SOLARIS)
+
+#define SANITIZER_INTERCEPTOR_HOOKS \
+  (SI_LINUX || SI_MAC || SI_WINDOWS || SI_NETBSD)
+#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
+#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
+#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
+
+#define SANITIZER_INTERCEPT_STAT \
+  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
+#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
+#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
+
+#define SANITIZER_INTERCEPT_UTMP \
+  (SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_UTMPX \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
+
+#define SANITIZER_INTERCEPT_GETLOADAVG \
+  (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_OPENBSD)
+
+#define SANITIZER_INTERCEPT_MMAP SI_POSIX
+#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
+  SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_MEMALIGN \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_PVALLOC \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
+  SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_CFREE \
+  (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
+  SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE \
+  (!SI_MAC && !SI_OPENBSD && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
+#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
+#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID
+
+#define SANITIZER_INTERCEPT_ACCT (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_USER_FROM_UID SI_NETBSD
+#define SANITIZER_INTERCEPT_UID_FROM_USER SI_NETBSD
+#define SANITIZER_INTERCEPT_GROUP_FROM_GID SI_NETBSD
+#define SANITIZER_INTERCEPT_GID_FROM_GROUP SI_NETBSD
+#define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_GETGROUPLIST (SI_NETBSD || SI_OPENBSD)
+#define SANITIZER_INTERCEPT_STRLCPY                                            \
+  (SI_NETBSD || SI_FREEBSD || SI_OPENBSD || SI_MAC || SI_ANDROID)
+
+#define SANITIZER_INTERCEPT_NAME_TO_HANDLE_AT SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_OPEN_BY_HANDLE_AT SI_LINUX_NOT_ANDROID
+
+#define SANITIZER_INTERCEPT_READLINK SI_POSIX
+#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
+    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000
+# define SI_MAC_DEPLOYMENT_BELOW_10_10 1
+#else
+# define SI_MAC_DEPLOYMENT_BELOW_10_10 0
+#endif
+#define SANITIZER_INTERCEPT_READLINKAT \
+  (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_10)
+
+#define SANITIZER_INTERCEPT_DEVNAME (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_DEVNAME_R (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
+#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_PROTOENT_R (SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_NETENT SI_NETBSD
+#define SANITIZER_INTERCEPT_SETVBUF (SI_NETBSD || SI_FREEBSD || \
+  SI_LINUX || SI_MAC)
+#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
+#define SANITIZER_INTERCEPT_MI_VECTOR_HASH SI_NETBSD
+#define SANITIZER_INTERCEPT_GETVFSSTAT SI_NETBSD
+#define SANITIZER_INTERCEPT_REGEX (SI_NETBSD || SI_FREEBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_REGEXSUB SI_NETBSD
+#define SANITIZER_INTERCEPT_FTS (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_SYSCTL (SI_NETBSD || SI_FREEBSD || SI_MAC)
+#define SANITIZER_INTERCEPT_ASYSCTL SI_NETBSD
+#define SANITIZER_INTERCEPT_SYSCTLGETMIBINFO SI_NETBSD
+#define SANITIZER_INTERCEPT_NL_LANGINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
+#define SANITIZER_INTERCEPT_MODCTL SI_NETBSD
+#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
+#define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD
+#define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD
+#define SANITIZER_INTERCEPT_STRTOI SI_NETBSD
+#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
+#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
+#define SANITIZER_INTERCEPT_MD4 SI_NETBSD
+#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
+#define SANITIZER_INTERCEPT_MD5 SI_NETBSD
+#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_MD2 SI_NETBSD
+#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD
+#define SANITIZER_INTERCEPT_CDB SI_NETBSD
+#define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_POPEN SI_POSIX
+#define SANITIZER_INTERCEPT_POPENVE SI_NETBSD
+#define SANITIZER_INTERCEPT_PCLOSE SI_POSIX
+#define SANITIZER_INTERCEPT_FUNOPEN (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_FUNOPEN2 SI_NETBSD
+#define SANITIZER_INTERCEPT_GETFSENT (SI_FREEBSD || SI_NETBSD || SI_MAC)
+#define SANITIZER_INTERCEPT_ARC4RANDOM (SI_FREEBSD || SI_NETBSD || SI_MAC)
+#define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD
+#define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID)
+#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
+#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID)
+#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID)
+
+#define SANITIZER_INTERCEPT_GETRANDOM \
+  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
+#define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
+#define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
+#define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
+#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD
+#define SANITIZER_INTERCEPT_QSORT \
+  (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
+#define SANITIZER_INTERCEPT_QSORT_R (SI_LINUX && !SI_ANDROID)
+// sigaltstack on i386 macOS cannot be intercepted due to setjmp()
+// calling it and assuming that it does not clobber registers.
+#define SANITIZER_INTERCEPT_SIGALTSTACK \
+  (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
+#define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
+#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
+
+#endif  // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
new file mode 100644
index 0000000000..dcc6c71c07
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
@@ -0,0 +1,531 @@
+//===-- sanitizer_platform_limits_freebsd.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific FreeBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD
+
+#include <sys/capsicum.h>
+#include <sys/consio.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/kbio.h>
+#include <sys/link_elf.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/mqueue.h>
+#include <sys/msg.h>
+#include <sys/mtio.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/signal.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/soundcard.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/timespec.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <sys/utsname.h>
+//
+#include <arpa/inet.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/ppp_defs.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip_mroute.h>
+//
+#include <dirent.h>
+#include <dlfcn.h>
+#include <fstab.h>
+#include <fts.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stringlist.h>
+#include <term.h>
+#include <termios.h>
+#include <time.h>
+#include <utime.h>
+#include <utmpx.h>
+#include <vis.h>
+#include <wchar.h>
+#include <wordexp.h>
+
+#define _KERNEL  // to declare 'shminfo' structure
+#include <sys/shm.h>
+#undef _KERNEL
+
+#undef INLINE  // to avoid clashes with sanitizers' definitions
+
+#undef IOC_DIRMASK
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_platform_limits_freebsd.h"
+
+namespace __sanitizer {
+void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) {
+  void *p = nullptr;
+  return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
+}
+
+unsigned struct_cap_rights_sz = sizeof(cap_rights_t);
+unsigned struct_utsname_sz = sizeof(struct utsname);
+unsigned struct_stat_sz = sizeof(struct stat);
+unsigned struct_rusage_sz = sizeof(struct rusage);
+unsigned struct_tm_sz = sizeof(struct tm);
+unsigned struct_passwd_sz = sizeof(struct passwd);
+unsigned struct_group_sz = sizeof(struct group);
+unsigned siginfo_t_sz = sizeof(siginfo_t);
+unsigned struct_sigaction_sz = sizeof(struct sigaction);
+unsigned struct_stack_t_sz = sizeof(stack_t);
+unsigned struct_itimerval_sz = sizeof(struct itimerval);
+unsigned pthread_t_sz = sizeof(pthread_t);
+unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+unsigned pid_t_sz = sizeof(pid_t);
+unsigned timeval_sz = sizeof(timeval);
+unsigned uid_t_sz = sizeof(uid_t);
+unsigned gid_t_sz = sizeof(gid_t);
+unsigned fpos_t_sz = sizeof(fpos_t);
+unsigned mbstate_t_sz = sizeof(mbstate_t);
+unsigned sigset_t_sz = sizeof(sigset_t);
+unsigned struct_timezone_sz = sizeof(struct timezone);
+unsigned struct_tms_sz = sizeof(struct tms);
+unsigned struct_sigevent_sz = sizeof(struct sigevent);
+unsigned struct_sched_param_sz = sizeof(struct sched_param);
+unsigned struct_statfs_sz = sizeof(struct statfs);
+unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned struct_rlimit_sz = sizeof(struct rlimit);
+unsigned struct_timespec_sz = sizeof(struct timespec);
+unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+unsigned struct_timeb_sz = sizeof(struct timeb);
+unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
+unsigned struct_statvfs_sz = sizeof(struct statvfs);
+unsigned struct_shminfo_sz = sizeof(struct shminfo);
+unsigned struct_shm_info_sz = sizeof(struct shm_info);
+unsigned struct_regmatch_sz = sizeof(regmatch_t);
+unsigned struct_regex_sz = sizeof(regex_t);
+unsigned struct_fstab_sz = sizeof(struct fstab);
+unsigned struct_FTS_sz = sizeof(FTS);
+unsigned struct_FTSENT_sz = sizeof(FTSENT);
+unsigned struct_StringList_sz = sizeof(StringList);
+
+const uptr sig_ign = (uptr)SIG_IGN;
+const uptr sig_dfl = (uptr)SIG_DFL;
+const uptr sig_err = (uptr)SIG_ERR;
+const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+int shmctl_ipc_stat = (int)IPC_STAT;
+int shmctl_ipc_info = (int)IPC_INFO;
+int shmctl_shm_info = (int)SHM_INFO;
+int shmctl_shm_stat = (int)SHM_STAT;
+unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+int map_fixed = MAP_FIXED;
+
+int af_inet = (int)AF_INET;
+int af_inet6 = (int)AF_INET6;
+
+uptr __sanitizer_in_addr_sz(int af) {
+  if (af == AF_INET)
+    return sizeof(struct in_addr);
+  else if (af == AF_INET6)
+    return sizeof(struct in6_addr);
+  else
+    return 0;
+}
+
+unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
+int glob_nomatch = GLOB_NOMATCH;
+int glob_altdirfunc = GLOB_ALTDIRFUNC;
+
+unsigned path_max = PATH_MAX;
+
+// ioctl arguments
+unsigned struct_ifreq_sz = sizeof(struct ifreq);
+unsigned struct_termios_sz = sizeof(struct termios);
+unsigned struct_winsize_sz = sizeof(struct winsize);
+#if SOUND_VERSION >= 0x040000
+unsigned struct_copr_buffer_sz = 0;
+unsigned struct_copr_debug_buf_sz = 0;
+unsigned struct_copr_msg_sz = 0;
+#else
+unsigned struct_copr_buffer_sz = sizeof(struct copr_buffer);
+unsigned struct_copr_debug_buf_sz = sizeof(struct copr_debug_buf);
+unsigned struct_copr_msg_sz = sizeof(struct copr_msg);
+#endif
+unsigned struct_midi_info_sz = sizeof(struct midi_info);
+unsigned struct_mtget_sz = sizeof(struct mtget);
+unsigned struct_mtop_sz = sizeof(struct mtop);
+unsigned struct_sbi_instrument_sz = sizeof(struct sbi_instrument);
+unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec);
+unsigned struct_synth_info_sz = sizeof(struct synth_info);
+unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
+unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
+unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+const unsigned long __sanitizer_bufsiz = BUFSIZ;
+
+const unsigned IOCTL_NOT_PRESENT = 0;
+
+unsigned IOCTL_FIOASYNC = FIOASYNC;
+unsigned IOCTL_FIOCLEX = FIOCLEX;
+unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+unsigned IOCTL_FIONBIO = FIONBIO;
+unsigned IOCTL_FIONCLEX = FIONCLEX;
+unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+unsigned IOCTL_TIOCCONS = TIOCCONS;
+unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+unsigned IOCTL_TIOCGETD = TIOCGETD;
+unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+unsigned IOCTL_TIOCMGET = TIOCMGET;
+unsigned IOCTL_TIOCMSET = TIOCMSET;
+unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+unsigned IOCTL_TIOCPKT = TIOCPKT;
+unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+unsigned IOCTL_TIOCSETD = TIOCSETD;
+unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+unsigned IOCTL_TIOCSTI = TIOCSTI;
+unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
+unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
+unsigned IOCTL_MTIOCGET = MTIOCGET;
+unsigned IOCTL_MTIOCTOP = MTIOCTOP;
+unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE;
+unsigned IOCTL_SNDCTL_DSP_GETFMTS = SNDCTL_DSP_GETFMTS;
+unsigned IOCTL_SNDCTL_DSP_NONBLOCK = SNDCTL_DSP_NONBLOCK;
+unsigned IOCTL_SNDCTL_DSP_POST = SNDCTL_DSP_POST;
+unsigned IOCTL_SNDCTL_DSP_RESET = SNDCTL_DSP_RESET;
+unsigned IOCTL_SNDCTL_DSP_SETFMT = SNDCTL_DSP_SETFMT;
+unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT = SNDCTL_DSP_SETFRAGMENT;
+unsigned IOCTL_SNDCTL_DSP_SPEED = SNDCTL_DSP_SPEED;
+unsigned IOCTL_SNDCTL_DSP_STEREO = SNDCTL_DSP_STEREO;
+unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE = SNDCTL_DSP_SUBDIVIDE;
+unsigned IOCTL_SNDCTL_DSP_SYNC = SNDCTL_DSP_SYNC;
+unsigned IOCTL_SNDCTL_FM_4OP_ENABLE = SNDCTL_FM_4OP_ENABLE;
+unsigned IOCTL_SNDCTL_FM_LOAD_INSTR = SNDCTL_FM_LOAD_INSTR;
+unsigned IOCTL_SNDCTL_MIDI_INFO = SNDCTL_MIDI_INFO;
+unsigned IOCTL_SNDCTL_MIDI_PRETIME = SNDCTL_MIDI_PRETIME;
+unsigned IOCTL_SNDCTL_SEQ_CTRLRATE = SNDCTL_SEQ_CTRLRATE;
+unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT = SNDCTL_SEQ_GETINCOUNT;
+unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT = SNDCTL_SEQ_GETOUTCOUNT;
+unsigned IOCTL_SNDCTL_SEQ_NRMIDIS = SNDCTL_SEQ_NRMIDIS;
+unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS = SNDCTL_SEQ_NRSYNTHS;
+unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND = SNDCTL_SEQ_OUTOFBAND;
+unsigned IOCTL_SNDCTL_SEQ_PANIC = SNDCTL_SEQ_PANIC;
+unsigned IOCTL_SNDCTL_SEQ_PERCMODE = SNDCTL_SEQ_PERCMODE;
+unsigned IOCTL_SNDCTL_SEQ_RESET = SNDCTL_SEQ_RESET;
+unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES = SNDCTL_SEQ_RESETSAMPLES;
+unsigned IOCTL_SNDCTL_SEQ_SYNC = SNDCTL_SEQ_SYNC;
+unsigned IOCTL_SNDCTL_SEQ_TESTMIDI = SNDCTL_SEQ_TESTMIDI;
+unsigned IOCTL_SNDCTL_SEQ_THRESHOLD = SNDCTL_SEQ_THRESHOLD;
+unsigned IOCTL_SNDCTL_SYNTH_INFO = SNDCTL_SYNTH_INFO;
+unsigned IOCTL_SNDCTL_SYNTH_MEMAVL = SNDCTL_SYNTH_MEMAVL;
+unsigned IOCTL_SNDCTL_TMR_CONTINUE = SNDCTL_TMR_CONTINUE;
+unsigned IOCTL_SNDCTL_TMR_METRONOME = SNDCTL_TMR_METRONOME;
+unsigned IOCTL_SNDCTL_TMR_SELECT = SNDCTL_TMR_SELECT;
+unsigned IOCTL_SNDCTL_TMR_SOURCE = SNDCTL_TMR_SOURCE;
+unsigned IOCTL_SNDCTL_TMR_START = SNDCTL_TMR_START;
+unsigned IOCTL_SNDCTL_TMR_STOP = SNDCTL_TMR_STOP;
+unsigned IOCTL_SNDCTL_TMR_TEMPO = SNDCTL_TMR_TEMPO;
+unsigned IOCTL_SNDCTL_TMR_TIMEBASE = SNDCTL_TMR_TIMEBASE;
+unsigned IOCTL_SOUND_MIXER_READ_ALTPCM = SOUND_MIXER_READ_ALTPCM;
+unsigned IOCTL_SOUND_MIXER_READ_BASS = SOUND_MIXER_READ_BASS;
+unsigned IOCTL_SOUND_MIXER_READ_CAPS = SOUND_MIXER_READ_CAPS;
+unsigned IOCTL_SOUND_MIXER_READ_CD = SOUND_MIXER_READ_CD;
+unsigned IOCTL_SOUND_MIXER_READ_DEVMASK = SOUND_MIXER_READ_DEVMASK;
+unsigned IOCTL_SOUND_MIXER_READ_ENHANCE = SOUND_MIXER_READ_ENHANCE;
+unsigned IOCTL_SOUND_MIXER_READ_IGAIN = SOUND_MIXER_READ_IGAIN;
+unsigned IOCTL_SOUND_MIXER_READ_IMIX = SOUND_MIXER_READ_IMIX;
+unsigned IOCTL_SOUND_MIXER_READ_LINE = SOUND_MIXER_READ_LINE;
+unsigned IOCTL_SOUND_MIXER_READ_LINE1 = SOUND_MIXER_READ_LINE1;
+unsigned IOCTL_SOUND_MIXER_READ_LINE2 = SOUND_MIXER_READ_LINE2;
+unsigned IOCTL_SOUND_MIXER_READ_LINE3 = SOUND_MIXER_READ_LINE3;
+unsigned IOCTL_SOUND_MIXER_READ_LOUD = SOUND_MIXER_READ_LOUD;
+unsigned IOCTL_SOUND_MIXER_READ_MIC = SOUND_MIXER_READ_MIC;
+unsigned IOCTL_SOUND_MIXER_READ_MUTE = SOUND_MIXER_READ_MUTE;
+unsigned IOCTL_SOUND_MIXER_READ_OGAIN = SOUND_MIXER_READ_OGAIN;
+unsigned IOCTL_SOUND_MIXER_READ_PCM = SOUND_MIXER_READ_PCM;
+unsigned IOCTL_SOUND_MIXER_READ_RECLEV = SOUND_MIXER_READ_RECLEV;
+unsigned IOCTL_SOUND_MIXER_READ_RECMASK = SOUND_MIXER_READ_RECMASK;
+unsigned IOCTL_SOUND_MIXER_READ_RECSRC = SOUND_MIXER_READ_RECSRC;
+unsigned IOCTL_SOUND_MIXER_READ_SPEAKER = SOUND_MIXER_READ_SPEAKER;
+unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS = SOUND_MIXER_READ_STEREODEVS;
+unsigned IOCTL_SOUND_MIXER_READ_SYNTH = SOUND_MIXER_READ_SYNTH;
+unsigned IOCTL_SOUND_MIXER_READ_TREBLE = SOUND_MIXER_READ_TREBLE;
+unsigned IOCTL_SOUND_MIXER_READ_VOLUME = SOUND_MIXER_READ_VOLUME;
+unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM = SOUND_MIXER_WRITE_ALTPCM;
+unsigned IOCTL_SOUND_MIXER_WRITE_BASS = SOUND_MIXER_WRITE_BASS;
+unsigned IOCTL_SOUND_MIXER_WRITE_CD = SOUND_MIXER_WRITE_CD;
+unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE = SOUND_MIXER_WRITE_ENHANCE;
+unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN = SOUND_MIXER_WRITE_IGAIN;
+unsigned IOCTL_SOUND_MIXER_WRITE_IMIX = SOUND_MIXER_WRITE_IMIX;
+unsigned IOCTL_SOUND_MIXER_WRITE_LINE = SOUND_MIXER_WRITE_LINE;
+unsigned IOCTL_SOUND_MIXER_WRITE_LINE1 = SOUND_MIXER_WRITE_LINE1;
+unsigned IOCTL_SOUND_MIXER_WRITE_LINE2 = SOUND_MIXER_WRITE_LINE2;
+unsigned IOCTL_SOUND_MIXER_WRITE_LINE3 = SOUND_MIXER_WRITE_LINE3;
+unsigned IOCTL_SOUND_MIXER_WRITE_LOUD = SOUND_MIXER_WRITE_LOUD;
+unsigned IOCTL_SOUND_MIXER_WRITE_MIC = SOUND_MIXER_WRITE_MIC;
+unsigned IOCTL_SOUND_MIXER_WRITE_MUTE = SOUND_MIXER_WRITE_MUTE;
+unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN = SOUND_MIXER_WRITE_OGAIN;
+unsigned IOCTL_SOUND_MIXER_WRITE_PCM = SOUND_MIXER_WRITE_PCM;
+unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV = SOUND_MIXER_WRITE_RECLEV;
+unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC = SOUND_MIXER_WRITE_RECSRC;
+unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER = SOUND_MIXER_WRITE_SPEAKER;
+unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH = SOUND_MIXER_WRITE_SYNTH;
+unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE = SOUND_MIXER_WRITE_TREBLE;
+unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME = SOUND_MIXER_WRITE_VOLUME;
+unsigned IOCTL_VT_ACTIVATE = VT_ACTIVATE;
+unsigned IOCTL_VT_GETMODE = VT_GETMODE;
+unsigned IOCTL_VT_OPENQRY = VT_OPENQRY;
+unsigned IOCTL_VT_RELDISP = VT_RELDISP;
+unsigned IOCTL_VT_SETMODE = VT_SETMODE;
+unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE;
+unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP;
+unsigned IOCTL_KDDISABIO = KDDISABIO;
+unsigned IOCTL_KDENABIO = KDENABIO;
+unsigned IOCTL_KDGETLED = KDGETLED;
+unsigned IOCTL_KDGETMODE = KDGETMODE;
+unsigned IOCTL_KDGKBMODE = KDGKBMODE;
+unsigned IOCTL_KDGKBTYPE = KDGKBTYPE;
+unsigned IOCTL_KDMKTONE = KDMKTONE;
+unsigned IOCTL_KDSETLED = KDSETLED;
+unsigned IOCTL_KDSETMODE = KDSETMODE;
+unsigned IOCTL_KDSKBMODE = KDSKBMODE;
+unsigned IOCTL_KIOCSOUND = KIOCSOUND;
+unsigned IOCTL_PIO_SCRNMAP = PIO_SCRNMAP;
+unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE;
+
+const int si_SEGV_MAPERR = SEGV_MAPERR;
+const int si_SEGV_ACCERR = SEGV_ACCERR;
+const int unvis_valid = UNVIS_VALID;
+const int unvis_validpush = UNVIS_VALIDPUSH;
+}  // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+#undef ifa_dstaddr
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+#if HAVE_RPC_XDR_H
+CHECK_TYPE_SIZE(XDR);
+CHECK_SIZE_AND_OFFSET(XDR, x_op);
+CHECK_SIZE_AND_OFFSET(XDR, x_ops);
+CHECK_SIZE_AND_OFFSET(XDR, x_public);
+CHECK_SIZE_AND_OFFSET(XDR, x_private);
+CHECK_SIZE_AND_OFFSET(XDR, x_base);
+CHECK_SIZE_AND_OFFSET(XDR, x_handy);
+COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE);
+COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE);
+COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
+#endif
+
+CHECK_TYPE_SIZE(sem_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t));
+#endif  // SANITIZER_FREEBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h
new file mode 100644
index 0000000000..5e0ca9c7d7
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h
@@ -0,0 +1,656 @@
+//===-- sanitizer_platform_limits_freebsd.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific FreeBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_FREEBSD_H
+#define SANITIZER_PLATFORM_LIMITS_FREEBSD_H
+
+#if SANITIZER_FREEBSD
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+#include "sanitizer_platform_limits_posix.h"
+
+// Get sys/_types.h, because that tells us whether 64-bit inodes are
+// used in struct dirent below.
+#include <sys/_types.h>
+
+namespace __sanitizer {
+void *__sanitizer_get_link_map_by_dlopen_handle(void *handle);
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+  (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
+
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+#if defined(__powerpc64__)
+const unsigned struct___old_kernel_stat_sz = 0;
+#else
+const unsigned struct___old_kernel_stat_sz = 32;
+#endif
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned fpos_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_stack_t_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs64_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+extern unsigned struct_regmatch_sz;
+extern unsigned struct_regex_sz;
+extern unsigned struct_FTS_sz;
+extern unsigned struct_FTSENT_sz;
+extern const int unvis_valid;
+extern const int unvis_validpush;
+
+struct __sanitizer_iocb {
+  u64 aio_data;
+  u32 aio_key_or_aio_reserved1;  // Simply crazy.
+  u32 aio_reserved1_or_aio_key;  // Luckily, we don't need these.
+  u16 aio_lio_opcode;
+  s16 aio_reqprio;
+  u32 aio_fildes;
+  u64 aio_buf;
+  u64 aio_nbytes;
+  s64 aio_offset;
+  u64 aio_reserved2;
+  u64 aio_reserved3;
+};
+
+struct __sanitizer_io_event {
+  u64 data;
+  u64 obj;
+  u64 res;
+  u64 res2;
+};
+
+const unsigned iocb_cmd_pread = 0;
+const unsigned iocb_cmd_pwrite = 1;
+const unsigned iocb_cmd_preadv = 7;
+const unsigned iocb_cmd_pwritev = 8;
+
+struct __sanitizer___sysctl_args {
+  int *name;
+  int nlen;
+  void *oldval;
+  uptr *oldlenp;
+  void *newval;
+  uptr newlen;
+  unsigned long ___unused[4];
+};
+
+struct __sanitizer_ipc_perm {
+  unsigned int cuid;
+  unsigned int cgid;
+  unsigned int uid;
+  unsigned int gid;
+  unsigned short mode;
+  unsigned short seq;
+  long key;
+};
+
+#if !defined(__i386__)
+typedef long long __sanitizer_time_t;
+#else
+typedef long __sanitizer_time_t;
+#endif
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+  unsigned long shm_segsz;
+  unsigned int shm_lpid;
+  unsigned int shm_cpid;
+  int shm_nattch;
+  __sanitizer_time_t shm_atime;
+  __sanitizer_time_t shm_dtime;
+  __sanitizer_time_t shm_ctime;
+};
+
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timeb_sz;
+extern unsigned struct_statvfs_sz;
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  unsigned int ifa_flags;
+  void *ifa_addr;     // (struct sockaddr *)
+  void *ifa_netmask;  // (struct sockaddr *)
+#undef ifa_dstaddr
+  void *ifa_dstaddr;  // (struct sockaddr *)
+  void *ifa_data;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  int pw_uid;
+  int pw_gid;
+  __sanitizer_time_t pw_change;
+  char *pw_class;
+  char *pw_gecos;
+  char *pw_dir;
+  char *pw_shell;
+  __sanitizer_time_t pw_expire;
+  int pw_fields;
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+typedef long __sanitizer_suseconds_t;
+
+struct __sanitizer_timeval {
+  __sanitizer_time_t tv_sec;
+  __sanitizer_suseconds_t tv_usec;
+};
+
+struct __sanitizer_itimerval {
+  struct __sanitizer_timeval it_interval;
+  struct __sanitizer_timeval it_value;
+};
+
+struct __sanitizer_timeb {
+  __sanitizer_time_t time;
+  unsigned short millitm;
+  short timezone;
+  short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+  long int tm_gmtoff;
+  const char *tm_zone;
+};
+
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+
+struct __sanitizer_dirent {
+#if defined(__INO64)
+  unsigned long long d_fileno;
+  unsigned long long d_off;
+#else
+  unsigned int d_fileno;
+#endif
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+
+// 'clock_t' is 32 bits wide on x64 FreeBSD
+typedef int __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
+    defined(__mips__)
+typedef unsigned __sanitizer___kernel_uid_t;
+typedef unsigned __sanitizer___kernel_gid_t;
+#else
+typedef unsigned short __sanitizer___kernel_uid_t;
+typedef unsigned short __sanitizer___kernel_gid_t;
+#endif
+typedef long long __sanitizer___kernel_off_t;
+
+#if defined(__powerpc__) || defined(__mips__)
+typedef unsigned int __sanitizer___kernel_old_uid_t;
+typedef unsigned int __sanitizer___kernel_old_gid_t;
+#else
+typedef unsigned short __sanitizer___kernel_old_uid_t;
+typedef unsigned short __sanitizer___kernel_old_gid_t;
+#endif
+
+typedef long long __sanitizer___kernel_loff_t;
+typedef struct {
+  unsigned long fds_bits[1024 / (8 * sizeof(long))];
+} __sanitizer___kernel_fd_set;
+
+// This thing depends on the platform. We are only interested in the upper
+// limit. Verified with a compiler assert in .cpp.
+union __sanitizer_pthread_attr_t {
+  char size[128];
+  void *align;
+};
+
+const unsigned old_sigset_t_sz = sizeof(unsigned long);
+
+struct __sanitizer_sigset_t {
+  // uint32_t * 4
+  unsigned int __bits[4];
+};
+
+typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
+
+struct __sanitizer_siginfo {
+  // The size is determined by looking at sizeof of real siginfo_t on linux.
+  u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
+                                                  __sanitizer_siginfo *siginfo,
+                                                  void *uctx);
+
+struct __sanitizer_sigaction {
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+  int sa_flags;
+  __sanitizer_sigset_t sa_mask;
+};
+
+struct __sanitizer_sem_t {
+  u32 data[4];
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+  uptr dlpi_addr;
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+  short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+  unsigned ai_addrlen;
+  char *ai_canonname;
+  void *ai_addr;
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+typedef unsigned __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+  uptr gl_pathc;
+  uptr gl_matchc;
+  uptr gl_offs;
+  int gl_flags;
+  char **gl_pathv;
+  int (*gl_errfunc)(const char *, int);
+  void (*gl_closedir)(void *dirp);
+  struct dirent *(*gl_readdir)(void *dirp);
+  void *(*gl_opendir)(const char *);
+  int (*gl_lstat)(const char *, void * /* struct stat* */);
+  int (*gl_stat)(const char *, void * /* struct stat* */);
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+  uptr we_wordc;
+  char **we_wordv;
+  uptr we_offs;
+  char *we_strings;
+  uptr we_nbytes;
+};
+
+typedef void __sanitizer_FILE;
+
+extern unsigned struct_shminfo_sz;
+extern unsigned struct_shm_info_sz;
+extern int shmctl_ipc_stat;
+extern int shmctl_ipc_info;
+extern int shmctl_shm_info;
+extern int shmctl_shm_stat;
+
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+};
+
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__)
+#define IOC_SIZEBITS 13
+#define IOC_DIRBITS 3
+#define IOC_NONE 1U
+#define IOC_WRITE 4U
+#define IOC_READ 2U
+#else
+#define IOC_SIZEBITS 14
+#define IOC_DIRBITS 2
+#define IOC_NONE 0U
+#define IOC_WRITE 1U
+#define IOC_READ 2U
+#endif
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#if defined(IOC_DIRMASK)
+#undef IOC_DIRMASK
+#endif
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+#define EVIOC_EV_MAX 0x1f
+#define EVIOC_ABS_MAX 0x3f
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+extern unsigned struct_copr_buffer_sz;
+extern unsigned struct_copr_debug_buf_sz;
+extern unsigned struct_copr_msg_sz;
+extern unsigned struct_midi_info_sz;
+extern unsigned struct_mtget_sz;
+extern unsigned struct_mtop_sz;
+extern unsigned struct_rtentry_sz;
+extern unsigned struct_sbi_instrument_sz;
+extern unsigned struct_seq_event_rec_sz;
+extern unsigned struct_synth_info_sz;
+extern unsigned struct_vt_mode_sz;
+
+extern const unsigned long __sanitizer_bufsiz;
+extern unsigned struct_audio_buf_info_sz;
+extern unsigned struct_ppp_stats_sz;
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCCONS;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_SIOCGETSGCNT;
+extern unsigned IOCTL_SIOCGETVIFCNT;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+extern unsigned IOCTL_SIOCADDRT;
+extern unsigned IOCTL_SIOCDELRT;
+extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE;
+extern unsigned IOCTL_SNDCTL_DSP_GETFMTS;
+extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK;
+extern unsigned IOCTL_SNDCTL_DSP_POST;
+extern unsigned IOCTL_SNDCTL_DSP_RESET;
+extern unsigned IOCTL_SNDCTL_DSP_SETFMT;
+extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT;
+extern unsigned IOCTL_SNDCTL_DSP_SPEED;
+extern unsigned IOCTL_SNDCTL_DSP_STEREO;
+extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE;
+extern unsigned IOCTL_SNDCTL_DSP_SYNC;
+extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE;
+extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR;
+extern unsigned IOCTL_SNDCTL_MIDI_INFO;
+extern unsigned IOCTL_SNDCTL_MIDI_PRETIME;
+extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE;
+extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS;
+extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS;
+extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND;
+extern unsigned IOCTL_SNDCTL_SEQ_PANIC;
+extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE;
+extern unsigned IOCTL_SNDCTL_SEQ_RESET;
+extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES;
+extern unsigned IOCTL_SNDCTL_SEQ_SYNC;
+extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI;
+extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD;
+extern unsigned IOCTL_SNDCTL_SYNTH_INFO;
+extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL;
+extern unsigned IOCTL_SNDCTL_TMR_CONTINUE;
+extern unsigned IOCTL_SNDCTL_TMR_METRONOME;
+extern unsigned IOCTL_SNDCTL_TMR_SELECT;
+extern unsigned IOCTL_SNDCTL_TMR_SOURCE;
+extern unsigned IOCTL_SNDCTL_TMR_START;
+extern unsigned IOCTL_SNDCTL_TMR_STOP;
+extern unsigned IOCTL_SNDCTL_TMR_TEMPO;
+extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE;
+extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_BASS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CAPS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CD;
+extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE;
+extern unsigned IOCTL_SOUND_MIXER_READ_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_READ_MIC;
+extern unsigned IOCTL_SOUND_MIXER_READ_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_PCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS;
+extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_CD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME;
+extern unsigned IOCTL_SOUND_PCM_READ_BITS;
+extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_READ_FILTER;
+extern unsigned IOCTL_SOUND_PCM_READ_RATE;
+extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER;
+extern unsigned IOCTL_VT_ACTIVATE;
+extern unsigned IOCTL_VT_GETMODE;
+extern unsigned IOCTL_VT_OPENQRY;
+extern unsigned IOCTL_VT_RELDISP;
+extern unsigned IOCTL_VT_SETMODE;
+extern unsigned IOCTL_VT_WAITACTIVE;
+extern unsigned IOCTL_GIO_SCRNMAP;
+extern unsigned IOCTL_KDDISABIO;
+extern unsigned IOCTL_KDENABIO;
+extern unsigned IOCTL_KDGETLED;
+extern unsigned IOCTL_KDGETMODE;
+extern unsigned IOCTL_KDGKBMODE;
+extern unsigned IOCTL_KDGKBTYPE;
+extern unsigned IOCTL_KDMKTONE;
+extern unsigned IOCTL_KDSETLED;
+extern unsigned IOCTL_KDSETMODE;
+extern unsigned IOCTL_KDSKBMODE;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+
+struct __sanitizer_cap_rights {
+  u64 cr_rights[2];
+};
+
+typedef struct __sanitizer_cap_rights __sanitizer_cap_rights_t;
+extern unsigned struct_cap_rights_sz;
+
+extern unsigned struct_fstab_sz;
+extern unsigned struct_StringList_sz;
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(struct CLASS, MEMBER))
+
+#define SIGACTION_SYMNAME sigaction
+
+#endif
+
+#endif  // SANITIZER_FREEBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp
new file mode 100644
index 0000000000..c51327e126
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp
@@ -0,0 +1,108 @@
+//===-- sanitizer_platform_limits_linux.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of linux kernel data structures.
+//===----------------------------------------------------------------------===//
+
+// This is a separate compilation unit for linux headers that conflict with
+// userspace headers.
+// Most "normal" includes go in sanitizer_platform_limits_posix.cpp
+
+#include "sanitizer_platform.h"
+#if SANITIZER_LINUX
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_posix.h"
+
+// For offsetof -> __builtin_offsetof definition.
+#include <stddef.h>
+
+// With old kernels (and even new kernels on powerpc) asm/stat.h uses types that
+// are not defined anywhere in userspace headers. Fake them. This seems to work
+// fine with newer headers, too.
+#include <linux/posix_types.h>
+#if defined(__x86_64__) ||  defined(__mips__)
+#include <sys/stat.h>
+#else
+#define ino_t __kernel_ino_t
+#define mode_t __kernel_mode_t
+#define nlink_t __kernel_nlink_t
+#define uid_t __kernel_uid_t
+#define gid_t __kernel_gid_t
+#define off_t __kernel_off_t
+#define time_t __kernel_time_t
+// This header seems to contain the definitions of _kernel_ stat* structs.
+#include <asm/stat.h>
+#undef ino_t
+#undef mode_t
+#undef nlink_t
+#undef uid_t
+#undef gid_t
+#undef off_t
+#endif
+
+#include <linux/aio_abi.h>
+
+#if !SANITIZER_ANDROID
+#include <sys/statfs.h>
+#include <linux/perf_event.h>
+#endif
+
+using namespace __sanitizer;
+
+namespace __sanitizer {
+#if !SANITIZER_ANDROID
+  unsigned struct_statfs64_sz = sizeof(struct statfs64);
+#endif
+}  // namespace __sanitizer
+
+#if !defined(__powerpc64__) && !defined(__x86_64__) && !defined(__aarch64__)\
+                            && !defined(__mips__) && !defined(__s390__)\
+                            && !defined(__sparc__) && !defined(__riscv)
+COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));
+#endif
+
+COMPILER_CHECK(struct_kernel_stat_sz == sizeof(struct stat));
+
+#if defined(__i386__)
+COMPILER_CHECK(struct_kernel_stat64_sz == sizeof(struct stat64));
+#endif
+
+CHECK_TYPE_SIZE(io_event);
+CHECK_SIZE_AND_OFFSET(io_event, data);
+CHECK_SIZE_AND_OFFSET(io_event, obj);
+CHECK_SIZE_AND_OFFSET(io_event, res);
+CHECK_SIZE_AND_OFFSET(io_event, res2);
+
+#if !SANITIZER_ANDROID
+COMPILER_CHECK(sizeof(struct __sanitizer_perf_event_attr) <=
+               sizeof(struct perf_event_attr));
+CHECK_SIZE_AND_OFFSET(perf_event_attr, type);
+CHECK_SIZE_AND_OFFSET(perf_event_attr, size);
+#endif
+
+COMPILER_CHECK(iocb_cmd_pread == IOCB_CMD_PREAD);
+COMPILER_CHECK(iocb_cmd_pwrite == IOCB_CMD_PWRITE);
+#if !SANITIZER_ANDROID
+COMPILER_CHECK(iocb_cmd_preadv == IOCB_CMD_PREADV);
+COMPILER_CHECK(iocb_cmd_pwritev == IOCB_CMD_PWRITEV);
+#endif
+
+CHECK_TYPE_SIZE(iocb);
+CHECK_SIZE_AND_OFFSET(iocb, aio_data);
+// Skip aio_key, it's weird.
+CHECK_SIZE_AND_OFFSET(iocb, aio_lio_opcode);
+CHECK_SIZE_AND_OFFSET(iocb, aio_reqprio);
+CHECK_SIZE_AND_OFFSET(iocb, aio_fildes);
+CHECK_SIZE_AND_OFFSET(iocb, aio_buf);
+CHECK_SIZE_AND_OFFSET(iocb, aio_nbytes);
+CHECK_SIZE_AND_OFFSET(iocb, aio_offset);
+
+#endif  // SANITIZER_LINUX
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
new file mode 100644
index 0000000000..25da334b63
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
@@ -0,0 +1,2586 @@
+//===-- sanitizer_platform_limits_netbsd.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_NETBSD
+
+#define _KMEMUSER
+#define RAY_DO_SIGLEV
+#define __LEGACY_PT_LWPINFO
+
+// clang-format off
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/disk.h>
+#include <sys/disklabel.h>
+#include <sys/mount.h>
+#include <sys/agpio.h>
+#include <sys/ataio.h>
+#include <sys/audioio.h>
+#include <sys/cdbr.h>
+#include <sys/cdio.h>
+#include <sys/chio.h>
+#include <sys/clockctl.h>
+#include <sys/cpuio.h>
+#include <sys/dkio.h>
+#include <sys/drvctlio.h>
+#include <sys/dvdio.h>
+#include <sys/envsys.h>
+#include <sys/event.h>
+#include <sys/fdio.h>
+#include <sys/filio.h>
+#include <sys/gpio.h>
+#include <sys/ioctl.h>
+#include <sys/ioctl_compat.h>
+#include <sys/joystick.h>
+#include <sys/ksyms.h>
+#include <sys/lua.h>
+#include <sys/midiio.h>
+#include <sys/mtio.h>
+#include <sys/power.h>
+#include <sys/radioio.h>
+#include <sys/rndio.h>
+#include <sys/scanio.h>
+#include <sys/scsiio.h>
+#include <sys/sockio.h>
+#include <sys/timepps.h>
+#include <sys/ttycom.h>
+#include <sys/verified_exec.h>
+#include <sys/videoio.h>
+#include <sys/wdog.h>
+#include <sys/event.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/ipmi.h>
+#include <sys/kcov.h>
+#include <sys/mman.h>
+#include <sys/module.h>
+#include <sys/mount.h>
+#include <sys/mqueue.h>
+#include <sys/msg.h>
+#include <sys/mtio.h>
+#include <sys/ptrace.h>
+
+// Compat for NetBSD < 9.99.30.
+#ifndef PT_LWPSTATUS
+#define PT_LWPSTATUS 24
+#endif
+#ifndef PT_LWPNEXT
+#define PT_LWPNEXT 25
+#endif
+
+#include <sys/resource.h>
+#include <sys/sem.h>
+#include <sys/sha1.h>
+#include <sys/sha2.h>
+#include <sys/shm.h>
+#include <sys/signal.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/soundcard.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/timespec.h>
+#include <sys/timex.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <sys/utsname.h>
+#include <altq/altq.h>
+#include <altq/altq_afmap.h>
+#include <altq/altq_blue.h>
+#include <altq/altq_cbq.h>
+#include <altq/altq_cdnr.h>
+#include <altq/altq_fifoq.h>
+#include <altq/altq_hfsc.h>
+#include <altq/altq_jobs.h>
+#include <altq/altq_priq.h>
+#include <altq/altq_red.h>
+#include <altq/altq_rio.h>
+#include <altq/altq_wfq.h>
+#include <arpa/inet.h>
+#include <crypto/cryptodev.h>
+#include <dev/apm/apmio.h>
+#include <dev/dm/netbsd-dm.h>
+#include <dev/dmover/dmover_io.h>
+#include <dev/dtv/dtvio_demux.h>
+#include <dev/dtv/dtvio_frontend.h>
+#if !__NetBSD_Prereq__(9, 99, 26)
+#include <dev/filemon/filemon.h>
+#else
+#define FILEMON_SET_FD          _IOWR('S', 1, int)
+#define FILEMON_SET_PID         _IOWR('S', 2, pid_t)
+#endif
+#include <dev/hdaudio/hdaudioio.h>
+#include <dev/hdmicec/hdmicecio.h>
+#include <dev/hpc/hpcfbio.h>
+#include <dev/i2o/iopio.h>
+#include <dev/ic/athioctl.h>
+#include <dev/ic/bt8xx.h>
+#include <dev/ic/icp_ioctl.h>
+#include <dev/ic/isp_ioctl.h>
+#include <dev/ic/mlxio.h>
+#include <dev/ic/qemufwcfgio.h>
+#include <dev/ic/nvmeio.h>
+#include <dev/ir/irdaio.h>
+#include <dev/isa/isvio.h>
+#include <dev/isa/wtreg.h>
+#include <dev/iscsi/iscsi_ioctl.h>
+#include <dev/ofw/openfirmio.h>
+#include <dev/pci/amrio.h>
+#include <dev/pci/mlyreg.h>
+#include <dev/pci/mlyio.h>
+#include <dev/pci/pciio.h>
+#include <dev/pci/tweio.h>
+#include <dev/pcmcia/if_cnwioctl.h>
+#include <net/bpf.h>
+#include <net/if_gre.h>
+#include <net/ppp_defs.h>
+#include <net/if_ppp.h>
+#include <net/if_pppoe.h>
+#include <net/if_sppp.h>
+#include <net/if_srt.h>
+#include <net/if_tap.h>
+#include <net/if_tun.h>
+#include <net/npf.h>
+#include <net/pfvar.h>
+#include <net/slip.h>
+#include <netbt/hci.h>
+#include <netinet/ip_compat.h>
+#if __has_include(<netinet/ip_fil.h>)
+#include <netinet/ip_fil.h>
+#include <netinet/ip_nat.h>
+#include <netinet/ip_proxy.h>
+#else
+/* Fallback for MKIPFILTER=no */
+
+typedef struct ap_control {
+  char apc_label[16];
+  char apc_config[16];
+  unsigned char apc_p;
+  unsigned long apc_cmd;
+  unsigned long apc_arg;
+  void *apc_data;
+  size_t apc_dsize;
+} ap_ctl_t;
+
+typedef struct ipftq {
+  ipfmutex_t ifq_lock;
+  unsigned int ifq_ttl;
+  void *ifq_head;
+  void **ifq_tail;
+  void *ifq_next;
+  void **ifq_pnext;
+  int ifq_ref;
+  unsigned int ifq_flags;
+} ipftq_t;
+
+typedef struct ipfobj {
+  uint32_t ipfo_rev;
+  uint32_t ipfo_size;
+  void *ipfo_ptr;
+  int ipfo_type;
+  int ipfo_offset;
+  int ipfo_retval;
+  unsigned char ipfo_xxxpad[28];
+} ipfobj_t;
+
+#define SIOCADNAT _IOW('r', 60, struct ipfobj)
+#define SIOCRMNAT _IOW('r', 61, struct ipfobj)
+#define SIOCGNATS _IOWR('r', 62, struct ipfobj)
+#define SIOCGNATL _IOWR('r', 63, struct ipfobj)
+#define SIOCPURGENAT _IOWR('r', 100, struct ipfobj)
+#endif
+#include <netinet6/in6_var.h>
+#include <netinet6/nd6.h>
+#if !__NetBSD_Prereq__(9, 99, 51)
+#include <netsmb/smb_dev.h>
+#else
+struct smbioc_flags {
+  int ioc_level;
+  int ioc_mask;
+  int ioc_flags;
+};
+struct smbioc_oshare {
+  int ioc_opt;
+  int ioc_stype;
+  char ioc_share[129];
+  char ioc_password[129];
+  uid_t ioc_owner;
+  gid_t ioc_group;
+  mode_t ioc_mode;
+  mode_t ioc_rights;
+};
+struct smbioc_ossn {
+  int ioc_opt;
+  uint32_t ioc_svlen;
+  struct sockaddr *ioc_server;
+  uint32_t ioc_lolen;
+  struct sockaddr *ioc_local;
+  char ioc_srvname[16];
+  int ioc_timeout;
+  int ioc_retrycount;
+  char ioc_localcs[16];
+  char ioc_servercs[16];
+  char ioc_user[129];
+  char ioc_workgroup[129];
+  char ioc_password[129];
+  uid_t ioc_owner;
+  gid_t ioc_group;
+  mode_t ioc_mode;
+  mode_t ioc_rights;
+};
+struct smbioc_lookup {
+  int ioc_level;
+  int ioc_flags;
+  struct smbioc_ossn ioc_ssn;
+  struct smbioc_oshare ioc_sh;
+};
+struct smbioc_rq {
+  u_char ioc_cmd;
+  u_char ioc_twc;
+  void *ioc_twords;
+  u_short ioc_tbc;
+  void *ioc_tbytes;
+  int ioc_rpbufsz;
+  char *ioc_rpbuf;
+  u_char ioc_rwc;
+  u_short ioc_rbc;
+};
+struct smbioc_rw {
+  u_int16_t ioc_fh;
+  char *ioc_base;
+  off_t ioc_offset;
+  int ioc_cnt;
+};
+#define SMBIOC_OPENSESSION _IOW('n', 100, struct smbioc_ossn)
+#define SMBIOC_OPENSHARE _IOW('n', 101, struct smbioc_oshare)
+#define SMBIOC_REQUEST _IOWR('n', 102, struct smbioc_rq)
+#define SMBIOC_T2RQ _IOWR('n', 103, struct smbioc_t2rq)
+#define SMBIOC_SETFLAGS _IOW('n', 104, struct smbioc_flags)
+#define SMBIOC_LOOKUP _IOW('n', 106, struct smbioc_lookup)
+#define SMBIOC_READ _IOWR('n', 107, struct smbioc_rw)
+#define SMBIOC_WRITE _IOWR('n', 108, struct smbioc_rw)
+#endif
+#include <dev/biovar.h>
+#include <dev/bluetooth/btdev.h>
+#include <dev/bluetooth/btsco.h>
+#include <dev/ccdvar.h>
+#include <dev/cgdvar.h>
+#include <dev/fssvar.h>
+#include <dev/kttcpio.h>
+#include <dev/lockstat.h>
+#include <dev/md.h>
+#include <net/if_ether.h>
+#include <dev/pcmcia/if_rayreg.h>
+#include <stdio.h>
+#include <dev/raidframe/raidframeio.h>
+#include <dev/sbus/mbppio.h>
+#include <dev/scsipi/ses.h>
+#include <dev/spi/spi_io.h>
+#include <dev/spkrio.h>
+#include <dev/sun/disklabel.h>
+#include <dev/sun/fbio.h>
+#include <dev/sun/kbio.h>
+#include <dev/sun/vuid_event.h>
+#include <dev/tc/sticio.h>
+#include <dev/usb/ukyopon.h>
+#if !__NetBSD_Prereq__(9, 99, 44)
+#include <dev/usb/urio.h>
+#else
+struct urio_command {
+  unsigned short length;
+  int request;
+  int requesttype;
+  int value;
+  int index;
+  void *buffer;
+  int timeout;
+};
+#define URIO_SEND_COMMAND      _IOWR('U', 200, struct urio_command)
+#define URIO_RECV_COMMAND      _IOWR('U', 201, struct urio_command)
+#endif
+#include <dev/usb/usb.h>
+#include <dev/usb/utoppy.h>
+#include <dev/vme/xio.h>
+#include <dev/vndvar.h>
+#include <dev/wscons/wsconsio.h>
+#include <dev/wscons/wsdisplay_usl_io.h>
+#include <fs/autofs/autofs_ioctl.h>
+#include <dirent.h>
+#include <dlfcn.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link_elf.h>
+#include <net/if.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip_mroute.h>
+#include <netinet/sctp_uio.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <md2.h>
+#include <md4.h>
+#include <md5.h>
+#include <rmd160.h>
+#include <soundcard.h>
+#include <term.h>
+#include <termios.h>
+#include <time.h>
+#include <ttyent.h>
+#include <utime.h>
+#include <utmp.h>
+#include <utmpx.h>
+#include <vis.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <ttyent.h>
+#include <fts.h>
+#include <regex.h>
+#include <fstab.h>
+#include <stringlist.h>
+
+#if defined(__x86_64__)
+#include <nvmm.h>
+#endif
+// clang-format on
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_platform_limits_netbsd.h"
+
+namespace __sanitizer {
+void *__sanitizer_get_link_map_by_dlopen_handle(void* handle) {
+  void *p = nullptr;
+  return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
+}
+
+unsigned struct_utsname_sz = sizeof(struct utsname);
+unsigned struct_stat_sz = sizeof(struct stat);
+unsigned struct_rusage_sz = sizeof(struct rusage);
+unsigned struct_tm_sz = sizeof(struct tm);
+unsigned struct_passwd_sz = sizeof(struct passwd);
+unsigned struct_group_sz = sizeof(struct group);
+unsigned siginfo_t_sz = sizeof(siginfo_t);
+unsigned struct_sigaction_sz = sizeof(struct sigaction);
+unsigned struct_stack_t_sz = sizeof(stack_t);
+unsigned struct_itimerval_sz = sizeof(struct itimerval);
+unsigned pthread_t_sz = sizeof(pthread_t);
+unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+unsigned pid_t_sz = sizeof(pid_t);
+unsigned timeval_sz = sizeof(timeval);
+unsigned uid_t_sz = sizeof(uid_t);
+unsigned gid_t_sz = sizeof(gid_t);
+unsigned mbstate_t_sz = sizeof(mbstate_t);
+unsigned sigset_t_sz = sizeof(sigset_t);
+unsigned struct_timezone_sz = sizeof(struct timezone);
+unsigned struct_tms_sz = sizeof(struct tms);
+unsigned struct_sigevent_sz = sizeof(struct sigevent);
+unsigned struct_sched_param_sz = sizeof(struct sched_param);
+unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned struct_rlimit_sz = sizeof(struct rlimit);
+unsigned struct_timespec_sz = sizeof(struct timespec);
+unsigned struct_sembuf_sz = sizeof(struct sembuf);
+unsigned struct_kevent_sz = sizeof(struct kevent);
+unsigned struct_FTS_sz = sizeof(FTS);
+unsigned struct_FTSENT_sz = sizeof(FTSENT);
+unsigned struct_regex_sz = sizeof(regex_t);
+unsigned struct_regmatch_sz = sizeof(regmatch_t);
+unsigned struct_fstab_sz = sizeof(struct fstab);
+unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+unsigned struct_timex_sz = sizeof(struct timex);
+unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
+unsigned struct_statvfs_sz = sizeof(struct statvfs);
+unsigned struct_sigaltstack_sz = sizeof(stack_t);
+
+const uptr sig_ign = (uptr)SIG_IGN;
+const uptr sig_dfl = (uptr)SIG_DFL;
+const uptr sig_err = (uptr)SIG_ERR;
+const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+const unsigned long __sanitizer_bufsiz = BUFSIZ;
+
+int ptrace_pt_io = PT_IO;
+int ptrace_pt_lwpinfo = PT_LWPINFO;
+int ptrace_pt_set_event_mask = PT_SET_EVENT_MASK;
+int ptrace_pt_get_event_mask = PT_GET_EVENT_MASK;
+int ptrace_pt_get_process_state = PT_GET_PROCESS_STATE;
+int ptrace_pt_set_siginfo = PT_SET_SIGINFO;
+int ptrace_pt_get_siginfo = PT_GET_SIGINFO;
+int ptrace_pt_lwpstatus = PT_LWPSTATUS;
+int ptrace_pt_lwpnext = PT_LWPNEXT;
+int ptrace_piod_read_d = PIOD_READ_D;
+int ptrace_piod_write_d = PIOD_WRITE_D;
+int ptrace_piod_read_i = PIOD_READ_I;
+int ptrace_piod_write_i = PIOD_WRITE_I;
+int ptrace_piod_read_auxv = PIOD_READ_AUXV;
+
+#if defined(PT_SETREGS) && defined(PT_GETREGS)
+int ptrace_pt_setregs = PT_SETREGS;
+int ptrace_pt_getregs = PT_GETREGS;
+#else
+int ptrace_pt_setregs = -1;
+int ptrace_pt_getregs = -1;
+#endif
+
+#if defined(PT_SETFPREGS) && defined(PT_GETFPREGS)
+int ptrace_pt_setfpregs = PT_SETFPREGS;
+int ptrace_pt_getfpregs = PT_GETFPREGS;
+#else
+int ptrace_pt_setfpregs = -1;
+int ptrace_pt_getfpregs = -1;
+#endif
+
+#if defined(PT_SETDBREGS) && defined(PT_GETDBREGS)
+int ptrace_pt_setdbregs = PT_SETDBREGS;
+int ptrace_pt_getdbregs = PT_GETDBREGS;
+#else
+int ptrace_pt_setdbregs = -1;
+int ptrace_pt_getdbregs = -1;
+#endif
+
+unsigned struct_ptrace_ptrace_io_desc_struct_sz = sizeof(struct ptrace_io_desc);
+unsigned struct_ptrace_ptrace_lwpinfo_struct_sz = sizeof(struct ptrace_lwpinfo);
+unsigned struct_ptrace_ptrace_lwpstatus_struct_sz =
+    sizeof(struct __sanitizer_ptrace_lwpstatus);
+unsigned struct_ptrace_ptrace_event_struct_sz = sizeof(ptrace_event_t);
+unsigned struct_ptrace_ptrace_siginfo_struct_sz = sizeof(ptrace_siginfo_t);
+
+#if defined(PT_SETREGS)
+unsigned struct_ptrace_reg_struct_sz = sizeof(struct reg);
+#else
+unsigned struct_ptrace_reg_struct_sz = -1;
+#endif
+
+#if defined(PT_SETFPREGS)
+unsigned struct_ptrace_fpreg_struct_sz = sizeof(struct fpreg);
+#else
+unsigned struct_ptrace_fpreg_struct_sz = -1;
+#endif
+
+#if defined(PT_SETDBREGS)
+unsigned struct_ptrace_dbreg_struct_sz = sizeof(struct dbreg);
+#else
+unsigned struct_ptrace_dbreg_struct_sz = -1;
+#endif
+
+int shmctl_ipc_stat = (int)IPC_STAT;
+
+unsigned struct_utmp_sz = sizeof(struct utmp);
+unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+int map_fixed = MAP_FIXED;
+
+int af_inet = (int)AF_INET;
+int af_inet6 = (int)AF_INET6;
+
+uptr __sanitizer_in_addr_sz(int af) {
+  if (af == AF_INET)
+    return sizeof(struct in_addr);
+  else if (af == AF_INET6)
+    return sizeof(struct in6_addr);
+  else
+    return 0;
+}
+
+unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
+
+int glob_nomatch = GLOB_NOMATCH;
+int glob_altdirfunc = GLOB_ALTDIRFUNC;
+
+unsigned path_max = PATH_MAX;
+
+int struct_ttyent_sz = sizeof(struct ttyent);
+
+struct __sanitizer_nvlist_ref_t {
+  void *buf;
+  uptr len;
+  int flags;
+};
+
+typedef __sanitizer_nvlist_ref_t nvlist_ref_t;
+
+// ioctl arguments
+unsigned struct_altqreq_sz = sizeof(altqreq);
+unsigned struct_amr_user_ioctl_sz = sizeof(amr_user_ioctl);
+unsigned struct_ap_control_sz = sizeof(ap_control);
+unsigned struct_apm_ctl_sz = sizeof(apm_ctl);
+unsigned struct_apm_event_info_sz = sizeof(apm_event_info);
+unsigned struct_apm_power_info_sz = sizeof(apm_power_info);
+unsigned struct_atabusiodetach_args_sz = sizeof(atabusiodetach_args);
+unsigned struct_atabusioscan_args_sz = sizeof(atabusioscan_args);
+unsigned struct_ath_diag_sz = sizeof(ath_diag);
+unsigned struct_atm_flowmap_sz = sizeof(atm_flowmap);
+unsigned struct_audio_buf_info_sz = sizeof(audio_buf_info);
+unsigned struct_audio_device_sz = sizeof(audio_device);
+unsigned struct_audio_encoding_sz = sizeof(audio_encoding);
+unsigned struct_audio_info_sz = sizeof(audio_info);
+unsigned struct_audio_offset_sz = sizeof(audio_offset);
+unsigned struct_bio_locate_sz = sizeof(bio_locate);
+unsigned struct_bioc_alarm_sz = sizeof(bioc_alarm);
+unsigned struct_bioc_blink_sz = sizeof(bioc_blink);
+unsigned struct_bioc_disk_sz = sizeof(bioc_disk);
+unsigned struct_bioc_inq_sz = sizeof(bioc_inq);
+unsigned struct_bioc_setstate_sz = sizeof(bioc_setstate);
+unsigned struct_bioc_vol_sz = sizeof(bioc_vol);
+unsigned struct_bioc_volops_sz = sizeof(bioc_volops);
+unsigned struct_bktr_chnlset_sz = sizeof(bktr_chnlset);
+unsigned struct_bktr_remote_sz = sizeof(bktr_remote);
+unsigned struct_blue_conf_sz = sizeof(blue_conf);
+unsigned struct_blue_interface_sz = sizeof(blue_interface);
+unsigned struct_blue_stats_sz = sizeof(blue_stats);
+unsigned struct_bpf_dltlist_sz = sizeof(bpf_dltlist);
+unsigned struct_bpf_program_sz = sizeof(bpf_program);
+unsigned struct_bpf_stat_old_sz = sizeof(bpf_stat_old);
+unsigned struct_bpf_stat_sz = sizeof(bpf_stat);
+unsigned struct_bpf_version_sz = sizeof(bpf_version);
+unsigned struct_btreq_sz = sizeof(btreq);
+unsigned struct_btsco_info_sz = sizeof(btsco_info);
+unsigned struct_buffmem_desc_sz = sizeof(buffmem_desc);
+unsigned struct_cbq_add_class_sz = sizeof(cbq_add_class);
+unsigned struct_cbq_add_filter_sz = sizeof(cbq_add_filter);
+unsigned struct_cbq_delete_class_sz = sizeof(cbq_delete_class);
+unsigned struct_cbq_delete_filter_sz = sizeof(cbq_delete_filter);
+unsigned struct_cbq_getstats_sz = sizeof(cbq_getstats);
+unsigned struct_cbq_interface_sz = sizeof(cbq_interface);
+unsigned struct_cbq_modify_class_sz = sizeof(cbq_modify_class);
+unsigned struct_ccd_ioctl_sz = sizeof(ccd_ioctl);
+unsigned struct_cdnr_add_element_sz = sizeof(cdnr_add_element);
+unsigned struct_cdnr_add_filter_sz = sizeof(cdnr_add_filter);
+unsigned struct_cdnr_add_tbmeter_sz = sizeof(cdnr_add_tbmeter);
+unsigned struct_cdnr_add_trtcm_sz = sizeof(cdnr_add_trtcm);
+unsigned struct_cdnr_add_tswtcm_sz = sizeof(cdnr_add_tswtcm);
+unsigned struct_cdnr_delete_element_sz = sizeof(cdnr_delete_element);
+unsigned struct_cdnr_delete_filter_sz = sizeof(cdnr_delete_filter);
+unsigned struct_cdnr_get_stats_sz = sizeof(cdnr_get_stats);
+unsigned struct_cdnr_interface_sz = sizeof(cdnr_interface);
+unsigned struct_cdnr_modify_tbmeter_sz = sizeof(cdnr_modify_tbmeter);
+unsigned struct_cdnr_modify_trtcm_sz = sizeof(cdnr_modify_trtcm);
+unsigned struct_cdnr_modify_tswtcm_sz = sizeof(cdnr_modify_tswtcm);
+unsigned struct_cdnr_tbmeter_stats_sz = sizeof(cdnr_tbmeter_stats);
+unsigned struct_cdnr_tcm_stats_sz = sizeof(cdnr_tcm_stats);
+unsigned struct_cgd_ioctl_sz = sizeof(cgd_ioctl);
+unsigned struct_cgd_user_sz = sizeof(cgd_user);
+unsigned struct_changer_element_status_request_sz =
+    sizeof(changer_element_status_request);
+unsigned struct_changer_exchange_request_sz = sizeof(changer_exchange_request);
+unsigned struct_changer_move_request_sz = sizeof(changer_move_request);
+unsigned struct_changer_params_sz = sizeof(changer_params);
+unsigned struct_changer_position_request_sz = sizeof(changer_position_request);
+unsigned struct_changer_set_voltag_request_sz =
+    sizeof(changer_set_voltag_request);
+unsigned struct_clockctl_adjtime_sz = sizeof(clockctl_adjtime);
+unsigned struct_clockctl_clock_settime_sz = sizeof(clockctl_clock_settime);
+unsigned struct_clockctl_ntp_adjtime_sz = sizeof(clockctl_ntp_adjtime);
+unsigned struct_clockctl_settimeofday_sz = sizeof(clockctl_settimeofday);
+unsigned struct_cnwistats_sz = sizeof(cnwistats);
+unsigned struct_cnwitrail_sz = sizeof(cnwitrail);
+unsigned struct_cnwstatus_sz = sizeof(cnwstatus);
+unsigned struct_count_info_sz = sizeof(count_info);
+unsigned struct_cpu_ucode_sz = sizeof(cpu_ucode);
+unsigned struct_cpu_ucode_version_sz = sizeof(cpu_ucode_version);
+unsigned struct_crypt_kop_sz = sizeof(crypt_kop);
+unsigned struct_crypt_mkop_sz = sizeof(crypt_mkop);
+unsigned struct_crypt_mop_sz = sizeof(crypt_mop);
+unsigned struct_crypt_op_sz = sizeof(crypt_op);
+unsigned struct_crypt_result_sz = sizeof(crypt_result);
+unsigned struct_crypt_sfop_sz = sizeof(crypt_sfop);
+unsigned struct_crypt_sgop_sz = sizeof(crypt_sgop);
+unsigned struct_cryptret_sz = sizeof(cryptret);
+unsigned struct_devdetachargs_sz = sizeof(devdetachargs);
+unsigned struct_devlistargs_sz = sizeof(devlistargs);
+unsigned struct_devpmargs_sz = sizeof(devpmargs);
+unsigned struct_devrescanargs_sz = sizeof(devrescanargs);
+unsigned struct_disk_badsecinfo_sz = sizeof(disk_badsecinfo);
+unsigned struct_disk_strategy_sz = sizeof(disk_strategy);
+unsigned struct_disklabel_sz = sizeof(disklabel);
+unsigned struct_dkbad_sz = sizeof(dkbad);
+unsigned struct_dkwedge_info_sz = sizeof(dkwedge_info);
+unsigned struct_dkwedge_list_sz = sizeof(dkwedge_list);
+unsigned struct_dmio_setfunc_sz = sizeof(dmio_setfunc);
+unsigned struct_dmx_pes_filter_params_sz = sizeof(dmx_pes_filter_params);
+unsigned struct_dmx_sct_filter_params_sz = sizeof(dmx_sct_filter_params);
+unsigned struct_dmx_stc_sz = sizeof(dmx_stc);
+unsigned struct_dvb_diseqc_master_cmd_sz = sizeof(dvb_diseqc_master_cmd);
+unsigned struct_dvb_diseqc_slave_reply_sz = sizeof(dvb_diseqc_slave_reply);
+unsigned struct_dvb_frontend_event_sz = sizeof(dvb_frontend_event);
+unsigned struct_dvb_frontend_info_sz = sizeof(dvb_frontend_info);
+unsigned struct_dvb_frontend_parameters_sz = sizeof(dvb_frontend_parameters);
+unsigned struct_eccapreq_sz = sizeof(eccapreq);
+unsigned struct_fbcmap_sz = sizeof(fbcmap);
+unsigned struct_fbcurpos_sz = sizeof(fbcurpos);
+unsigned struct_fbcursor_sz = sizeof(fbcursor);
+unsigned struct_fbgattr_sz = sizeof(fbgattr);
+unsigned struct_fbsattr_sz = sizeof(fbsattr);
+unsigned struct_fbtype_sz = sizeof(fbtype);
+unsigned struct_fdformat_cmd_sz = sizeof(fdformat_cmd);
+unsigned struct_fdformat_parms_sz = sizeof(fdformat_parms);
+unsigned struct_fifoq_conf_sz = sizeof(fifoq_conf);
+unsigned struct_fifoq_getstats_sz = sizeof(fifoq_getstats);
+unsigned struct_fifoq_interface_sz = sizeof(fifoq_interface);
+unsigned struct_format_op_sz = sizeof(format_op);
+unsigned struct_fss_get_sz = sizeof(fss_get);
+unsigned struct_fss_set_sz = sizeof(fss_set);
+unsigned struct_gpio_attach_sz = sizeof(gpio_attach);
+unsigned struct_gpio_info_sz = sizeof(gpio_info);
+unsigned struct_gpio_req_sz = sizeof(gpio_req);
+unsigned struct_gpio_set_sz = sizeof(gpio_set);
+unsigned struct_hfsc_add_class_sz = sizeof(hfsc_add_class);
+unsigned struct_hfsc_add_filter_sz = sizeof(hfsc_add_filter);
+unsigned struct_hfsc_attach_sz = sizeof(hfsc_attach);
+unsigned struct_hfsc_class_stats_sz = sizeof(hfsc_class_stats);
+unsigned struct_hfsc_delete_class_sz = sizeof(hfsc_delete_class);
+unsigned struct_hfsc_delete_filter_sz = sizeof(hfsc_delete_filter);
+unsigned struct_hfsc_interface_sz = sizeof(hfsc_interface);
+unsigned struct_hfsc_modify_class_sz = sizeof(hfsc_modify_class);
+unsigned struct_hpcfb_dsp_op_sz = sizeof(hpcfb_dsp_op);
+unsigned struct_hpcfb_dspconf_sz = sizeof(hpcfb_dspconf);
+unsigned struct_hpcfb_fbconf_sz = sizeof(hpcfb_fbconf);
+unsigned struct_if_addrprefreq_sz = sizeof(if_addrprefreq);
+unsigned struct_if_clonereq_sz = sizeof(if_clonereq);
+unsigned struct_if_laddrreq_sz = sizeof(if_laddrreq);
+unsigned struct_ifaddr_sz = sizeof(ifaddr);
+unsigned struct_ifaliasreq_sz = sizeof(ifaliasreq);
+unsigned struct_ifcapreq_sz = sizeof(ifcapreq);
+unsigned struct_ifconf_sz = sizeof(ifconf);
+unsigned struct_ifdatareq_sz = sizeof(ifdatareq);
+unsigned struct_ifdrv_sz = sizeof(ifdrv);
+unsigned struct_ifmediareq_sz = sizeof(ifmediareq);
+unsigned struct_ifpppcstatsreq_sz = sizeof(ifpppcstatsreq);
+unsigned struct_ifpppstatsreq_sz = sizeof(ifpppstatsreq);
+unsigned struct_ifreq_sz = sizeof(ifreq);
+unsigned struct_in6_addrpolicy_sz = sizeof(in6_addrpolicy);
+unsigned struct_in6_ndireq_sz = sizeof(in6_ndireq);
+unsigned struct_ioc_load_unload_sz = sizeof(ioc_load_unload);
+unsigned struct_ioc_patch_sz = sizeof(ioc_patch);
+unsigned struct_ioc_play_blocks_sz = sizeof(ioc_play_blocks);
+unsigned struct_ioc_play_msf_sz = sizeof(ioc_play_msf);
+unsigned struct_ioc_play_track_sz = sizeof(ioc_play_track);
+unsigned struct_ioc_read_subchannel_sz = sizeof(ioc_read_subchannel);
+unsigned struct_ioc_read_toc_entry_sz = sizeof(ioc_read_toc_entry);
+unsigned struct_ioc_toc_header_sz = sizeof(ioc_toc_header);
+unsigned struct_ioc_vol_sz = sizeof(ioc_vol);
+unsigned struct_ioctl_pt_sz = sizeof(ioctl_pt);
+unsigned struct_ioppt_sz = sizeof(ioppt);
+unsigned struct_iovec_sz = sizeof(iovec);
+unsigned struct_ipfobj_sz = sizeof(ipfobj);
+unsigned struct_irda_params_sz = sizeof(irda_params);
+unsigned struct_isp_fc_device_sz = sizeof(isp_fc_device);
+unsigned struct_isp_fc_tsk_mgmt_sz = sizeof(isp_fc_tsk_mgmt);
+unsigned struct_isp_hba_device_sz = sizeof(isp_hba_device);
+unsigned struct_isv_cmd_sz = sizeof(isv_cmd);
+unsigned struct_jobs_add_class_sz = sizeof(jobs_add_class);
+unsigned struct_jobs_add_filter_sz = sizeof(jobs_add_filter);
+unsigned struct_jobs_attach_sz = sizeof(jobs_attach);
+unsigned struct_jobs_class_stats_sz = sizeof(jobs_class_stats);
+unsigned struct_jobs_delete_class_sz = sizeof(jobs_delete_class);
+unsigned struct_jobs_delete_filter_sz = sizeof(jobs_delete_filter);
+unsigned struct_jobs_interface_sz = sizeof(jobs_interface);
+unsigned struct_jobs_modify_class_sz = sizeof(jobs_modify_class);
+unsigned struct_kbentry_sz = sizeof(kbentry);
+unsigned struct_kfilter_mapping_sz = sizeof(kfilter_mapping);
+unsigned struct_kiockeymap_sz = sizeof(kiockeymap);
+unsigned struct_ksyms_gsymbol_sz = sizeof(ksyms_gsymbol);
+unsigned struct_ksyms_gvalue_sz = sizeof(ksyms_gvalue);
+unsigned struct_ksyms_ogsymbol_sz = sizeof(ksyms_ogsymbol);
+unsigned struct_kttcp_io_args_sz = sizeof(kttcp_io_args);
+unsigned struct_ltchars_sz = sizeof(ltchars);
+unsigned struct_lua_create_sz = sizeof(struct lua_create);
+unsigned struct_lua_info_sz = sizeof(struct lua_info);
+unsigned struct_lua_load_sz = sizeof(struct lua_load);
+unsigned struct_lua_require_sz = sizeof(lua_require);
+unsigned struct_mbpp_param_sz = sizeof(mbpp_param);
+unsigned struct_md_conf_sz = sizeof(md_conf);
+unsigned struct_meteor_capframe_sz = sizeof(meteor_capframe);
+unsigned struct_meteor_counts_sz = sizeof(meteor_counts);
+unsigned struct_meteor_geomet_sz = sizeof(meteor_geomet);
+unsigned struct_meteor_pixfmt_sz = sizeof(meteor_pixfmt);
+unsigned struct_meteor_video_sz = sizeof(meteor_video);
+unsigned struct_mlx_cinfo_sz = sizeof(mlx_cinfo);
+unsigned struct_mlx_pause_sz = sizeof(mlx_pause);
+unsigned struct_mlx_rebuild_request_sz = sizeof(mlx_rebuild_request);
+unsigned struct_mlx_rebuild_status_sz = sizeof(mlx_rebuild_status);
+unsigned struct_mlx_usercommand_sz = sizeof(mlx_usercommand);
+unsigned struct_mly_user_command_sz = sizeof(mly_user_command);
+unsigned struct_mly_user_health_sz = sizeof(mly_user_health);
+unsigned struct_mtget_sz = sizeof(mtget);
+unsigned struct_mtop_sz = sizeof(mtop);
+unsigned struct_npf_ioctl_table_sz = sizeof(npf_ioctl_table);
+unsigned struct_npioctl_sz = sizeof(npioctl);
+unsigned struct_nvme_pt_command_sz = sizeof(nvme_pt_command);
+unsigned struct_ochanger_element_status_request_sz =
+    sizeof(ochanger_element_status_request);
+unsigned struct_ofiocdesc_sz = sizeof(ofiocdesc);
+unsigned struct_okiockey_sz = sizeof(okiockey);
+unsigned struct_ortentry_sz = sizeof(ortentry);
+unsigned struct_oscsi_addr_sz = sizeof(oscsi_addr);
+unsigned struct_oss_audioinfo_sz = sizeof(oss_audioinfo);
+unsigned struct_oss_sysinfo_sz = sizeof(oss_sysinfo);
+unsigned struct_pciio_bdf_cfgreg_sz = sizeof(pciio_bdf_cfgreg);
+unsigned struct_pciio_businfo_sz = sizeof(pciio_businfo);
+unsigned struct_pciio_cfgreg_sz = sizeof(pciio_cfgreg);
+unsigned struct_pciio_drvname_sz = sizeof(pciio_drvname);
+unsigned struct_pciio_drvnameonbus_sz = sizeof(pciio_drvnameonbus);
+unsigned struct_pcvtid_sz = sizeof(pcvtid);
+unsigned struct_pf_osfp_ioctl_sz = sizeof(pf_osfp_ioctl);
+unsigned struct_pf_status_sz = sizeof(pf_status);
+unsigned struct_pfioc_altq_sz = sizeof(pfioc_altq);
+unsigned struct_pfioc_if_sz = sizeof(pfioc_if);
+unsigned struct_pfioc_iface_sz = sizeof(pfioc_iface);
+unsigned struct_pfioc_limit_sz = sizeof(pfioc_limit);
+unsigned struct_pfioc_natlook_sz = sizeof(pfioc_natlook);
+unsigned struct_pfioc_pooladdr_sz = sizeof(pfioc_pooladdr);
+unsigned struct_pfioc_qstats_sz = sizeof(pfioc_qstats);
+unsigned struct_pfioc_rule_sz = sizeof(pfioc_rule);
+unsigned struct_pfioc_ruleset_sz = sizeof(pfioc_ruleset);
+unsigned struct_pfioc_src_node_kill_sz = sizeof(pfioc_src_node_kill);
+unsigned struct_pfioc_src_nodes_sz = sizeof(pfioc_src_nodes);
+unsigned struct_pfioc_state_kill_sz = sizeof(pfioc_state_kill);
+unsigned struct_pfioc_state_sz = sizeof(pfioc_state);
+unsigned struct_pfioc_states_sz = sizeof(pfioc_states);
+unsigned struct_pfioc_table_sz = sizeof(pfioc_table);
+unsigned struct_pfioc_tm_sz = sizeof(pfioc_tm);
+unsigned struct_pfioc_trans_sz = sizeof(pfioc_trans);
+unsigned struct_plistref_sz = sizeof(plistref);
+unsigned struct_power_type_sz = sizeof(power_type);
+unsigned struct_ppp_idle_sz = sizeof(ppp_idle);
+unsigned struct_ppp_option_data_sz = sizeof(ppp_option_data);
+unsigned struct_ppp_rawin_sz = sizeof(ppp_rawin);
+unsigned struct_pppoeconnectionstate_sz = sizeof(pppoeconnectionstate);
+unsigned struct_pppoediscparms_sz = sizeof(pppoediscparms);
+unsigned struct_priq_add_class_sz = sizeof(priq_add_class);
+unsigned struct_priq_add_filter_sz = sizeof(priq_add_filter);
+unsigned struct_priq_class_stats_sz = sizeof(priq_class_stats);
+unsigned struct_priq_delete_class_sz = sizeof(priq_delete_class);
+unsigned struct_priq_delete_filter_sz = sizeof(priq_delete_filter);
+unsigned struct_priq_interface_sz = sizeof(priq_interface);
+unsigned struct_priq_modify_class_sz = sizeof(priq_modify_class);
+unsigned struct_ptmget_sz = sizeof(ptmget);
+unsigned struct_radio_info_sz = sizeof(radio_info);
+unsigned struct_red_conf_sz = sizeof(red_conf);
+unsigned struct_red_interface_sz = sizeof(red_interface);
+unsigned struct_red_stats_sz = sizeof(red_stats);
+unsigned struct_redparams_sz = sizeof(redparams);
+unsigned struct_rf_pmparams_sz = sizeof(rf_pmparams);
+unsigned struct_rf_pmstat_sz = sizeof(rf_pmstat);
+unsigned struct_rf_recon_req_sz = sizeof(rf_recon_req);
+unsigned struct_rio_conf_sz = sizeof(rio_conf);
+unsigned struct_rio_interface_sz = sizeof(rio_interface);
+unsigned struct_rio_stats_sz = sizeof(rio_stats);
+unsigned struct_scan_io_sz = sizeof(scan_io);
+unsigned struct_scbusaccel_args_sz = sizeof(scbusaccel_args);
+unsigned struct_scbusiodetach_args_sz = sizeof(scbusiodetach_args);
+unsigned struct_scbusioscan_args_sz = sizeof(scbusioscan_args);
+unsigned struct_scsi_addr_sz = sizeof(scsi_addr);
+unsigned struct_seq_event_rec_sz = sizeof(seq_event_rec);
+unsigned struct_session_op_sz = sizeof(session_op);
+unsigned struct_sgttyb_sz = sizeof(sgttyb);
+unsigned struct_sioc_sg_req_sz = sizeof(sioc_sg_req);
+unsigned struct_sioc_vif_req_sz = sizeof(sioc_vif_req);
+unsigned struct_smbioc_flags_sz = sizeof(smbioc_flags);
+unsigned struct_smbioc_lookup_sz = sizeof(smbioc_lookup);
+unsigned struct_smbioc_oshare_sz = sizeof(smbioc_oshare);
+unsigned struct_smbioc_ossn_sz = sizeof(smbioc_ossn);
+unsigned struct_smbioc_rq_sz = sizeof(smbioc_rq);
+unsigned struct_smbioc_rw_sz = sizeof(smbioc_rw);
+unsigned struct_spppauthcfg_sz = sizeof(spppauthcfg);
+unsigned struct_spppauthfailuresettings_sz = sizeof(spppauthfailuresettings);
+unsigned struct_spppauthfailurestats_sz = sizeof(spppauthfailurestats);
+unsigned struct_spppdnsaddrs_sz = sizeof(spppdnsaddrs);
+unsigned struct_spppdnssettings_sz = sizeof(spppdnssettings);
+unsigned struct_spppidletimeout_sz = sizeof(spppidletimeout);
+unsigned struct_spppkeepalivesettings_sz = sizeof(spppkeepalivesettings);
+unsigned struct_sppplcpcfg_sz = sizeof(sppplcpcfg);
+unsigned struct_spppstatus_sz = sizeof(spppstatus);
+unsigned struct_spppstatusncp_sz = sizeof(spppstatusncp);
+unsigned struct_srt_rt_sz = sizeof(srt_rt);
+unsigned struct_stic_xinfo_sz = sizeof(stic_xinfo);
+unsigned struct_sun_dkctlr_sz = sizeof(sun_dkctlr);
+unsigned struct_sun_dkgeom_sz = sizeof(sun_dkgeom);
+unsigned struct_sun_dkpart_sz = sizeof(sun_dkpart);
+unsigned struct_synth_info_sz = sizeof(synth_info);
+unsigned struct_tbrreq_sz = sizeof(tbrreq);
+unsigned struct_tchars_sz = sizeof(tchars);
+unsigned struct_termios_sz = sizeof(termios);
+unsigned struct_timeval_sz = sizeof(timeval);
+unsigned struct_twe_drivecommand_sz = sizeof(twe_drivecommand);
+unsigned struct_twe_paramcommand_sz = sizeof(twe_paramcommand);
+unsigned struct_twe_usercommand_sz = sizeof(twe_usercommand);
+unsigned struct_ukyopon_identify_sz = sizeof(ukyopon_identify);
+unsigned struct_urio_command_sz = sizeof(urio_command);
+unsigned struct_usb_alt_interface_sz = sizeof(usb_alt_interface);
+unsigned struct_usb_bulk_ra_wb_opt_sz = sizeof(usb_bulk_ra_wb_opt);
+unsigned struct_usb_config_desc_sz = sizeof(usb_config_desc);
+unsigned struct_usb_ctl_report_desc_sz = sizeof(usb_ctl_report_desc);
+unsigned struct_usb_ctl_report_sz = sizeof(usb_ctl_report);
+unsigned struct_usb_ctl_request_sz = sizeof(usb_ctl_request);
+#if defined(__x86_64__)
+unsigned struct_nvmm_ioc_capability_sz = sizeof(nvmm_ioc_capability);
+unsigned struct_nvmm_ioc_machine_create_sz = sizeof(nvmm_ioc_machine_create);
+unsigned struct_nvmm_ioc_machine_destroy_sz = sizeof(nvmm_ioc_machine_destroy);
+unsigned struct_nvmm_ioc_machine_configure_sz =
+    sizeof(nvmm_ioc_machine_configure);
+unsigned struct_nvmm_ioc_vcpu_create_sz = sizeof(nvmm_ioc_vcpu_create);
+unsigned struct_nvmm_ioc_vcpu_destroy_sz = sizeof(nvmm_ioc_vcpu_destroy);
+unsigned struct_nvmm_ioc_vcpu_configure_sz = sizeof(nvmm_ioc_vcpu_configure);
+unsigned struct_nvmm_ioc_vcpu_setstate_sz = sizeof(nvmm_ioc_vcpu_destroy);
+unsigned struct_nvmm_ioc_vcpu_getstate_sz = sizeof(nvmm_ioc_vcpu_getstate);
+unsigned struct_nvmm_ioc_vcpu_inject_sz = sizeof(nvmm_ioc_vcpu_inject);
+unsigned struct_nvmm_ioc_vcpu_run_sz = sizeof(nvmm_ioc_vcpu_run);
+unsigned struct_nvmm_ioc_gpa_map_sz = sizeof(nvmm_ioc_gpa_map);
+unsigned struct_nvmm_ioc_gpa_unmap_sz = sizeof(nvmm_ioc_gpa_unmap);
+unsigned struct_nvmm_ioc_hva_map_sz = sizeof(nvmm_ioc_hva_map);
+unsigned struct_nvmm_ioc_hva_unmap_sz = sizeof(nvmm_ioc_hva_unmap);
+unsigned struct_nvmm_ioc_ctl_sz = sizeof(nvmm_ioc_ctl);
+#endif
+unsigned struct_spi_ioctl_configure_sz = sizeof(spi_ioctl_configure);
+unsigned struct_spi_ioctl_transfer_sz = sizeof(spi_ioctl_transfer);
+unsigned struct_autofs_daemon_request_sz = sizeof(autofs_daemon_request);
+unsigned struct_autofs_daemon_done_sz = sizeof(autofs_daemon_done);
+unsigned struct_sctp_connectx_addrs_sz = sizeof(sctp_connectx_addrs);
+unsigned struct_usb_device_info_old_sz = sizeof(usb_device_info_old);
+unsigned struct_usb_device_info_sz = sizeof(usb_device_info);
+unsigned struct_usb_device_stats_sz = sizeof(usb_device_stats);
+unsigned struct_usb_endpoint_desc_sz = sizeof(usb_endpoint_desc);
+unsigned struct_usb_full_desc_sz = sizeof(usb_full_desc);
+unsigned struct_usb_interface_desc_sz = sizeof(usb_interface_desc);
+unsigned struct_usb_string_desc_sz = sizeof(usb_string_desc);
+unsigned struct_utoppy_readfile_sz = sizeof(utoppy_readfile);
+unsigned struct_utoppy_rename_sz = sizeof(utoppy_rename);
+unsigned struct_utoppy_stats_sz = sizeof(utoppy_stats);
+unsigned struct_utoppy_writefile_sz = sizeof(utoppy_writefile);
+unsigned struct_v4l2_audio_sz = sizeof(v4l2_audio);
+unsigned struct_v4l2_audioout_sz = sizeof(v4l2_audioout);
+unsigned struct_v4l2_buffer_sz = sizeof(v4l2_buffer);
+unsigned struct_v4l2_capability_sz = sizeof(v4l2_capability);
+unsigned struct_v4l2_control_sz = sizeof(v4l2_control);
+unsigned struct_v4l2_crop_sz = sizeof(v4l2_crop);
+unsigned struct_v4l2_cropcap_sz = sizeof(v4l2_cropcap);
+unsigned struct_v4l2_fmtdesc_sz = sizeof(v4l2_fmtdesc);
+unsigned struct_v4l2_format_sz = sizeof(v4l2_format);
+unsigned struct_v4l2_framebuffer_sz = sizeof(v4l2_framebuffer);
+unsigned struct_v4l2_frequency_sz = sizeof(v4l2_frequency);
+unsigned struct_v4l2_frmivalenum_sz = sizeof(v4l2_frmivalenum);
+unsigned struct_v4l2_frmsizeenum_sz = sizeof(v4l2_frmsizeenum);
+unsigned struct_v4l2_input_sz = sizeof(v4l2_input);
+unsigned struct_v4l2_jpegcompression_sz = sizeof(v4l2_jpegcompression);
+unsigned struct_v4l2_modulator_sz = sizeof(v4l2_modulator);
+unsigned struct_v4l2_output_sz = sizeof(v4l2_output);
+unsigned struct_v4l2_queryctrl_sz = sizeof(v4l2_queryctrl);
+unsigned struct_v4l2_querymenu_sz = sizeof(v4l2_querymenu);
+unsigned struct_v4l2_requestbuffers_sz = sizeof(v4l2_requestbuffers);
+unsigned struct_v4l2_standard_sz = sizeof(v4l2_standard);
+unsigned struct_v4l2_streamparm_sz = sizeof(v4l2_streamparm);
+unsigned struct_v4l2_tuner_sz = sizeof(v4l2_tuner);
+unsigned struct_vnd_ioctl_sz = sizeof(vnd_ioctl);
+unsigned struct_vnd_user_sz = sizeof(vnd_user);
+unsigned struct_vt_stat_sz = sizeof(vt_stat);
+unsigned struct_wdog_conf_sz = sizeof(wdog_conf);
+unsigned struct_wdog_mode_sz = sizeof(wdog_mode);
+unsigned struct_ipmi_recv_sz = sizeof(ipmi_recv);
+unsigned struct_ipmi_req_sz = sizeof(ipmi_req);
+unsigned struct_ipmi_cmdspec_sz = sizeof(ipmi_cmdspec);
+unsigned struct_wfq_conf_sz = sizeof(wfq_conf);
+unsigned struct_wfq_getqid_sz = sizeof(wfq_getqid);
+unsigned struct_wfq_getstats_sz = sizeof(wfq_getstats);
+unsigned struct_wfq_interface_sz = sizeof(wfq_interface);
+unsigned struct_wfq_setweight_sz = sizeof(wfq_setweight);
+unsigned struct_winsize_sz = sizeof(winsize);
+unsigned struct_wscons_event_sz = sizeof(wscons_event);
+unsigned struct_wsdisplay_addscreendata_sz = sizeof(wsdisplay_addscreendata);
+unsigned struct_wsdisplay_char_sz = sizeof(wsdisplay_char);
+unsigned struct_wsdisplay_cmap_sz = sizeof(wsdisplay_cmap);
+unsigned struct_wsdisplay_curpos_sz = sizeof(wsdisplay_curpos);
+unsigned struct_wsdisplay_cursor_sz = sizeof(wsdisplay_cursor);
+unsigned struct_wsdisplay_delscreendata_sz = sizeof(wsdisplay_delscreendata);
+unsigned struct_wsdisplay_fbinfo_sz = sizeof(wsdisplay_fbinfo);
+unsigned struct_wsdisplay_font_sz = sizeof(wsdisplay_font);
+unsigned struct_wsdisplay_kbddata_sz = sizeof(wsdisplay_kbddata);
+unsigned struct_wsdisplay_msgattrs_sz = sizeof(wsdisplay_msgattrs);
+unsigned struct_wsdisplay_param_sz = sizeof(wsdisplay_param);
+unsigned struct_wsdisplay_scroll_data_sz = sizeof(wsdisplay_scroll_data);
+unsigned struct_wsdisplay_usefontdata_sz = sizeof(wsdisplay_usefontdata);
+unsigned struct_wsdisplayio_blit_sz = sizeof(wsdisplayio_blit);
+unsigned struct_wsdisplayio_bus_id_sz = sizeof(wsdisplayio_bus_id);
+unsigned struct_wsdisplayio_edid_info_sz = sizeof(wsdisplayio_edid_info);
+unsigned struct_wsdisplayio_fbinfo_sz = sizeof(wsdisplayio_fbinfo);
+unsigned struct_wskbd_bell_data_sz = sizeof(wskbd_bell_data);
+unsigned struct_wskbd_keyrepeat_data_sz = sizeof(wskbd_keyrepeat_data);
+unsigned struct_wskbd_map_data_sz = sizeof(wskbd_map_data);
+unsigned struct_wskbd_scroll_data_sz = sizeof(wskbd_scroll_data);
+unsigned struct_wsmouse_calibcoords_sz = sizeof(wsmouse_calibcoords);
+unsigned struct_wsmouse_id_sz = sizeof(wsmouse_id);
+unsigned struct_wsmouse_repeat_sz = sizeof(wsmouse_repeat);
+unsigned struct_wsmux_device_list_sz = sizeof(wsmux_device_list);
+unsigned struct_wsmux_device_sz = sizeof(wsmux_device);
+unsigned struct_xd_iocmd_sz = sizeof(xd_iocmd);
+
+unsigned struct_scsireq_sz = sizeof(struct scsireq);
+unsigned struct_tone_sz = sizeof(tone_t);
+unsigned union_twe_statrequest_sz = sizeof(union twe_statrequest);
+unsigned struct_usb_device_descriptor_sz = sizeof(usb_device_descriptor_t);
+unsigned struct_vt_mode_sz = sizeof(struct vt_mode);
+unsigned struct__old_mixer_info_sz = sizeof(struct _old_mixer_info);
+unsigned struct__agp_allocate_sz = sizeof(struct _agp_allocate);
+unsigned struct__agp_bind_sz = sizeof(struct _agp_bind);
+unsigned struct__agp_info_sz = sizeof(struct _agp_info);
+unsigned struct__agp_setup_sz = sizeof(struct _agp_setup);
+unsigned struct__agp_unbind_sz = sizeof(struct _agp_unbind);
+unsigned struct_atareq_sz = sizeof(struct atareq);
+unsigned struct_cpustate_sz = sizeof(struct cpustate);
+unsigned struct_dmx_caps_sz = sizeof(struct dmx_caps);
+unsigned enum_dmx_source_sz = sizeof(dmx_source_t);
+unsigned union_dvd_authinfo_sz = sizeof(dvd_authinfo);
+unsigned union_dvd_struct_sz = sizeof(dvd_struct);
+unsigned enum_v4l2_priority_sz = sizeof(enum v4l2_priority);
+unsigned struct_envsys_basic_info_sz = sizeof(struct envsys_basic_info);
+unsigned struct_envsys_tre_data_sz = sizeof(struct envsys_tre_data);
+unsigned enum_fe_sec_mini_cmd_sz = sizeof(enum fe_sec_mini_cmd);
+unsigned enum_fe_sec_tone_mode_sz = sizeof(enum fe_sec_tone_mode);
+unsigned enum_fe_sec_voltage_sz = sizeof(enum fe_sec_voltage);
+unsigned enum_fe_status_sz = sizeof(enum fe_status);
+unsigned struct_gdt_ctrt_sz = sizeof(struct gdt_ctrt);
+unsigned struct_gdt_event_sz = sizeof(struct gdt_event);
+unsigned struct_gdt_osv_sz = sizeof(struct gdt_osv);
+unsigned struct_gdt_rescan_sz = sizeof(struct gdt_rescan);
+unsigned struct_gdt_statist_sz = sizeof(struct gdt_statist);
+unsigned struct_gdt_ucmd_sz = sizeof(struct gdt_ucmd);
+unsigned struct_iscsi_conn_status_parameters_sz =
+    sizeof(iscsi_conn_status_parameters_t);
+unsigned struct_iscsi_get_version_parameters_sz =
+    sizeof(iscsi_get_version_parameters_t);
+unsigned struct_iscsi_iocommand_parameters_sz =
+    sizeof(iscsi_iocommand_parameters_t);
+unsigned struct_iscsi_login_parameters_sz = sizeof(iscsi_login_parameters_t);
+unsigned struct_iscsi_logout_parameters_sz = sizeof(iscsi_logout_parameters_t);
+unsigned struct_iscsi_register_event_parameters_sz =
+    sizeof(iscsi_register_event_parameters_t);
+unsigned struct_iscsi_remove_parameters_sz = sizeof(iscsi_remove_parameters_t);
+unsigned struct_iscsi_send_targets_parameters_sz =
+    sizeof(iscsi_send_targets_parameters_t);
+unsigned struct_iscsi_set_node_name_parameters_sz =
+    sizeof(iscsi_set_node_name_parameters_t);
+unsigned struct_iscsi_wait_event_parameters_sz =
+    sizeof(iscsi_wait_event_parameters_t);
+unsigned struct_isp_stats_sz = sizeof(isp_stats_t);
+unsigned struct_lsenable_sz = sizeof(struct lsenable);
+unsigned struct_lsdisable_sz = sizeof(struct lsdisable);
+unsigned struct_audio_format_query_sz = sizeof(audio_format_query);
+unsigned struct_mixer_ctrl_sz = sizeof(struct mixer_ctrl);
+unsigned struct_mixer_devinfo_sz = sizeof(struct mixer_devinfo);
+unsigned struct_mpu_command_rec_sz = sizeof(mpu_command_rec);
+unsigned struct_rndstat_sz = sizeof(rndstat_t);
+unsigned struct_rndstat_name_sz = sizeof(rndstat_name_t);
+unsigned struct_rndctl_sz = sizeof(rndctl_t);
+unsigned struct_rnddata_sz = sizeof(rnddata_t);
+unsigned struct_rndpoolstat_sz = sizeof(rndpoolstat_t);
+unsigned struct_rndstat_est_sz = sizeof(rndstat_est_t);
+unsigned struct_rndstat_est_name_sz = sizeof(rndstat_est_name_t);
+unsigned struct_pps_params_sz = sizeof(pps_params_t);
+unsigned struct_pps_info_sz = sizeof(pps_info_t);
+unsigned struct_mixer_info_sz = sizeof(struct mixer_info);
+unsigned struct_RF_SparetWait_sz = sizeof(RF_SparetWait_t);
+unsigned struct_RF_ComponentLabel_sz = sizeof(RF_ComponentLabel_t);
+unsigned struct_RF_SingleComponent_sz = sizeof(RF_SingleComponent_t);
+unsigned struct_RF_ProgressInfo_sz = sizeof(RF_ProgressInfo_t);
+unsigned struct_nvlist_ref_sz = sizeof(struct __sanitizer_nvlist_ref_t);
+unsigned struct_StringList_sz = sizeof(StringList);
+
+const unsigned IOCTL_NOT_PRESENT = 0;
+
+unsigned IOCTL_AFM_ADDFMAP = AFM_ADDFMAP;
+unsigned IOCTL_AFM_DELFMAP = AFM_DELFMAP;
+unsigned IOCTL_AFM_CLEANFMAP = AFM_CLEANFMAP;
+unsigned IOCTL_AFM_GETFMAP = AFM_GETFMAP;
+unsigned IOCTL_ALTQGTYPE = ALTQGTYPE;
+unsigned IOCTL_ALTQTBRSET = ALTQTBRSET;
+unsigned IOCTL_ALTQTBRGET = ALTQTBRGET;
+unsigned IOCTL_BLUE_IF_ATTACH = BLUE_IF_ATTACH;
+unsigned IOCTL_BLUE_IF_DETACH = BLUE_IF_DETACH;
+unsigned IOCTL_BLUE_ENABLE = BLUE_ENABLE;
+unsigned IOCTL_BLUE_DISABLE = BLUE_DISABLE;
+unsigned IOCTL_BLUE_CONFIG = BLUE_CONFIG;
+unsigned IOCTL_BLUE_GETSTATS = BLUE_GETSTATS;
+unsigned IOCTL_CBQ_IF_ATTACH = CBQ_IF_ATTACH;
+unsigned IOCTL_CBQ_IF_DETACH = CBQ_IF_DETACH;
+unsigned IOCTL_CBQ_ENABLE = CBQ_ENABLE;
+unsigned IOCTL_CBQ_DISABLE = CBQ_DISABLE;
+unsigned IOCTL_CBQ_CLEAR_HIERARCHY = CBQ_CLEAR_HIERARCHY;
+unsigned IOCTL_CBQ_ADD_CLASS = CBQ_ADD_CLASS;
+unsigned IOCTL_CBQ_DEL_CLASS = CBQ_DEL_CLASS;
+unsigned IOCTL_CBQ_MODIFY_CLASS = CBQ_MODIFY_CLASS;
+unsigned IOCTL_CBQ_ADD_FILTER = CBQ_ADD_FILTER;
+unsigned IOCTL_CBQ_DEL_FILTER = CBQ_DEL_FILTER;
+unsigned IOCTL_CBQ_GETSTATS = CBQ_GETSTATS;
+unsigned IOCTL_CDNR_IF_ATTACH = CDNR_IF_ATTACH;
+unsigned IOCTL_CDNR_IF_DETACH = CDNR_IF_DETACH;
+unsigned IOCTL_CDNR_ENABLE = CDNR_ENABLE;
+unsigned IOCTL_CDNR_DISABLE = CDNR_DISABLE;
+unsigned IOCTL_CDNR_ADD_FILTER = CDNR_ADD_FILTER;
+unsigned IOCTL_CDNR_DEL_FILTER = CDNR_DEL_FILTER;
+unsigned IOCTL_CDNR_GETSTATS = CDNR_GETSTATS;
+unsigned IOCTL_CDNR_ADD_ELEM = CDNR_ADD_ELEM;
+unsigned IOCTL_CDNR_DEL_ELEM = CDNR_DEL_ELEM;
+unsigned IOCTL_CDNR_ADD_TBM = CDNR_ADD_TBM;
+unsigned IOCTL_CDNR_MOD_TBM = CDNR_MOD_TBM;
+unsigned IOCTL_CDNR_TBM_STATS = CDNR_TBM_STATS;
+unsigned IOCTL_CDNR_ADD_TCM = CDNR_ADD_TCM;
+unsigned IOCTL_CDNR_MOD_TCM = CDNR_MOD_TCM;
+unsigned IOCTL_CDNR_TCM_STATS = CDNR_TCM_STATS;
+unsigned IOCTL_CDNR_ADD_TSW = CDNR_ADD_TSW;
+unsigned IOCTL_CDNR_MOD_TSW = CDNR_MOD_TSW;
+unsigned IOCTL_FIFOQ_IF_ATTACH = FIFOQ_IF_ATTACH;
+unsigned IOCTL_FIFOQ_IF_DETACH = FIFOQ_IF_DETACH;
+unsigned IOCTL_FIFOQ_ENABLE = FIFOQ_ENABLE;
+unsigned IOCTL_FIFOQ_DISABLE = FIFOQ_DISABLE;
+unsigned IOCTL_FIFOQ_CONFIG = FIFOQ_CONFIG;
+unsigned IOCTL_FIFOQ_GETSTATS = FIFOQ_GETSTATS;
+unsigned IOCTL_HFSC_IF_ATTACH = HFSC_IF_ATTACH;
+unsigned IOCTL_HFSC_IF_DETACH = HFSC_IF_DETACH;
+unsigned IOCTL_HFSC_ENABLE = HFSC_ENABLE;
+unsigned IOCTL_HFSC_DISABLE = HFSC_DISABLE;
+unsigned IOCTL_HFSC_CLEAR_HIERARCHY = HFSC_CLEAR_HIERARCHY;
+unsigned IOCTL_HFSC_ADD_CLASS = HFSC_ADD_CLASS;
+unsigned IOCTL_HFSC_DEL_CLASS = HFSC_DEL_CLASS;
+unsigned IOCTL_HFSC_MOD_CLASS = HFSC_MOD_CLASS;
+unsigned IOCTL_HFSC_ADD_FILTER = HFSC_ADD_FILTER;
+unsigned IOCTL_HFSC_DEL_FILTER = HFSC_DEL_FILTER;
+unsigned IOCTL_HFSC_GETSTATS = HFSC_GETSTATS;
+unsigned IOCTL_JOBS_IF_ATTACH = JOBS_IF_ATTACH;
+unsigned IOCTL_JOBS_IF_DETACH = JOBS_IF_DETACH;
+unsigned IOCTL_JOBS_ENABLE = JOBS_ENABLE;
+unsigned IOCTL_JOBS_DISABLE = JOBS_DISABLE;
+unsigned IOCTL_JOBS_CLEAR = JOBS_CLEAR;
+unsigned IOCTL_JOBS_ADD_CLASS = JOBS_ADD_CLASS;
+unsigned IOCTL_JOBS_DEL_CLASS = JOBS_DEL_CLASS;
+unsigned IOCTL_JOBS_MOD_CLASS = JOBS_MOD_CLASS;
+unsigned IOCTL_JOBS_ADD_FILTER = JOBS_ADD_FILTER;
+unsigned IOCTL_JOBS_DEL_FILTER = JOBS_DEL_FILTER;
+unsigned IOCTL_JOBS_GETSTATS = JOBS_GETSTATS;
+unsigned IOCTL_PRIQ_IF_ATTACH = PRIQ_IF_ATTACH;
+unsigned IOCTL_PRIQ_IF_DETACH = PRIQ_IF_DETACH;
+unsigned IOCTL_PRIQ_ENABLE = PRIQ_ENABLE;
+unsigned IOCTL_PRIQ_DISABLE = PRIQ_DISABLE;
+unsigned IOCTL_PRIQ_CLEAR = PRIQ_CLEAR;
+unsigned IOCTL_PRIQ_ADD_CLASS = PRIQ_ADD_CLASS;
+unsigned IOCTL_PRIQ_DEL_CLASS = PRIQ_DEL_CLASS;
+unsigned IOCTL_PRIQ_MOD_CLASS = PRIQ_MOD_CLASS;
+unsigned IOCTL_PRIQ_ADD_FILTER = PRIQ_ADD_FILTER;
+unsigned IOCTL_PRIQ_DEL_FILTER = PRIQ_DEL_FILTER;
+unsigned IOCTL_PRIQ_GETSTATS = PRIQ_GETSTATS;
+unsigned IOCTL_RED_IF_ATTACH = RED_IF_ATTACH;
+unsigned IOCTL_RED_IF_DETACH = RED_IF_DETACH;
+unsigned IOCTL_RED_ENABLE = RED_ENABLE;
+unsigned IOCTL_RED_DISABLE = RED_DISABLE;
+unsigned IOCTL_RED_CONFIG = RED_CONFIG;
+unsigned IOCTL_RED_GETSTATS = RED_GETSTATS;
+unsigned IOCTL_RED_SETDEFAULTS = RED_SETDEFAULTS;
+unsigned IOCTL_RIO_IF_ATTACH = RIO_IF_ATTACH;
+unsigned IOCTL_RIO_IF_DETACH = RIO_IF_DETACH;
+unsigned IOCTL_RIO_ENABLE = RIO_ENABLE;
+unsigned IOCTL_RIO_DISABLE = RIO_DISABLE;
+unsigned IOCTL_RIO_CONFIG = RIO_CONFIG;
+unsigned IOCTL_RIO_GETSTATS = RIO_GETSTATS;
+unsigned IOCTL_RIO_SETDEFAULTS = RIO_SETDEFAULTS;
+unsigned IOCTL_WFQ_IF_ATTACH = WFQ_IF_ATTACH;
+unsigned IOCTL_WFQ_IF_DETACH = WFQ_IF_DETACH;
+unsigned IOCTL_WFQ_ENABLE = WFQ_ENABLE;
+unsigned IOCTL_WFQ_DISABLE = WFQ_DISABLE;
+unsigned IOCTL_WFQ_CONFIG = WFQ_CONFIG;
+unsigned IOCTL_WFQ_GET_STATS = WFQ_GET_STATS;
+unsigned IOCTL_WFQ_GET_QID = WFQ_GET_QID;
+unsigned IOCTL_WFQ_SET_WEIGHT = WFQ_SET_WEIGHT;
+unsigned IOCTL_CRIOGET = CRIOGET;
+unsigned IOCTL_CIOCFSESSION = CIOCFSESSION;
+unsigned IOCTL_CIOCKEY = CIOCKEY;
+unsigned IOCTL_CIOCNFKEYM = CIOCNFKEYM;
+unsigned IOCTL_CIOCNFSESSION = CIOCNFSESSION;
+unsigned IOCTL_CIOCNCRYPTRETM = CIOCNCRYPTRETM;
+unsigned IOCTL_CIOCNCRYPTRET = CIOCNCRYPTRET;
+unsigned IOCTL_CIOCGSESSION = CIOCGSESSION;
+unsigned IOCTL_CIOCNGSESSION = CIOCNGSESSION;
+unsigned IOCTL_CIOCCRYPT = CIOCCRYPT;
+unsigned IOCTL_CIOCNCRYPTM = CIOCNCRYPTM;
+unsigned IOCTL_CIOCASYMFEAT = CIOCASYMFEAT;
+unsigned IOCTL_APM_IOC_REJECT = APM_IOC_REJECT;
+unsigned IOCTL_APM_IOC_STANDBY = APM_IOC_STANDBY;
+unsigned IOCTL_APM_IOC_SUSPEND = APM_IOC_SUSPEND;
+unsigned IOCTL_OAPM_IOC_GETPOWER = OAPM_IOC_GETPOWER;
+unsigned IOCTL_APM_IOC_GETPOWER = APM_IOC_GETPOWER;
+unsigned IOCTL_APM_IOC_NEXTEVENT = APM_IOC_NEXTEVENT;
+unsigned IOCTL_APM_IOC_DEV_CTL = APM_IOC_DEV_CTL;
+unsigned IOCTL_NETBSD_DM_IOCTL = NETBSD_DM_IOCTL;
+unsigned IOCTL_DMIO_SETFUNC = DMIO_SETFUNC;
+unsigned IOCTL_DMX_START = DMX_START;
+unsigned IOCTL_DMX_STOP = DMX_STOP;
+unsigned IOCTL_DMX_SET_FILTER = DMX_SET_FILTER;
+unsigned IOCTL_DMX_SET_PES_FILTER = DMX_SET_PES_FILTER;
+unsigned IOCTL_DMX_SET_BUFFER_SIZE = DMX_SET_BUFFER_SIZE;
+unsigned IOCTL_DMX_GET_STC = DMX_GET_STC;
+unsigned IOCTL_DMX_ADD_PID = DMX_ADD_PID;
+unsigned IOCTL_DMX_REMOVE_PID = DMX_REMOVE_PID;
+unsigned IOCTL_DMX_GET_CAPS = DMX_GET_CAPS;
+unsigned IOCTL_DMX_SET_SOURCE = DMX_SET_SOURCE;
+unsigned IOCTL_FE_READ_STATUS = FE_READ_STATUS;
+unsigned IOCTL_FE_READ_BER = FE_READ_BER;
+unsigned IOCTL_FE_READ_SNR = FE_READ_SNR;
+unsigned IOCTL_FE_READ_SIGNAL_STRENGTH = FE_READ_SIGNAL_STRENGTH;
+unsigned IOCTL_FE_READ_UNCORRECTED_BLOCKS = FE_READ_UNCORRECTED_BLOCKS;
+unsigned IOCTL_FE_SET_FRONTEND = FE_SET_FRONTEND;
+unsigned IOCTL_FE_GET_FRONTEND = FE_GET_FRONTEND;
+unsigned IOCTL_FE_GET_EVENT = FE_GET_EVENT;
+unsigned IOCTL_FE_GET_INFO = FE_GET_INFO;
+unsigned IOCTL_FE_DISEQC_RESET_OVERLOAD = FE_DISEQC_RESET_OVERLOAD;
+unsigned IOCTL_FE_DISEQC_SEND_MASTER_CMD = FE_DISEQC_SEND_MASTER_CMD;
+unsigned IOCTL_FE_DISEQC_RECV_SLAVE_REPLY = FE_DISEQC_RECV_SLAVE_REPLY;
+unsigned IOCTL_FE_DISEQC_SEND_BURST = FE_DISEQC_SEND_BURST;
+unsigned IOCTL_FE_SET_TONE = FE_SET_TONE;
+unsigned IOCTL_FE_SET_VOLTAGE = FE_SET_VOLTAGE;
+unsigned IOCTL_FE_ENABLE_HIGH_LNB_VOLTAGE = FE_ENABLE_HIGH_LNB_VOLTAGE;
+unsigned IOCTL_FE_SET_FRONTEND_TUNE_MODE = FE_SET_FRONTEND_TUNE_MODE;
+unsigned IOCTL_FE_DISHNETWORK_SEND_LEGACY_CMD = FE_DISHNETWORK_SEND_LEGACY_CMD;
+unsigned IOCTL_FILEMON_SET_FD = FILEMON_SET_FD;
+unsigned IOCTL_FILEMON_SET_PID = FILEMON_SET_PID;
+unsigned IOCTL_HDAUDIO_FGRP_INFO = HDAUDIO_FGRP_INFO;
+unsigned IOCTL_HDAUDIO_FGRP_GETCONFIG = HDAUDIO_FGRP_GETCONFIG;
+unsigned IOCTL_HDAUDIO_FGRP_SETCONFIG = HDAUDIO_FGRP_SETCONFIG;
+unsigned IOCTL_HDAUDIO_FGRP_WIDGET_INFO = HDAUDIO_FGRP_WIDGET_INFO;
+unsigned IOCTL_HDAUDIO_FGRP_CODEC_INFO = HDAUDIO_FGRP_CODEC_INFO;
+unsigned IOCTL_HDAUDIO_AFG_WIDGET_INFO = HDAUDIO_AFG_WIDGET_INFO;
+unsigned IOCTL_HDAUDIO_AFG_CODEC_INFO = HDAUDIO_AFG_CODEC_INFO;
+unsigned IOCTL_CEC_GET_PHYS_ADDR = CEC_GET_PHYS_ADDR;
+unsigned IOCTL_CEC_GET_LOG_ADDRS = CEC_GET_LOG_ADDRS;
+unsigned IOCTL_CEC_SET_LOG_ADDRS = CEC_SET_LOG_ADDRS;
+unsigned IOCTL_CEC_GET_VENDOR_ID = CEC_GET_VENDOR_ID;
+unsigned IOCTL_HPCFBIO_GCONF = HPCFBIO_GCONF;
+unsigned IOCTL_HPCFBIO_SCONF = HPCFBIO_SCONF;
+unsigned IOCTL_HPCFBIO_GDSPCONF = HPCFBIO_GDSPCONF;
+unsigned IOCTL_HPCFBIO_SDSPCONF = HPCFBIO_SDSPCONF;
+unsigned IOCTL_HPCFBIO_GOP = HPCFBIO_GOP;
+unsigned IOCTL_HPCFBIO_SOP = HPCFBIO_SOP;
+unsigned IOCTL_IOPIOCPT = IOPIOCPT;
+unsigned IOCTL_IOPIOCGLCT = IOPIOCGLCT;
+unsigned IOCTL_IOPIOCGSTATUS = IOPIOCGSTATUS;
+unsigned IOCTL_IOPIOCRECONFIG = IOPIOCRECONFIG;
+unsigned IOCTL_IOPIOCGTIDMAP = IOPIOCGTIDMAP;
+unsigned IOCTL_SIOCGATHSTATS = SIOCGATHSTATS;
+unsigned IOCTL_SIOCGATHDIAG = SIOCGATHDIAG;
+unsigned IOCTL_METEORCAPTUR = METEORCAPTUR;
+unsigned IOCTL_METEORCAPFRM = METEORCAPFRM;
+unsigned IOCTL_METEORSETGEO = METEORSETGEO;
+unsigned IOCTL_METEORGETGEO = METEORGETGEO;
+unsigned IOCTL_METEORSTATUS = METEORSTATUS;
+unsigned IOCTL_METEORSHUE = METEORSHUE;
+unsigned IOCTL_METEORGHUE = METEORGHUE;
+unsigned IOCTL_METEORSFMT = METEORSFMT;
+unsigned IOCTL_METEORGFMT = METEORGFMT;
+unsigned IOCTL_METEORSINPUT = METEORSINPUT;
+unsigned IOCTL_METEORGINPUT = METEORGINPUT;
+unsigned IOCTL_METEORSCHCV = METEORSCHCV;
+unsigned IOCTL_METEORGCHCV = METEORGCHCV;
+unsigned IOCTL_METEORSCOUNT = METEORSCOUNT;
+unsigned IOCTL_METEORGCOUNT = METEORGCOUNT;
+unsigned IOCTL_METEORSFPS = METEORSFPS;
+unsigned IOCTL_METEORGFPS = METEORGFPS;
+unsigned IOCTL_METEORSSIGNAL = METEORSSIGNAL;
+unsigned IOCTL_METEORGSIGNAL = METEORGSIGNAL;
+unsigned IOCTL_METEORSVIDEO = METEORSVIDEO;
+unsigned IOCTL_METEORGVIDEO = METEORGVIDEO;
+unsigned IOCTL_METEORSBRIG = METEORSBRIG;
+unsigned IOCTL_METEORGBRIG = METEORGBRIG;
+unsigned IOCTL_METEORSCSAT = METEORSCSAT;
+unsigned IOCTL_METEORGCSAT = METEORGCSAT;
+unsigned IOCTL_METEORSCONT = METEORSCONT;
+unsigned IOCTL_METEORGCONT = METEORGCONT;
+unsigned IOCTL_METEORSHWS = METEORSHWS;
+unsigned IOCTL_METEORGHWS = METEORGHWS;
+unsigned IOCTL_METEORSVWS = METEORSVWS;
+unsigned IOCTL_METEORGVWS = METEORGVWS;
+unsigned IOCTL_METEORSTS = METEORSTS;
+unsigned IOCTL_METEORGTS = METEORGTS;
+unsigned IOCTL_TVTUNER_SETCHNL = TVTUNER_SETCHNL;
+unsigned IOCTL_TVTUNER_GETCHNL = TVTUNER_GETCHNL;
+unsigned IOCTL_TVTUNER_SETTYPE = TVTUNER_SETTYPE;
+unsigned IOCTL_TVTUNER_GETTYPE = TVTUNER_GETTYPE;
+unsigned IOCTL_TVTUNER_GETSTATUS = TVTUNER_GETSTATUS;
+unsigned IOCTL_TVTUNER_SETFREQ = TVTUNER_SETFREQ;
+unsigned IOCTL_TVTUNER_GETFREQ = TVTUNER_GETFREQ;
+unsigned IOCTL_TVTUNER_SETAFC = TVTUNER_SETAFC;
+unsigned IOCTL_TVTUNER_GETAFC = TVTUNER_GETAFC;
+unsigned IOCTL_RADIO_SETMODE = RADIO_SETMODE;
+unsigned IOCTL_RADIO_GETMODE = RADIO_GETMODE;
+unsigned IOCTL_RADIO_SETFREQ = RADIO_SETFREQ;
+unsigned IOCTL_RADIO_GETFREQ = RADIO_GETFREQ;
+unsigned IOCTL_METEORSACTPIXFMT = METEORSACTPIXFMT;
+unsigned IOCTL_METEORGACTPIXFMT = METEORGACTPIXFMT;
+unsigned IOCTL_METEORGSUPPIXFMT = METEORGSUPPIXFMT;
+unsigned IOCTL_TVTUNER_GETCHNLSET = TVTUNER_GETCHNLSET;
+unsigned IOCTL_REMOTE_GETKEY = REMOTE_GETKEY;
+unsigned IOCTL_GDT_IOCTL_GENERAL = GDT_IOCTL_GENERAL;
+unsigned IOCTL_GDT_IOCTL_DRVERS = GDT_IOCTL_DRVERS;
+unsigned IOCTL_GDT_IOCTL_CTRTYPE = GDT_IOCTL_CTRTYPE;
+unsigned IOCTL_GDT_IOCTL_OSVERS = GDT_IOCTL_OSVERS;
+unsigned IOCTL_GDT_IOCTL_CTRCNT = GDT_IOCTL_CTRCNT;
+unsigned IOCTL_GDT_IOCTL_EVENT = GDT_IOCTL_EVENT;
+unsigned IOCTL_GDT_IOCTL_STATIST = GDT_IOCTL_STATIST;
+unsigned IOCTL_GDT_IOCTL_RESCAN = GDT_IOCTL_RESCAN;
+unsigned IOCTL_ISP_SDBLEV = ISP_SDBLEV;
+unsigned IOCTL_ISP_RESETHBA = ISP_RESETHBA;
+unsigned IOCTL_ISP_RESCAN = ISP_RESCAN;
+unsigned IOCTL_ISP_SETROLE = ISP_SETROLE;
+unsigned IOCTL_ISP_GETROLE = ISP_GETROLE;
+unsigned IOCTL_ISP_GET_STATS = ISP_GET_STATS;
+unsigned IOCTL_ISP_CLR_STATS = ISP_CLR_STATS;
+unsigned IOCTL_ISP_FC_LIP = ISP_FC_LIP;
+unsigned IOCTL_ISP_FC_GETDINFO = ISP_FC_GETDINFO;
+unsigned IOCTL_ISP_GET_FW_CRASH_DUMP = ISP_GET_FW_CRASH_DUMP;
+unsigned IOCTL_ISP_FORCE_CRASH_DUMP = ISP_FORCE_CRASH_DUMP;
+unsigned IOCTL_ISP_FC_GETHINFO = ISP_FC_GETHINFO;
+unsigned IOCTL_ISP_TSK_MGMT = ISP_TSK_MGMT;
+unsigned IOCTL_ISP_FC_GETDLIST = ISP_FC_GETDLIST;
+unsigned IOCTL_MLXD_STATUS = MLXD_STATUS;
+unsigned IOCTL_MLXD_CHECKASYNC = MLXD_CHECKASYNC;
+unsigned IOCTL_MLXD_DETACH = MLXD_DETACH;
+unsigned IOCTL_MLX_RESCAN_DRIVES = MLX_RESCAN_DRIVES;
+unsigned IOCTL_MLX_PAUSE_CHANNEL = MLX_PAUSE_CHANNEL;
+unsigned IOCTL_MLX_COMMAND = MLX_COMMAND;
+unsigned IOCTL_MLX_REBUILDASYNC = MLX_REBUILDASYNC;
+unsigned IOCTL_MLX_REBUILDSTAT = MLX_REBUILDSTAT;
+unsigned IOCTL_MLX_GET_SYSDRIVE = MLX_GET_SYSDRIVE;
+unsigned IOCTL_MLX_GET_CINFO = MLX_GET_CINFO;
+unsigned IOCTL_NVME_PASSTHROUGH_CMD = NVME_PASSTHROUGH_CMD;
+unsigned IOCTL_FWCFGIO_SET_INDEX = FWCFGIO_SET_INDEX;
+unsigned IOCTL_IRDA_RESET_PARAMS = IRDA_RESET_PARAMS;
+unsigned IOCTL_IRDA_SET_PARAMS = IRDA_SET_PARAMS;
+unsigned IOCTL_IRDA_GET_SPEEDMASK = IRDA_GET_SPEEDMASK;
+unsigned IOCTL_IRDA_GET_TURNAROUNDMASK = IRDA_GET_TURNAROUNDMASK;
+unsigned IOCTL_IRFRAMETTY_GET_DEVICE = IRFRAMETTY_GET_DEVICE;
+unsigned IOCTL_IRFRAMETTY_GET_DONGLE = IRFRAMETTY_GET_DONGLE;
+unsigned IOCTL_IRFRAMETTY_SET_DONGLE = IRFRAMETTY_SET_DONGLE;
+unsigned IOCTL_ISV_CMD = ISV_CMD;
+unsigned IOCTL_WTQICMD = WTQICMD;
+unsigned IOCTL_ISCSI_GET_VERSION = ISCSI_GET_VERSION;
+unsigned IOCTL_ISCSI_LOGIN = ISCSI_LOGIN;
+unsigned IOCTL_ISCSI_LOGOUT = ISCSI_LOGOUT;
+unsigned IOCTL_ISCSI_ADD_CONNECTION = ISCSI_ADD_CONNECTION;
+unsigned IOCTL_ISCSI_RESTORE_CONNECTION = ISCSI_RESTORE_CONNECTION;
+unsigned IOCTL_ISCSI_REMOVE_CONNECTION = ISCSI_REMOVE_CONNECTION;
+unsigned IOCTL_ISCSI_CONNECTION_STATUS = ISCSI_CONNECTION_STATUS;
+unsigned IOCTL_ISCSI_SEND_TARGETS = ISCSI_SEND_TARGETS;
+unsigned IOCTL_ISCSI_SET_NODE_NAME = ISCSI_SET_NODE_NAME;
+unsigned IOCTL_ISCSI_IO_COMMAND = ISCSI_IO_COMMAND;
+unsigned IOCTL_ISCSI_REGISTER_EVENT = ISCSI_REGISTER_EVENT;
+unsigned IOCTL_ISCSI_DEREGISTER_EVENT = ISCSI_DEREGISTER_EVENT;
+unsigned IOCTL_ISCSI_WAIT_EVENT = ISCSI_WAIT_EVENT;
+unsigned IOCTL_ISCSI_POLL_EVENT = ISCSI_POLL_EVENT;
+unsigned IOCTL_OFIOCGET = OFIOCGET;
+unsigned IOCTL_OFIOCSET = OFIOCSET;
+unsigned IOCTL_OFIOCNEXTPROP = OFIOCNEXTPROP;
+unsigned IOCTL_OFIOCGETOPTNODE = OFIOCGETOPTNODE;
+unsigned IOCTL_OFIOCGETNEXT = OFIOCGETNEXT;
+unsigned IOCTL_OFIOCGETCHILD = OFIOCGETCHILD;
+unsigned IOCTL_OFIOCFINDDEVICE = OFIOCFINDDEVICE;
+unsigned IOCTL_AMR_IO_VERSION = AMR_IO_VERSION;
+unsigned IOCTL_AMR_IO_COMMAND = AMR_IO_COMMAND;
+unsigned IOCTL_MLYIO_COMMAND = MLYIO_COMMAND;
+unsigned IOCTL_MLYIO_HEALTH = MLYIO_HEALTH;
+unsigned IOCTL_PCI_IOC_CFGREAD = PCI_IOC_CFGREAD;
+unsigned IOCTL_PCI_IOC_CFGWRITE = PCI_IOC_CFGWRITE;
+unsigned IOCTL_PCI_IOC_BDF_CFGREAD = PCI_IOC_BDF_CFGREAD;
+unsigned IOCTL_PCI_IOC_BDF_CFGWRITE = PCI_IOC_BDF_CFGWRITE;
+unsigned IOCTL_PCI_IOC_BUSINFO = PCI_IOC_BUSINFO;
+unsigned IOCTL_PCI_IOC_DRVNAME = PCI_IOC_DRVNAME;
+unsigned IOCTL_PCI_IOC_DRVNAMEONBUS = PCI_IOC_DRVNAMEONBUS;
+unsigned IOCTL_TWEIO_COMMAND = TWEIO_COMMAND;
+unsigned IOCTL_TWEIO_STATS = TWEIO_STATS;
+unsigned IOCTL_TWEIO_AEN_POLL = TWEIO_AEN_POLL;
+unsigned IOCTL_TWEIO_AEN_WAIT = TWEIO_AEN_WAIT;
+unsigned IOCTL_TWEIO_SET_PARAM = TWEIO_SET_PARAM;
+unsigned IOCTL_TWEIO_GET_PARAM = TWEIO_GET_PARAM;
+unsigned IOCTL_TWEIO_RESET = TWEIO_RESET;
+unsigned IOCTL_TWEIO_ADD_UNIT = TWEIO_ADD_UNIT;
+unsigned IOCTL_TWEIO_DEL_UNIT = TWEIO_DEL_UNIT;
+unsigned IOCTL_SIOCSCNWDOMAIN = SIOCSCNWDOMAIN;
+unsigned IOCTL_SIOCGCNWDOMAIN = SIOCGCNWDOMAIN;
+unsigned IOCTL_SIOCSCNWKEY = SIOCSCNWKEY;
+unsigned IOCTL_SIOCGCNWSTATUS = SIOCGCNWSTATUS;
+unsigned IOCTL_SIOCGCNWSTATS = SIOCGCNWSTATS;
+unsigned IOCTL_SIOCGCNWTRAIL = SIOCGCNWTRAIL;
+unsigned IOCTL_SIOCGRAYSIGLEV = SIOCGRAYSIGLEV;
+unsigned IOCTL_RAIDFRAME_SHUTDOWN = RAIDFRAME_SHUTDOWN;
+unsigned IOCTL_RAIDFRAME_TUR = RAIDFRAME_TUR;
+unsigned IOCTL_RAIDFRAME_FAIL_DISK = RAIDFRAME_FAIL_DISK;
+unsigned IOCTL_RAIDFRAME_CHECK_RECON_STATUS = RAIDFRAME_CHECK_RECON_STATUS;
+unsigned IOCTL_RAIDFRAME_REWRITEPARITY = RAIDFRAME_REWRITEPARITY;
+unsigned IOCTL_RAIDFRAME_COPYBACK = RAIDFRAME_COPYBACK;
+unsigned IOCTL_RAIDFRAME_SPARET_WAIT = RAIDFRAME_SPARET_WAIT;
+unsigned IOCTL_RAIDFRAME_SEND_SPARET = RAIDFRAME_SEND_SPARET;
+unsigned IOCTL_RAIDFRAME_ABORT_SPARET_WAIT = RAIDFRAME_ABORT_SPARET_WAIT;
+unsigned IOCTL_RAIDFRAME_START_ATRACE = RAIDFRAME_START_ATRACE;
+unsigned IOCTL_RAIDFRAME_STOP_ATRACE = RAIDFRAME_STOP_ATRACE;
+unsigned IOCTL_RAIDFRAME_GET_SIZE = RAIDFRAME_GET_SIZE;
+unsigned IOCTL_RAIDFRAME_RESET_ACCTOTALS = RAIDFRAME_RESET_ACCTOTALS;
+unsigned IOCTL_RAIDFRAME_KEEP_ACCTOTALS = RAIDFRAME_KEEP_ACCTOTALS;
+unsigned IOCTL_RAIDFRAME_GET_COMPONENT_LABEL = RAIDFRAME_GET_COMPONENT_LABEL;
+unsigned IOCTL_RAIDFRAME_SET_COMPONENT_LABEL = RAIDFRAME_SET_COMPONENT_LABEL;
+unsigned IOCTL_RAIDFRAME_INIT_LABELS = RAIDFRAME_INIT_LABELS;
+unsigned IOCTL_RAIDFRAME_ADD_HOT_SPARE = RAIDFRAME_ADD_HOT_SPARE;
+unsigned IOCTL_RAIDFRAME_REMOVE_HOT_SPARE = RAIDFRAME_REMOVE_HOT_SPARE;
+unsigned IOCTL_RAIDFRAME_REBUILD_IN_PLACE = RAIDFRAME_REBUILD_IN_PLACE;
+unsigned IOCTL_RAIDFRAME_CHECK_PARITY = RAIDFRAME_CHECK_PARITY;
+unsigned IOCTL_RAIDFRAME_CHECK_PARITYREWRITE_STATUS =
+    RAIDFRAME_CHECK_PARITYREWRITE_STATUS;
+unsigned IOCTL_RAIDFRAME_CHECK_COPYBACK_STATUS =
+    RAIDFRAME_CHECK_COPYBACK_STATUS;
+unsigned IOCTL_RAIDFRAME_SET_AUTOCONFIG = RAIDFRAME_SET_AUTOCONFIG;
+unsigned IOCTL_RAIDFRAME_SET_ROOT = RAIDFRAME_SET_ROOT;
+unsigned IOCTL_RAIDFRAME_DELETE_COMPONENT = RAIDFRAME_DELETE_COMPONENT;
+unsigned IOCTL_RAIDFRAME_INCORPORATE_HOT_SPARE =
+    RAIDFRAME_INCORPORATE_HOT_SPARE;
+unsigned IOCTL_RAIDFRAME_CHECK_RECON_STATUS_EXT =
+    RAIDFRAME_CHECK_RECON_STATUS_EXT;
+unsigned IOCTL_RAIDFRAME_CHECK_PARITYREWRITE_STATUS_EXT =
+    RAIDFRAME_CHECK_PARITYREWRITE_STATUS_EXT;
+unsigned IOCTL_RAIDFRAME_CHECK_COPYBACK_STATUS_EXT =
+    RAIDFRAME_CHECK_COPYBACK_STATUS_EXT;
+unsigned IOCTL_RAIDFRAME_CONFIGURE = RAIDFRAME_CONFIGURE;
+unsigned IOCTL_RAIDFRAME_GET_INFO = RAIDFRAME_GET_INFO;
+unsigned IOCTL_RAIDFRAME_PARITYMAP_STATUS = RAIDFRAME_PARITYMAP_STATUS;
+unsigned IOCTL_RAIDFRAME_PARITYMAP_GET_DISABLE =
+    RAIDFRAME_PARITYMAP_GET_DISABLE;
+unsigned IOCTL_RAIDFRAME_PARITYMAP_SET_DISABLE =
+    RAIDFRAME_PARITYMAP_SET_DISABLE;
+unsigned IOCTL_RAIDFRAME_PARITYMAP_SET_PARAMS = RAIDFRAME_PARITYMAP_SET_PARAMS;
+unsigned IOCTL_RAIDFRAME_SET_LAST_UNIT = RAIDFRAME_SET_LAST_UNIT;
+unsigned IOCTL_MBPPIOCSPARAM = MBPPIOCSPARAM;
+unsigned IOCTL_MBPPIOCGPARAM = MBPPIOCGPARAM;
+unsigned IOCTL_MBPPIOCGSTAT = MBPPIOCGSTAT;
+unsigned IOCTL_SESIOC_GETNOBJ = SESIOC_GETNOBJ;
+unsigned IOCTL_SESIOC_GETOBJMAP = SESIOC_GETOBJMAP;
+unsigned IOCTL_SESIOC_GETENCSTAT = SESIOC_GETENCSTAT;
+unsigned IOCTL_SESIOC_SETENCSTAT = SESIOC_SETENCSTAT;
+unsigned IOCTL_SESIOC_GETOBJSTAT = SESIOC_GETOBJSTAT;
+unsigned IOCTL_SESIOC_SETOBJSTAT = SESIOC_SETOBJSTAT;
+unsigned IOCTL_SESIOC_GETTEXT = SESIOC_GETTEXT;
+unsigned IOCTL_SESIOC_INIT = SESIOC_INIT;
+unsigned IOCTL_SUN_DKIOCGGEOM = SUN_DKIOCGGEOM;
+unsigned IOCTL_SUN_DKIOCINFO = SUN_DKIOCINFO;
+unsigned IOCTL_SUN_DKIOCGPART = SUN_DKIOCGPART;
+unsigned IOCTL_FBIOGTYPE = FBIOGTYPE;
+unsigned IOCTL_FBIOPUTCMAP = FBIOPUTCMAP;
+unsigned IOCTL_FBIOGETCMAP = FBIOGETCMAP;
+unsigned IOCTL_FBIOGATTR = FBIOGATTR;
+unsigned IOCTL_FBIOSVIDEO = FBIOSVIDEO;
+unsigned IOCTL_FBIOGVIDEO = FBIOGVIDEO;
+unsigned IOCTL_FBIOSCURSOR = FBIOSCURSOR;
+unsigned IOCTL_FBIOGCURSOR = FBIOGCURSOR;
+unsigned IOCTL_FBIOSCURPOS = FBIOSCURPOS;
+unsigned IOCTL_FBIOGCURPOS = FBIOGCURPOS;
+unsigned IOCTL_FBIOGCURMAX = FBIOGCURMAX;
+unsigned IOCTL_KIOCTRANS = KIOCTRANS;
+unsigned IOCTL_KIOCSETKEY = KIOCSETKEY;
+unsigned IOCTL_KIOCGETKEY = KIOCGETKEY;
+unsigned IOCTL_KIOCGTRANS = KIOCGTRANS;
+unsigned IOCTL_KIOCCMD = KIOCCMD;
+unsigned IOCTL_KIOCTYPE = KIOCTYPE;
+unsigned IOCTL_KIOCSDIRECT = KIOCSDIRECT;
+unsigned IOCTL_KIOCSKEY = KIOCSKEY;
+unsigned IOCTL_KIOCGKEY = KIOCGKEY;
+unsigned IOCTL_KIOCSLED = KIOCSLED;
+unsigned IOCTL_KIOCGLED = KIOCGLED;
+unsigned IOCTL_KIOCLAYOUT = KIOCLAYOUT;
+unsigned IOCTL_VUIDSFORMAT = VUIDSFORMAT;
+unsigned IOCTL_VUIDGFORMAT = VUIDGFORMAT;
+unsigned IOCTL_STICIO_GXINFO = STICIO_GXINFO;
+unsigned IOCTL_STICIO_RESET = STICIO_RESET;
+unsigned IOCTL_STICIO_STARTQ = STICIO_STARTQ;
+unsigned IOCTL_STICIO_STOPQ = STICIO_STOPQ;
+unsigned IOCTL_UKYOPON_IDENTIFY = UKYOPON_IDENTIFY;
+unsigned IOCTL_URIO_SEND_COMMAND = URIO_SEND_COMMAND;
+unsigned IOCTL_URIO_RECV_COMMAND = URIO_RECV_COMMAND;
+unsigned IOCTL_USB_REQUEST = USB_REQUEST;
+unsigned IOCTL_USB_SETDEBUG = USB_SETDEBUG;
+unsigned IOCTL_USB_DISCOVER = USB_DISCOVER;
+unsigned IOCTL_USB_DEVICEINFO = USB_DEVICEINFO;
+unsigned IOCTL_USB_DEVICEINFO_OLD = USB_DEVICEINFO_OLD;
+unsigned IOCTL_USB_DEVICESTATS = USB_DEVICESTATS;
+unsigned IOCTL_USB_GET_REPORT_DESC = USB_GET_REPORT_DESC;
+unsigned IOCTL_USB_SET_IMMED = USB_SET_IMMED;
+unsigned IOCTL_USB_GET_REPORT = USB_GET_REPORT;
+unsigned IOCTL_USB_SET_REPORT = USB_SET_REPORT;
+unsigned IOCTL_USB_GET_REPORT_ID = USB_GET_REPORT_ID;
+unsigned IOCTL_USB_GET_CONFIG = USB_GET_CONFIG;
+unsigned IOCTL_USB_SET_CONFIG = USB_SET_CONFIG;
+unsigned IOCTL_USB_GET_ALTINTERFACE = USB_GET_ALTINTERFACE;
+unsigned IOCTL_USB_SET_ALTINTERFACE = USB_SET_ALTINTERFACE;
+unsigned IOCTL_USB_GET_NO_ALT = USB_GET_NO_ALT;
+unsigned IOCTL_USB_GET_DEVICE_DESC = USB_GET_DEVICE_DESC;
+unsigned IOCTL_USB_GET_CONFIG_DESC = USB_GET_CONFIG_DESC;
+unsigned IOCTL_USB_GET_INTERFACE_DESC = USB_GET_INTERFACE_DESC;
+unsigned IOCTL_USB_GET_ENDPOINT_DESC = USB_GET_ENDPOINT_DESC;
+unsigned IOCTL_USB_GET_FULL_DESC = USB_GET_FULL_DESC;
+unsigned IOCTL_USB_GET_STRING_DESC = USB_GET_STRING_DESC;
+unsigned IOCTL_USB_DO_REQUEST = USB_DO_REQUEST;
+unsigned IOCTL_USB_GET_DEVICEINFO = USB_GET_DEVICEINFO;
+unsigned IOCTL_USB_GET_DEVICEINFO_OLD = USB_GET_DEVICEINFO_OLD;
+unsigned IOCTL_USB_SET_SHORT_XFER = USB_SET_SHORT_XFER;
+unsigned IOCTL_USB_SET_TIMEOUT = USB_SET_TIMEOUT;
+unsigned IOCTL_USB_SET_BULK_RA = USB_SET_BULK_RA;
+unsigned IOCTL_USB_SET_BULK_WB = USB_SET_BULK_WB;
+unsigned IOCTL_USB_SET_BULK_RA_OPT = USB_SET_BULK_RA_OPT;
+unsigned IOCTL_USB_SET_BULK_WB_OPT = USB_SET_BULK_WB_OPT;
+unsigned IOCTL_USB_GET_CM_OVER_DATA = USB_GET_CM_OVER_DATA;
+unsigned IOCTL_USB_SET_CM_OVER_DATA = USB_SET_CM_OVER_DATA;
+unsigned IOCTL_UTOPPYIOTURBO = UTOPPYIOTURBO;
+unsigned IOCTL_UTOPPYIOCANCEL = UTOPPYIOCANCEL;
+unsigned IOCTL_UTOPPYIOREBOOT = UTOPPYIOREBOOT;
+unsigned IOCTL_UTOPPYIOSTATS = UTOPPYIOSTATS;
+unsigned IOCTL_UTOPPYIORENAME = UTOPPYIORENAME;
+unsigned IOCTL_UTOPPYIOMKDIR = UTOPPYIOMKDIR;
+unsigned IOCTL_UTOPPYIODELETE = UTOPPYIODELETE;
+unsigned IOCTL_UTOPPYIOREADDIR = UTOPPYIOREADDIR;
+unsigned IOCTL_UTOPPYIOREADFILE = UTOPPYIOREADFILE;
+unsigned IOCTL_UTOPPYIOWRITEFILE = UTOPPYIOWRITEFILE;
+unsigned IOCTL_DIOSXDCMD = DIOSXDCMD;
+unsigned IOCTL_VT_OPENQRY = VT_OPENQRY;
+unsigned IOCTL_VT_SETMODE = VT_SETMODE;
+unsigned IOCTL_VT_GETMODE = VT_GETMODE;
+unsigned IOCTL_VT_RELDISP = VT_RELDISP;
+unsigned IOCTL_VT_ACTIVATE = VT_ACTIVATE;
+unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE;
+unsigned IOCTL_VT_GETACTIVE = VT_GETACTIVE;
+unsigned IOCTL_VT_GETSTATE = VT_GETSTATE;
+unsigned IOCTL_KDGETKBENT = KDGETKBENT;
+unsigned IOCTL_KDGKBMODE = KDGKBMODE;
+unsigned IOCTL_KDSKBMODE = KDSKBMODE;
+unsigned IOCTL_KDMKTONE = KDMKTONE;
+unsigned IOCTL_KDSETMODE = KDSETMODE;
+unsigned IOCTL_KDENABIO = KDENABIO;
+unsigned IOCTL_KDDISABIO = KDDISABIO;
+unsigned IOCTL_KDGKBTYPE = KDGKBTYPE;
+unsigned IOCTL_KDGETLED = KDGETLED;
+unsigned IOCTL_KDSETLED = KDSETLED;
+unsigned IOCTL_KDSETRAD = KDSETRAD;
+unsigned IOCTL_VGAPCVTID = VGAPCVTID;
+unsigned IOCTL_CONS_GETVERS = CONS_GETVERS;
+unsigned IOCTL_WSKBDIO_GTYPE = WSKBDIO_GTYPE;
+unsigned IOCTL_WSKBDIO_BELL = WSKBDIO_BELL;
+unsigned IOCTL_WSKBDIO_COMPLEXBELL = WSKBDIO_COMPLEXBELL;
+unsigned IOCTL_WSKBDIO_SETBELL = WSKBDIO_SETBELL;
+unsigned IOCTL_WSKBDIO_GETBELL = WSKBDIO_GETBELL;
+unsigned IOCTL_WSKBDIO_SETDEFAULTBELL = WSKBDIO_SETDEFAULTBELL;
+unsigned IOCTL_WSKBDIO_GETDEFAULTBELL = WSKBDIO_GETDEFAULTBELL;
+unsigned IOCTL_WSKBDIO_SETKEYREPEAT = WSKBDIO_SETKEYREPEAT;
+unsigned IOCTL_WSKBDIO_GETKEYREPEAT = WSKBDIO_GETKEYREPEAT;
+unsigned IOCTL_WSKBDIO_SETDEFAULTKEYREPEAT = WSKBDIO_SETDEFAULTKEYREPEAT;
+unsigned IOCTL_WSKBDIO_GETDEFAULTKEYREPEAT = WSKBDIO_GETDEFAULTKEYREPEAT;
+unsigned IOCTL_WSKBDIO_SETLEDS = WSKBDIO_SETLEDS;
+unsigned IOCTL_WSKBDIO_GETLEDS = WSKBDIO_GETLEDS;
+unsigned IOCTL_WSKBDIO_GETMAP = WSKBDIO_GETMAP;
+unsigned IOCTL_WSKBDIO_SETMAP = WSKBDIO_SETMAP;
+unsigned IOCTL_WSKBDIO_GETENCODING = WSKBDIO_GETENCODING;
+unsigned IOCTL_WSKBDIO_SETENCODING = WSKBDIO_SETENCODING;
+unsigned IOCTL_WSKBDIO_SETMODE = WSKBDIO_SETMODE;
+unsigned IOCTL_WSKBDIO_GETMODE = WSKBDIO_GETMODE;
+unsigned IOCTL_WSKBDIO_SETKEYCLICK = WSKBDIO_SETKEYCLICK;
+unsigned IOCTL_WSKBDIO_GETKEYCLICK = WSKBDIO_GETKEYCLICK;
+unsigned IOCTL_WSKBDIO_GETSCROLL = WSKBDIO_GETSCROLL;
+unsigned IOCTL_WSKBDIO_SETSCROLL = WSKBDIO_SETSCROLL;
+unsigned IOCTL_WSKBDIO_SETVERSION = WSKBDIO_SETVERSION;
+unsigned IOCTL_WSMOUSEIO_GTYPE = WSMOUSEIO_GTYPE;
+unsigned IOCTL_WSMOUSEIO_SRES = WSMOUSEIO_SRES;
+unsigned IOCTL_WSMOUSEIO_SSCALE = WSMOUSEIO_SSCALE;
+unsigned IOCTL_WSMOUSEIO_SRATE = WSMOUSEIO_SRATE;
+unsigned IOCTL_WSMOUSEIO_SCALIBCOORDS = WSMOUSEIO_SCALIBCOORDS;
+unsigned IOCTL_WSMOUSEIO_GCALIBCOORDS = WSMOUSEIO_GCALIBCOORDS;
+unsigned IOCTL_WSMOUSEIO_GETID = WSMOUSEIO_GETID;
+unsigned IOCTL_WSMOUSEIO_GETREPEAT = WSMOUSEIO_GETREPEAT;
+unsigned IOCTL_WSMOUSEIO_SETREPEAT = WSMOUSEIO_SETREPEAT;
+unsigned IOCTL_WSMOUSEIO_SETVERSION = WSMOUSEIO_SETVERSION;
+unsigned IOCTL_WSDISPLAYIO_GTYPE = WSDISPLAYIO_GTYPE;
+unsigned IOCTL_WSDISPLAYIO_GINFO = WSDISPLAYIO_GINFO;
+unsigned IOCTL_WSDISPLAYIO_GETCMAP = WSDISPLAYIO_GETCMAP;
+unsigned IOCTL_WSDISPLAYIO_PUTCMAP = WSDISPLAYIO_PUTCMAP;
+unsigned IOCTL_WSDISPLAYIO_GVIDEO = WSDISPLAYIO_GVIDEO;
+unsigned IOCTL_WSDISPLAYIO_SVIDEO = WSDISPLAYIO_SVIDEO;
+unsigned IOCTL_WSDISPLAYIO_GCURPOS = WSDISPLAYIO_GCURPOS;
+unsigned IOCTL_WSDISPLAYIO_SCURPOS = WSDISPLAYIO_SCURPOS;
+unsigned IOCTL_WSDISPLAYIO_GCURMAX = WSDISPLAYIO_GCURMAX;
+unsigned IOCTL_WSDISPLAYIO_GCURSOR = WSDISPLAYIO_GCURSOR;
+unsigned IOCTL_WSDISPLAYIO_SCURSOR = WSDISPLAYIO_SCURSOR;
+unsigned IOCTL_WSDISPLAYIO_GMODE = WSDISPLAYIO_GMODE;
+unsigned IOCTL_WSDISPLAYIO_SMODE = WSDISPLAYIO_SMODE;
+unsigned IOCTL_WSDISPLAYIO_LDFONT = WSDISPLAYIO_LDFONT;
+unsigned IOCTL_WSDISPLAYIO_ADDSCREEN = WSDISPLAYIO_ADDSCREEN;
+unsigned IOCTL_WSDISPLAYIO_DELSCREEN = WSDISPLAYIO_DELSCREEN;
+unsigned IOCTL_WSDISPLAYIO_SFONT = WSDISPLAYIO_SFONT;
+unsigned IOCTL__O_WSDISPLAYIO_SETKEYBOARD = _O_WSDISPLAYIO_SETKEYBOARD;
+unsigned IOCTL_WSDISPLAYIO_GETPARAM = WSDISPLAYIO_GETPARAM;
+unsigned IOCTL_WSDISPLAYIO_SETPARAM = WSDISPLAYIO_SETPARAM;
+unsigned IOCTL_WSDISPLAYIO_GETACTIVESCREEN = WSDISPLAYIO_GETACTIVESCREEN;
+unsigned IOCTL_WSDISPLAYIO_GETWSCHAR = WSDISPLAYIO_GETWSCHAR;
+unsigned IOCTL_WSDISPLAYIO_PUTWSCHAR = WSDISPLAYIO_PUTWSCHAR;
+unsigned IOCTL_WSDISPLAYIO_DGSCROLL = WSDISPLAYIO_DGSCROLL;
+unsigned IOCTL_WSDISPLAYIO_DSSCROLL = WSDISPLAYIO_DSSCROLL;
+unsigned IOCTL_WSDISPLAYIO_GMSGATTRS = WSDISPLAYIO_GMSGATTRS;
+unsigned IOCTL_WSDISPLAYIO_SMSGATTRS = WSDISPLAYIO_SMSGATTRS;
+unsigned IOCTL_WSDISPLAYIO_GBORDER = WSDISPLAYIO_GBORDER;
+unsigned IOCTL_WSDISPLAYIO_SBORDER = WSDISPLAYIO_SBORDER;
+unsigned IOCTL_WSDISPLAYIO_SSPLASH = WSDISPLAYIO_SSPLASH;
+unsigned IOCTL_WSDISPLAYIO_SPROGRESS = WSDISPLAYIO_SPROGRESS;
+unsigned IOCTL_WSDISPLAYIO_LINEBYTES = WSDISPLAYIO_LINEBYTES;
+unsigned IOCTL_WSDISPLAYIO_SETVERSION = WSDISPLAYIO_SETVERSION;
+unsigned IOCTL_WSMUXIO_ADD_DEVICE = WSMUXIO_ADD_DEVICE;
+unsigned IOCTL_WSMUXIO_REMOVE_DEVICE = WSMUXIO_REMOVE_DEVICE;
+unsigned IOCTL_WSMUXIO_LIST_DEVICES = WSMUXIO_LIST_DEVICES;
+unsigned IOCTL_WSMUXIO_INJECTEVENT = WSMUXIO_INJECTEVENT;
+unsigned IOCTL_WSDISPLAYIO_GET_BUSID = WSDISPLAYIO_GET_BUSID;
+unsigned IOCTL_WSDISPLAYIO_GET_EDID = WSDISPLAYIO_GET_EDID;
+unsigned IOCTL_WSDISPLAYIO_SET_POLLING = WSDISPLAYIO_SET_POLLING;
+unsigned IOCTL_WSDISPLAYIO_GET_FBINFO = WSDISPLAYIO_GET_FBINFO;
+unsigned IOCTL_WSDISPLAYIO_DOBLIT = WSDISPLAYIO_DOBLIT;
+unsigned IOCTL_WSDISPLAYIO_WAITBLIT = WSDISPLAYIO_WAITBLIT;
+unsigned IOCTL_BIOCLOCATE = BIOCLOCATE;
+unsigned IOCTL_BIOCINQ = BIOCINQ;
+unsigned IOCTL_BIOCDISK_NOVOL = BIOCDISK_NOVOL;
+unsigned IOCTL_BIOCDISK = BIOCDISK;
+unsigned IOCTL_BIOCVOL = BIOCVOL;
+unsigned IOCTL_BIOCALARM = BIOCALARM;
+unsigned IOCTL_BIOCBLINK = BIOCBLINK;
+unsigned IOCTL_BIOCSETSTATE = BIOCSETSTATE;
+unsigned IOCTL_BIOCVOLOPS = BIOCVOLOPS;
+unsigned IOCTL_MD_GETCONF = MD_GETCONF;
+unsigned IOCTL_MD_SETCONF = MD_SETCONF;
+unsigned IOCTL_CCDIOCSET = CCDIOCSET;
+unsigned IOCTL_CCDIOCCLR = CCDIOCCLR;
+unsigned IOCTL_CGDIOCSET = CGDIOCSET;
+unsigned IOCTL_CGDIOCCLR = CGDIOCCLR;
+unsigned IOCTL_CGDIOCGET = CGDIOCGET;
+unsigned IOCTL_FSSIOCSET = FSSIOCSET;
+unsigned IOCTL_FSSIOCGET = FSSIOCGET;
+unsigned IOCTL_FSSIOCCLR = FSSIOCCLR;
+unsigned IOCTL_FSSIOFSET = FSSIOFSET;
+unsigned IOCTL_FSSIOFGET = FSSIOFGET;
+unsigned IOCTL_BTDEV_ATTACH = BTDEV_ATTACH;
+unsigned IOCTL_BTDEV_DETACH = BTDEV_DETACH;
+unsigned IOCTL_BTSCO_GETINFO = BTSCO_GETINFO;
+unsigned IOCTL_KTTCP_IO_SEND = KTTCP_IO_SEND;
+unsigned IOCTL_KTTCP_IO_RECV = KTTCP_IO_RECV;
+unsigned IOCTL_IOC_LOCKSTAT_GVERSION = IOC_LOCKSTAT_GVERSION;
+unsigned IOCTL_IOC_LOCKSTAT_ENABLE = IOC_LOCKSTAT_ENABLE;
+unsigned IOCTL_IOC_LOCKSTAT_DISABLE = IOC_LOCKSTAT_DISABLE;
+unsigned IOCTL_VNDIOCSET = VNDIOCSET;
+unsigned IOCTL_VNDIOCCLR = VNDIOCCLR;
+unsigned IOCTL_VNDIOCGET = VNDIOCGET;
+unsigned IOCTL_SPKRTONE = SPKRTONE;
+unsigned IOCTL_SPKRTUNE = SPKRTUNE;
+unsigned IOCTL_SPKRGETVOL = SPKRGETVOL;
+unsigned IOCTL_SPKRSETVOL = SPKRSETVOL;
+#if defined(__x86_64__)
+unsigned IOCTL_NVMM_IOC_CAPABILITY = NVMM_IOC_CAPABILITY;
+unsigned IOCTL_NVMM_IOC_MACHINE_CREATE = NVMM_IOC_MACHINE_CREATE;
+unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY = NVMM_IOC_MACHINE_DESTROY;
+unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE = NVMM_IOC_MACHINE_CONFIGURE;
+unsigned IOCTL_NVMM_IOC_VCPU_CREATE = NVMM_IOC_VCPU_CREATE;
+unsigned IOCTL_NVMM_IOC_VCPU_DESTROY = NVMM_IOC_VCPU_DESTROY;
+unsigned IOCTL_NVMM_IOC_VCPU_CONFIGURE = NVMM_IOC_VCPU_CONFIGURE;
+unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE = NVMM_IOC_VCPU_SETSTATE;
+unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE = NVMM_IOC_VCPU_GETSTATE;
+unsigned IOCTL_NVMM_IOC_VCPU_INJECT = NVMM_IOC_VCPU_INJECT;
+unsigned IOCTL_NVMM_IOC_VCPU_RUN = NVMM_IOC_VCPU_RUN;
+unsigned IOCTL_NVMM_IOC_GPA_MAP = NVMM_IOC_GPA_MAP;
+unsigned IOCTL_NVMM_IOC_GPA_UNMAP = NVMM_IOC_GPA_UNMAP;
+unsigned IOCTL_NVMM_IOC_HVA_MAP = NVMM_IOC_HVA_MAP;
+unsigned IOCTL_NVMM_IOC_HVA_UNMAP = NVMM_IOC_HVA_UNMAP;
+unsigned IOCTL_NVMM_IOC_CTL = NVMM_IOC_CTL;
+#endif
+unsigned IOCTL_SPI_IOCTL_CONFIGURE = SPI_IOCTL_CONFIGURE;
+unsigned IOCTL_SPI_IOCTL_TRANSFER = SPI_IOCTL_TRANSFER;
+unsigned IOCTL_AUTOFSREQUEST = AUTOFSREQUEST;
+unsigned IOCTL_AUTOFSDONE = AUTOFSDONE;
+unsigned IOCTL_BIOCGBLEN = BIOCGBLEN;
+unsigned IOCTL_BIOCSBLEN = BIOCSBLEN;
+unsigned IOCTL_BIOCSETF = BIOCSETF;
+unsigned IOCTL_BIOCFLUSH = BIOCFLUSH;
+unsigned IOCTL_BIOCPROMISC = BIOCPROMISC;
+unsigned IOCTL_BIOCGDLT = BIOCGDLT;
+unsigned IOCTL_BIOCGETIF = BIOCGETIF;
+unsigned IOCTL_BIOCSETIF = BIOCSETIF;
+unsigned IOCTL_BIOCGSTATS = BIOCGSTATS;
+unsigned IOCTL_BIOCGSTATSOLD = BIOCGSTATSOLD;
+unsigned IOCTL_BIOCIMMEDIATE = BIOCIMMEDIATE;
+unsigned IOCTL_BIOCVERSION = BIOCVERSION;
+unsigned IOCTL_BIOCSTCPF = BIOCSTCPF;
+unsigned IOCTL_BIOCSUDPF = BIOCSUDPF;
+unsigned IOCTL_BIOCGHDRCMPLT = BIOCGHDRCMPLT;
+unsigned IOCTL_BIOCSHDRCMPLT = BIOCSHDRCMPLT;
+unsigned IOCTL_BIOCSDLT = BIOCSDLT;
+unsigned IOCTL_BIOCGDLTLIST = BIOCGDLTLIST;
+unsigned IOCTL_BIOCGDIRECTION = BIOCGDIRECTION;
+unsigned IOCTL_BIOCSDIRECTION = BIOCSDIRECTION;
+unsigned IOCTL_BIOCSRTIMEOUT = BIOCSRTIMEOUT;
+unsigned IOCTL_BIOCGRTIMEOUT = BIOCGRTIMEOUT;
+unsigned IOCTL_BIOCGFEEDBACK = BIOCGFEEDBACK;
+unsigned IOCTL_BIOCSFEEDBACK = BIOCSFEEDBACK;
+unsigned IOCTL_GRESADDRS = GRESADDRS;
+unsigned IOCTL_GRESADDRD = GRESADDRD;
+unsigned IOCTL_GREGADDRS = GREGADDRS;
+unsigned IOCTL_GREGADDRD = GREGADDRD;
+unsigned IOCTL_GRESPROTO = GRESPROTO;
+unsigned IOCTL_GREGPROTO = GREGPROTO;
+unsigned IOCTL_GRESSOCK = GRESSOCK;
+unsigned IOCTL_GREDSOCK = GREDSOCK;
+unsigned IOCTL_PPPIOCGRAWIN = PPPIOCGRAWIN;
+unsigned IOCTL_PPPIOCGFLAGS = PPPIOCGFLAGS;
+unsigned IOCTL_PPPIOCSFLAGS = PPPIOCSFLAGS;
+unsigned IOCTL_PPPIOCGASYNCMAP = PPPIOCGASYNCMAP;
+unsigned IOCTL_PPPIOCSASYNCMAP = PPPIOCSASYNCMAP;
+unsigned IOCTL_PPPIOCGUNIT = PPPIOCGUNIT;
+unsigned IOCTL_PPPIOCGRASYNCMAP = PPPIOCGRASYNCMAP;
+unsigned IOCTL_PPPIOCSRASYNCMAP = PPPIOCSRASYNCMAP;
+unsigned IOCTL_PPPIOCGMRU = PPPIOCGMRU;
+unsigned IOCTL_PPPIOCSMRU = PPPIOCSMRU;
+unsigned IOCTL_PPPIOCSMAXCID = PPPIOCSMAXCID;
+unsigned IOCTL_PPPIOCGXASYNCMAP = PPPIOCGXASYNCMAP;
+unsigned IOCTL_PPPIOCSXASYNCMAP = PPPIOCSXASYNCMAP;
+unsigned IOCTL_PPPIOCXFERUNIT = PPPIOCXFERUNIT;
+unsigned IOCTL_PPPIOCSCOMPRESS = PPPIOCSCOMPRESS;
+unsigned IOCTL_PPPIOCGNPMODE = PPPIOCGNPMODE;
+unsigned IOCTL_PPPIOCSNPMODE = PPPIOCSNPMODE;
+unsigned IOCTL_PPPIOCGIDLE = PPPIOCGIDLE;
+unsigned IOCTL_PPPIOCGMTU = PPPIOCGMTU;
+unsigned IOCTL_PPPIOCSMTU = PPPIOCSMTU;
+unsigned IOCTL_SIOCGPPPSTATS = SIOCGPPPSTATS;
+unsigned IOCTL_SIOCGPPPCSTATS = SIOCGPPPCSTATS;
+unsigned IOCTL_IOC_NPF_VERSION = IOC_NPF_VERSION;
+unsigned IOCTL_IOC_NPF_SWITCH = IOC_NPF_SWITCH;
+unsigned IOCTL_IOC_NPF_LOAD = IOC_NPF_LOAD;
+unsigned IOCTL_IOC_NPF_TABLE = IOC_NPF_TABLE;
+unsigned IOCTL_IOC_NPF_STATS = IOC_NPF_STATS;
+unsigned IOCTL_IOC_NPF_SAVE = IOC_NPF_SAVE;
+unsigned IOCTL_IOC_NPF_RULE = IOC_NPF_RULE;
+unsigned IOCTL_IOC_NPF_CONN_LOOKUP = IOC_NPF_CONN_LOOKUP;
+unsigned IOCTL_IOC_NPF_TABLE_REPLACE = IOC_NPF_TABLE_REPLACE;
+unsigned IOCTL_PPPOESETPARMS = PPPOESETPARMS;
+unsigned IOCTL_PPPOEGETPARMS = PPPOEGETPARMS;
+unsigned IOCTL_PPPOEGETSESSION = PPPOEGETSESSION;
+unsigned IOCTL_SPPPGETAUTHCFG = SPPPGETAUTHCFG;
+unsigned IOCTL_SPPPSETAUTHCFG = SPPPSETAUTHCFG;
+unsigned IOCTL_SPPPGETLCPCFG = SPPPGETLCPCFG;
+unsigned IOCTL_SPPPSETLCPCFG = SPPPSETLCPCFG;
+unsigned IOCTL_SPPPGETSTATUS = SPPPGETSTATUS;
+unsigned IOCTL_SPPPGETSTATUSNCP = SPPPGETSTATUSNCP;
+unsigned IOCTL_SPPPGETIDLETO = SPPPGETIDLETO;
+unsigned IOCTL_SPPPSETIDLETO = SPPPSETIDLETO;
+unsigned IOCTL_SPPPGETAUTHFAILURES = SPPPGETAUTHFAILURES;
+unsigned IOCTL_SPPPSETAUTHFAILURE = SPPPSETAUTHFAILURE;
+unsigned IOCTL_SPPPSETDNSOPTS = SPPPSETDNSOPTS;
+unsigned IOCTL_SPPPGETDNSOPTS = SPPPGETDNSOPTS;
+unsigned IOCTL_SPPPGETDNSADDRS = SPPPGETDNSADDRS;
+unsigned IOCTL_SPPPSETKEEPALIVE = SPPPSETKEEPALIVE;
+unsigned IOCTL_SPPPGETKEEPALIVE = SPPPGETKEEPALIVE;
+unsigned IOCTL_SRT_GETNRT = SRT_GETNRT;
+unsigned IOCTL_SRT_GETRT = SRT_GETRT;
+unsigned IOCTL_SRT_SETRT = SRT_SETRT;
+unsigned IOCTL_SRT_DELRT = SRT_DELRT;
+unsigned IOCTL_SRT_SFLAGS = SRT_SFLAGS;
+unsigned IOCTL_SRT_GFLAGS = SRT_GFLAGS;
+unsigned IOCTL_SRT_SGFLAGS = SRT_SGFLAGS;
+unsigned IOCTL_SRT_DEBUG = SRT_DEBUG;
+unsigned IOCTL_TAPGIFNAME = TAPGIFNAME;
+unsigned IOCTL_TUNSDEBUG = TUNSDEBUG;
+unsigned IOCTL_TUNGDEBUG = TUNGDEBUG;
+unsigned IOCTL_TUNSIFMODE = TUNSIFMODE;
+unsigned IOCTL_TUNSLMODE = TUNSLMODE;
+unsigned IOCTL_TUNSIFHEAD = TUNSIFHEAD;
+unsigned IOCTL_TUNGIFHEAD = TUNGIFHEAD;
+unsigned IOCTL_DIOCSTART = DIOCSTART;
+unsigned IOCTL_DIOCSTOP = DIOCSTOP;
+unsigned IOCTL_DIOCADDRULE = DIOCADDRULE;
+unsigned IOCTL_DIOCGETRULES = DIOCGETRULES;
+unsigned IOCTL_DIOCGETRULE = DIOCGETRULE;
+unsigned IOCTL_DIOCSETLCK = DIOCSETLCK;
+unsigned IOCTL_DIOCCLRSTATES = DIOCCLRSTATES;
+unsigned IOCTL_DIOCGETSTATE = DIOCGETSTATE;
+unsigned IOCTL_DIOCSETSTATUSIF = DIOCSETSTATUSIF;
+unsigned IOCTL_DIOCGETSTATUS = DIOCGETSTATUS;
+unsigned IOCTL_DIOCCLRSTATUS = DIOCCLRSTATUS;
+unsigned IOCTL_DIOCNATLOOK = DIOCNATLOOK;
+unsigned IOCTL_DIOCSETDEBUG = DIOCSETDEBUG;
+unsigned IOCTL_DIOCGETSTATES = DIOCGETSTATES;
+unsigned IOCTL_DIOCCHANGERULE = DIOCCHANGERULE;
+unsigned IOCTL_DIOCSETTIMEOUT = DIOCSETTIMEOUT;
+unsigned IOCTL_DIOCGETTIMEOUT = DIOCGETTIMEOUT;
+unsigned IOCTL_DIOCADDSTATE = DIOCADDSTATE;
+unsigned IOCTL_DIOCCLRRULECTRS = DIOCCLRRULECTRS;
+unsigned IOCTL_DIOCGETLIMIT = DIOCGETLIMIT;
+unsigned IOCTL_DIOCSETLIMIT = DIOCSETLIMIT;
+unsigned IOCTL_DIOCKILLSTATES = DIOCKILLSTATES;
+unsigned IOCTL_DIOCSTARTALTQ = DIOCSTARTALTQ;
+unsigned IOCTL_DIOCSTOPALTQ = DIOCSTOPALTQ;
+unsigned IOCTL_DIOCADDALTQ = DIOCADDALTQ;
+unsigned IOCTL_DIOCGETALTQS = DIOCGETALTQS;
+unsigned IOCTL_DIOCGETALTQ = DIOCGETALTQ;
+unsigned IOCTL_DIOCCHANGEALTQ = DIOCCHANGEALTQ;
+unsigned IOCTL_DIOCGETQSTATS = DIOCGETQSTATS;
+unsigned IOCTL_DIOCBEGINADDRS = DIOCBEGINADDRS;
+unsigned IOCTL_DIOCADDADDR = DIOCADDADDR;
+unsigned IOCTL_DIOCGETADDRS = DIOCGETADDRS;
+unsigned IOCTL_DIOCGETADDR = DIOCGETADDR;
+unsigned IOCTL_DIOCCHANGEADDR = DIOCCHANGEADDR;
+unsigned IOCTL_DIOCADDSTATES = DIOCADDSTATES;
+unsigned IOCTL_DIOCGETRULESETS = DIOCGETRULESETS;
+unsigned IOCTL_DIOCGETRULESET = DIOCGETRULESET;
+unsigned IOCTL_DIOCRCLRTABLES = DIOCRCLRTABLES;
+unsigned IOCTL_DIOCRADDTABLES = DIOCRADDTABLES;
+unsigned IOCTL_DIOCRDELTABLES = DIOCRDELTABLES;
+unsigned IOCTL_DIOCRGETTABLES = DIOCRGETTABLES;
+unsigned IOCTL_DIOCRGETTSTATS = DIOCRGETTSTATS;
+unsigned IOCTL_DIOCRCLRTSTATS = DIOCRCLRTSTATS;
+unsigned IOCTL_DIOCRCLRADDRS = DIOCRCLRADDRS;
+unsigned IOCTL_DIOCRADDADDRS = DIOCRADDADDRS;
+unsigned IOCTL_DIOCRDELADDRS = DIOCRDELADDRS;
+unsigned IOCTL_DIOCRSETADDRS = DIOCRSETADDRS;
+unsigned IOCTL_DIOCRGETADDRS = DIOCRGETADDRS;
+unsigned IOCTL_DIOCRGETASTATS = DIOCRGETASTATS;
+unsigned IOCTL_DIOCRCLRASTATS = DIOCRCLRASTATS;
+unsigned IOCTL_DIOCRTSTADDRS = DIOCRTSTADDRS;
+unsigned IOCTL_DIOCRSETTFLAGS = DIOCRSETTFLAGS;
+unsigned IOCTL_DIOCRINADEFINE = DIOCRINADEFINE;
+unsigned IOCTL_DIOCOSFPFLUSH = DIOCOSFPFLUSH;
+unsigned IOCTL_DIOCOSFPADD = DIOCOSFPADD;
+unsigned IOCTL_DIOCOSFPGET = DIOCOSFPGET;
+unsigned IOCTL_DIOCXBEGIN = DIOCXBEGIN;
+unsigned IOCTL_DIOCXCOMMIT = DIOCXCOMMIT;
+unsigned IOCTL_DIOCXROLLBACK = DIOCXROLLBACK;
+unsigned IOCTL_DIOCGETSRCNODES = DIOCGETSRCNODES;
+unsigned IOCTL_DIOCCLRSRCNODES = DIOCCLRSRCNODES;
+unsigned IOCTL_DIOCSETHOSTID = DIOCSETHOSTID;
+unsigned IOCTL_DIOCIGETIFACES = DIOCIGETIFACES;
+unsigned IOCTL_DIOCSETIFFLAG = DIOCSETIFFLAG;
+unsigned IOCTL_DIOCCLRIFFLAG = DIOCCLRIFFLAG;
+unsigned IOCTL_DIOCKILLSRCNODES = DIOCKILLSRCNODES;
+unsigned IOCTL_SLIOCGUNIT = SLIOCGUNIT;
+unsigned IOCTL_SIOCGBTINFO = SIOCGBTINFO;
+unsigned IOCTL_SIOCGBTINFOA = SIOCGBTINFOA;
+unsigned IOCTL_SIOCNBTINFO = SIOCNBTINFO;
+unsigned IOCTL_SIOCSBTFLAGS = SIOCSBTFLAGS;
+unsigned IOCTL_SIOCSBTPOLICY = SIOCSBTPOLICY;
+unsigned IOCTL_SIOCSBTPTYPE = SIOCSBTPTYPE;
+unsigned IOCTL_SIOCGBTSTATS = SIOCGBTSTATS;
+unsigned IOCTL_SIOCZBTSTATS = SIOCZBTSTATS;
+unsigned IOCTL_SIOCBTDUMP = SIOCBTDUMP;
+unsigned IOCTL_SIOCSBTSCOMTU = SIOCSBTSCOMTU;
+unsigned IOCTL_SIOCGBTFEAT = SIOCGBTFEAT;
+unsigned IOCTL_SIOCADNAT = SIOCADNAT;
+unsigned IOCTL_SIOCRMNAT = SIOCRMNAT;
+unsigned IOCTL_SIOCGNATS = SIOCGNATS;
+unsigned IOCTL_SIOCGNATL = SIOCGNATL;
+unsigned IOCTL_SIOCPURGENAT = SIOCPURGENAT;
+unsigned IOCTL_SIOCCONNECTX = SIOCCONNECTX;
+unsigned IOCTL_SIOCCONNECTXDEL = SIOCCONNECTXDEL;
+unsigned IOCTL_SIOCSIFINFO_FLAGS = SIOCSIFINFO_FLAGS;
+unsigned IOCTL_SIOCAADDRCTL_POLICY = SIOCAADDRCTL_POLICY;
+unsigned IOCTL_SIOCDADDRCTL_POLICY = SIOCDADDRCTL_POLICY;
+unsigned IOCTL_SMBIOC_OPENSESSION = SMBIOC_OPENSESSION;
+unsigned IOCTL_SMBIOC_OPENSHARE = SMBIOC_OPENSHARE;
+unsigned IOCTL_SMBIOC_REQUEST = SMBIOC_REQUEST;
+unsigned IOCTL_SMBIOC_SETFLAGS = SMBIOC_SETFLAGS;
+unsigned IOCTL_SMBIOC_LOOKUP = SMBIOC_LOOKUP;
+unsigned IOCTL_SMBIOC_READ = SMBIOC_READ;
+unsigned IOCTL_SMBIOC_WRITE = SMBIOC_WRITE;
+unsigned IOCTL_AGPIOC_INFO = AGPIOC_INFO;
+unsigned IOCTL_AGPIOC_ACQUIRE = AGPIOC_ACQUIRE;
+unsigned IOCTL_AGPIOC_RELEASE = AGPIOC_RELEASE;
+unsigned IOCTL_AGPIOC_SETUP = AGPIOC_SETUP;
+unsigned IOCTL_AGPIOC_ALLOCATE = AGPIOC_ALLOCATE;
+unsigned IOCTL_AGPIOC_DEALLOCATE = AGPIOC_DEALLOCATE;
+unsigned IOCTL_AGPIOC_BIND = AGPIOC_BIND;
+unsigned IOCTL_AGPIOC_UNBIND = AGPIOC_UNBIND;
+unsigned IOCTL_AUDIO_GETINFO = AUDIO_GETINFO;
+unsigned IOCTL_AUDIO_SETINFO = AUDIO_SETINFO;
+unsigned IOCTL_AUDIO_DRAIN = AUDIO_DRAIN;
+unsigned IOCTL_AUDIO_FLUSH = AUDIO_FLUSH;
+unsigned IOCTL_AUDIO_WSEEK = AUDIO_WSEEK;
+unsigned IOCTL_AUDIO_RERROR = AUDIO_RERROR;
+unsigned IOCTL_AUDIO_GETDEV = AUDIO_GETDEV;
+unsigned IOCTL_AUDIO_GETENC = AUDIO_GETENC;
+unsigned IOCTL_AUDIO_GETFD = AUDIO_GETFD;
+unsigned IOCTL_AUDIO_SETFD = AUDIO_SETFD;
+unsigned IOCTL_AUDIO_PERROR = AUDIO_PERROR;
+unsigned IOCTL_AUDIO_GETIOFFS = AUDIO_GETIOFFS;
+unsigned IOCTL_AUDIO_GETOOFFS = AUDIO_GETOOFFS;
+unsigned IOCTL_AUDIO_GETPROPS = AUDIO_GETPROPS;
+unsigned IOCTL_AUDIO_GETBUFINFO = AUDIO_GETBUFINFO;
+unsigned IOCTL_AUDIO_SETCHAN = AUDIO_SETCHAN;
+unsigned IOCTL_AUDIO_GETCHAN = AUDIO_GETCHAN;
+unsigned IOCTL_AUDIO_QUERYFORMAT = AUDIO_QUERYFORMAT;
+unsigned IOCTL_AUDIO_GETFORMAT = AUDIO_GETFORMAT;
+unsigned IOCTL_AUDIO_SETFORMAT = AUDIO_SETFORMAT;
+unsigned IOCTL_AUDIO_MIXER_READ = AUDIO_MIXER_READ;
+unsigned IOCTL_AUDIO_MIXER_WRITE = AUDIO_MIXER_WRITE;
+unsigned IOCTL_AUDIO_MIXER_DEVINFO = AUDIO_MIXER_DEVINFO;
+unsigned IOCTL_ATAIOCCOMMAND = ATAIOCCOMMAND;
+unsigned IOCTL_ATABUSIOSCAN = ATABUSIOSCAN;
+unsigned IOCTL_ATABUSIORESET = ATABUSIORESET;
+unsigned IOCTL_ATABUSIODETACH = ATABUSIODETACH;
+unsigned IOCTL_CDIOCPLAYTRACKS = CDIOCPLAYTRACKS;
+unsigned IOCTL_CDIOCPLAYBLOCKS = CDIOCPLAYBLOCKS;
+unsigned IOCTL_CDIOCREADSUBCHANNEL = CDIOCREADSUBCHANNEL;
+unsigned IOCTL_CDIOREADTOCHEADER = CDIOREADTOCHEADER;
+unsigned IOCTL_CDIOREADTOCENTRIES = CDIOREADTOCENTRIES;
+unsigned IOCTL_CDIOREADMSADDR = CDIOREADMSADDR;
+unsigned IOCTL_CDIOCSETPATCH = CDIOCSETPATCH;
+unsigned IOCTL_CDIOCGETVOL = CDIOCGETVOL;
+unsigned IOCTL_CDIOCSETVOL = CDIOCSETVOL;
+unsigned IOCTL_CDIOCSETMONO = CDIOCSETMONO;
+unsigned IOCTL_CDIOCSETSTEREO = CDIOCSETSTEREO;
+unsigned IOCTL_CDIOCSETMUTE = CDIOCSETMUTE;
+unsigned IOCTL_CDIOCSETLEFT = CDIOCSETLEFT;
+unsigned IOCTL_CDIOCSETRIGHT = CDIOCSETRIGHT;
+unsigned IOCTL_CDIOCSETDEBUG = CDIOCSETDEBUG;
+unsigned IOCTL_CDIOCCLRDEBUG = CDIOCCLRDEBUG;
+unsigned IOCTL_CDIOCPAUSE = CDIOCPAUSE;
+unsigned IOCTL_CDIOCRESUME = CDIOCRESUME;
+unsigned IOCTL_CDIOCRESET = CDIOCRESET;
+unsigned IOCTL_CDIOCSTART = CDIOCSTART;
+unsigned IOCTL_CDIOCSTOP = CDIOCSTOP;
+unsigned IOCTL_CDIOCEJECT = CDIOCEJECT;
+unsigned IOCTL_CDIOCALLOW = CDIOCALLOW;
+unsigned IOCTL_CDIOCPREVENT = CDIOCPREVENT;
+unsigned IOCTL_CDIOCCLOSE = CDIOCCLOSE;
+unsigned IOCTL_CDIOCPLAYMSF = CDIOCPLAYMSF;
+unsigned IOCTL_CDIOCLOADUNLOAD = CDIOCLOADUNLOAD;
+unsigned IOCTL_CHIOMOVE = CHIOMOVE;
+unsigned IOCTL_CHIOEXCHANGE = CHIOEXCHANGE;
+unsigned IOCTL_CHIOPOSITION = CHIOPOSITION;
+unsigned IOCTL_CHIOGPICKER = CHIOGPICKER;
+unsigned IOCTL_CHIOSPICKER = CHIOSPICKER;
+unsigned IOCTL_CHIOGPARAMS = CHIOGPARAMS;
+unsigned IOCTL_CHIOIELEM = CHIOIELEM;
+unsigned IOCTL_OCHIOGSTATUS = OCHIOGSTATUS;
+unsigned IOCTL_CHIOGSTATUS = CHIOGSTATUS;
+unsigned IOCTL_CHIOSVOLTAG = CHIOSVOLTAG;
+unsigned IOCTL_CLOCKCTL_SETTIMEOFDAY = CLOCKCTL_SETTIMEOFDAY;
+unsigned IOCTL_CLOCKCTL_ADJTIME = CLOCKCTL_ADJTIME;
+unsigned IOCTL_CLOCKCTL_CLOCK_SETTIME = CLOCKCTL_CLOCK_SETTIME;
+unsigned IOCTL_CLOCKCTL_NTP_ADJTIME = CLOCKCTL_NTP_ADJTIME;
+unsigned IOCTL_IOC_CPU_SETSTATE = IOC_CPU_SETSTATE;
+unsigned IOCTL_IOC_CPU_GETSTATE = IOC_CPU_GETSTATE;
+unsigned IOCTL_IOC_CPU_GETCOUNT = IOC_CPU_GETCOUNT;
+unsigned IOCTL_IOC_CPU_MAPID = IOC_CPU_MAPID;
+unsigned IOCTL_IOC_CPU_UCODE_GET_VERSION = IOC_CPU_UCODE_GET_VERSION;
+unsigned IOCTL_IOC_CPU_UCODE_APPLY = IOC_CPU_UCODE_APPLY;
+unsigned IOCTL_DIOCGDINFO = DIOCGDINFO;
+unsigned IOCTL_DIOCSDINFO = DIOCSDINFO;
+unsigned IOCTL_DIOCWDINFO = DIOCWDINFO;
+unsigned IOCTL_DIOCRFORMAT = DIOCRFORMAT;
+unsigned IOCTL_DIOCWFORMAT = DIOCWFORMAT;
+unsigned IOCTL_DIOCSSTEP = DIOCSSTEP;
+unsigned IOCTL_DIOCSRETRIES = DIOCSRETRIES;
+unsigned IOCTL_DIOCKLABEL = DIOCKLABEL;
+unsigned IOCTL_DIOCWLABEL = DIOCWLABEL;
+unsigned IOCTL_DIOCSBAD = DIOCSBAD;
+unsigned IOCTL_DIOCEJECT = DIOCEJECT;
+unsigned IOCTL_ODIOCEJECT = ODIOCEJECT;
+unsigned IOCTL_DIOCLOCK = DIOCLOCK;
+unsigned IOCTL_DIOCGDEFLABEL = DIOCGDEFLABEL;
+unsigned IOCTL_DIOCCLRLABEL = DIOCCLRLABEL;
+unsigned IOCTL_DIOCGCACHE = DIOCGCACHE;
+unsigned IOCTL_DIOCSCACHE = DIOCSCACHE;
+unsigned IOCTL_DIOCCACHESYNC = DIOCCACHESYNC;
+unsigned IOCTL_DIOCBSLIST = DIOCBSLIST;
+unsigned IOCTL_DIOCBSFLUSH = DIOCBSFLUSH;
+unsigned IOCTL_DIOCAWEDGE = DIOCAWEDGE;
+unsigned IOCTL_DIOCGWEDGEINFO = DIOCGWEDGEINFO;
+unsigned IOCTL_DIOCDWEDGE = DIOCDWEDGE;
+unsigned IOCTL_DIOCLWEDGES = DIOCLWEDGES;
+unsigned IOCTL_DIOCGSTRATEGY = DIOCGSTRATEGY;
+unsigned IOCTL_DIOCSSTRATEGY = DIOCSSTRATEGY;
+unsigned IOCTL_DIOCGDISKINFO = DIOCGDISKINFO;
+unsigned IOCTL_DIOCTUR = DIOCTUR;
+unsigned IOCTL_DIOCMWEDGES = DIOCMWEDGES;
+unsigned IOCTL_DIOCGSECTORSIZE = DIOCGSECTORSIZE;
+unsigned IOCTL_DIOCGMEDIASIZE = DIOCGMEDIASIZE;
+unsigned IOCTL_DIOCRMWEDGES = DIOCRMWEDGES;
+unsigned IOCTL_DRVDETACHDEV = DRVDETACHDEV;
+unsigned IOCTL_DRVRESCANBUS = DRVRESCANBUS;
+unsigned IOCTL_DRVCTLCOMMAND = DRVCTLCOMMAND;
+unsigned IOCTL_DRVRESUMEDEV = DRVRESUMEDEV;
+unsigned IOCTL_DRVLISTDEV = DRVLISTDEV;
+unsigned IOCTL_DRVGETEVENT = DRVGETEVENT;
+unsigned IOCTL_DRVSUSPENDDEV = DRVSUSPENDDEV;
+unsigned IOCTL_DVD_READ_STRUCT = DVD_READ_STRUCT;
+unsigned IOCTL_DVD_WRITE_STRUCT = DVD_WRITE_STRUCT;
+unsigned IOCTL_DVD_AUTH = DVD_AUTH;
+unsigned IOCTL_ENVSYS_GETDICTIONARY = ENVSYS_GETDICTIONARY;
+unsigned IOCTL_ENVSYS_SETDICTIONARY = ENVSYS_SETDICTIONARY;
+unsigned IOCTL_ENVSYS_REMOVEPROPS = ENVSYS_REMOVEPROPS;
+unsigned IOCTL_ENVSYS_GTREDATA = ENVSYS_GTREDATA;
+unsigned IOCTL_ENVSYS_GTREINFO = ENVSYS_GTREINFO;
+unsigned IOCTL_KFILTER_BYFILTER = KFILTER_BYFILTER;
+unsigned IOCTL_KFILTER_BYNAME = KFILTER_BYNAME;
+unsigned IOCTL_FDIOCGETOPTS = FDIOCGETOPTS;
+unsigned IOCTL_FDIOCSETOPTS = FDIOCSETOPTS;
+unsigned IOCTL_FDIOCSETFORMAT = FDIOCSETFORMAT;
+unsigned IOCTL_FDIOCGETFORMAT = FDIOCGETFORMAT;
+unsigned IOCTL_FDIOCFORMAT_TRACK = FDIOCFORMAT_TRACK;
+unsigned IOCTL_FIOCLEX = FIOCLEX;
+unsigned IOCTL_FIONCLEX = FIONCLEX;
+unsigned IOCTL_FIOSEEKDATA = FIOSEEKDATA;
+unsigned IOCTL_FIOSEEKHOLE = FIOSEEKHOLE;
+unsigned IOCTL_FIONREAD = FIONREAD;
+unsigned IOCTL_FIONBIO = FIONBIO;
+unsigned IOCTL_FIOASYNC = FIOASYNC;
+unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+unsigned IOCTL_OFIOGETBMAP = OFIOGETBMAP;
+unsigned IOCTL_FIOGETBMAP = FIOGETBMAP;
+unsigned IOCTL_FIONWRITE = FIONWRITE;
+unsigned IOCTL_FIONSPACE = FIONSPACE;
+unsigned IOCTL_GPIOINFO = GPIOINFO;
+unsigned IOCTL_GPIOSET = GPIOSET;
+unsigned IOCTL_GPIOUNSET = GPIOUNSET;
+unsigned IOCTL_GPIOREAD = GPIOREAD;
+unsigned IOCTL_GPIOWRITE = GPIOWRITE;
+unsigned IOCTL_GPIOTOGGLE = GPIOTOGGLE;
+unsigned IOCTL_GPIOATTACH = GPIOATTACH;
+unsigned IOCTL_PTIOCNETBSD = PTIOCNETBSD;
+unsigned IOCTL_PTIOCSUNOS = PTIOCSUNOS;
+unsigned IOCTL_PTIOCLINUX = PTIOCLINUX;
+unsigned IOCTL_PTIOCFREEBSD = PTIOCFREEBSD;
+unsigned IOCTL_PTIOCULTRIX = PTIOCULTRIX;
+unsigned IOCTL_TIOCHPCL = TIOCHPCL;
+unsigned IOCTL_TIOCGETP = TIOCGETP;
+unsigned IOCTL_TIOCSETP = TIOCSETP;
+unsigned IOCTL_TIOCSETN = TIOCSETN;
+unsigned IOCTL_TIOCSETC = TIOCSETC;
+unsigned IOCTL_TIOCGETC = TIOCGETC;
+unsigned IOCTL_TIOCLBIS = TIOCLBIS;
+unsigned IOCTL_TIOCLBIC = TIOCLBIC;
+unsigned IOCTL_TIOCLSET = TIOCLSET;
+unsigned IOCTL_TIOCLGET = TIOCLGET;
+unsigned IOCTL_TIOCSLTC = TIOCSLTC;
+unsigned IOCTL_TIOCGLTC = TIOCGLTC;
+unsigned IOCTL_OTIOCCONS = OTIOCCONS;
+unsigned IOCTL_JOY_SETTIMEOUT = JOY_SETTIMEOUT;
+unsigned IOCTL_JOY_GETTIMEOUT = JOY_GETTIMEOUT;
+unsigned IOCTL_JOY_SET_X_OFFSET = JOY_SET_X_OFFSET;
+unsigned IOCTL_JOY_SET_Y_OFFSET = JOY_SET_Y_OFFSET;
+unsigned IOCTL_JOY_GET_X_OFFSET = JOY_GET_X_OFFSET;
+unsigned IOCTL_JOY_GET_Y_OFFSET = JOY_GET_Y_OFFSET;
+unsigned IOCTL_OKIOCGSYMBOL = OKIOCGSYMBOL;
+unsigned IOCTL_OKIOCGVALUE = OKIOCGVALUE;
+unsigned IOCTL_KIOCGSIZE = KIOCGSIZE;
+unsigned IOCTL_KIOCGVALUE = KIOCGVALUE;
+unsigned IOCTL_KIOCGSYMBOL = KIOCGSYMBOL;
+unsigned IOCTL_LUAINFO = LUAINFO;
+unsigned IOCTL_LUACREATE = LUACREATE;
+unsigned IOCTL_LUADESTROY = LUADESTROY;
+unsigned IOCTL_LUAREQUIRE = LUAREQUIRE;
+unsigned IOCTL_LUALOAD = LUALOAD;
+unsigned IOCTL_MIDI_PRETIME = MIDI_PRETIME;
+unsigned IOCTL_MIDI_MPUMODE = MIDI_MPUMODE;
+unsigned IOCTL_MIDI_MPUCMD = MIDI_MPUCMD;
+unsigned IOCTL_SEQUENCER_RESET = SEQUENCER_RESET;
+unsigned IOCTL_SEQUENCER_SYNC = SEQUENCER_SYNC;
+unsigned IOCTL_SEQUENCER_INFO = SEQUENCER_INFO;
+unsigned IOCTL_SEQUENCER_CTRLRATE = SEQUENCER_CTRLRATE;
+unsigned IOCTL_SEQUENCER_GETOUTCOUNT = SEQUENCER_GETOUTCOUNT;
+unsigned IOCTL_SEQUENCER_GETINCOUNT = SEQUENCER_GETINCOUNT;
+unsigned IOCTL_SEQUENCER_RESETSAMPLES = SEQUENCER_RESETSAMPLES;
+unsigned IOCTL_SEQUENCER_NRSYNTHS = SEQUENCER_NRSYNTHS;
+unsigned IOCTL_SEQUENCER_NRMIDIS = SEQUENCER_NRMIDIS;
+unsigned IOCTL_SEQUENCER_THRESHOLD = SEQUENCER_THRESHOLD;
+unsigned IOCTL_SEQUENCER_MEMAVL = SEQUENCER_MEMAVL;
+unsigned IOCTL_SEQUENCER_PANIC = SEQUENCER_PANIC;
+unsigned IOCTL_SEQUENCER_OUTOFBAND = SEQUENCER_OUTOFBAND;
+unsigned IOCTL_SEQUENCER_GETTIME = SEQUENCER_GETTIME;
+unsigned IOCTL_SEQUENCER_TMR_TIMEBASE = SEQUENCER_TMR_TIMEBASE;
+unsigned IOCTL_SEQUENCER_TMR_START = SEQUENCER_TMR_START;
+unsigned IOCTL_SEQUENCER_TMR_STOP = SEQUENCER_TMR_STOP;
+unsigned IOCTL_SEQUENCER_TMR_CONTINUE = SEQUENCER_TMR_CONTINUE;
+unsigned IOCTL_SEQUENCER_TMR_TEMPO = SEQUENCER_TMR_TEMPO;
+unsigned IOCTL_SEQUENCER_TMR_SOURCE = SEQUENCER_TMR_SOURCE;
+unsigned IOCTL_SEQUENCER_TMR_METRONOME = SEQUENCER_TMR_METRONOME;
+unsigned IOCTL_SEQUENCER_TMR_SELECT = SEQUENCER_TMR_SELECT;
+unsigned IOCTL_MTIOCTOP = MTIOCTOP;
+unsigned IOCTL_MTIOCGET = MTIOCGET;
+unsigned IOCTL_MTIOCIEOT = MTIOCIEOT;
+unsigned IOCTL_MTIOCEEOT = MTIOCEEOT;
+unsigned IOCTL_MTIOCRDSPOS = MTIOCRDSPOS;
+unsigned IOCTL_MTIOCRDHPOS = MTIOCRDHPOS;
+unsigned IOCTL_MTIOCSLOCATE = MTIOCSLOCATE;
+unsigned IOCTL_MTIOCHLOCATE = MTIOCHLOCATE;
+unsigned IOCTL_POWER_EVENT_RECVDICT = POWER_EVENT_RECVDICT;
+unsigned IOCTL_POWER_IOC_GET_TYPE = POWER_IOC_GET_TYPE;
+unsigned IOCTL_RIOCGINFO = RIOCGINFO;
+unsigned IOCTL_RIOCSINFO = RIOCSINFO;
+unsigned IOCTL_RIOCSSRCH = RIOCSSRCH;
+unsigned IOCTL_RNDGETENTCNT = RNDGETENTCNT;
+unsigned IOCTL_RNDGETSRCNUM = RNDGETSRCNUM;
+unsigned IOCTL_RNDGETSRCNAME = RNDGETSRCNAME;
+unsigned IOCTL_RNDCTL = RNDCTL;
+unsigned IOCTL_RNDADDDATA = RNDADDDATA;
+unsigned IOCTL_RNDGETPOOLSTAT = RNDGETPOOLSTAT;
+unsigned IOCTL_RNDGETESTNUM = RNDGETESTNUM;
+unsigned IOCTL_RNDGETESTNAME = RNDGETESTNAME;
+unsigned IOCTL_SCIOCGET = SCIOCGET;
+unsigned IOCTL_SCIOCSET = SCIOCSET;
+unsigned IOCTL_SCIOCRESTART = SCIOCRESTART;
+unsigned IOCTL_SCIOC_USE_ADF = SCIOC_USE_ADF;
+unsigned IOCTL_SCIOCCOMMAND = SCIOCCOMMAND;
+unsigned IOCTL_SCIOCDEBUG = SCIOCDEBUG;
+unsigned IOCTL_SCIOCIDENTIFY = SCIOCIDENTIFY;
+unsigned IOCTL_OSCIOCIDENTIFY = OSCIOCIDENTIFY;
+unsigned IOCTL_SCIOCDECONFIG = SCIOCDECONFIG;
+unsigned IOCTL_SCIOCRECONFIG = SCIOCRECONFIG;
+unsigned IOCTL_SCIOCRESET = SCIOCRESET;
+unsigned IOCTL_SCBUSIOSCAN = SCBUSIOSCAN;
+unsigned IOCTL_SCBUSIORESET = SCBUSIORESET;
+unsigned IOCTL_SCBUSIODETACH = SCBUSIODETACH;
+unsigned IOCTL_SCBUSACCEL = SCBUSACCEL;
+unsigned IOCTL_SCBUSIOLLSCAN = SCBUSIOLLSCAN;
+unsigned IOCTL_SIOCSHIWAT = SIOCSHIWAT;
+unsigned IOCTL_SIOCGHIWAT = SIOCGHIWAT;
+unsigned IOCTL_SIOCSLOWAT = SIOCSLOWAT;
+unsigned IOCTL_SIOCGLOWAT = SIOCGLOWAT;
+unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+unsigned IOCTL_SIOCPEELOFF = SIOCPEELOFF;
+unsigned IOCTL_SIOCADDRT = SIOCADDRT;
+unsigned IOCTL_SIOCDELRT = SIOCDELRT;
+unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+unsigned IOCTL_SIOCDIFADDR = SIOCDIFADDR;
+unsigned IOCTL_SIOCAIFADDR = SIOCAIFADDR;
+unsigned IOCTL_SIOCGIFALIAS = SIOCGIFALIAS;
+unsigned IOCTL_SIOCGIFAFLAG_IN = SIOCGIFAFLAG_IN;
+unsigned IOCTL_SIOCALIFADDR = SIOCALIFADDR;
+unsigned IOCTL_SIOCGLIFADDR = SIOCGLIFADDR;
+unsigned IOCTL_SIOCDLIFADDR = SIOCDLIFADDR;
+unsigned IOCTL_SIOCSIFADDRPREF = SIOCSIFADDRPREF;
+unsigned IOCTL_SIOCGIFADDRPREF = SIOCGIFADDRPREF;
+unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
+unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
+unsigned IOCTL_SIOCSIFMEDIA = SIOCSIFMEDIA;
+unsigned IOCTL_SIOCGIFMEDIA = SIOCGIFMEDIA;
+unsigned IOCTL_SIOCSIFGENERIC = SIOCSIFGENERIC;
+unsigned IOCTL_SIOCGIFGENERIC = SIOCGIFGENERIC;
+unsigned IOCTL_SIOCSIFPHYADDR = SIOCSIFPHYADDR;
+unsigned IOCTL_SIOCGIFPSRCADDR = SIOCGIFPSRCADDR;
+unsigned IOCTL_SIOCGIFPDSTADDR = SIOCGIFPDSTADDR;
+unsigned IOCTL_SIOCDIFPHYADDR = SIOCDIFPHYADDR;
+unsigned IOCTL_SIOCSLIFPHYADDR = SIOCSLIFPHYADDR;
+unsigned IOCTL_SIOCGLIFPHYADDR = SIOCGLIFPHYADDR;
+unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+unsigned IOCTL_SIOCSDRVSPEC = SIOCSDRVSPEC;
+unsigned IOCTL_SIOCGDRVSPEC = SIOCGDRVSPEC;
+unsigned IOCTL_SIOCIFCREATE = SIOCIFCREATE;
+unsigned IOCTL_SIOCIFDESTROY = SIOCIFDESTROY;
+unsigned IOCTL_SIOCIFGCLONERS = SIOCIFGCLONERS;
+unsigned IOCTL_SIOCGIFDLT = SIOCGIFDLT;
+unsigned IOCTL_SIOCGIFCAP = SIOCGIFCAP;
+unsigned IOCTL_SIOCSIFCAP = SIOCSIFCAP;
+unsigned IOCTL_SIOCSVH = SIOCSVH;
+unsigned IOCTL_SIOCGVH = SIOCGVH;
+unsigned IOCTL_SIOCINITIFADDR = SIOCINITIFADDR;
+unsigned IOCTL_SIOCGIFDATA = SIOCGIFDATA;
+unsigned IOCTL_SIOCZIFDATA = SIOCZIFDATA;
+unsigned IOCTL_SIOCGLINKSTR = SIOCGLINKSTR;
+unsigned IOCTL_SIOCSLINKSTR = SIOCSLINKSTR;
+unsigned IOCTL_SIOCGETHERCAP = SIOCGETHERCAP;
+unsigned IOCTL_SIOCGIFINDEX = SIOCGIFINDEX;
+unsigned IOCTL_SIOCSETHERCAP = SIOCSETHERCAP;
+unsigned IOCTL_SIOCSIFDESCR = SIOCSIFDESCR;
+unsigned IOCTL_SIOCGIFDESCR = SIOCGIFDESCR;
+unsigned IOCTL_SIOCGUMBINFO = SIOCGUMBINFO;
+unsigned IOCTL_SIOCSUMBPARAM = SIOCSUMBPARAM;
+unsigned IOCTL_SIOCGUMBPARAM = SIOCGUMBPARAM;
+unsigned IOCTL_SIOCSETPFSYNC = SIOCSETPFSYNC;
+unsigned IOCTL_SIOCGETPFSYNC = SIOCGETPFSYNC;
+unsigned IOCTL_PPS_IOC_CREATE = PPS_IOC_CREATE;
+unsigned IOCTL_PPS_IOC_DESTROY = PPS_IOC_DESTROY;
+unsigned IOCTL_PPS_IOC_SETPARAMS = PPS_IOC_SETPARAMS;
+unsigned IOCTL_PPS_IOC_GETPARAMS = PPS_IOC_GETPARAMS;
+unsigned IOCTL_PPS_IOC_GETCAP = PPS_IOC_GETCAP;
+unsigned IOCTL_PPS_IOC_FETCH = PPS_IOC_FETCH;
+unsigned IOCTL_PPS_IOC_KCBIND = PPS_IOC_KCBIND;
+unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+unsigned IOCTL_TIOCFLUSH = TIOCFLUSH;
+unsigned IOCTL_TIOCGETA = TIOCGETA;
+unsigned IOCTL_TIOCSETA = TIOCSETA;
+unsigned IOCTL_TIOCSETAW = TIOCSETAW;
+unsigned IOCTL_TIOCSETAF = TIOCSETAF;
+unsigned IOCTL_TIOCGETD = TIOCGETD;
+unsigned IOCTL_TIOCSETD = TIOCSETD;
+unsigned IOCTL_TIOCGLINED = TIOCGLINED;
+unsigned IOCTL_TIOCSLINED = TIOCSLINED;
+unsigned IOCTL_TIOCSBRK = TIOCSBRK;
+unsigned IOCTL_TIOCCBRK = TIOCCBRK;
+unsigned IOCTL_TIOCSDTR = TIOCSDTR;
+unsigned IOCTL_TIOCCDTR = TIOCCDTR;
+unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+unsigned IOCTL_TIOCSTI = TIOCSTI;
+unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+unsigned IOCTL_TIOCPKT = TIOCPKT;
+unsigned IOCTL_TIOCSTOP = TIOCSTOP;
+unsigned IOCTL_TIOCSTART = TIOCSTART;
+unsigned IOCTL_TIOCMSET = TIOCMSET;
+unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+unsigned IOCTL_TIOCMGET = TIOCMGET;
+unsigned IOCTL_TIOCREMOTE = TIOCREMOTE;
+unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+unsigned IOCTL_TIOCUCNTL = TIOCUCNTL;
+unsigned IOCTL_TIOCSTAT = TIOCSTAT;
+unsigned IOCTL_TIOCGSID = TIOCGSID;
+unsigned IOCTL_TIOCCONS = TIOCCONS;
+unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+unsigned IOCTL_TIOCEXT = TIOCEXT;
+unsigned IOCTL_TIOCSIG = TIOCSIG;
+unsigned IOCTL_TIOCDRAIN = TIOCDRAIN;
+unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS;
+unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS;
+unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP;
+unsigned IOCTL_TIOCRCVFRAME = TIOCRCVFRAME;
+unsigned IOCTL_TIOCXMTFRAME = TIOCXMTFRAME;
+unsigned IOCTL_TIOCPTMGET = TIOCPTMGET;
+unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT;
+unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME;
+unsigned IOCTL_TIOCSQSIZE = TIOCSQSIZE;
+unsigned IOCTL_TIOCGQSIZE = TIOCGQSIZE;
+unsigned IOCTL_VERIEXEC_LOAD = VERIEXEC_LOAD;
+unsigned IOCTL_VERIEXEC_TABLESIZE = VERIEXEC_TABLESIZE;
+unsigned IOCTL_VERIEXEC_DELETE = VERIEXEC_DELETE;
+unsigned IOCTL_VERIEXEC_QUERY = VERIEXEC_QUERY;
+unsigned IOCTL_VERIEXEC_DUMP = VERIEXEC_DUMP;
+unsigned IOCTL_VERIEXEC_FLUSH = VERIEXEC_FLUSH;
+unsigned IOCTL_VIDIOC_QUERYCAP = VIDIOC_QUERYCAP;
+unsigned IOCTL_VIDIOC_RESERVED = VIDIOC_RESERVED;
+unsigned IOCTL_VIDIOC_ENUM_FMT = VIDIOC_ENUM_FMT;
+unsigned IOCTL_VIDIOC_G_FMT = VIDIOC_G_FMT;
+unsigned IOCTL_VIDIOC_S_FMT = VIDIOC_S_FMT;
+unsigned IOCTL_VIDIOC_REQBUFS = VIDIOC_REQBUFS;
+unsigned IOCTL_VIDIOC_QUERYBUF = VIDIOC_QUERYBUF;
+unsigned IOCTL_VIDIOC_G_FBUF = VIDIOC_G_FBUF;
+unsigned IOCTL_VIDIOC_S_FBUF = VIDIOC_S_FBUF;
+unsigned IOCTL_VIDIOC_OVERLAY = VIDIOC_OVERLAY;
+unsigned IOCTL_VIDIOC_QBUF = VIDIOC_QBUF;
+unsigned IOCTL_VIDIOC_DQBUF = VIDIOC_DQBUF;
+unsigned IOCTL_VIDIOC_STREAMON = VIDIOC_STREAMON;
+unsigned IOCTL_VIDIOC_STREAMOFF = VIDIOC_STREAMOFF;
+unsigned IOCTL_VIDIOC_G_PARM = VIDIOC_G_PARM;
+unsigned IOCTL_VIDIOC_S_PARM = VIDIOC_S_PARM;
+unsigned IOCTL_VIDIOC_G_STD = VIDIOC_G_STD;
+unsigned IOCTL_VIDIOC_S_STD = VIDIOC_S_STD;
+unsigned IOCTL_VIDIOC_ENUMSTD = VIDIOC_ENUMSTD;
+unsigned IOCTL_VIDIOC_ENUMINPUT = VIDIOC_ENUMINPUT;
+unsigned IOCTL_VIDIOC_G_CTRL = VIDIOC_G_CTRL;
+unsigned IOCTL_VIDIOC_S_CTRL = VIDIOC_S_CTRL;
+unsigned IOCTL_VIDIOC_G_TUNER = VIDIOC_G_TUNER;
+unsigned IOCTL_VIDIOC_S_TUNER = VIDIOC_S_TUNER;
+unsigned IOCTL_VIDIOC_G_AUDIO = VIDIOC_G_AUDIO;
+unsigned IOCTL_VIDIOC_S_AUDIO = VIDIOC_S_AUDIO;
+unsigned IOCTL_VIDIOC_QUERYCTRL = VIDIOC_QUERYCTRL;
+unsigned IOCTL_VIDIOC_QUERYMENU = VIDIOC_QUERYMENU;
+unsigned IOCTL_VIDIOC_G_INPUT = VIDIOC_G_INPUT;
+unsigned IOCTL_VIDIOC_S_INPUT = VIDIOC_S_INPUT;
+unsigned IOCTL_VIDIOC_G_OUTPUT = VIDIOC_G_OUTPUT;
+unsigned IOCTL_VIDIOC_S_OUTPUT = VIDIOC_S_OUTPUT;
+unsigned IOCTL_VIDIOC_ENUMOUTPUT = VIDIOC_ENUMOUTPUT;
+unsigned IOCTL_VIDIOC_G_AUDOUT = VIDIOC_G_AUDOUT;
+unsigned IOCTL_VIDIOC_S_AUDOUT = VIDIOC_S_AUDOUT;
+unsigned IOCTL_VIDIOC_G_MODULATOR = VIDIOC_G_MODULATOR;
+unsigned IOCTL_VIDIOC_S_MODULATOR = VIDIOC_S_MODULATOR;
+unsigned IOCTL_VIDIOC_G_FREQUENCY = VIDIOC_G_FREQUENCY;
+unsigned IOCTL_VIDIOC_S_FREQUENCY = VIDIOC_S_FREQUENCY;
+unsigned IOCTL_VIDIOC_CROPCAP = VIDIOC_CROPCAP;
+unsigned IOCTL_VIDIOC_G_CROP = VIDIOC_G_CROP;
+unsigned IOCTL_VIDIOC_S_CROP = VIDIOC_S_CROP;
+unsigned IOCTL_VIDIOC_G_JPEGCOMP = VIDIOC_G_JPEGCOMP;
+unsigned IOCTL_VIDIOC_S_JPEGCOMP = VIDIOC_S_JPEGCOMP;
+unsigned IOCTL_VIDIOC_QUERYSTD = VIDIOC_QUERYSTD;
+unsigned IOCTL_VIDIOC_TRY_FMT = VIDIOC_TRY_FMT;
+unsigned IOCTL_VIDIOC_ENUMAUDIO = VIDIOC_ENUMAUDIO;
+unsigned IOCTL_VIDIOC_ENUMAUDOUT = VIDIOC_ENUMAUDOUT;
+unsigned IOCTL_VIDIOC_G_PRIORITY = VIDIOC_G_PRIORITY;
+unsigned IOCTL_VIDIOC_S_PRIORITY = VIDIOC_S_PRIORITY;
+unsigned IOCTL_VIDIOC_ENUM_FRAMESIZES = VIDIOC_ENUM_FRAMESIZES;
+unsigned IOCTL_VIDIOC_ENUM_FRAMEINTERVALS = VIDIOC_ENUM_FRAMEINTERVALS;
+unsigned IOCTL_WDOGIOC_GMODE = WDOGIOC_GMODE;
+unsigned IOCTL_WDOGIOC_SMODE = WDOGIOC_SMODE;
+unsigned IOCTL_WDOGIOC_WHICH = WDOGIOC_WHICH;
+unsigned IOCTL_WDOGIOC_TICKLE = WDOGIOC_TICKLE;
+unsigned IOCTL_WDOGIOC_GTICKLER = WDOGIOC_GTICKLER;
+unsigned IOCTL_WDOGIOC_GWDOGS = WDOGIOC_GWDOGS;
+unsigned IOCTL_KCOV_IOC_SETBUFSIZE = KCOV_IOC_SETBUFSIZE;
+unsigned IOCTL_KCOV_IOC_ENABLE = KCOV_IOC_ENABLE;
+unsigned IOCTL_KCOV_IOC_DISABLE = KCOV_IOC_DISABLE;
+unsigned IOCTL_IPMICTL_RECEIVE_MSG_TRUNC = IPMICTL_RECEIVE_MSG_TRUNC;
+unsigned IOCTL_IPMICTL_RECEIVE_MSG = IPMICTL_RECEIVE_MSG;
+unsigned IOCTL_IPMICTL_SEND_COMMAND = IPMICTL_SEND_COMMAND;
+unsigned IOCTL_IPMICTL_REGISTER_FOR_CMD = IPMICTL_REGISTER_FOR_CMD;
+unsigned IOCTL_IPMICTL_UNREGISTER_FOR_CMD = IPMICTL_UNREGISTER_FOR_CMD;
+unsigned IOCTL_IPMICTL_SET_GETS_EVENTS_CMD = IPMICTL_SET_GETS_EVENTS_CMD;
+unsigned IOCTL_IPMICTL_SET_MY_ADDRESS_CMD = IPMICTL_SET_MY_ADDRESS_CMD;
+unsigned IOCTL_IPMICTL_GET_MY_ADDRESS_CMD = IPMICTL_GET_MY_ADDRESS_CMD;
+unsigned IOCTL_IPMICTL_SET_MY_LUN_CMD = IPMICTL_SET_MY_LUN_CMD;
+unsigned IOCTL_IPMICTL_GET_MY_LUN_CMD = IPMICTL_GET_MY_LUN_CMD;
+unsigned IOCTL_SNDCTL_DSP_RESET = SNDCTL_DSP_RESET;
+unsigned IOCTL_SNDCTL_DSP_SYNC = SNDCTL_DSP_SYNC;
+unsigned IOCTL_SNDCTL_DSP_SPEED = SNDCTL_DSP_SPEED;
+unsigned IOCTL_SOUND_PCM_READ_RATE = SOUND_PCM_READ_RATE;
+unsigned IOCTL_SNDCTL_DSP_STEREO = SNDCTL_DSP_STEREO;
+unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE;
+unsigned IOCTL_SNDCTL_DSP_SETFMT = SNDCTL_DSP_SETFMT;
+unsigned IOCTL_SOUND_PCM_READ_BITS = SOUND_PCM_READ_BITS;
+unsigned IOCTL_SNDCTL_DSP_CHANNELS = SNDCTL_DSP_CHANNELS;
+unsigned IOCTL_SOUND_PCM_READ_CHANNELS = SOUND_PCM_READ_CHANNELS;
+unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER;
+unsigned IOCTL_SOUND_PCM_READ_FILTER = SOUND_PCM_READ_FILTER;
+unsigned IOCTL_SNDCTL_DSP_POST = SNDCTL_DSP_POST;
+unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE = SNDCTL_DSP_SUBDIVIDE;
+unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT = SNDCTL_DSP_SETFRAGMENT;
+unsigned IOCTL_SNDCTL_DSP_GETFMTS = SNDCTL_DSP_GETFMTS;
+unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE;
+unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE;
+unsigned IOCTL_SNDCTL_DSP_NONBLOCK = SNDCTL_DSP_NONBLOCK;
+unsigned IOCTL_SNDCTL_DSP_GETCAPS = SNDCTL_DSP_GETCAPS;
+unsigned IOCTL_SNDCTL_DSP_GETTRIGGER = SNDCTL_DSP_GETTRIGGER;
+unsigned IOCTL_SNDCTL_DSP_SETTRIGGER = SNDCTL_DSP_SETTRIGGER;
+unsigned IOCTL_SNDCTL_DSP_GETIPTR = SNDCTL_DSP_GETIPTR;
+unsigned IOCTL_SNDCTL_DSP_GETOPTR = SNDCTL_DSP_GETOPTR;
+unsigned IOCTL_SNDCTL_DSP_MAPINBUF = SNDCTL_DSP_MAPINBUF;
+unsigned IOCTL_SNDCTL_DSP_MAPOUTBUF = SNDCTL_DSP_MAPOUTBUF;
+unsigned IOCTL_SNDCTL_DSP_SETSYNCRO = SNDCTL_DSP_SETSYNCRO;
+unsigned IOCTL_SNDCTL_DSP_SETDUPLEX = SNDCTL_DSP_SETDUPLEX;
+unsigned IOCTL_SNDCTL_DSP_PROFILE = SNDCTL_DSP_PROFILE;
+unsigned IOCTL_SNDCTL_DSP_GETODELAY = SNDCTL_DSP_GETODELAY;
+unsigned IOCTL_SOUND_MIXER_INFO = SOUND_MIXER_INFO;
+unsigned IOCTL_SOUND_OLD_MIXER_INFO = SOUND_OLD_MIXER_INFO;
+unsigned IOCTL_OSS_GETVERSION = OSS_GETVERSION;
+unsigned IOCTL_SNDCTL_SYSINFO = SNDCTL_SYSINFO;
+unsigned IOCTL_SNDCTL_AUDIOINFO = SNDCTL_AUDIOINFO;
+unsigned IOCTL_SNDCTL_ENGINEINFO = SNDCTL_ENGINEINFO;
+unsigned IOCTL_SNDCTL_DSP_GETPLAYVOL = SNDCTL_DSP_GETPLAYVOL;
+unsigned IOCTL_SNDCTL_DSP_SETPLAYVOL = SNDCTL_DSP_SETPLAYVOL;
+unsigned IOCTL_SNDCTL_DSP_GETRECVOL = SNDCTL_DSP_GETRECVOL;
+unsigned IOCTL_SNDCTL_DSP_SETRECVOL = SNDCTL_DSP_SETRECVOL;
+unsigned IOCTL_SNDCTL_DSP_SKIP = SNDCTL_DSP_SKIP;
+unsigned IOCTL_SNDCTL_DSP_SILENCE = SNDCTL_DSP_SILENCE;
+
+const int si_SEGV_MAPERR = SEGV_MAPERR;
+const int si_SEGV_ACCERR = SEGV_ACCERR;
+
+const int modctl_load = MODCTL_LOAD;
+const int modctl_unload = MODCTL_UNLOAD;
+const int modctl_stat = MODCTL_STAT;
+const int modctl_exists = MODCTL_EXISTS;
+
+const unsigned SHA1_CTX_sz = sizeof(SHA1_CTX);
+const unsigned SHA1_return_length = SHA1_DIGEST_STRING_LENGTH;
+
+const unsigned MD4_CTX_sz = sizeof(MD4_CTX);
+const unsigned MD4_return_length = MD4_DIGEST_STRING_LENGTH;
+
+const unsigned RMD160_CTX_sz = sizeof(RMD160_CTX);
+const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH;
+
+const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
+const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
+
+const unsigned fpos_t_sz = sizeof(fpos_t);
+
+const unsigned MD2_CTX_sz = sizeof(MD2_CTX);
+const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH;
+
+#define SHA2_CONST(LEN)                                                      \
+  const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX);                 \
+  const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \
+  const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH;          \
+  const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH
+
+SHA2_CONST(224);
+SHA2_CONST(256);
+SHA2_CONST(384);
+SHA2_CONST(512);
+
+#undef SHA2_CONST
+
+const int unvis_valid = UNVIS_VALID;
+const int unvis_validpush = UNVIS_VALIDPUSH;
+}  // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_fileno);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+COMPILER_CHECK(sizeof(__sanitizer_FILE) <= sizeof(FILE));
+CHECK_SIZE_AND_OFFSET(FILE, _p);
+CHECK_SIZE_AND_OFFSET(FILE, _r);
+CHECK_SIZE_AND_OFFSET(FILE, _w);
+CHECK_SIZE_AND_OFFSET(FILE, _flags);
+CHECK_SIZE_AND_OFFSET(FILE, _file);
+CHECK_SIZE_AND_OFFSET(FILE, _bf);
+CHECK_SIZE_AND_OFFSET(FILE, _lbfsize);
+CHECK_SIZE_AND_OFFSET(FILE, _cookie);
+CHECK_SIZE_AND_OFFSET(FILE, _close);
+CHECK_SIZE_AND_OFFSET(FILE, _read);
+CHECK_SIZE_AND_OFFSET(FILE, _seek);
+CHECK_SIZE_AND_OFFSET(FILE, _write);
+CHECK_SIZE_AND_OFFSET(FILE, _ext);
+CHECK_SIZE_AND_OFFSET(FILE, _up);
+CHECK_SIZE_AND_OFFSET(FILE, _ur);
+CHECK_SIZE_AND_OFFSET(FILE, _ubuf);
+CHECK_SIZE_AND_OFFSET(FILE, _nbuf);
+CHECK_SIZE_AND_OFFSET(FILE, _flush);
+CHECK_SIZE_AND_OFFSET(FILE, _lb_unused);
+CHECK_SIZE_AND_OFFSET(FILE, _blksize);
+CHECK_SIZE_AND_OFFSET(FILE, _offset);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+CHECK_TYPE_SIZE(modctl_load_t);
+CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_filename);
+CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_flags);
+CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_props);
+CHECK_SIZE_AND_OFFSET(modctl_load_t, ml_propslen);
+
+// Compat with 9.0
+struct statvfs90 {
+  unsigned long f_flag;
+  unsigned long f_bsize;
+  unsigned long f_frsize;
+  unsigned long f_iosize;
+
+  u64 f_blocks;
+  u64 f_bfree;
+  u64 f_bavail;
+  u64 f_bresvd;
+
+  u64 f_files;
+  u64 f_ffree;
+  u64 f_favail;
+  u64 f_fresvd;
+
+  u64 f_syncreads;
+  u64 f_syncwrites;
+
+  u64 f_asyncreads;
+  u64 f_asyncwrites;
+
+  struct {
+    s32 __fsid_val[2];
+  } f_fsidx;
+  unsigned long f_fsid;
+  unsigned long f_namemax;
+  u32 f_owner;
+
+  u32 f_spare[4];
+
+  char f_fstypename[32];
+  char f_mntonname[32];
+  char f_mntfromname[32];
+};
+unsigned struct_statvfs90_sz = sizeof(struct statvfs90);
+
+#endif  // SANITIZER_NETBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h
new file mode 100644
index 0000000000..d80280d9bf
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h
@@ -0,0 +1,2422 @@
+//===-- sanitizer_platform_limits_netbsd.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_NETBSD_H
+#define SANITIZER_PLATFORM_LIMITS_NETBSD_H
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+void *__sanitizer_get_link_map_by_dlopen_handle(void *handle);
+# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+    (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
+
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_stack_t_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+extern unsigned struct_sembuf_sz;
+
+extern unsigned struct_kevent_sz;
+extern unsigned struct_FTS_sz;
+extern unsigned struct_FTSENT_sz;
+
+extern unsigned struct_regex_sz;
+extern unsigned struct_regmatch_sz;
+
+extern unsigned struct_fstab_sz;
+
+struct __sanitizer_regmatch {
+  OFF_T rm_so;
+  OFF_T rm_eo;
+};
+
+typedef struct __sanitizer_modctl_load {
+  const char *ml_filename;
+  int ml_flags;
+  const char *ml_props;
+  uptr ml_propslen;
+} __sanitizer_modctl_load_t;
+extern const int modctl_load;
+extern const int modctl_unload;
+extern const int modctl_stat;
+extern const int modctl_exists;
+
+union __sanitizer_sigval {
+  int sival_int;
+  uptr sival_ptr;
+};
+
+struct __sanitizer_sigevent {
+  int sigev_notify;
+  int sigev_signo;
+  union __sanitizer_sigval sigev_value;
+  uptr sigev_notify_function;
+  uptr sigev_notify_attributes;
+};
+
+struct __sanitizer_aiocb {
+  u64 aio_offset;
+  uptr aio_buf;
+  uptr aio_nbytes;
+  int aio_fildes;
+  int aio_lio_opcode;
+  int aio_reqprio;
+  struct __sanitizer_sigevent aio_sigevent;
+  int _state;
+  int _errno;
+  long _retval;
+};
+
+struct __sanitizer_sem_t {
+  uptr data[5];
+};
+
+struct __sanitizer_ipc_perm {
+  u32 uid;
+  u32 gid;
+  u32 cuid;
+  u32 cgid;
+  u32 mode;
+  unsigned short _seq;
+  long _key;
+};
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+  unsigned long shm_segsz;
+  u32 shm_lpid;
+  u32 shm_cpid;
+  unsigned int shm_nattch;
+  u64 shm_atime;
+  u64 shm_dtime;
+  u64 shm_ctime;
+  void *_shm_internal;
+};
+
+struct __sanitizer_protoent {
+  char *p_name;
+  char **p_aliases;
+  int p_proto;
+};
+
+struct __sanitizer_netent {
+  char *n_name;
+  char **n_aliases;
+  int n_addrtype;
+  u32 n_net;
+};
+
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timex_sz;
+extern unsigned struct_statvfs_sz;
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  unsigned int ifa_flags;
+  void *ifa_addr;     // (struct sockaddr *)
+  void *ifa_netmask;  // (struct sockaddr *)
+  void *ifa_dstaddr;  // (struct sockaddr *)
+  void *ifa_data;
+  unsigned int ifa_addrflags;
+};
+
+typedef unsigned int __sanitizer_socklen_t;
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+typedef long long __sanitizer_time_t;
+typedef int __sanitizer_suseconds_t;
+
+struct __sanitizer_timeval {
+  __sanitizer_time_t tv_sec;
+  __sanitizer_suseconds_t tv_usec;
+};
+
+struct __sanitizer_itimerval {
+  struct __sanitizer_timeval it_interval;
+  struct __sanitizer_timeval it_value;
+};
+
+struct __sanitizer_timespec {
+  __sanitizer_time_t tv_sec;
+  long tv_nsec;
+};
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  int pw_uid;
+  int pw_gid;
+  __sanitizer_time_t pw_change;
+  char *pw_class;
+  char *pw_gecos;
+  char *pw_dir;
+  char *pw_shell;
+  __sanitizer_time_t pw_expire;
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+struct __sanitizer_timeb {
+  __sanitizer_time_t time;
+  unsigned short millitm;
+  short timezone;
+  short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+  long int tm_gmtoff;
+  const char *tm_zone;
+};
+
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+
+struct __sanitizer_mmsghdr {
+  struct __sanitizer_msghdr msg_hdr;
+  unsigned int msg_len;
+};
+
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+
+struct __sanitizer_dirent {
+  u64 d_fileno;
+  u16 d_reclen;
+  // more fields that we don't care about
+};
+
+typedef int __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+typedef u32 __sanitizer___kernel_uid_t;
+typedef u32 __sanitizer___kernel_gid_t;
+typedef u64 __sanitizer___kernel_off_t;
+typedef struct {
+  u32 fds_bits[8];
+} __sanitizer___kernel_fd_set;
+
+typedef struct {
+  unsigned int pta_magic;
+  int pta_flags;
+  void *pta_private;
+} __sanitizer_pthread_attr_t;
+
+struct __sanitizer_sigset_t {
+  // uint32_t * 4
+  unsigned int __bits[4];
+};
+
+struct __sanitizer_siginfo {
+  // The size is determined by looking at sizeof of real siginfo_t on linux.
+  u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
+                                                  __sanitizer_siginfo *siginfo,
+                                                  void *uctx);
+
+struct __sanitizer_sigaction {
+  union {
+    __sanitizer_sighandler_ptr handler;
+    __sanitizer_sigactionhandler_ptr sigaction;
+  };
+  __sanitizer_sigset_t sa_mask;
+  int sa_flags;
+};
+
+extern unsigned struct_sigaltstack_sz;
+
+typedef unsigned int __sanitizer_sigset13_t;
+
+struct __sanitizer_sigaction13 {
+  __sanitizer_sighandler_ptr osa_handler;
+  __sanitizer_sigset13_t osa_mask;
+  int osa_flags;
+};
+
+struct __sanitizer_sigaltstack {
+  void *ss_sp;
+  uptr ss_size;
+  int ss_flags;
+};
+
+typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
+
+struct __sanitizer_kernel_sigaction_t {
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, void *info, void *ctx);
+  };
+  unsigned long sa_flags;
+  void (*sa_restorer)(void);
+  __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+  uptr dlpi_addr;
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+  short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+#if defined(__sparc__) && defined(_LP64)
+  int __ai_pad0;
+#endif
+  unsigned ai_addrlen;
+#if defined(__alpha__) || (defined(__i386__) && defined(_LP64))
+  int __ai_pad0;
+#endif
+  char *ai_canonname;
+  void *ai_addr;
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+typedef unsigned __sanitizer_nfds_t;
+
+typedef int __sanitizer_lwpid_t;
+
+struct __sanitizer_glob_t {
+  uptr gl_pathc;
+  uptr gl_matchc;
+  uptr gl_offs;
+  int gl_flags;
+  char **gl_pathv;
+  int (*gl_errfunc)(const char *, int);
+  void (*gl_closedir)(void *dirp);
+  struct dirent *(*gl_readdir)(void *dirp);
+  void *(*gl_opendir)(const char *);
+  int (*gl_lstat)(const char *, void * /* struct stat* */);
+  int (*gl_stat)(const char *, void * /* struct stat* */);
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+extern int struct_ttyent_sz;
+
+extern int ptrace_pt_io;
+extern int ptrace_pt_lwpinfo;
+extern int ptrace_pt_set_event_mask;
+extern int ptrace_pt_get_event_mask;
+extern int ptrace_pt_get_process_state;
+extern int ptrace_pt_set_siginfo;
+extern int ptrace_pt_get_siginfo;
+extern int ptrace_pt_lwpstatus;
+extern int ptrace_pt_lwpnext;
+extern int ptrace_piod_read_d;
+extern int ptrace_piod_write_d;
+extern int ptrace_piod_read_i;
+extern int ptrace_piod_write_i;
+extern int ptrace_piod_read_auxv;
+extern int ptrace_pt_setregs;
+extern int ptrace_pt_getregs;
+extern int ptrace_pt_setfpregs;
+extern int ptrace_pt_getfpregs;
+extern int ptrace_pt_setdbregs;
+extern int ptrace_pt_getdbregs;
+
+struct __sanitizer_ptrace_io_desc {
+  int piod_op;
+  void *piod_offs;
+  void *piod_addr;
+  uptr piod_len;
+};
+
+struct __sanitizer_ptrace_lwpinfo {
+  __sanitizer_lwpid_t pl_lwpid;
+  int pl_event;
+};
+
+struct __sanitizer_ptrace_lwpstatus {
+  __sanitizer_lwpid_t pl_lwpid;
+  __sanitizer_sigset_t pl_sigpend;
+  __sanitizer_sigset_t pl_sigmask;
+  char pl_name[20];
+  void *pl_private;
+};
+
+extern unsigned struct_ptrace_ptrace_io_desc_struct_sz;
+extern unsigned struct_ptrace_ptrace_lwpinfo_struct_sz;
+extern unsigned struct_ptrace_ptrace_lwpstatus_struct_sz;
+extern unsigned struct_ptrace_ptrace_event_struct_sz;
+extern unsigned struct_ptrace_ptrace_siginfo_struct_sz;
+
+extern unsigned struct_ptrace_reg_struct_sz;
+extern unsigned struct_ptrace_fpreg_struct_sz;
+extern unsigned struct_ptrace_dbreg_struct_sz;
+
+struct __sanitizer_wordexp_t {
+  uptr we_wordc;
+  char **we_wordv;
+  uptr we_offs;
+  char *we_strings;
+  uptr we_nbytes;
+};
+
+struct __sanitizer_FILE {
+  unsigned char *_p;
+  int _r;
+  int _w;
+  unsigned short _flags;
+  short _file;
+  struct {
+    unsigned char *_base;
+    int _size;
+  } _bf;
+  int _lbfsize;
+  void *_cookie;
+  int (*_close)(void *ptr);
+  u64 (*_read)(void *, void *, uptr);
+  u64 (*_seek)(void *, u64, int);
+  uptr (*_write)(void *, const void *, uptr);
+  struct {
+    unsigned char *_base;
+    int _size;
+  } _ext;
+  unsigned char *_up;
+  int _ur;
+  unsigned char _ubuf[3];
+  unsigned char _nbuf[1];
+  int (*_flush)(void *ptr);
+  char _lb_unused[sizeof(uptr)];
+  int _blksize;
+  u64 _offset;
+};
+#define SANITIZER_HAS_STRUCT_FILE 1
+
+extern int shmctl_ipc_stat;
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+};
+
+struct __sanitizer_ttyent {
+  char *ty_name;
+  char *ty_getty;
+  char *ty_type;
+  int ty_status;
+  char *ty_window;
+  char *ty_comment;
+  char *ty_class;
+};
+
+extern const unsigned long __sanitizer_bufsiz;
+
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#define IOC_SIZEBITS 14
+#define IOC_DIRBITS 2
+#define IOC_NONE 0U
+#define IOC_WRITE 1U
+#define IOC_READ 2U
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#undef IOC_DIRMASK
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+#define EVIOC_EV_MAX 0x1f
+#define EVIOC_ABS_MAX 0x3f
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+
+// ioctl request identifiers
+
+extern unsigned struct_altqreq_sz;
+extern unsigned struct_amr_user_ioctl_sz;
+extern unsigned struct_ap_control_sz;
+extern unsigned struct_apm_ctl_sz;
+extern unsigned struct_apm_event_info_sz;
+extern unsigned struct_apm_power_info_sz;
+extern unsigned struct_atabusiodetach_args_sz;
+extern unsigned struct_atabusioscan_args_sz;
+extern unsigned struct_ath_diag_sz;
+extern unsigned struct_atm_flowmap_sz;
+extern unsigned struct_audio_buf_info_sz;
+extern unsigned struct_audio_device_sz;
+extern unsigned struct_audio_encoding_sz;
+extern unsigned struct_audio_info_sz;
+extern unsigned struct_audio_offset_sz;
+extern unsigned struct_bio_locate_sz;
+extern unsigned struct_bioc_alarm_sz;
+extern unsigned struct_bioc_blink_sz;
+extern unsigned struct_bioc_disk_sz;
+extern unsigned struct_bioc_inq_sz;
+extern unsigned struct_bioc_setstate_sz;
+extern unsigned struct_bioc_vol_sz;
+extern unsigned struct_bioc_volops_sz;
+extern unsigned struct_bktr_chnlset_sz;
+extern unsigned struct_bktr_remote_sz;
+extern unsigned struct_blue_conf_sz;
+extern unsigned struct_blue_interface_sz;
+extern unsigned struct_blue_stats_sz;
+extern unsigned struct_bpf_dltlist_sz;
+extern unsigned struct_bpf_program_sz;
+extern unsigned struct_bpf_stat_old_sz;
+extern unsigned struct_bpf_stat_sz;
+extern unsigned struct_bpf_version_sz;
+extern unsigned struct_btreq_sz;
+extern unsigned struct_btsco_info_sz;
+extern unsigned struct_buffmem_desc_sz;
+extern unsigned struct_cbq_add_class_sz;
+extern unsigned struct_cbq_add_filter_sz;
+extern unsigned struct_cbq_delete_class_sz;
+extern unsigned struct_cbq_delete_filter_sz;
+extern unsigned struct_cbq_getstats_sz;
+extern unsigned struct_cbq_interface_sz;
+extern unsigned struct_cbq_modify_class_sz;
+extern unsigned struct_ccd_ioctl_sz;
+extern unsigned struct_cdnr_add_element_sz;
+extern unsigned struct_cdnr_add_filter_sz;
+extern unsigned struct_cdnr_add_tbmeter_sz;
+extern unsigned struct_cdnr_add_trtcm_sz;
+extern unsigned struct_cdnr_add_tswtcm_sz;
+extern unsigned struct_cdnr_delete_element_sz;
+extern unsigned struct_cdnr_delete_filter_sz;
+extern unsigned struct_cdnr_get_stats_sz;
+extern unsigned struct_cdnr_interface_sz;
+extern unsigned struct_cdnr_modify_tbmeter_sz;
+extern unsigned struct_cdnr_modify_trtcm_sz;
+extern unsigned struct_cdnr_modify_tswtcm_sz;
+extern unsigned struct_cdnr_tbmeter_stats_sz;
+extern unsigned struct_cdnr_tcm_stats_sz;
+extern unsigned struct_cgd_ioctl_sz;
+extern unsigned struct_cgd_user_sz;
+extern unsigned struct_changer_element_status_request_sz;
+extern unsigned struct_changer_exchange_request_sz;
+extern unsigned struct_changer_move_request_sz;
+extern unsigned struct_changer_params_sz;
+extern unsigned struct_changer_position_request_sz;
+extern unsigned struct_changer_set_voltag_request_sz;
+extern unsigned struct_clockctl_adjtime_sz;
+extern unsigned struct_clockctl_clock_settime_sz;
+extern unsigned struct_clockctl_ntp_adjtime_sz;
+extern unsigned struct_clockctl_settimeofday_sz;
+extern unsigned struct_cnwistats_sz;
+extern unsigned struct_cnwitrail_sz;
+extern unsigned struct_cnwstatus_sz;
+extern unsigned struct_count_info_sz;
+extern unsigned struct_cpu_ucode_sz;
+extern unsigned struct_cpu_ucode_version_sz;
+extern unsigned struct_crypt_kop_sz;
+extern unsigned struct_crypt_mkop_sz;
+extern unsigned struct_crypt_mop_sz;
+extern unsigned struct_crypt_op_sz;
+extern unsigned struct_crypt_result_sz;
+extern unsigned struct_crypt_sfop_sz;
+extern unsigned struct_crypt_sgop_sz;
+extern unsigned struct_cryptret_sz;
+extern unsigned struct_devdetachargs_sz;
+extern unsigned struct_devlistargs_sz;
+extern unsigned struct_devpmargs_sz;
+extern unsigned struct_devrescanargs_sz;
+extern unsigned struct_disk_badsecinfo_sz;
+extern unsigned struct_disk_strategy_sz;
+extern unsigned struct_disklabel_sz;
+extern unsigned struct_dkbad_sz;
+extern unsigned struct_dkwedge_info_sz;
+extern unsigned struct_dkwedge_list_sz;
+extern unsigned struct_dmio_setfunc_sz;
+extern unsigned struct_dmx_pes_filter_params_sz;
+extern unsigned struct_dmx_sct_filter_params_sz;
+extern unsigned struct_dmx_stc_sz;
+extern unsigned struct_dvb_diseqc_master_cmd_sz;
+extern unsigned struct_dvb_diseqc_slave_reply_sz;
+extern unsigned struct_dvb_frontend_event_sz;
+extern unsigned struct_dvb_frontend_info_sz;
+extern unsigned struct_dvb_frontend_parameters_sz;
+extern unsigned struct_eccapreq_sz;
+extern unsigned struct_fbcmap_sz;
+extern unsigned struct_fbcurpos_sz;
+extern unsigned struct_fbcursor_sz;
+extern unsigned struct_fbgattr_sz;
+extern unsigned struct_fbsattr_sz;
+extern unsigned struct_fbtype_sz;
+extern unsigned struct_fdformat_cmd_sz;
+extern unsigned struct_fdformat_parms_sz;
+extern unsigned struct_fifoq_conf_sz;
+extern unsigned struct_fifoq_getstats_sz;
+extern unsigned struct_fifoq_interface_sz;
+extern unsigned struct_format_op_sz;
+extern unsigned struct_fss_get_sz;
+extern unsigned struct_fss_set_sz;
+extern unsigned struct_gpio_attach_sz;
+extern unsigned struct_gpio_info_sz;
+extern unsigned struct_gpio_req_sz;
+extern unsigned struct_gpio_set_sz;
+extern unsigned struct_hfsc_add_class_sz;
+extern unsigned struct_hfsc_add_filter_sz;
+extern unsigned struct_hfsc_attach_sz;
+extern unsigned struct_hfsc_class_stats_sz;
+extern unsigned struct_hfsc_delete_class_sz;
+extern unsigned struct_hfsc_delete_filter_sz;
+extern unsigned struct_hfsc_interface_sz;
+extern unsigned struct_hfsc_modify_class_sz;
+extern unsigned struct_hpcfb_dsp_op_sz;
+extern unsigned struct_hpcfb_dspconf_sz;
+extern unsigned struct_hpcfb_fbconf_sz;
+extern unsigned struct_if_addrprefreq_sz;
+extern unsigned struct_if_clonereq_sz;
+extern unsigned struct_if_laddrreq_sz;
+extern unsigned struct_ifaddr_sz;
+extern unsigned struct_ifaliasreq_sz;
+extern unsigned struct_ifcapreq_sz;
+extern unsigned struct_ifconf_sz;
+extern unsigned struct_ifdatareq_sz;
+extern unsigned struct_ifdrv_sz;
+extern unsigned struct_ifmediareq_sz;
+extern unsigned struct_ifpppcstatsreq_sz;
+extern unsigned struct_ifpppstatsreq_sz;
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_in6_addrpolicy_sz;
+extern unsigned struct_in6_ndireq_sz;
+extern unsigned struct_ioc_load_unload_sz;
+extern unsigned struct_ioc_patch_sz;
+extern unsigned struct_ioc_play_blocks_sz;
+extern unsigned struct_ioc_play_msf_sz;
+extern unsigned struct_ioc_play_track_sz;
+extern unsigned struct_ioc_read_subchannel_sz;
+extern unsigned struct_ioc_read_toc_entry_sz;
+extern unsigned struct_ioc_toc_header_sz;
+extern unsigned struct_ioc_vol_sz;
+extern unsigned struct_ioctl_pt_sz;
+extern unsigned struct_ioppt_sz;
+extern unsigned struct_iovec_sz;
+extern unsigned struct_ipfobj_sz;
+extern unsigned struct_irda_params_sz;
+extern unsigned struct_isp_fc_device_sz;
+extern unsigned struct_isp_fc_tsk_mgmt_sz;
+extern unsigned struct_isp_hba_device_sz;
+extern unsigned struct_isv_cmd_sz;
+extern unsigned struct_jobs_add_class_sz;
+extern unsigned struct_jobs_add_filter_sz;
+extern unsigned struct_jobs_attach_sz;
+extern unsigned struct_jobs_class_stats_sz;
+extern unsigned struct_jobs_delete_class_sz;
+extern unsigned struct_jobs_delete_filter_sz;
+extern unsigned struct_jobs_interface_sz;
+extern unsigned struct_jobs_modify_class_sz;
+extern unsigned struct_kbentry_sz;
+extern unsigned struct_kfilter_mapping_sz;
+extern unsigned struct_kiockeymap_sz;
+extern unsigned struct_ksyms_gsymbol_sz;
+extern unsigned struct_ksyms_gvalue_sz;
+extern unsigned struct_ksyms_ogsymbol_sz;
+extern unsigned struct_kttcp_io_args_sz;
+extern unsigned struct_ltchars_sz;
+extern unsigned struct_lua_create_sz;
+extern unsigned struct_lua_info_sz;
+extern unsigned struct_lua_load_sz;
+extern unsigned struct_lua_require_sz;
+extern unsigned struct_mbpp_param_sz;
+extern unsigned struct_md_conf_sz;
+extern unsigned struct_meteor_capframe_sz;
+extern unsigned struct_meteor_counts_sz;
+extern unsigned struct_meteor_geomet_sz;
+extern unsigned struct_meteor_pixfmt_sz;
+extern unsigned struct_meteor_video_sz;
+extern unsigned struct_mlx_cinfo_sz;
+extern unsigned struct_mlx_pause_sz;
+extern unsigned struct_mlx_rebuild_request_sz;
+extern unsigned struct_mlx_rebuild_status_sz;
+extern unsigned struct_mlx_usercommand_sz;
+extern unsigned struct_mly_user_command_sz;
+extern unsigned struct_mly_user_health_sz;
+extern unsigned struct_mtget_sz;
+extern unsigned struct_mtop_sz;
+extern unsigned struct_npf_ioctl_table_sz;
+extern unsigned struct_npioctl_sz;
+extern unsigned struct_nvme_pt_command_sz;
+extern unsigned struct_ochanger_element_status_request_sz;
+extern unsigned struct_ofiocdesc_sz;
+extern unsigned struct_okiockey_sz;
+extern unsigned struct_ortentry_sz;
+extern unsigned struct_oscsi_addr_sz;
+extern unsigned struct_oss_audioinfo_sz;
+extern unsigned struct_oss_sysinfo_sz;
+extern unsigned struct_pciio_bdf_cfgreg_sz;
+extern unsigned struct_pciio_businfo_sz;
+extern unsigned struct_pciio_cfgreg_sz;
+extern unsigned struct_pciio_drvname_sz;
+extern unsigned struct_pciio_drvnameonbus_sz;
+extern unsigned struct_pcvtid_sz;
+extern unsigned struct_pf_osfp_ioctl_sz;
+extern unsigned struct_pf_status_sz;
+extern unsigned struct_pfioc_altq_sz;
+extern unsigned struct_pfioc_if_sz;
+extern unsigned struct_pfioc_iface_sz;
+extern unsigned struct_pfioc_limit_sz;
+extern unsigned struct_pfioc_natlook_sz;
+extern unsigned struct_pfioc_pooladdr_sz;
+extern unsigned struct_pfioc_qstats_sz;
+extern unsigned struct_pfioc_rule_sz;
+extern unsigned struct_pfioc_ruleset_sz;
+extern unsigned struct_pfioc_src_node_kill_sz;
+extern unsigned struct_pfioc_src_nodes_sz;
+extern unsigned struct_pfioc_state_kill_sz;
+extern unsigned struct_pfioc_state_sz;
+extern unsigned struct_pfioc_states_sz;
+extern unsigned struct_pfioc_table_sz;
+extern unsigned struct_pfioc_tm_sz;
+extern unsigned struct_pfioc_trans_sz;
+extern unsigned struct_plistref_sz;
+extern unsigned struct_power_type_sz;
+extern unsigned struct_ppp_idle_sz;
+extern unsigned struct_ppp_option_data_sz;
+extern unsigned struct_ppp_rawin_sz;
+extern unsigned struct_pppoeconnectionstate_sz;
+extern unsigned struct_pppoediscparms_sz;
+extern unsigned struct_priq_add_class_sz;
+extern unsigned struct_priq_add_filter_sz;
+extern unsigned struct_priq_class_stats_sz;
+extern unsigned struct_priq_delete_class_sz;
+extern unsigned struct_priq_delete_filter_sz;
+extern unsigned struct_priq_interface_sz;
+extern unsigned struct_priq_modify_class_sz;
+extern unsigned struct_ptmget_sz;
+extern unsigned struct_pvctxreq_sz;
+extern unsigned struct_radio_info_sz;
+extern unsigned struct_red_conf_sz;
+extern unsigned struct_red_interface_sz;
+extern unsigned struct_red_stats_sz;
+extern unsigned struct_redparams_sz;
+extern unsigned struct_rf_pmparams_sz;
+extern unsigned struct_rf_pmstat_sz;
+extern unsigned struct_rf_recon_req_sz;
+extern unsigned struct_rio_conf_sz;
+extern unsigned struct_rio_interface_sz;
+extern unsigned struct_rio_stats_sz;
+extern unsigned struct_scan_io_sz;
+extern unsigned struct_scbusaccel_args_sz;
+extern unsigned struct_scbusiodetach_args_sz;
+extern unsigned struct_scbusioscan_args_sz;
+extern unsigned struct_scsi_addr_sz;
+extern unsigned struct_seq_event_rec_sz;
+extern unsigned struct_session_op_sz;
+extern unsigned struct_sgttyb_sz;
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+extern unsigned struct_smbioc_flags_sz;
+extern unsigned struct_smbioc_lookup_sz;
+extern unsigned struct_smbioc_oshare_sz;
+extern unsigned struct_smbioc_ossn_sz;
+extern unsigned struct_smbioc_rq_sz;
+extern unsigned struct_smbioc_rw_sz;
+extern unsigned struct_spppauthcfg_sz;
+extern unsigned struct_spppauthfailuresettings_sz;
+extern unsigned struct_spppauthfailurestats_sz;
+extern unsigned struct_spppdnsaddrs_sz;
+extern unsigned struct_spppdnssettings_sz;
+extern unsigned struct_spppidletimeout_sz;
+extern unsigned struct_spppkeepalivesettings_sz;
+extern unsigned struct_sppplcpcfg_sz;
+extern unsigned struct_spppstatus_sz;
+extern unsigned struct_spppstatusncp_sz;
+extern unsigned struct_srt_rt_sz;
+extern unsigned struct_stic_xinfo_sz;
+extern unsigned struct_sun_dkctlr_sz;
+extern unsigned struct_sun_dkgeom_sz;
+extern unsigned struct_sun_dkpart_sz;
+extern unsigned struct_synth_info_sz;
+extern unsigned struct_tbrreq_sz;
+extern unsigned struct_tchars_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_timeval_sz;
+extern unsigned struct_twe_drivecommand_sz;
+extern unsigned struct_twe_paramcommand_sz;
+extern unsigned struct_twe_usercommand_sz;
+extern unsigned struct_ukyopon_identify_sz;
+extern unsigned struct_urio_command_sz;
+extern unsigned struct_usb_alt_interface_sz;
+extern unsigned struct_usb_bulk_ra_wb_opt_sz;
+extern unsigned struct_usb_config_desc_sz;
+extern unsigned struct_usb_ctl_report_desc_sz;
+extern unsigned struct_usb_ctl_report_sz;
+extern unsigned struct_usb_ctl_request_sz;
+#if defined(__x86_64__)
+extern unsigned struct_nvmm_ioc_capability_sz;
+extern unsigned struct_nvmm_ioc_machine_create_sz;
+extern unsigned struct_nvmm_ioc_machine_destroy_sz;
+extern unsigned struct_nvmm_ioc_machine_configure_sz;
+extern unsigned struct_nvmm_ioc_vcpu_create_sz;
+extern unsigned struct_nvmm_ioc_vcpu_destroy_sz;
+extern unsigned struct_nvmm_ioc_vcpu_configure_sz;
+extern unsigned struct_nvmm_ioc_vcpu_setstate_sz;
+extern unsigned struct_nvmm_ioc_vcpu_getstate_sz;
+extern unsigned struct_nvmm_ioc_vcpu_inject_sz;
+extern unsigned struct_nvmm_ioc_vcpu_run_sz;
+extern unsigned struct_nvmm_ioc_gpa_map_sz;
+extern unsigned struct_nvmm_ioc_gpa_unmap_sz;
+extern unsigned struct_nvmm_ioc_hva_map_sz;
+extern unsigned struct_nvmm_ioc_hva_unmap_sz;
+extern unsigned struct_nvmm_ioc_ctl_sz;
+#endif
+extern unsigned struct_spi_ioctl_configure_sz;
+extern unsigned struct_spi_ioctl_transfer_sz;
+extern unsigned struct_autofs_daemon_request_sz;
+extern unsigned struct_autofs_daemon_done_sz;
+extern unsigned struct_sctp_connectx_addrs_sz;
+extern unsigned struct_usb_device_info_old_sz;
+extern unsigned struct_usb_device_info_sz;
+extern unsigned struct_usb_device_stats_sz;
+extern unsigned struct_usb_endpoint_desc_sz;
+extern unsigned struct_usb_full_desc_sz;
+extern unsigned struct_usb_interface_desc_sz;
+extern unsigned struct_usb_string_desc_sz;
+extern unsigned struct_utoppy_readfile_sz;
+extern unsigned struct_utoppy_rename_sz;
+extern unsigned struct_utoppy_stats_sz;
+extern unsigned struct_utoppy_writefile_sz;
+extern unsigned struct_v4l2_audio_sz;
+extern unsigned struct_v4l2_audioout_sz;
+extern unsigned struct_v4l2_buffer_sz;
+extern unsigned struct_v4l2_capability_sz;
+extern unsigned struct_v4l2_control_sz;
+extern unsigned struct_v4l2_crop_sz;
+extern unsigned struct_v4l2_cropcap_sz;
+extern unsigned struct_v4l2_fmtdesc_sz;
+extern unsigned struct_v4l2_format_sz;
+extern unsigned struct_v4l2_framebuffer_sz;
+extern unsigned struct_v4l2_frequency_sz;
+extern unsigned struct_v4l2_frmivalenum_sz;
+extern unsigned struct_v4l2_frmsizeenum_sz;
+extern unsigned struct_v4l2_input_sz;
+extern unsigned struct_v4l2_jpegcompression_sz;
+extern unsigned struct_v4l2_modulator_sz;
+extern unsigned struct_v4l2_output_sz;
+extern unsigned struct_v4l2_queryctrl_sz;
+extern unsigned struct_v4l2_querymenu_sz;
+extern unsigned struct_v4l2_requestbuffers_sz;
+extern unsigned struct_v4l2_standard_sz;
+extern unsigned struct_v4l2_streamparm_sz;
+extern unsigned struct_v4l2_tuner_sz;
+extern unsigned struct_vnd_ioctl_sz;
+extern unsigned struct_vnd_user_sz;
+extern unsigned struct_vt_stat_sz;
+extern unsigned struct_wdog_conf_sz;
+extern unsigned struct_wdog_mode_sz;
+extern unsigned struct_ipmi_recv_sz;
+extern unsigned struct_ipmi_req_sz;
+extern unsigned struct_ipmi_cmdspec_sz;
+extern unsigned struct_wfq_conf_sz;
+extern unsigned struct_wfq_getqid_sz;
+extern unsigned struct_wfq_getstats_sz;
+extern unsigned struct_wfq_interface_sz;
+extern unsigned struct_wfq_setweight_sz;
+extern unsigned struct_winsize_sz;
+extern unsigned struct_wscons_event_sz;
+extern unsigned struct_wsdisplay_addscreendata_sz;
+extern unsigned struct_wsdisplay_char_sz;
+extern unsigned struct_wsdisplay_cmap_sz;
+extern unsigned struct_wsdisplay_curpos_sz;
+extern unsigned struct_wsdisplay_cursor_sz;
+extern unsigned struct_wsdisplay_delscreendata_sz;
+extern unsigned struct_wsdisplay_fbinfo_sz;
+extern unsigned struct_wsdisplay_font_sz;
+extern unsigned struct_wsdisplay_kbddata_sz;
+extern unsigned struct_wsdisplay_msgattrs_sz;
+extern unsigned struct_wsdisplay_param_sz;
+extern unsigned struct_wsdisplay_scroll_data_sz;
+extern unsigned struct_wsdisplay_usefontdata_sz;
+extern unsigned struct_wsdisplayio_blit_sz;
+extern unsigned struct_wsdisplayio_bus_id_sz;
+extern unsigned struct_wsdisplayio_edid_info_sz;
+extern unsigned struct_wsdisplayio_fbinfo_sz;
+extern unsigned struct_wskbd_bell_data_sz;
+extern unsigned struct_wskbd_keyrepeat_data_sz;
+extern unsigned struct_wskbd_map_data_sz;
+extern unsigned struct_wskbd_scroll_data_sz;
+extern unsigned struct_wsmouse_calibcoords_sz;
+extern unsigned struct_wsmouse_id_sz;
+extern unsigned struct_wsmouse_repeat_sz;
+extern unsigned struct_wsmux_device_list_sz;
+extern unsigned struct_wsmux_device_sz;
+extern unsigned struct_xd_iocmd_sz;
+
+extern unsigned struct_scsireq_sz;
+extern unsigned struct_tone_sz;
+extern unsigned union_twe_statrequest_sz;
+extern unsigned struct_usb_device_descriptor_sz;
+extern unsigned struct_vt_mode_sz;
+extern unsigned struct__old_mixer_info_sz;
+extern unsigned struct__agp_allocate_sz;
+extern unsigned struct__agp_bind_sz;
+extern unsigned struct__agp_info_sz;
+extern unsigned struct__agp_setup_sz;
+extern unsigned struct__agp_unbind_sz;
+extern unsigned struct_atareq_sz;
+extern unsigned struct_cpustate_sz;
+extern unsigned struct_dmx_caps_sz;
+extern unsigned enum_dmx_source_sz;
+extern unsigned union_dvd_authinfo_sz;
+extern unsigned union_dvd_struct_sz;
+extern unsigned enum_v4l2_priority_sz;
+extern unsigned struct_envsys_basic_info_sz;
+extern unsigned struct_envsys_tre_data_sz;
+extern unsigned enum_fe_sec_mini_cmd_sz;
+extern unsigned enum_fe_sec_tone_mode_sz;
+extern unsigned enum_fe_sec_voltage_sz;
+extern unsigned enum_fe_status_sz;
+extern unsigned struct_gdt_ctrt_sz;
+extern unsigned struct_gdt_event_sz;
+extern unsigned struct_gdt_osv_sz;
+extern unsigned struct_gdt_rescan_sz;
+extern unsigned struct_gdt_statist_sz;
+extern unsigned struct_gdt_ucmd_sz;
+extern unsigned struct_iscsi_conn_status_parameters_sz;
+extern unsigned struct_iscsi_get_version_parameters_sz;
+extern unsigned struct_iscsi_iocommand_parameters_sz;
+extern unsigned struct_iscsi_login_parameters_sz;
+extern unsigned struct_iscsi_logout_parameters_sz;
+extern unsigned struct_iscsi_register_event_parameters_sz;
+extern unsigned struct_iscsi_remove_parameters_sz;
+extern unsigned struct_iscsi_send_targets_parameters_sz;
+extern unsigned struct_iscsi_set_node_name_parameters_sz;
+extern unsigned struct_iscsi_wait_event_parameters_sz;
+extern unsigned struct_isp_stats_sz;
+extern unsigned struct_lsenable_sz;
+extern unsigned struct_lsdisable_sz;
+extern unsigned struct_audio_format_query_sz;
+extern unsigned struct_mixer_ctrl_sz;
+extern unsigned struct_mixer_devinfo_sz;
+extern unsigned struct_mpu_command_rec_sz;
+extern unsigned struct_rndstat_sz;
+extern unsigned struct_rndstat_name_sz;
+extern unsigned struct_rndctl_sz;
+extern unsigned struct_rnddata_sz;
+extern unsigned struct_rndpoolstat_sz;
+extern unsigned struct_rndstat_est_sz;
+extern unsigned struct_rndstat_est_name_sz;
+extern unsigned struct_pps_params_sz;
+extern unsigned struct_pps_info_sz;
+extern unsigned struct_mixer_info_sz;
+extern unsigned struct_RF_SparetWait_sz;
+extern unsigned struct_RF_ComponentLabel_sz;
+extern unsigned struct_RF_SingleComponent_sz;
+extern unsigned struct_RF_ProgressInfo_sz;
+extern unsigned struct_nvlist_ref_sz;
+extern unsigned struct_StringList_sz;
+
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+
+extern unsigned IOCTL_AFM_ADDFMAP;
+extern unsigned IOCTL_AFM_DELFMAP;
+extern unsigned IOCTL_AFM_CLEANFMAP;
+extern unsigned IOCTL_AFM_GETFMAP;
+extern unsigned IOCTL_ALTQGTYPE;
+extern unsigned IOCTL_ALTQTBRSET;
+extern unsigned IOCTL_ALTQTBRGET;
+extern unsigned IOCTL_BLUE_IF_ATTACH;
+extern unsigned IOCTL_BLUE_IF_DETACH;
+extern unsigned IOCTL_BLUE_ENABLE;
+extern unsigned IOCTL_BLUE_DISABLE;
+extern unsigned IOCTL_BLUE_CONFIG;
+extern unsigned IOCTL_BLUE_GETSTATS;
+extern unsigned IOCTL_CBQ_IF_ATTACH;
+extern unsigned IOCTL_CBQ_IF_DETACH;
+extern unsigned IOCTL_CBQ_ENABLE;
+extern unsigned IOCTL_CBQ_DISABLE;
+extern unsigned IOCTL_CBQ_CLEAR_HIERARCHY;
+extern unsigned IOCTL_CBQ_ADD_CLASS;
+extern unsigned IOCTL_CBQ_DEL_CLASS;
+extern unsigned IOCTL_CBQ_MODIFY_CLASS;
+extern unsigned IOCTL_CBQ_ADD_FILTER;
+extern unsigned IOCTL_CBQ_DEL_FILTER;
+extern unsigned IOCTL_CBQ_GETSTATS;
+extern unsigned IOCTL_CDNR_IF_ATTACH;
+extern unsigned IOCTL_CDNR_IF_DETACH;
+extern unsigned IOCTL_CDNR_ENABLE;
+extern unsigned IOCTL_CDNR_DISABLE;
+extern unsigned IOCTL_CDNR_ADD_FILTER;
+extern unsigned IOCTL_CDNR_DEL_FILTER;
+extern unsigned IOCTL_CDNR_GETSTATS;
+extern unsigned IOCTL_CDNR_ADD_ELEM;
+extern unsigned IOCTL_CDNR_DEL_ELEM;
+extern unsigned IOCTL_CDNR_ADD_TBM;
+extern unsigned IOCTL_CDNR_MOD_TBM;
+extern unsigned IOCTL_CDNR_TBM_STATS;
+extern unsigned IOCTL_CDNR_ADD_TCM;
+extern unsigned IOCTL_CDNR_MOD_TCM;
+extern unsigned IOCTL_CDNR_TCM_STATS;
+extern unsigned IOCTL_CDNR_ADD_TSW;
+extern unsigned IOCTL_CDNR_MOD_TSW;
+extern unsigned IOCTL_FIFOQ_IF_ATTACH;
+extern unsigned IOCTL_FIFOQ_IF_DETACH;
+extern unsigned IOCTL_FIFOQ_ENABLE;
+extern unsigned IOCTL_FIFOQ_DISABLE;
+extern unsigned IOCTL_FIFOQ_CONFIG;
+extern unsigned IOCTL_FIFOQ_GETSTATS;
+extern unsigned IOCTL_HFSC_IF_ATTACH;
+extern unsigned IOCTL_HFSC_IF_DETACH;
+extern unsigned IOCTL_HFSC_ENABLE;
+extern unsigned IOCTL_HFSC_DISABLE;
+extern unsigned IOCTL_HFSC_CLEAR_HIERARCHY;
+extern unsigned IOCTL_HFSC_ADD_CLASS;
+extern unsigned IOCTL_HFSC_DEL_CLASS;
+extern unsigned IOCTL_HFSC_MOD_CLASS;
+extern unsigned IOCTL_HFSC_ADD_FILTER;
+extern unsigned IOCTL_HFSC_DEL_FILTER;
+extern unsigned IOCTL_HFSC_GETSTATS;
+extern unsigned IOCTL_JOBS_IF_ATTACH;
+extern unsigned IOCTL_JOBS_IF_DETACH;
+extern unsigned IOCTL_JOBS_ENABLE;
+extern unsigned IOCTL_JOBS_DISABLE;
+extern unsigned IOCTL_JOBS_CLEAR;
+extern unsigned IOCTL_JOBS_ADD_CLASS;
+extern unsigned IOCTL_JOBS_DEL_CLASS;
+extern unsigned IOCTL_JOBS_MOD_CLASS;
+extern unsigned IOCTL_JOBS_ADD_FILTER;
+extern unsigned IOCTL_JOBS_DEL_FILTER;
+extern unsigned IOCTL_JOBS_GETSTATS;
+extern unsigned IOCTL_PRIQ_IF_ATTACH;
+extern unsigned IOCTL_PRIQ_IF_DETACH;
+extern unsigned IOCTL_PRIQ_ENABLE;
+extern unsigned IOCTL_PRIQ_DISABLE;
+extern unsigned IOCTL_PRIQ_CLEAR;
+extern unsigned IOCTL_PRIQ_ADD_CLASS;
+extern unsigned IOCTL_PRIQ_DEL_CLASS;
+extern unsigned IOCTL_PRIQ_MOD_CLASS;
+extern unsigned IOCTL_PRIQ_ADD_FILTER;
+extern unsigned IOCTL_PRIQ_DEL_FILTER;
+extern unsigned IOCTL_PRIQ_GETSTATS;
+extern unsigned IOCTL_RED_IF_ATTACH;
+extern unsigned IOCTL_RED_IF_DETACH;
+extern unsigned IOCTL_RED_ENABLE;
+extern unsigned IOCTL_RED_DISABLE;
+extern unsigned IOCTL_RED_CONFIG;
+extern unsigned IOCTL_RED_GETSTATS;
+extern unsigned IOCTL_RED_SETDEFAULTS;
+extern unsigned IOCTL_RIO_IF_ATTACH;
+extern unsigned IOCTL_RIO_IF_DETACH;
+extern unsigned IOCTL_RIO_ENABLE;
+extern unsigned IOCTL_RIO_DISABLE;
+extern unsigned IOCTL_RIO_CONFIG;
+extern unsigned IOCTL_RIO_GETSTATS;
+extern unsigned IOCTL_RIO_SETDEFAULTS;
+extern unsigned IOCTL_WFQ_IF_ATTACH;
+extern unsigned IOCTL_WFQ_IF_DETACH;
+extern unsigned IOCTL_WFQ_ENABLE;
+extern unsigned IOCTL_WFQ_DISABLE;
+extern unsigned IOCTL_WFQ_CONFIG;
+extern unsigned IOCTL_WFQ_GET_STATS;
+extern unsigned IOCTL_WFQ_GET_QID;
+extern unsigned IOCTL_WFQ_SET_WEIGHT;
+extern unsigned IOCTL_CRIOGET;
+extern unsigned IOCTL_CIOCFSESSION;
+extern unsigned IOCTL_CIOCKEY;
+extern unsigned IOCTL_CIOCNFKEYM;
+extern unsigned IOCTL_CIOCNFSESSION;
+extern unsigned IOCTL_CIOCNCRYPTRETM;
+extern unsigned IOCTL_CIOCNCRYPTRET;
+extern unsigned IOCTL_CIOCGSESSION;
+extern unsigned IOCTL_CIOCNGSESSION;
+extern unsigned IOCTL_CIOCCRYPT;
+extern unsigned IOCTL_CIOCNCRYPTM;
+extern unsigned IOCTL_CIOCASYMFEAT;
+extern unsigned IOCTL_APM_IOC_REJECT;
+extern unsigned IOCTL_APM_IOC_STANDBY;
+extern unsigned IOCTL_APM_IOC_SUSPEND;
+extern unsigned IOCTL_OAPM_IOC_GETPOWER;
+extern unsigned IOCTL_APM_IOC_GETPOWER;
+extern unsigned IOCTL_APM_IOC_NEXTEVENT;
+extern unsigned IOCTL_APM_IOC_DEV_CTL;
+extern unsigned IOCTL_NETBSD_DM_IOCTL;
+extern unsigned IOCTL_DMIO_SETFUNC;
+extern unsigned IOCTL_DMX_START;
+extern unsigned IOCTL_DMX_STOP;
+extern unsigned IOCTL_DMX_SET_FILTER;
+extern unsigned IOCTL_DMX_SET_PES_FILTER;
+extern unsigned IOCTL_DMX_SET_BUFFER_SIZE;
+extern unsigned IOCTL_DMX_GET_STC;
+extern unsigned IOCTL_DMX_ADD_PID;
+extern unsigned IOCTL_DMX_REMOVE_PID;
+extern unsigned IOCTL_DMX_GET_CAPS;
+extern unsigned IOCTL_DMX_SET_SOURCE;
+extern unsigned IOCTL_FE_READ_STATUS;
+extern unsigned IOCTL_FE_READ_BER;
+extern unsigned IOCTL_FE_READ_SNR;
+extern unsigned IOCTL_FE_READ_SIGNAL_STRENGTH;
+extern unsigned IOCTL_FE_READ_UNCORRECTED_BLOCKS;
+extern unsigned IOCTL_FE_SET_FRONTEND;
+extern unsigned IOCTL_FE_GET_FRONTEND;
+extern unsigned IOCTL_FE_GET_EVENT;
+extern unsigned IOCTL_FE_GET_INFO;
+extern unsigned IOCTL_FE_DISEQC_RESET_OVERLOAD;
+extern unsigned IOCTL_FE_DISEQC_SEND_MASTER_CMD;
+extern unsigned IOCTL_FE_DISEQC_RECV_SLAVE_REPLY;
+extern unsigned IOCTL_FE_DISEQC_SEND_BURST;
+extern unsigned IOCTL_FE_SET_TONE;
+extern unsigned IOCTL_FE_SET_VOLTAGE;
+extern unsigned IOCTL_FE_ENABLE_HIGH_LNB_VOLTAGE;
+extern unsigned IOCTL_FE_SET_FRONTEND_TUNE_MODE;
+extern unsigned IOCTL_FE_DISHNETWORK_SEND_LEGACY_CMD;
+extern unsigned IOCTL_FILEMON_SET_FD;
+extern unsigned IOCTL_FILEMON_SET_PID;
+extern unsigned IOCTL_HDAUDIO_FGRP_INFO;
+extern unsigned IOCTL_HDAUDIO_FGRP_GETCONFIG;
+extern unsigned IOCTL_HDAUDIO_FGRP_SETCONFIG;
+extern unsigned IOCTL_HDAUDIO_FGRP_WIDGET_INFO;
+extern unsigned IOCTL_HDAUDIO_FGRP_CODEC_INFO;
+extern unsigned IOCTL_HDAUDIO_AFG_WIDGET_INFO;
+extern unsigned IOCTL_HDAUDIO_AFG_CODEC_INFO;
+extern unsigned IOCTL_CEC_GET_PHYS_ADDR;
+extern unsigned IOCTL_CEC_GET_LOG_ADDRS;
+extern unsigned IOCTL_CEC_SET_LOG_ADDRS;
+extern unsigned IOCTL_CEC_GET_VENDOR_ID;
+extern unsigned IOCTL_HPCFBIO_GCONF;
+extern unsigned IOCTL_HPCFBIO_SCONF;
+extern unsigned IOCTL_HPCFBIO_GDSPCONF;
+extern unsigned IOCTL_HPCFBIO_SDSPCONF;
+extern unsigned IOCTL_HPCFBIO_GOP;
+extern unsigned IOCTL_HPCFBIO_SOP;
+extern unsigned IOCTL_IOPIOCPT;
+extern unsigned IOCTL_IOPIOCGLCT;
+extern unsigned IOCTL_IOPIOCGSTATUS;
+extern unsigned IOCTL_IOPIOCRECONFIG;
+extern unsigned IOCTL_IOPIOCGTIDMAP;
+extern unsigned IOCTL_SIOCGATHSTATS;
+extern unsigned IOCTL_SIOCGATHDIAG;
+extern unsigned IOCTL_METEORCAPTUR;
+extern unsigned IOCTL_METEORCAPFRM;
+extern unsigned IOCTL_METEORSETGEO;
+extern unsigned IOCTL_METEORGETGEO;
+extern unsigned IOCTL_METEORSTATUS;
+extern unsigned IOCTL_METEORSHUE;
+extern unsigned IOCTL_METEORGHUE;
+extern unsigned IOCTL_METEORSFMT;
+extern unsigned IOCTL_METEORGFMT;
+extern unsigned IOCTL_METEORSINPUT;
+extern unsigned IOCTL_METEORGINPUT;
+extern unsigned IOCTL_METEORSCHCV;
+extern unsigned IOCTL_METEORGCHCV;
+extern unsigned IOCTL_METEORSCOUNT;
+extern unsigned IOCTL_METEORGCOUNT;
+extern unsigned IOCTL_METEORSFPS;
+extern unsigned IOCTL_METEORGFPS;
+extern unsigned IOCTL_METEORSSIGNAL;
+extern unsigned IOCTL_METEORGSIGNAL;
+extern unsigned IOCTL_METEORSVIDEO;
+extern unsigned IOCTL_METEORGVIDEO;
+extern unsigned IOCTL_METEORSBRIG;
+extern unsigned IOCTL_METEORGBRIG;
+extern unsigned IOCTL_METEORSCSAT;
+extern unsigned IOCTL_METEORGCSAT;
+extern unsigned IOCTL_METEORSCONT;
+extern unsigned IOCTL_METEORGCONT;
+extern unsigned IOCTL_METEORSHWS;
+extern unsigned IOCTL_METEORGHWS;
+extern unsigned IOCTL_METEORSVWS;
+extern unsigned IOCTL_METEORGVWS;
+extern unsigned IOCTL_METEORSTS;
+extern unsigned IOCTL_METEORGTS;
+extern unsigned IOCTL_TVTUNER_SETCHNL;
+extern unsigned IOCTL_TVTUNER_GETCHNL;
+extern unsigned IOCTL_TVTUNER_SETTYPE;
+extern unsigned IOCTL_TVTUNER_GETTYPE;
+extern unsigned IOCTL_TVTUNER_GETSTATUS;
+extern unsigned IOCTL_TVTUNER_SETFREQ;
+extern unsigned IOCTL_TVTUNER_GETFREQ;
+extern unsigned IOCTL_TVTUNER_SETAFC;
+extern unsigned IOCTL_TVTUNER_GETAFC;
+extern unsigned IOCTL_RADIO_SETMODE;
+extern unsigned IOCTL_RADIO_GETMODE;
+extern unsigned IOCTL_RADIO_SETFREQ;
+extern unsigned IOCTL_RADIO_GETFREQ;
+extern unsigned IOCTL_METEORSACTPIXFMT;
+extern unsigned IOCTL_METEORGACTPIXFMT;
+extern unsigned IOCTL_METEORGSUPPIXFMT;
+extern unsigned IOCTL_TVTUNER_GETCHNLSET;
+extern unsigned IOCTL_REMOTE_GETKEY;
+extern unsigned IOCTL_GDT_IOCTL_GENERAL;
+extern unsigned IOCTL_GDT_IOCTL_DRVERS;
+extern unsigned IOCTL_GDT_IOCTL_CTRTYPE;
+extern unsigned IOCTL_GDT_IOCTL_OSVERS;
+extern unsigned IOCTL_GDT_IOCTL_CTRCNT;
+extern unsigned IOCTL_GDT_IOCTL_EVENT;
+extern unsigned IOCTL_GDT_IOCTL_STATIST;
+extern unsigned IOCTL_GDT_IOCTL_RESCAN;
+extern unsigned IOCTL_ISP_SDBLEV;
+extern unsigned IOCTL_ISP_RESETHBA;
+extern unsigned IOCTL_ISP_RESCAN;
+extern unsigned IOCTL_ISP_SETROLE;
+extern unsigned IOCTL_ISP_GETROLE;
+extern unsigned IOCTL_ISP_GET_STATS;
+extern unsigned IOCTL_ISP_CLR_STATS;
+extern unsigned IOCTL_ISP_FC_LIP;
+extern unsigned IOCTL_ISP_FC_GETDINFO;
+extern unsigned IOCTL_ISP_GET_FW_CRASH_DUMP;
+extern unsigned IOCTL_ISP_FORCE_CRASH_DUMP;
+extern unsigned IOCTL_ISP_FC_GETHINFO;
+extern unsigned IOCTL_ISP_TSK_MGMT;
+extern unsigned IOCTL_ISP_FC_GETDLIST;
+extern unsigned IOCTL_MLXD_STATUS;
+extern unsigned IOCTL_MLXD_CHECKASYNC;
+extern unsigned IOCTL_MLXD_DETACH;
+extern unsigned IOCTL_MLX_RESCAN_DRIVES;
+extern unsigned IOCTL_MLX_PAUSE_CHANNEL;
+extern unsigned IOCTL_MLX_COMMAND;
+extern unsigned IOCTL_MLX_REBUILDASYNC;
+extern unsigned IOCTL_MLX_REBUILDSTAT;
+extern unsigned IOCTL_MLX_GET_SYSDRIVE;
+extern unsigned IOCTL_MLX_GET_CINFO;
+extern unsigned IOCTL_NVME_PASSTHROUGH_CMD;
+extern unsigned IOCTL_FWCFGIO_SET_INDEX;
+extern unsigned IOCTL_IRDA_RESET_PARAMS;
+extern unsigned IOCTL_IRDA_SET_PARAMS;
+extern unsigned IOCTL_IRDA_GET_SPEEDMASK;
+extern unsigned IOCTL_IRDA_GET_TURNAROUNDMASK;
+extern unsigned IOCTL_IRFRAMETTY_GET_DEVICE;
+extern unsigned IOCTL_IRFRAMETTY_GET_DONGLE;
+extern unsigned IOCTL_IRFRAMETTY_SET_DONGLE;
+extern unsigned IOCTL_ISV_CMD;
+extern unsigned IOCTL_WTQICMD;
+extern unsigned IOCTL_ISCSI_GET_VERSION;
+extern unsigned IOCTL_ISCSI_LOGIN;
+extern unsigned IOCTL_ISCSI_LOGOUT;
+extern unsigned IOCTL_ISCSI_ADD_CONNECTION;
+extern unsigned IOCTL_ISCSI_RESTORE_CONNECTION;
+extern unsigned IOCTL_ISCSI_REMOVE_CONNECTION;
+extern unsigned IOCTL_ISCSI_CONNECTION_STATUS;
+extern unsigned IOCTL_ISCSI_SEND_TARGETS;
+extern unsigned IOCTL_ISCSI_SET_NODE_NAME;
+extern unsigned IOCTL_ISCSI_IO_COMMAND;
+extern unsigned IOCTL_ISCSI_REGISTER_EVENT;
+extern unsigned IOCTL_ISCSI_DEREGISTER_EVENT;
+extern unsigned IOCTL_ISCSI_WAIT_EVENT;
+extern unsigned IOCTL_ISCSI_POLL_EVENT;
+extern unsigned IOCTL_OFIOCGET;
+extern unsigned IOCTL_OFIOCSET;
+extern unsigned IOCTL_OFIOCNEXTPROP;
+extern unsigned IOCTL_OFIOCGETOPTNODE;
+extern unsigned IOCTL_OFIOCGETNEXT;
+extern unsigned IOCTL_OFIOCGETCHILD;
+extern unsigned IOCTL_OFIOCFINDDEVICE;
+extern unsigned IOCTL_AMR_IO_VERSION;
+extern unsigned IOCTL_AMR_IO_COMMAND;
+extern unsigned IOCTL_MLYIO_COMMAND;
+extern unsigned IOCTL_MLYIO_HEALTH;
+extern unsigned IOCTL_PCI_IOC_CFGREAD;
+extern unsigned IOCTL_PCI_IOC_CFGWRITE;
+extern unsigned IOCTL_PCI_IOC_BDF_CFGREAD;
+extern unsigned IOCTL_PCI_IOC_BDF_CFGWRITE;
+extern unsigned IOCTL_PCI_IOC_BUSINFO;
+extern unsigned IOCTL_PCI_IOC_DRVNAME;
+extern unsigned IOCTL_PCI_IOC_DRVNAMEONBUS;
+extern unsigned IOCTL_TWEIO_COMMAND;
+extern unsigned IOCTL_TWEIO_STATS;
+extern unsigned IOCTL_TWEIO_AEN_POLL;
+extern unsigned IOCTL_TWEIO_AEN_WAIT;
+extern unsigned IOCTL_TWEIO_SET_PARAM;
+extern unsigned IOCTL_TWEIO_GET_PARAM;
+extern unsigned IOCTL_TWEIO_RESET;
+extern unsigned IOCTL_TWEIO_ADD_UNIT;
+extern unsigned IOCTL_TWEIO_DEL_UNIT;
+extern unsigned IOCTL_SIOCSCNWDOMAIN;
+extern unsigned IOCTL_SIOCGCNWDOMAIN;
+extern unsigned IOCTL_SIOCSCNWKEY;
+extern unsigned IOCTL_SIOCGCNWSTATUS;
+extern unsigned IOCTL_SIOCGCNWSTATS;
+extern unsigned IOCTL_SIOCGCNWTRAIL;
+extern unsigned IOCTL_SIOCGRAYSIGLEV;
+extern unsigned IOCTL_RAIDFRAME_SHUTDOWN;
+extern unsigned IOCTL_RAIDFRAME_TUR;
+extern unsigned IOCTL_RAIDFRAME_FAIL_DISK;
+extern unsigned IOCTL_RAIDFRAME_CHECK_RECON_STATUS;
+extern unsigned IOCTL_RAIDFRAME_REWRITEPARITY;
+extern unsigned IOCTL_RAIDFRAME_COPYBACK;
+extern unsigned IOCTL_RAIDFRAME_SPARET_WAIT;
+extern unsigned IOCTL_RAIDFRAME_SEND_SPARET;
+extern unsigned IOCTL_RAIDFRAME_ABORT_SPARET_WAIT;
+extern unsigned IOCTL_RAIDFRAME_START_ATRACE;
+extern unsigned IOCTL_RAIDFRAME_STOP_ATRACE;
+extern unsigned IOCTL_RAIDFRAME_GET_SIZE;
+extern unsigned IOCTL_RAIDFRAME_RESET_ACCTOTALS;
+extern unsigned IOCTL_RAIDFRAME_KEEP_ACCTOTALS;
+extern unsigned IOCTL_RAIDFRAME_GET_COMPONENT_LABEL;
+extern unsigned IOCTL_RAIDFRAME_SET_COMPONENT_LABEL;
+extern unsigned IOCTL_RAIDFRAME_INIT_LABELS;
+extern unsigned IOCTL_RAIDFRAME_ADD_HOT_SPARE;
+extern unsigned IOCTL_RAIDFRAME_REMOVE_HOT_SPARE;
+extern unsigned IOCTL_RAIDFRAME_REBUILD_IN_PLACE;
+extern unsigned IOCTL_RAIDFRAME_CHECK_PARITY;
+extern unsigned IOCTL_RAIDFRAME_CHECK_PARITYREWRITE_STATUS;
+extern unsigned IOCTL_RAIDFRAME_CHECK_COPYBACK_STATUS;
+extern unsigned IOCTL_RAIDFRAME_SET_AUTOCONFIG;
+extern unsigned IOCTL_RAIDFRAME_SET_ROOT;
+extern unsigned IOCTL_RAIDFRAME_DELETE_COMPONENT;
+extern unsigned IOCTL_RAIDFRAME_INCORPORATE_HOT_SPARE;
+extern unsigned IOCTL_RAIDFRAME_CHECK_RECON_STATUS_EXT;
+extern unsigned IOCTL_RAIDFRAME_CHECK_PARITYREWRITE_STATUS_EXT;
+extern unsigned IOCTL_RAIDFRAME_CHECK_COPYBACK_STATUS_EXT;
+extern unsigned IOCTL_RAIDFRAME_CONFIGURE;
+extern unsigned IOCTL_RAIDFRAME_GET_INFO;
+extern unsigned IOCTL_RAIDFRAME_PARITYMAP_STATUS;
+extern unsigned IOCTL_RAIDFRAME_PARITYMAP_GET_DISABLE;
+extern unsigned IOCTL_RAIDFRAME_PARITYMAP_SET_DISABLE;
+extern unsigned IOCTL_RAIDFRAME_PARITYMAP_SET_PARAMS;
+extern unsigned IOCTL_RAIDFRAME_SET_LAST_UNIT;
+extern unsigned IOCTL_MBPPIOCSPARAM;
+extern unsigned IOCTL_MBPPIOCGPARAM;
+extern unsigned IOCTL_MBPPIOCGSTAT;
+extern unsigned IOCTL_SESIOC_GETNOBJ;
+extern unsigned IOCTL_SESIOC_GETOBJMAP;
+extern unsigned IOCTL_SESIOC_GETENCSTAT;
+extern unsigned IOCTL_SESIOC_SETENCSTAT;
+extern unsigned IOCTL_SESIOC_GETOBJSTAT;
+extern unsigned IOCTL_SESIOC_SETOBJSTAT;
+extern unsigned IOCTL_SESIOC_GETTEXT;
+extern unsigned IOCTL_SESIOC_INIT;
+extern unsigned IOCTL_SUN_DKIOCGGEOM;
+extern unsigned IOCTL_SUN_DKIOCINFO;
+extern unsigned IOCTL_SUN_DKIOCGPART;
+extern unsigned IOCTL_FBIOGTYPE;
+extern unsigned IOCTL_FBIOPUTCMAP;
+extern unsigned IOCTL_FBIOGETCMAP;
+extern unsigned IOCTL_FBIOGATTR;
+extern unsigned IOCTL_FBIOSVIDEO;
+extern unsigned IOCTL_FBIOGVIDEO;
+extern unsigned IOCTL_FBIOSCURSOR;
+extern unsigned IOCTL_FBIOGCURSOR;
+extern unsigned IOCTL_FBIOSCURPOS;
+extern unsigned IOCTL_FBIOGCURPOS;
+extern unsigned IOCTL_FBIOGCURMAX;
+extern unsigned IOCTL_KIOCTRANS;
+extern unsigned IOCTL_KIOCSETKEY;
+extern unsigned IOCTL_KIOCGETKEY;
+extern unsigned IOCTL_KIOCGTRANS;
+extern unsigned IOCTL_KIOCCMD;
+extern unsigned IOCTL_KIOCTYPE;
+extern unsigned IOCTL_KIOCSDIRECT;
+extern unsigned IOCTL_KIOCSKEY;
+extern unsigned IOCTL_KIOCGKEY;
+extern unsigned IOCTL_KIOCSLED;
+extern unsigned IOCTL_KIOCGLED;
+extern unsigned IOCTL_KIOCLAYOUT;
+extern unsigned IOCTL_VUIDSFORMAT;
+extern unsigned IOCTL_VUIDGFORMAT;
+extern unsigned IOCTL_STICIO_GXINFO;
+extern unsigned IOCTL_STICIO_RESET;
+extern unsigned IOCTL_STICIO_STARTQ;
+extern unsigned IOCTL_STICIO_STOPQ;
+extern unsigned IOCTL_UKYOPON_IDENTIFY;
+extern unsigned IOCTL_URIO_SEND_COMMAND;
+extern unsigned IOCTL_URIO_RECV_COMMAND;
+extern unsigned IOCTL_USB_REQUEST;
+extern unsigned IOCTL_USB_SETDEBUG;
+extern unsigned IOCTL_USB_DISCOVER;
+extern unsigned IOCTL_USB_DEVICEINFO;
+extern unsigned IOCTL_USB_DEVICEINFO_OLD;
+extern unsigned IOCTL_USB_DEVICESTATS;
+extern unsigned IOCTL_USB_GET_REPORT_DESC;
+extern unsigned IOCTL_USB_SET_IMMED;
+extern unsigned IOCTL_USB_GET_REPORT;
+extern unsigned IOCTL_USB_SET_REPORT;
+extern unsigned IOCTL_USB_GET_REPORT_ID;
+extern unsigned IOCTL_USB_GET_CONFIG;
+extern unsigned IOCTL_USB_SET_CONFIG;
+extern unsigned IOCTL_USB_GET_ALTINTERFACE;
+extern unsigned IOCTL_USB_SET_ALTINTERFACE;
+extern unsigned IOCTL_USB_GET_NO_ALT;
+extern unsigned IOCTL_USB_GET_DEVICE_DESC;
+extern unsigned IOCTL_USB_GET_CONFIG_DESC;
+extern unsigned IOCTL_USB_GET_INTERFACE_DESC;
+extern unsigned IOCTL_USB_GET_ENDPOINT_DESC;
+extern unsigned IOCTL_USB_GET_FULL_DESC;
+extern unsigned IOCTL_USB_GET_STRING_DESC;
+extern unsigned IOCTL_USB_DO_REQUEST;
+extern unsigned IOCTL_USB_GET_DEVICEINFO;
+extern unsigned IOCTL_USB_GET_DEVICEINFO_OLD;
+extern unsigned IOCTL_USB_SET_SHORT_XFER;
+extern unsigned IOCTL_USB_SET_TIMEOUT;
+extern unsigned IOCTL_USB_SET_BULK_RA;
+extern unsigned IOCTL_USB_SET_BULK_WB;
+extern unsigned IOCTL_USB_SET_BULK_RA_OPT;
+extern unsigned IOCTL_USB_SET_BULK_WB_OPT;
+extern unsigned IOCTL_USB_GET_CM_OVER_DATA;
+extern unsigned IOCTL_USB_SET_CM_OVER_DATA;
+extern unsigned IOCTL_UTOPPYIOTURBO;
+extern unsigned IOCTL_UTOPPYIOCANCEL;
+extern unsigned IOCTL_UTOPPYIOREBOOT;
+extern unsigned IOCTL_UTOPPYIOSTATS;
+extern unsigned IOCTL_UTOPPYIORENAME;
+extern unsigned IOCTL_UTOPPYIOMKDIR;
+extern unsigned IOCTL_UTOPPYIODELETE;
+extern unsigned IOCTL_UTOPPYIOREADDIR;
+extern unsigned IOCTL_UTOPPYIOREADFILE;
+extern unsigned IOCTL_UTOPPYIOWRITEFILE;
+extern unsigned IOCTL_DIOSXDCMD;
+extern unsigned IOCTL_VT_OPENQRY;
+extern unsigned IOCTL_VT_SETMODE;
+extern unsigned IOCTL_VT_GETMODE;
+extern unsigned IOCTL_VT_RELDISP;
+extern unsigned IOCTL_VT_ACTIVATE;
+extern unsigned IOCTL_VT_WAITACTIVE;
+extern unsigned IOCTL_VT_GETACTIVE;
+extern unsigned IOCTL_VT_GETSTATE;
+extern unsigned IOCTL_KDGETKBENT;
+extern unsigned IOCTL_KDGKBMODE;
+extern unsigned IOCTL_KDSKBMODE;
+extern unsigned IOCTL_KDMKTONE;
+extern unsigned IOCTL_KDSETMODE;
+extern unsigned IOCTL_KDENABIO;
+extern unsigned IOCTL_KDDISABIO;
+extern unsigned IOCTL_KDGKBTYPE;
+extern unsigned IOCTL_KDGETLED;
+extern unsigned IOCTL_KDSETLED;
+extern unsigned IOCTL_KDSETRAD;
+extern unsigned IOCTL_VGAPCVTID;
+extern unsigned IOCTL_CONS_GETVERS;
+extern unsigned IOCTL_WSKBDIO_GTYPE;
+extern unsigned IOCTL_WSKBDIO_BELL;
+extern unsigned IOCTL_WSKBDIO_COMPLEXBELL;
+extern unsigned IOCTL_WSKBDIO_SETBELL;
+extern unsigned IOCTL_WSKBDIO_GETBELL;
+extern unsigned IOCTL_WSKBDIO_SETDEFAULTBELL;
+extern unsigned IOCTL_WSKBDIO_GETDEFAULTBELL;
+extern unsigned IOCTL_WSKBDIO_SETKEYREPEAT;
+extern unsigned IOCTL_WSKBDIO_GETKEYREPEAT;
+extern unsigned IOCTL_WSKBDIO_SETDEFAULTKEYREPEAT;
+extern unsigned IOCTL_WSKBDIO_GETDEFAULTKEYREPEAT;
+extern unsigned IOCTL_WSKBDIO_SETLEDS;
+extern unsigned IOCTL_WSKBDIO_GETLEDS;
+extern unsigned IOCTL_WSKBDIO_GETMAP;
+extern unsigned IOCTL_WSKBDIO_SETMAP;
+extern unsigned IOCTL_WSKBDIO_GETENCODING;
+extern unsigned IOCTL_WSKBDIO_SETENCODING;
+extern unsigned IOCTL_WSKBDIO_SETMODE;
+extern unsigned IOCTL_WSKBDIO_GETMODE;
+extern unsigned IOCTL_WSKBDIO_SETKEYCLICK;
+extern unsigned IOCTL_WSKBDIO_GETKEYCLICK;
+extern unsigned IOCTL_WSKBDIO_GETSCROLL;
+extern unsigned IOCTL_WSKBDIO_SETSCROLL;
+extern unsigned IOCTL_WSKBDIO_SETVERSION;
+extern unsigned IOCTL_WSMOUSEIO_GTYPE;
+extern unsigned IOCTL_WSMOUSEIO_SRES;
+extern unsigned IOCTL_WSMOUSEIO_SSCALE;
+extern unsigned IOCTL_WSMOUSEIO_SRATE;
+extern unsigned IOCTL_WSMOUSEIO_SCALIBCOORDS;
+extern unsigned IOCTL_WSMOUSEIO_GCALIBCOORDS;
+extern unsigned IOCTL_WSMOUSEIO_GETID;
+extern unsigned IOCTL_WSMOUSEIO_GETREPEAT;
+extern unsigned IOCTL_WSMOUSEIO_SETREPEAT;
+extern unsigned IOCTL_WSMOUSEIO_SETVERSION;
+extern unsigned IOCTL_WSDISPLAYIO_GTYPE;
+extern unsigned IOCTL_WSDISPLAYIO_GINFO;
+extern unsigned IOCTL_WSDISPLAYIO_GETCMAP;
+extern unsigned IOCTL_WSDISPLAYIO_PUTCMAP;
+extern unsigned IOCTL_WSDISPLAYIO_GVIDEO;
+extern unsigned IOCTL_WSDISPLAYIO_SVIDEO;
+extern unsigned IOCTL_WSDISPLAYIO_GCURPOS;
+extern unsigned IOCTL_WSDISPLAYIO_SCURPOS;
+extern unsigned IOCTL_WSDISPLAYIO_GCURMAX;
+extern unsigned IOCTL_WSDISPLAYIO_GCURSOR;
+extern unsigned IOCTL_WSDISPLAYIO_SCURSOR;
+extern unsigned IOCTL_WSDISPLAYIO_GMODE;
+extern unsigned IOCTL_WSDISPLAYIO_SMODE;
+extern unsigned IOCTL_WSDISPLAYIO_LDFONT;
+extern unsigned IOCTL_WSDISPLAYIO_ADDSCREEN;
+extern unsigned IOCTL_WSDISPLAYIO_DELSCREEN;
+extern unsigned IOCTL_WSDISPLAYIO_SFONT;
+extern unsigned IOCTL__O_WSDISPLAYIO_SETKEYBOARD;
+extern unsigned IOCTL_WSDISPLAYIO_GETPARAM;
+extern unsigned IOCTL_WSDISPLAYIO_SETPARAM;
+extern unsigned IOCTL_WSDISPLAYIO_GETACTIVESCREEN;
+extern unsigned IOCTL_WSDISPLAYIO_GETWSCHAR;
+extern unsigned IOCTL_WSDISPLAYIO_PUTWSCHAR;
+extern unsigned IOCTL_WSDISPLAYIO_DGSCROLL;
+extern unsigned IOCTL_WSDISPLAYIO_DSSCROLL;
+extern unsigned IOCTL_WSDISPLAYIO_GMSGATTRS;
+extern unsigned IOCTL_WSDISPLAYIO_SMSGATTRS;
+extern unsigned IOCTL_WSDISPLAYIO_GBORDER;
+extern unsigned IOCTL_WSDISPLAYIO_SBORDER;
+extern unsigned IOCTL_WSDISPLAYIO_SSPLASH;
+extern unsigned IOCTL_WSDISPLAYIO_SPROGRESS;
+extern unsigned IOCTL_WSDISPLAYIO_LINEBYTES;
+extern unsigned IOCTL_WSDISPLAYIO_SETVERSION;
+extern unsigned IOCTL_WSMUXIO_ADD_DEVICE;
+extern unsigned IOCTL_WSMUXIO_REMOVE_DEVICE;
+extern unsigned IOCTL_WSMUXIO_LIST_DEVICES;
+extern unsigned IOCTL_WSMUXIO_INJECTEVENT;
+extern unsigned IOCTL_WSDISPLAYIO_GET_BUSID;
+extern unsigned IOCTL_WSDISPLAYIO_GET_EDID;
+extern unsigned IOCTL_WSDISPLAYIO_SET_POLLING;
+extern unsigned IOCTL_WSDISPLAYIO_GET_FBINFO;
+extern unsigned IOCTL_WSDISPLAYIO_DOBLIT;
+extern unsigned IOCTL_WSDISPLAYIO_WAITBLIT;
+extern unsigned IOCTL_BIOCLOCATE;
+extern unsigned IOCTL_BIOCINQ;
+extern unsigned IOCTL_BIOCDISK_NOVOL;
+extern unsigned IOCTL_BIOCDISK;
+extern unsigned IOCTL_BIOCVOL;
+extern unsigned IOCTL_BIOCALARM;
+extern unsigned IOCTL_BIOCBLINK;
+extern unsigned IOCTL_BIOCSETSTATE;
+extern unsigned IOCTL_BIOCVOLOPS;
+extern unsigned IOCTL_MD_GETCONF;
+extern unsigned IOCTL_MD_SETCONF;
+extern unsigned IOCTL_CCDIOCSET;
+extern unsigned IOCTL_CCDIOCCLR;
+extern unsigned IOCTL_CGDIOCSET;
+extern unsigned IOCTL_CGDIOCCLR;
+extern unsigned IOCTL_CGDIOCGET;
+extern unsigned IOCTL_FSSIOCSET;
+extern unsigned IOCTL_FSSIOCGET;
+extern unsigned IOCTL_FSSIOCCLR;
+extern unsigned IOCTL_FSSIOFSET;
+extern unsigned IOCTL_FSSIOFGET;
+extern unsigned IOCTL_BTDEV_ATTACH;
+extern unsigned IOCTL_BTDEV_DETACH;
+extern unsigned IOCTL_BTSCO_GETINFO;
+extern unsigned IOCTL_KTTCP_IO_SEND;
+extern unsigned IOCTL_KTTCP_IO_RECV;
+extern unsigned IOCTL_IOC_LOCKSTAT_GVERSION;
+extern unsigned IOCTL_IOC_LOCKSTAT_ENABLE;
+extern unsigned IOCTL_IOC_LOCKSTAT_DISABLE;
+extern unsigned IOCTL_VNDIOCSET;
+extern unsigned IOCTL_VNDIOCCLR;
+extern unsigned IOCTL_VNDIOCGET;
+extern unsigned IOCTL_SPKRTONE;
+extern unsigned IOCTL_SPKRTUNE;
+extern unsigned IOCTL_SPKRGETVOL;
+extern unsigned IOCTL_SPKRSETVOL;
+#if defined(__x86_64__)
+extern unsigned IOCTL_NVMM_IOC_CAPABILITY;
+extern unsigned IOCTL_NVMM_IOC_MACHINE_CREATE;
+extern unsigned IOCTL_NVMM_IOC_MACHINE_DESTROY;
+extern unsigned IOCTL_NVMM_IOC_MACHINE_CONFIGURE;
+extern unsigned IOCTL_NVMM_IOC_VCPU_CREATE;
+extern unsigned IOCTL_NVMM_IOC_VCPU_DESTROY;
+extern unsigned IOCTL_NVMM_IOC_VCPU_CONFIGURE;
+extern unsigned IOCTL_NVMM_IOC_VCPU_SETSTATE;
+extern unsigned IOCTL_NVMM_IOC_VCPU_GETSTATE;
+extern unsigned IOCTL_NVMM_IOC_VCPU_INJECT;
+extern unsigned IOCTL_NVMM_IOC_VCPU_RUN;
+extern unsigned IOCTL_NVMM_IOC_GPA_MAP;
+extern unsigned IOCTL_NVMM_IOC_GPA_UNMAP;
+extern unsigned IOCTL_NVMM_IOC_HVA_MAP;
+extern unsigned IOCTL_NVMM_IOC_HVA_UNMAP;
+extern unsigned IOCTL_NVMM_IOC_CTL;
+#endif
+extern unsigned IOCTL_AUTOFSREQUEST;
+extern unsigned IOCTL_AUTOFSDONE;
+extern unsigned IOCTL_BIOCGBLEN;
+extern unsigned IOCTL_BIOCSBLEN;
+extern unsigned IOCTL_BIOCSETF;
+extern unsigned IOCTL_BIOCFLUSH;
+extern unsigned IOCTL_BIOCPROMISC;
+extern unsigned IOCTL_BIOCGDLT;
+extern unsigned IOCTL_BIOCGETIF;
+extern unsigned IOCTL_BIOCSETIF;
+extern unsigned IOCTL_BIOCGSTATS;
+extern unsigned IOCTL_BIOCGSTATSOLD;
+extern unsigned IOCTL_BIOCIMMEDIATE;
+extern unsigned IOCTL_BIOCVERSION;
+extern unsigned IOCTL_BIOCSTCPF;
+extern unsigned IOCTL_BIOCSUDPF;
+extern unsigned IOCTL_BIOCGHDRCMPLT;
+extern unsigned IOCTL_BIOCSHDRCMPLT;
+extern unsigned IOCTL_BIOCSDLT;
+extern unsigned IOCTL_BIOCGDLTLIST;
+extern unsigned IOCTL_BIOCGDIRECTION;
+extern unsigned IOCTL_BIOCSDIRECTION;
+extern unsigned IOCTL_BIOCSRTIMEOUT;
+extern unsigned IOCTL_BIOCGRTIMEOUT;
+extern unsigned IOCTL_BIOCGFEEDBACK;
+extern unsigned IOCTL_BIOCSFEEDBACK;
+extern unsigned IOCTL_GRESADDRS;
+extern unsigned IOCTL_GRESADDRD;
+extern unsigned IOCTL_GREGADDRS;
+extern unsigned IOCTL_GREGADDRD;
+extern unsigned IOCTL_GRESPROTO;
+extern unsigned IOCTL_GREGPROTO;
+extern unsigned IOCTL_GRESSOCK;
+extern unsigned IOCTL_GREDSOCK;
+extern unsigned IOCTL_PPPIOCGRAWIN;
+extern unsigned IOCTL_PPPIOCGFLAGS;
+extern unsigned IOCTL_PPPIOCSFLAGS;
+extern unsigned IOCTL_PPPIOCGASYNCMAP;
+extern unsigned IOCTL_PPPIOCSASYNCMAP;
+extern unsigned IOCTL_PPPIOCGUNIT;
+extern unsigned IOCTL_PPPIOCGRASYNCMAP;
+extern unsigned IOCTL_PPPIOCSRASYNCMAP;
+extern unsigned IOCTL_PPPIOCGMRU;
+extern unsigned IOCTL_PPPIOCSMRU;
+extern unsigned IOCTL_PPPIOCSMAXCID;
+extern unsigned IOCTL_PPPIOCGXASYNCMAP;
+extern unsigned IOCTL_PPPIOCSXASYNCMAP;
+extern unsigned IOCTL_PPPIOCXFERUNIT;
+extern unsigned IOCTL_PPPIOCSCOMPRESS;
+extern unsigned IOCTL_PPPIOCGNPMODE;
+extern unsigned IOCTL_PPPIOCSNPMODE;
+extern unsigned IOCTL_PPPIOCGIDLE;
+extern unsigned IOCTL_PPPIOCGMTU;
+extern unsigned IOCTL_PPPIOCSMTU;
+extern unsigned IOCTL_SIOCGPPPSTATS;
+extern unsigned IOCTL_SIOCGPPPCSTATS;
+extern unsigned IOCTL_IOC_NPF_VERSION;
+extern unsigned IOCTL_IOC_NPF_SWITCH;
+extern unsigned IOCTL_IOC_NPF_LOAD;
+extern unsigned IOCTL_IOC_NPF_TABLE;
+extern unsigned IOCTL_IOC_NPF_STATS;
+extern unsigned IOCTL_IOC_NPF_SAVE;
+extern unsigned IOCTL_IOC_NPF_RULE;
+extern unsigned IOCTL_IOC_NPF_CONN_LOOKUP;
+extern unsigned IOCTL_IOC_NPF_TABLE_REPLACE;
+extern unsigned IOCTL_PPPOESETPARMS;
+extern unsigned IOCTL_PPPOEGETPARMS;
+extern unsigned IOCTL_PPPOEGETSESSION;
+extern unsigned IOCTL_SPPPGETAUTHCFG;
+extern unsigned IOCTL_SPPPSETAUTHCFG;
+extern unsigned IOCTL_SPPPGETLCPCFG;
+extern unsigned IOCTL_SPPPSETLCPCFG;
+extern unsigned IOCTL_SPPPGETSTATUS;
+extern unsigned IOCTL_SPPPGETSTATUSNCP;
+extern unsigned IOCTL_SPPPGETIDLETO;
+extern unsigned IOCTL_SPPPSETIDLETO;
+extern unsigned IOCTL_SPPPGETAUTHFAILURES;
+extern unsigned IOCTL_SPPPSETAUTHFAILURE;
+extern unsigned IOCTL_SPPPSETDNSOPTS;
+extern unsigned IOCTL_SPPPGETDNSOPTS;
+extern unsigned IOCTL_SPPPGETDNSADDRS;
+extern unsigned IOCTL_SPPPSETKEEPALIVE;
+extern unsigned IOCTL_SPPPGETKEEPALIVE;
+extern unsigned IOCTL_SRT_GETNRT;
+extern unsigned IOCTL_SRT_GETRT;
+extern unsigned IOCTL_SRT_SETRT;
+extern unsigned IOCTL_SRT_DELRT;
+extern unsigned IOCTL_SRT_SFLAGS;
+extern unsigned IOCTL_SRT_GFLAGS;
+extern unsigned IOCTL_SRT_SGFLAGS;
+extern unsigned IOCTL_SRT_DEBUG;
+extern unsigned IOCTL_TAPGIFNAME;
+extern unsigned IOCTL_TUNSDEBUG;
+extern unsigned IOCTL_TUNGDEBUG;
+extern unsigned IOCTL_TUNSIFMODE;
+extern unsigned IOCTL_TUNSLMODE;
+extern unsigned IOCTL_TUNSIFHEAD;
+extern unsigned IOCTL_TUNGIFHEAD;
+extern unsigned IOCTL_DIOCSTART;
+extern unsigned IOCTL_DIOCSTOP;
+extern unsigned IOCTL_DIOCADDRULE;
+extern unsigned IOCTL_DIOCGETRULES;
+extern unsigned IOCTL_DIOCGETRULE;
+extern unsigned IOCTL_DIOCSETLCK;
+extern unsigned IOCTL_DIOCCLRSTATES;
+extern unsigned IOCTL_DIOCGETSTATE;
+extern unsigned IOCTL_DIOCSETSTATUSIF;
+extern unsigned IOCTL_DIOCGETSTATUS;
+extern unsigned IOCTL_DIOCCLRSTATUS;
+extern unsigned IOCTL_DIOCNATLOOK;
+extern unsigned IOCTL_DIOCSETDEBUG;
+extern unsigned IOCTL_DIOCGETSTATES;
+extern unsigned IOCTL_DIOCCHANGERULE;
+extern unsigned IOCTL_DIOCSETTIMEOUT;
+extern unsigned IOCTL_DIOCGETTIMEOUT;
+extern unsigned IOCTL_DIOCADDSTATE;
+extern unsigned IOCTL_DIOCCLRRULECTRS;
+extern unsigned IOCTL_DIOCGETLIMIT;
+extern unsigned IOCTL_DIOCSETLIMIT;
+extern unsigned IOCTL_DIOCKILLSTATES;
+extern unsigned IOCTL_DIOCSTARTALTQ;
+extern unsigned IOCTL_DIOCSTOPALTQ;
+extern unsigned IOCTL_DIOCADDALTQ;
+extern unsigned IOCTL_DIOCGETALTQS;
+extern unsigned IOCTL_DIOCGETALTQ;
+extern unsigned IOCTL_DIOCCHANGEALTQ;
+extern unsigned IOCTL_DIOCGETQSTATS;
+extern unsigned IOCTL_DIOCBEGINADDRS;
+extern unsigned IOCTL_DIOCADDADDR;
+extern unsigned IOCTL_DIOCGETADDRS;
+extern unsigned IOCTL_DIOCGETADDR;
+extern unsigned IOCTL_DIOCCHANGEADDR;
+extern unsigned IOCTL_DIOCADDSTATES;
+extern unsigned IOCTL_DIOCGETRULESETS;
+extern unsigned IOCTL_DIOCGETRULESET;
+extern unsigned IOCTL_DIOCRCLRTABLES;
+extern unsigned IOCTL_DIOCRADDTABLES;
+extern unsigned IOCTL_DIOCRDELTABLES;
+extern unsigned IOCTL_DIOCRGETTABLES;
+extern unsigned IOCTL_DIOCRGETTSTATS;
+extern unsigned IOCTL_DIOCRCLRTSTATS;
+extern unsigned IOCTL_DIOCRCLRADDRS;
+extern unsigned IOCTL_DIOCRADDADDRS;
+extern unsigned IOCTL_DIOCRDELADDRS;
+extern unsigned IOCTL_DIOCRSETADDRS;
+extern unsigned IOCTL_DIOCRGETADDRS;
+extern unsigned IOCTL_DIOCRGETASTATS;
+extern unsigned IOCTL_DIOCRCLRASTATS;
+extern unsigned IOCTL_DIOCRTSTADDRS;
+extern unsigned IOCTL_DIOCRSETTFLAGS;
+extern unsigned IOCTL_DIOCRINADEFINE;
+extern unsigned IOCTL_DIOCOSFPFLUSH;
+extern unsigned IOCTL_DIOCOSFPADD;
+extern unsigned IOCTL_DIOCOSFPGET;
+extern unsigned IOCTL_DIOCXBEGIN;
+extern unsigned IOCTL_DIOCXCOMMIT;
+extern unsigned IOCTL_DIOCXROLLBACK;
+extern unsigned IOCTL_DIOCGETSRCNODES;
+extern unsigned IOCTL_DIOCCLRSRCNODES;
+extern unsigned IOCTL_DIOCSETHOSTID;
+extern unsigned IOCTL_DIOCIGETIFACES;
+extern unsigned IOCTL_DIOCSETIFFLAG;
+extern unsigned IOCTL_DIOCCLRIFFLAG;
+extern unsigned IOCTL_DIOCKILLSRCNODES;
+extern unsigned IOCTL_SLIOCGUNIT;
+extern unsigned IOCTL_SIOCGBTINFO;
+extern unsigned IOCTL_SIOCGBTINFOA;
+extern unsigned IOCTL_SIOCNBTINFO;
+extern unsigned IOCTL_SIOCSBTFLAGS;
+extern unsigned IOCTL_SIOCSBTPOLICY;
+extern unsigned IOCTL_SIOCSBTPTYPE;
+extern unsigned IOCTL_SIOCGBTSTATS;
+extern unsigned IOCTL_SIOCZBTSTATS;
+extern unsigned IOCTL_SIOCBTDUMP;
+extern unsigned IOCTL_SIOCSBTSCOMTU;
+extern unsigned IOCTL_SIOCGBTFEAT;
+extern unsigned IOCTL_SIOCADNAT;
+extern unsigned IOCTL_SIOCRMNAT;
+extern unsigned IOCTL_SIOCGNATS;
+extern unsigned IOCTL_SIOCGNATL;
+extern unsigned IOCTL_SIOCPURGENAT;
+extern unsigned IOCTL_SIOCCONNECTX;
+extern unsigned IOCTL_SIOCCONNECTXDEL;
+extern unsigned IOCTL_SIOCSIFINFO_FLAGS;
+extern unsigned IOCTL_SIOCAADDRCTL_POLICY;
+extern unsigned IOCTL_SIOCDADDRCTL_POLICY;
+extern unsigned IOCTL_SMBIOC_OPENSESSION;
+extern unsigned IOCTL_SMBIOC_OPENSHARE;
+extern unsigned IOCTL_SMBIOC_REQUEST;
+extern unsigned IOCTL_SMBIOC_SETFLAGS;
+extern unsigned IOCTL_SMBIOC_LOOKUP;
+extern unsigned IOCTL_SMBIOC_READ;
+extern unsigned IOCTL_SMBIOC_WRITE;
+extern unsigned IOCTL_AGPIOC_INFO;
+extern unsigned IOCTL_AGPIOC_ACQUIRE;
+extern unsigned IOCTL_AGPIOC_RELEASE;
+extern unsigned IOCTL_AGPIOC_SETUP;
+extern unsigned IOCTL_AGPIOC_ALLOCATE;
+extern unsigned IOCTL_AGPIOC_DEALLOCATE;
+extern unsigned IOCTL_AGPIOC_BIND;
+extern unsigned IOCTL_AGPIOC_UNBIND;
+extern unsigned IOCTL_AUDIO_GETINFO;
+extern unsigned IOCTL_AUDIO_SETINFO;
+extern unsigned IOCTL_AUDIO_DRAIN;
+extern unsigned IOCTL_AUDIO_FLUSH;
+extern unsigned IOCTL_AUDIO_WSEEK;
+extern unsigned IOCTL_AUDIO_RERROR;
+extern unsigned IOCTL_AUDIO_GETDEV;
+extern unsigned IOCTL_AUDIO_GETENC;
+extern unsigned IOCTL_AUDIO_GETFD;
+extern unsigned IOCTL_AUDIO_SETFD;
+extern unsigned IOCTL_AUDIO_PERROR;
+extern unsigned IOCTL_AUDIO_GETIOFFS;
+extern unsigned IOCTL_AUDIO_GETOOFFS;
+extern unsigned IOCTL_AUDIO_GETPROPS;
+extern unsigned IOCTL_AUDIO_GETBUFINFO;
+extern unsigned IOCTL_AUDIO_SETCHAN;
+extern unsigned IOCTL_AUDIO_GETCHAN;
+extern unsigned IOCTL_AUDIO_QUERYFORMAT;
+extern unsigned IOCTL_AUDIO_GETFORMAT;
+extern unsigned IOCTL_AUDIO_SETFORMAT;
+extern unsigned IOCTL_AUDIO_MIXER_READ;
+extern unsigned IOCTL_AUDIO_MIXER_WRITE;
+extern unsigned IOCTL_AUDIO_MIXER_DEVINFO;
+extern unsigned IOCTL_ATAIOCCOMMAND;
+extern unsigned IOCTL_ATABUSIOSCAN;
+extern unsigned IOCTL_ATABUSIORESET;
+extern unsigned IOCTL_ATABUSIODETACH;
+extern unsigned IOCTL_CDIOCPLAYTRACKS;
+extern unsigned IOCTL_CDIOCPLAYBLOCKS;
+extern unsigned IOCTL_CDIOCREADSUBCHANNEL;
+extern unsigned IOCTL_CDIOREADTOCHEADER;
+extern unsigned IOCTL_CDIOREADTOCENTRIES;
+extern unsigned IOCTL_CDIOREADMSADDR;
+extern unsigned IOCTL_CDIOCSETPATCH;
+extern unsigned IOCTL_CDIOCGETVOL;
+extern unsigned IOCTL_CDIOCSETVOL;
+extern unsigned IOCTL_CDIOCSETMONO;
+extern unsigned IOCTL_CDIOCSETSTEREO;
+extern unsigned IOCTL_CDIOCSETMUTE;
+extern unsigned IOCTL_CDIOCSETLEFT;
+extern unsigned IOCTL_CDIOCSETRIGHT;
+extern unsigned IOCTL_CDIOCSETDEBUG;
+extern unsigned IOCTL_CDIOCCLRDEBUG;
+extern unsigned IOCTL_CDIOCPAUSE;
+extern unsigned IOCTL_CDIOCRESUME;
+extern unsigned IOCTL_CDIOCRESET;
+extern unsigned IOCTL_CDIOCSTART;
+extern unsigned IOCTL_CDIOCSTOP;
+extern unsigned IOCTL_CDIOCEJECT;
+extern unsigned IOCTL_CDIOCALLOW;
+extern unsigned IOCTL_CDIOCPREVENT;
+extern unsigned IOCTL_CDIOCCLOSE;
+extern unsigned IOCTL_CDIOCPLAYMSF;
+extern unsigned IOCTL_CDIOCLOADUNLOAD;
+extern unsigned IOCTL_CHIOMOVE;
+extern unsigned IOCTL_CHIOEXCHANGE;
+extern unsigned IOCTL_CHIOPOSITION;
+extern unsigned IOCTL_CHIOGPICKER;
+extern unsigned IOCTL_CHIOSPICKER;
+extern unsigned IOCTL_CHIOGPARAMS;
+extern unsigned IOCTL_CHIOIELEM;
+extern unsigned IOCTL_OCHIOGSTATUS;
+extern unsigned IOCTL_CHIOGSTATUS;
+extern unsigned IOCTL_CHIOSVOLTAG;
+extern unsigned IOCTL_CLOCKCTL_SETTIMEOFDAY;
+extern unsigned IOCTL_CLOCKCTL_ADJTIME;
+extern unsigned IOCTL_CLOCKCTL_CLOCK_SETTIME;
+extern unsigned IOCTL_CLOCKCTL_NTP_ADJTIME;
+extern unsigned IOCTL_IOC_CPU_SETSTATE;
+extern unsigned IOCTL_IOC_CPU_GETSTATE;
+extern unsigned IOCTL_IOC_CPU_GETCOUNT;
+extern unsigned IOCTL_IOC_CPU_MAPID;
+extern unsigned IOCTL_IOC_CPU_UCODE_GET_VERSION;
+extern unsigned IOCTL_IOC_CPU_UCODE_APPLY;
+extern unsigned IOCTL_DIOCGDINFO;
+extern unsigned IOCTL_DIOCSDINFO;
+extern unsigned IOCTL_DIOCWDINFO;
+extern unsigned IOCTL_DIOCRFORMAT;
+extern unsigned IOCTL_DIOCWFORMAT;
+extern unsigned IOCTL_DIOCSSTEP;
+extern unsigned IOCTL_DIOCSRETRIES;
+extern unsigned IOCTL_DIOCKLABEL;
+extern unsigned IOCTL_DIOCWLABEL;
+extern unsigned IOCTL_DIOCSBAD;
+extern unsigned IOCTL_DIOCEJECT;
+extern unsigned IOCTL_ODIOCEJECT;
+extern unsigned IOCTL_DIOCLOCK;
+extern unsigned IOCTL_DIOCGDEFLABEL;
+extern unsigned IOCTL_DIOCCLRLABEL;
+extern unsigned IOCTL_DIOCGCACHE;
+extern unsigned IOCTL_DIOCSCACHE;
+extern unsigned IOCTL_DIOCCACHESYNC;
+extern unsigned IOCTL_DIOCBSLIST;
+extern unsigned IOCTL_DIOCBSFLUSH;
+extern unsigned IOCTL_DIOCAWEDGE;
+extern unsigned IOCTL_DIOCGWEDGEINFO;
+extern unsigned IOCTL_DIOCDWEDGE;
+extern unsigned IOCTL_DIOCLWEDGES;
+extern unsigned IOCTL_DIOCGSTRATEGY;
+extern unsigned IOCTL_DIOCSSTRATEGY;
+extern unsigned IOCTL_DIOCGDISKINFO;
+extern unsigned IOCTL_DIOCTUR;
+extern unsigned IOCTL_DIOCMWEDGES;
+extern unsigned IOCTL_DIOCGSECTORSIZE;
+extern unsigned IOCTL_DIOCGMEDIASIZE;
+extern unsigned IOCTL_DIOCRMWEDGES;
+extern unsigned IOCTL_DRVDETACHDEV;
+extern unsigned IOCTL_DRVRESCANBUS;
+extern unsigned IOCTL_DRVCTLCOMMAND;
+extern unsigned IOCTL_DRVRESUMEDEV;
+extern unsigned IOCTL_DRVLISTDEV;
+extern unsigned IOCTL_DRVGETEVENT;
+extern unsigned IOCTL_DRVSUSPENDDEV;
+extern unsigned IOCTL_DVD_READ_STRUCT;
+extern unsigned IOCTL_DVD_WRITE_STRUCT;
+extern unsigned IOCTL_DVD_AUTH;
+extern unsigned IOCTL_ENVSYS_GETDICTIONARY;
+extern unsigned IOCTL_ENVSYS_SETDICTIONARY;
+extern unsigned IOCTL_ENVSYS_REMOVEPROPS;
+extern unsigned IOCTL_ENVSYS_GTREDATA;
+extern unsigned IOCTL_ENVSYS_GTREINFO;
+extern unsigned IOCTL_KFILTER_BYFILTER;
+extern unsigned IOCTL_KFILTER_BYNAME;
+extern unsigned IOCTL_FDIOCGETOPTS;
+extern unsigned IOCTL_FDIOCSETOPTS;
+extern unsigned IOCTL_FDIOCSETFORMAT;
+extern unsigned IOCTL_FDIOCGETFORMAT;
+extern unsigned IOCTL_FDIOCFORMAT_TRACK;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSEEKDATA;
+extern unsigned IOCTL_FIOSEEKHOLE;
+extern unsigned IOCTL_FIONREAD;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_OFIOGETBMAP;
+extern unsigned IOCTL_FIOGETBMAP;
+extern unsigned IOCTL_FIONWRITE;
+extern unsigned IOCTL_FIONSPACE;
+extern unsigned IOCTL_GPIOINFO;
+extern unsigned IOCTL_GPIOSET;
+extern unsigned IOCTL_GPIOUNSET;
+extern unsigned IOCTL_GPIOREAD;
+extern unsigned IOCTL_GPIOWRITE;
+extern unsigned IOCTL_GPIOTOGGLE;
+extern unsigned IOCTL_GPIOATTACH;
+extern unsigned IOCTL_PTIOCNETBSD;
+extern unsigned IOCTL_PTIOCSUNOS;
+extern unsigned IOCTL_PTIOCLINUX;
+extern unsigned IOCTL_PTIOCFREEBSD;
+extern unsigned IOCTL_PTIOCULTRIX;
+extern unsigned IOCTL_TIOCHPCL;
+extern unsigned IOCTL_TIOCGETP;
+extern unsigned IOCTL_TIOCSETP;
+extern unsigned IOCTL_TIOCSETN;
+extern unsigned IOCTL_TIOCSETC;
+extern unsigned IOCTL_TIOCGETC;
+extern unsigned IOCTL_TIOCLBIS;
+extern unsigned IOCTL_TIOCLBIC;
+extern unsigned IOCTL_TIOCLSET;
+extern unsigned IOCTL_TIOCLGET;
+extern unsigned IOCTL_TIOCSLTC;
+extern unsigned IOCTL_TIOCGLTC;
+extern unsigned IOCTL_OTIOCCONS;
+extern unsigned IOCTL_JOY_SETTIMEOUT;
+extern unsigned IOCTL_JOY_GETTIMEOUT;
+extern unsigned IOCTL_JOY_SET_X_OFFSET;
+extern unsigned IOCTL_JOY_SET_Y_OFFSET;
+extern unsigned IOCTL_JOY_GET_X_OFFSET;
+extern unsigned IOCTL_JOY_GET_Y_OFFSET;
+extern unsigned IOCTL_OKIOCGSYMBOL;
+extern unsigned IOCTL_OKIOCGVALUE;
+extern unsigned IOCTL_KIOCGSIZE;
+extern unsigned IOCTL_KIOCGVALUE;
+extern unsigned IOCTL_KIOCGSYMBOL;
+extern unsigned IOCTL_LUAINFO;
+extern unsigned IOCTL_LUACREATE;
+extern unsigned IOCTL_LUADESTROY;
+extern unsigned IOCTL_LUAREQUIRE;
+extern unsigned IOCTL_LUALOAD;
+extern unsigned IOCTL_MIDI_PRETIME;
+extern unsigned IOCTL_MIDI_MPUMODE;
+extern unsigned IOCTL_MIDI_MPUCMD;
+extern unsigned IOCTL_SEQUENCER_RESET;
+extern unsigned IOCTL_SEQUENCER_SYNC;
+extern unsigned IOCTL_SEQUENCER_INFO;
+extern unsigned IOCTL_SEQUENCER_CTRLRATE;
+extern unsigned IOCTL_SEQUENCER_GETOUTCOUNT;
+extern unsigned IOCTL_SEQUENCER_GETINCOUNT;
+extern unsigned IOCTL_SEQUENCER_RESETSAMPLES;
+extern unsigned IOCTL_SEQUENCER_NRSYNTHS;
+extern unsigned IOCTL_SEQUENCER_NRMIDIS;
+extern unsigned IOCTL_SEQUENCER_THRESHOLD;
+extern unsigned IOCTL_SEQUENCER_MEMAVL;
+extern unsigned IOCTL_SEQUENCER_PANIC;
+extern unsigned IOCTL_SEQUENCER_OUTOFBAND;
+extern unsigned IOCTL_SEQUENCER_GETTIME;
+extern unsigned IOCTL_SEQUENCER_TMR_TIMEBASE;
+extern unsigned IOCTL_SEQUENCER_TMR_START;
+extern unsigned IOCTL_SEQUENCER_TMR_STOP;
+extern unsigned IOCTL_SEQUENCER_TMR_CONTINUE;
+extern unsigned IOCTL_SEQUENCER_TMR_TEMPO;
+extern unsigned IOCTL_SEQUENCER_TMR_SOURCE;
+extern unsigned IOCTL_SEQUENCER_TMR_METRONOME;
+extern unsigned IOCTL_SEQUENCER_TMR_SELECT;
+extern unsigned IOCTL_SPI_IOCTL_CONFIGURE;
+extern unsigned IOCTL_SPI_IOCTL_TRANSFER;
+extern unsigned IOCTL_MTIOCTOP;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCIEOT;
+extern unsigned IOCTL_MTIOCEEOT;
+extern unsigned IOCTL_MTIOCRDSPOS;
+extern unsigned IOCTL_MTIOCRDHPOS;
+extern unsigned IOCTL_MTIOCSLOCATE;
+extern unsigned IOCTL_MTIOCHLOCATE;
+extern unsigned IOCTL_POWER_EVENT_RECVDICT;
+extern unsigned IOCTL_POWER_IOC_GET_TYPE;
+extern unsigned IOCTL_RIOCGINFO;
+extern unsigned IOCTL_RIOCSINFO;
+extern unsigned IOCTL_RIOCSSRCH;
+extern unsigned IOCTL_RNDGETENTCNT;
+extern unsigned IOCTL_RNDGETSRCNUM;
+extern unsigned IOCTL_RNDGETSRCNAME;
+extern unsigned IOCTL_RNDCTL;
+extern unsigned IOCTL_RNDADDDATA;
+extern unsigned IOCTL_RNDGETPOOLSTAT;
+extern unsigned IOCTL_RNDGETESTNUM;
+extern unsigned IOCTL_RNDGETESTNAME;
+extern unsigned IOCTL_SCIOCGET;
+extern unsigned IOCTL_SCIOCSET;
+extern unsigned IOCTL_SCIOCRESTART;
+extern unsigned IOCTL_SCIOC_USE_ADF;
+extern unsigned IOCTL_SCIOCCOMMAND;
+extern unsigned IOCTL_SCIOCDEBUG;
+extern unsigned IOCTL_SCIOCIDENTIFY;
+extern unsigned IOCTL_OSCIOCIDENTIFY;
+extern unsigned IOCTL_SCIOCDECONFIG;
+extern unsigned IOCTL_SCIOCRECONFIG;
+extern unsigned IOCTL_SCIOCRESET;
+extern unsigned IOCTL_SCBUSIOSCAN;
+extern unsigned IOCTL_SCBUSIORESET;
+extern unsigned IOCTL_SCBUSIODETACH;
+extern unsigned IOCTL_SCBUSACCEL;
+extern unsigned IOCTL_SCBUSIOLLSCAN;
+extern unsigned IOCTL_SIOCSHIWAT;
+extern unsigned IOCTL_SIOCGHIWAT;
+extern unsigned IOCTL_SIOCSLOWAT;
+extern unsigned IOCTL_SIOCGLOWAT;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCPEELOFF;
+extern unsigned IOCTL_SIOCADDRT;
+extern unsigned IOCTL_SIOCDELRT;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCDIFADDR;
+extern unsigned IOCTL_SIOCAIFADDR;
+extern unsigned IOCTL_SIOCGIFALIAS;
+extern unsigned IOCTL_SIOCGIFAFLAG_IN;
+extern unsigned IOCTL_SIOCALIFADDR;
+extern unsigned IOCTL_SIOCGLIFADDR;
+extern unsigned IOCTL_SIOCDLIFADDR;
+extern unsigned IOCTL_SIOCSIFADDRPREF;
+extern unsigned IOCTL_SIOCGIFADDRPREF;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGETVIFCNT;
+extern unsigned IOCTL_SIOCGETSGCNT;
+extern unsigned IOCTL_SIOCSIFMEDIA;
+extern unsigned IOCTL_SIOCGIFMEDIA;
+extern unsigned IOCTL_SIOCSIFGENERIC;
+extern unsigned IOCTL_SIOCGIFGENERIC;
+extern unsigned IOCTL_SIOCSIFPHYADDR;
+extern unsigned IOCTL_SIOCGIFPSRCADDR;
+extern unsigned IOCTL_SIOCGIFPDSTADDR;
+extern unsigned IOCTL_SIOCDIFPHYADDR;
+extern unsigned IOCTL_SIOCSLIFPHYADDR;
+extern unsigned IOCTL_SIOCGLIFPHYADDR;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCSDRVSPEC;
+extern unsigned IOCTL_SIOCGDRVSPEC;
+extern unsigned IOCTL_SIOCIFCREATE;
+extern unsigned IOCTL_SIOCIFDESTROY;
+extern unsigned IOCTL_SIOCIFGCLONERS;
+extern unsigned IOCTL_SIOCGIFDLT;
+extern unsigned IOCTL_SIOCGIFCAP;
+extern unsigned IOCTL_SIOCSIFCAP;
+extern unsigned IOCTL_SIOCSVH;
+extern unsigned IOCTL_SIOCGVH;
+extern unsigned IOCTL_SIOCINITIFADDR;
+extern unsigned IOCTL_SIOCGIFDATA;
+extern unsigned IOCTL_SIOCZIFDATA;
+extern unsigned IOCTL_SIOCGLINKSTR;
+extern unsigned IOCTL_SIOCSLINKSTR;
+extern unsigned IOCTL_SIOCGETHERCAP;
+extern unsigned IOCTL_SIOCGIFINDEX;
+extern unsigned IOCTL_SIOCSETHERCAP;
+extern unsigned IOCTL_SIOCSIFDESCR;
+extern unsigned IOCTL_SIOCGIFDESCR;
+extern unsigned IOCTL_SIOCGUMBINFO;
+extern unsigned IOCTL_SIOCSUMBPARAM;
+extern unsigned IOCTL_SIOCGUMBPARAM;
+extern unsigned IOCTL_SIOCSETPFSYNC;
+extern unsigned IOCTL_SIOCGETPFSYNC;
+extern unsigned IOCTL_PPS_IOC_CREATE;
+extern unsigned IOCTL_PPS_IOC_DESTROY;
+extern unsigned IOCTL_PPS_IOC_SETPARAMS;
+extern unsigned IOCTL_PPS_IOC_GETPARAMS;
+extern unsigned IOCTL_PPS_IOC_GETCAP;
+extern unsigned IOCTL_PPS_IOC_FETCH;
+extern unsigned IOCTL_PPS_IOC_KCBIND;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCFLUSH;
+extern unsigned IOCTL_TIOCGETA;
+extern unsigned IOCTL_TIOCSETA;
+extern unsigned IOCTL_TIOCSETAW;
+extern unsigned IOCTL_TIOCSETAF;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCGLINED;
+extern unsigned IOCTL_TIOCSLINED;
+extern unsigned IOCTL_TIOCSBRK;
+extern unsigned IOCTL_TIOCCBRK;
+extern unsigned IOCTL_TIOCSDTR;
+extern unsigned IOCTL_TIOCCDTR;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSTOP;
+extern unsigned IOCTL_TIOCSTART;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCREMOTE;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_TIOCUCNTL;
+extern unsigned IOCTL_TIOCSTAT;
+extern unsigned IOCTL_TIOCGSID;
+extern unsigned IOCTL_TIOCCONS;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCEXT;
+extern unsigned IOCTL_TIOCSIG;
+extern unsigned IOCTL_TIOCDRAIN;
+extern unsigned IOCTL_TIOCGFLAGS;
+extern unsigned IOCTL_TIOCSFLAGS;
+extern unsigned IOCTL_TIOCDCDTIMESTAMP;
+extern unsigned IOCTL_TIOCRCVFRAME;
+extern unsigned IOCTL_TIOCXMTFRAME;
+extern unsigned IOCTL_TIOCPTMGET;
+extern unsigned IOCTL_TIOCGRANTPT;
+extern unsigned IOCTL_TIOCPTSNAME;
+extern unsigned IOCTL_TIOCSQSIZE;
+extern unsigned IOCTL_TIOCGQSIZE;
+extern unsigned IOCTL_VERIEXEC_LOAD;
+extern unsigned IOCTL_VERIEXEC_TABLESIZE;
+extern unsigned IOCTL_VERIEXEC_DELETE;
+extern unsigned IOCTL_VERIEXEC_QUERY;
+extern unsigned IOCTL_VERIEXEC_DUMP;
+extern unsigned IOCTL_VERIEXEC_FLUSH;
+extern unsigned IOCTL_VIDIOC_QUERYCAP;
+extern unsigned IOCTL_VIDIOC_RESERVED;
+extern unsigned IOCTL_VIDIOC_ENUM_FMT;
+extern unsigned IOCTL_VIDIOC_G_FMT;
+extern unsigned IOCTL_VIDIOC_S_FMT;
+extern unsigned IOCTL_VIDIOC_REQBUFS;
+extern unsigned IOCTL_VIDIOC_QUERYBUF;
+extern unsigned IOCTL_VIDIOC_G_FBUF;
+extern unsigned IOCTL_VIDIOC_S_FBUF;
+extern unsigned IOCTL_VIDIOC_OVERLAY;
+extern unsigned IOCTL_VIDIOC_QBUF;
+extern unsigned IOCTL_VIDIOC_DQBUF;
+extern unsigned IOCTL_VIDIOC_STREAMON;
+extern unsigned IOCTL_VIDIOC_STREAMOFF;
+extern unsigned IOCTL_VIDIOC_G_PARM;
+extern unsigned IOCTL_VIDIOC_S_PARM;
+extern unsigned IOCTL_VIDIOC_G_STD;
+extern unsigned IOCTL_VIDIOC_S_STD;
+extern unsigned IOCTL_VIDIOC_ENUMSTD;
+extern unsigned IOCTL_VIDIOC_ENUMINPUT;
+extern unsigned IOCTL_VIDIOC_G_CTRL;
+extern unsigned IOCTL_VIDIOC_S_CTRL;
+extern unsigned IOCTL_VIDIOC_G_TUNER;
+extern unsigned IOCTL_VIDIOC_S_TUNER;
+extern unsigned IOCTL_VIDIOC_G_AUDIO;
+extern unsigned IOCTL_VIDIOC_S_AUDIO;
+extern unsigned IOCTL_VIDIOC_QUERYCTRL;
+extern unsigned IOCTL_VIDIOC_QUERYMENU;
+extern unsigned IOCTL_VIDIOC_G_INPUT;
+extern unsigned IOCTL_VIDIOC_S_INPUT;
+extern unsigned IOCTL_VIDIOC_G_OUTPUT;
+extern unsigned IOCTL_VIDIOC_S_OUTPUT;
+extern unsigned IOCTL_VIDIOC_ENUMOUTPUT;
+extern unsigned IOCTL_VIDIOC_G_AUDOUT;
+extern unsigned IOCTL_VIDIOC_S_AUDOUT;
+extern unsigned IOCTL_VIDIOC_G_MODULATOR;
+extern unsigned IOCTL_VIDIOC_S_MODULATOR;
+extern unsigned IOCTL_VIDIOC_G_FREQUENCY;
+extern unsigned IOCTL_VIDIOC_S_FREQUENCY;
+extern unsigned IOCTL_VIDIOC_CROPCAP;
+extern unsigned IOCTL_VIDIOC_G_CROP;
+extern unsigned IOCTL_VIDIOC_S_CROP;
+extern unsigned IOCTL_VIDIOC_G_JPEGCOMP;
+extern unsigned IOCTL_VIDIOC_S_JPEGCOMP;
+extern unsigned IOCTL_VIDIOC_QUERYSTD;
+extern unsigned IOCTL_VIDIOC_TRY_FMT;
+extern unsigned IOCTL_VIDIOC_ENUMAUDIO;
+extern unsigned IOCTL_VIDIOC_ENUMAUDOUT;
+extern unsigned IOCTL_VIDIOC_G_PRIORITY;
+extern unsigned IOCTL_VIDIOC_S_PRIORITY;
+extern unsigned IOCTL_VIDIOC_ENUM_FRAMESIZES;
+extern unsigned IOCTL_VIDIOC_ENUM_FRAMEINTERVALS;
+extern unsigned IOCTL_WDOGIOC_GMODE;
+extern unsigned IOCTL_WDOGIOC_SMODE;
+extern unsigned IOCTL_WDOGIOC_WHICH;
+extern unsigned IOCTL_WDOGIOC_TICKLE;
+extern unsigned IOCTL_WDOGIOC_GTICKLER;
+extern unsigned IOCTL_WDOGIOC_GWDOGS;
+extern unsigned IOCTL_KCOV_IOC_SETBUFSIZE;
+extern unsigned IOCTL_KCOV_IOC_ENABLE;
+extern unsigned IOCTL_KCOV_IOC_DISABLE;
+extern unsigned IOCTL_IPMICTL_RECEIVE_MSG_TRUNC;
+extern unsigned IOCTL_IPMICTL_RECEIVE_MSG;
+extern unsigned IOCTL_IPMICTL_SEND_COMMAND;
+extern unsigned IOCTL_IPMICTL_REGISTER_FOR_CMD;
+extern unsigned IOCTL_IPMICTL_UNREGISTER_FOR_CMD;
+extern unsigned IOCTL_IPMICTL_SET_GETS_EVENTS_CMD;
+extern unsigned IOCTL_IPMICTL_SET_MY_ADDRESS_CMD;
+extern unsigned IOCTL_IPMICTL_GET_MY_ADDRESS_CMD;
+extern unsigned IOCTL_IPMICTL_SET_MY_LUN_CMD;
+extern unsigned IOCTL_IPMICTL_GET_MY_LUN_CMD;
+extern unsigned IOCTL_SNDCTL_DSP_RESET;
+extern unsigned IOCTL_SNDCTL_DSP_SYNC;
+extern unsigned IOCTL_SNDCTL_DSP_SPEED;
+extern unsigned IOCTL_SOUND_PCM_READ_RATE;
+extern unsigned IOCTL_SNDCTL_DSP_STEREO;
+extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE;
+extern unsigned IOCTL_SNDCTL_DSP_SETFMT;
+extern unsigned IOCTL_SOUND_PCM_READ_BITS;
+extern unsigned IOCTL_SNDCTL_DSP_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER;
+extern unsigned IOCTL_SOUND_PCM_READ_FILTER;
+extern unsigned IOCTL_SNDCTL_DSP_POST;
+extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE;
+extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT;
+extern unsigned IOCTL_SNDCTL_DSP_GETFMTS;
+extern unsigned IOCTL_SNDCTL_DSP_GETOSPACE;
+extern unsigned IOCTL_SNDCTL_DSP_GETISPACE;
+extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK;
+extern unsigned IOCTL_SNDCTL_DSP_GETCAPS;
+extern unsigned IOCTL_SNDCTL_DSP_GETTRIGGER;
+extern unsigned IOCTL_SNDCTL_DSP_SETTRIGGER;
+extern unsigned IOCTL_SNDCTL_DSP_GETIPTR;
+extern unsigned IOCTL_SNDCTL_DSP_GETOPTR;
+extern unsigned IOCTL_SNDCTL_DSP_MAPINBUF;
+extern unsigned IOCTL_SNDCTL_DSP_MAPOUTBUF;
+extern unsigned IOCTL_SNDCTL_DSP_SETSYNCRO;
+extern unsigned IOCTL_SNDCTL_DSP_SETDUPLEX;
+extern unsigned IOCTL_SNDCTL_DSP_PROFILE;
+extern unsigned IOCTL_SNDCTL_DSP_GETODELAY;
+extern unsigned IOCTL_SOUND_MIXER_INFO;
+extern unsigned IOCTL_SOUND_OLD_MIXER_INFO;
+extern unsigned IOCTL_OSS_GETVERSION;
+extern unsigned IOCTL_SNDCTL_SYSINFO;
+extern unsigned IOCTL_SNDCTL_AUDIOINFO;
+extern unsigned IOCTL_SNDCTL_ENGINEINFO;
+extern unsigned IOCTL_SNDCTL_DSP_GETPLAYVOL;
+extern unsigned IOCTL_SNDCTL_DSP_SETPLAYVOL;
+extern unsigned IOCTL_SNDCTL_DSP_GETRECVOL;
+extern unsigned IOCTL_SNDCTL_DSP_SETRECVOL;
+extern unsigned IOCTL_SNDCTL_DSP_SKIP;
+extern unsigned IOCTL_SNDCTL_DSP_SILENCE;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+
+extern const unsigned SHA1_CTX_sz;
+extern const unsigned SHA1_return_length;
+
+extern const unsigned MD4_CTX_sz;
+extern const unsigned MD4_return_length;
+
+extern const unsigned RMD160_CTX_sz;
+extern const unsigned RMD160_return_length;
+
+extern const unsigned MD5_CTX_sz;
+extern const unsigned MD5_return_length;
+
+extern const unsigned fpos_t_sz;
+
+extern const unsigned MD2_CTX_sz;
+extern const unsigned MD2_return_length;
+
+#define SHA2_EXTERN(LEN)                          \
+  extern const unsigned SHA##LEN##_CTX_sz;        \
+  extern const unsigned SHA##LEN##_return_length; \
+  extern const unsigned SHA##LEN##_block_length;  \
+  extern const unsigned SHA##LEN##_digest_length
+
+SHA2_EXTERN(224);
+SHA2_EXTERN(256);
+SHA2_EXTERN(384);
+SHA2_EXTERN(512);
+
+#undef SHA2_EXTERN
+
+extern const int unvis_valid;
+extern const int unvis_validpush;
+
+struct __sanitizer_cdbr {
+  void (*unmap)(void *, void *, uptr);
+  void *cookie;
+  u8 *mmap_base;
+  uptr mmap_size;
+
+  u8 *hash_base;
+  u8 *offset_base;
+  u8 *data_base;
+
+  u32 data_size;
+  u32 entries;
+  u32 entries_index;
+  u32 seed;
+
+  u8 offset_size;
+  u8 index_size;
+
+  u32 entries_m;
+  u32 entries_index_m;
+  u8 entries_s1, entries_s2;
+  u8 entries_index_s1, entries_index_s2;
+};
+
+struct __sanitizer_cdbw {
+  uptr data_counter;
+  uptr data_allocated;
+  uptr data_size;
+  uptr *data_len;
+  void **data_ptr;
+  uptr hash_size;
+  void *hash;
+  uptr key_counter;
+};
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(struct CLASS, MEMBER))
+
+#define SIGACTION_SYMNAME __sigaction14
+
+// Compat with 9.0
+extern unsigned struct_statvfs90_sz;
+
+#endif  // SANITIZER_NETBSD
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp
new file mode 100644
index 0000000000..1420ecbfa5
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.cpp
@@ -0,0 +1,279 @@
+//===-- sanitizer_platform_limits_openbsd.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_OPENBSD
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link_elf.h>
+#include <sys/socket.h>
+#include <net/if.h>
+#include <net/ppp_defs.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip_mroute.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <soundcard.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/msg.h>
+#include <sys/mtio.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/shm.h>
+#include <sys/signal.h>
+#include <sys/sockio.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/times.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <term.h>
+#include <time.h>
+#include <utime.h>
+#include <utmp.h>
+#include <wchar.h>
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_openbsd.h"
+
+namespace __sanitizer {
+unsigned struct_utsname_sz = sizeof(struct utsname);
+unsigned struct_stat_sz = sizeof(struct stat);
+unsigned struct_rusage_sz = sizeof(struct rusage);
+unsigned struct_tm_sz = sizeof(struct tm);
+unsigned struct_passwd_sz = sizeof(struct passwd);
+unsigned struct_group_sz = sizeof(struct group);
+unsigned siginfo_t_sz = sizeof(siginfo_t);
+unsigned struct_sigaction_sz = sizeof(struct sigaction);
+unsigned struct_stack_t_sz = sizeof(stack_t);
+unsigned struct_itimerval_sz = sizeof(struct itimerval);
+unsigned pthread_t_sz = sizeof(pthread_t);
+unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+unsigned pid_t_sz = sizeof(pid_t);
+unsigned timeval_sz = sizeof(timeval);
+unsigned uid_t_sz = sizeof(uid_t);
+unsigned gid_t_sz = sizeof(gid_t);
+unsigned mbstate_t_sz = sizeof(mbstate_t);
+unsigned sigset_t_sz = sizeof(sigset_t);
+unsigned struct_timezone_sz = sizeof(struct timezone);
+unsigned struct_tms_sz = sizeof(struct tms);
+unsigned struct_sched_param_sz = sizeof(struct sched_param);
+unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+unsigned struct_rlimit_sz = sizeof(struct rlimit);
+unsigned struct_timespec_sz = sizeof(struct timespec);
+unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+unsigned struct_statvfs_sz = sizeof(struct statvfs);
+
+const uptr sig_ign = (uptr)SIG_IGN;
+const uptr sig_dfl = (uptr)SIG_DFL;
+const uptr sig_err = (uptr)SIG_ERR;
+const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+int shmctl_ipc_stat = (int)IPC_STAT;
+
+unsigned struct_utmp_sz = sizeof(struct utmp);
+
+int map_fixed = MAP_FIXED;
+
+int af_inet = (int)AF_INET;
+int af_inet6 = (int)AF_INET6;
+
+uptr __sanitizer_in_addr_sz(int af) {
+  if (af == AF_INET)
+    return sizeof(struct in_addr);
+  else if (af == AF_INET6)
+    return sizeof(struct in6_addr);
+  else
+    return 0;
+}
+
+unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
+
+int glob_nomatch = GLOB_NOMATCH;
+int glob_altdirfunc = GLOB_ALTDIRFUNC;
+
+unsigned path_max = PATH_MAX;
+
+const int si_SEGV_MAPERR = SEGV_MAPERR;
+const int si_SEGV_ACCERR = SEGV_ACCERR;
+}  // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+CHECK_TYPE_SIZE(dl_phdr_info);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_next);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_fileno);
+CHECK_SIZE_AND_OFFSET(dirent, d_off);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, key);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_atimensec);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_dtimensec);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_ctimensec);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+#endif  // SANITIZER_OPENBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h
new file mode 100644
index 0000000000..8a19487236
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_openbsd.h
@@ -0,0 +1,382 @@
+//===-- sanitizer_platform_limits_openbsd.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific OpenBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_OPENBSD_H
+#define SANITIZER_PLATFORM_LIMITS_OPENBSD_H
+
+#if SANITIZER_OPENBSD
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \
+  ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift))))
+
+#if defined(__x86_64__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+  _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312)
+#elif defined(__i386__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+  _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164)
+#endif
+
+#define RLIMIT_AS RLIMIT_DATA
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_stack_t_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+
+struct __sanitizer_iocb {
+  u64 aio_offset;
+  uptr aio_buf;
+  long aio_nbytes;
+  u32 aio_fildes;
+  u32 aio_lio_opcode;
+  long aio_reqprio;
+#if SANITIZER_WORDSIZE == 64
+  u8 aio_sigevent[32];
+#else
+  u8 aio_sigevent[20];
+#endif
+  u32 _state;
+  u32 _errno;
+  long _retval;
+};
+
+struct __sanitizer___sysctl_args {
+  int *name;
+  int nlen;
+  void *oldval;
+  uptr *oldlenp;
+  void *newval;
+  uptr newlen;
+};
+
+struct __sanitizer_sem_t {
+  uptr data[5];
+};
+
+struct __sanitizer_ipc_perm {
+  u32 cuid;
+  u32 cgid;
+  u32 uid;
+  u32 gid;
+  u32 mode;
+  unsigned short seq;
+  long key;
+};
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+  int shm_segsz;
+  u32 shm_lpid;
+  u32 shm_cpid;
+  short shm_nattch;
+  u64 shm_atime;
+  long __shm_atimensec;
+  u64 shm_dtime;
+  long __shm_dtimensec;
+  u64 shm_ctime;
+  long __shm_ctimensec;
+  void *_shm_internal;
+};
+
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timex_sz;
+extern unsigned struct_statvfs_sz;
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  unsigned int ifa_flags;
+  struct __sanitizer_sockaddr *ifa_addr;     // (struct sockaddr *)
+  struct __sanitizer_sockaddr *ifa_netmask;  // (struct sockaddr *)
+  struct __sanitizer_sockaddr *ifa_dstaddr;  // (struct sockaddr *)
+  void *ifa_data;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+typedef long long __sanitizer_time_t;
+typedef int __sanitizer_suseconds_t;
+
+struct __sanitizer_timeval {
+  __sanitizer_time_t tv_sec;
+  __sanitizer_suseconds_t tv_usec;
+};
+
+struct __sanitizer_itimerval {
+  struct __sanitizer_timeval it_interval;
+  struct __sanitizer_timeval it_value;
+};
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  int pw_uid;
+  int pw_gid;
+  __sanitizer_time_t pw_change;
+  char *pw_class;
+  char *pw_gecos;
+  char *pw_dir;
+  char *pw_shell;
+  __sanitizer_time_t pw_expire;
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+  long int tm_gmtoff;
+  const char *tm_zone;
+};
+
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+
+struct __sanitizer_dirent {
+  u64 d_fileno;
+  u64 d_off;
+  u16 d_reclen;
+};
+
+typedef u64 __sanitizer_clock_t;
+typedef u32 __sanitizer_clockid_t;
+
+typedef u32 __sanitizer___kernel_uid_t;
+typedef u32 __sanitizer___kernel_gid_t;
+typedef u64 __sanitizer___kernel_off_t;
+typedef struct {
+  u32 fds_bits[8];
+} __sanitizer___kernel_fd_set;
+
+typedef struct {
+  unsigned int pta_magic;
+  int pta_flags;
+  void *pta_private;
+} __sanitizer_pthread_attr_t;
+
+typedef unsigned int __sanitizer_sigset_t;
+
+struct __sanitizer_siginfo {
+  // The size is determined by looking at sizeof of real siginfo_t on linux.
+  u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
+                                                  __sanitizer_siginfo *siginfo,
+                                                  void *uctx);
+
+struct __sanitizer_sigaction {
+  union {
+    __sanitizer_sighandler_ptr handler;
+    __sanitizer_sigactionhandler_ptr sigaction;
+  };
+  __sanitizer_sigset_t sa_mask;
+  int sa_flags;
+};
+
+typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
+
+struct __sanitizer_kernel_sigaction_t {
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, void *info, void *ctx);
+  };
+  unsigned long sa_flags;
+  void (*sa_restorer)(void);
+  __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+#if SANITIZER_WORDSIZE == 64
+  u64 dlpi_addr;
+#else
+  u32 dlpi_addr;
+#endif
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+#if SANITIZER_WORDSIZE == 64
+  u32 dlpi_phnum;
+#else
+  u16 dlpi_phnum;
+#endif
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+  unsigned ai_addrlen;
+  struct __sanitizer_sockaddr *ai_addr;
+  char *ai_canonname;
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+typedef unsigned __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+  int gl_pathc;
+  int gl_matchc;
+  int gl_offs;
+  int gl_flags;
+  char **gl_pathv;
+  void **gl_statv;
+  int (*gl_errfunc)(const char *, int);
+  void (*gl_closedir)(void *dirp);
+  struct dirent *(*gl_readdir)(void *dirp);
+  void *(*gl_opendir)(const char *);
+  int (*gl_lstat)(const char *, void * /* struct stat* */);
+  int (*gl_stat)(const char *, void * /* struct stat* */);
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+typedef char __sanitizer_FILE;
+#define SANITIZER_HAS_STRUCT_FILE 0
+
+extern int shmctl_ipc_stat;
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+};
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(struct CLASS, MEMBER))
+
+#define SIGACTION_SYMNAME __sigaction14
+
+#endif  // SANITIZER_OPENBSD
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp
new file mode 100644
index 0000000000..c052aa2bc9
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp
@@ -0,0 +1,1275 @@
+//===-- sanitizer_platform_limits_posix.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific POSIX data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_MAC
+// Tests in this file assume that off_t-dependent data structures match the
+// libc ABI. For example, struct dirent here is what readdir() function (as
+// exported from libc) returns, and not the user-facing "dirent", which
+// depends on _FILE_OFFSET_BITS setting.
+// To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below.
+#ifdef _FILE_OFFSET_BITS
+#undef _FILE_OFFSET_BITS
+#endif
+
+// Must go after undef _FILE_OFFSET_BITS.
+#include "sanitizer_glibc_version.h"
+
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <grp.h>
+#include <limits.h>
+#include <net/if.h>
+#include <netdb.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/times.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <termios.h>
+#include <time.h>
+#include <wchar.h>
+#include <regex.h>
+#if !SANITIZER_MAC
+#include <utmp.h>
+#endif
+
+#if !SANITIZER_IOS
+#include <net/route.h>
+#endif
+
+#if !SANITIZER_ANDROID
+#include <fstab.h>
+#include <sys/mount.h>
+#include <sys/timeb.h>
+#include <utmpx.h>
+#endif
+
+#if SANITIZER_LINUX
+#include <malloc.h>
+#include <mntent.h>
+#include <netinet/ether.h>
+#include <sys/sysinfo.h>
+#include <sys/vt.h>
+#include <linux/cdrom.h>
+#include <linux/fd.h>
+#include <linux/fs.h>
+#include <linux/hdreg.h>
+#include <linux/input.h>
+#include <linux/ioctl.h>
+#include <linux/soundcard.h>
+#include <linux/sysctl.h>
+#include <linux/utsname.h>
+#include <linux/posix_types.h>
+#include <net/if_arp.h>
+#endif
+
+#if SANITIZER_IOS
+#undef IOC_DIRMASK
+#endif
+
+#if SANITIZER_LINUX
+# include <utime.h>
+# include <sys/ptrace.h>
+# if defined(__mips64) || defined(__aarch64__) || defined(__arm__)
+#  include <asm/ptrace.h>
+#  ifdef __arm__
+typedef struct user_fpregs elf_fpregset_t;
+#   define ARM_VFPREGS_SIZE_ASAN (32 * 8 /*fpregs*/ + 4 /*fpscr*/)
+#   if !defined(ARM_VFPREGS_SIZE)
+#     define ARM_VFPREGS_SIZE ARM_VFPREGS_SIZE_ASAN
+#   endif
+#  endif
+# endif
+# include <semaphore.h>
+#endif
+
+#if !SANITIZER_ANDROID
+#include <ifaddrs.h>
+#include <sys/ucontext.h>
+#include <wordexp.h>
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#include <glob.h>
+#include <obstack.h>
+#include <mqueue.h>
+#include <net/if_ppp.h>
+#include <netax25/ax25.h>
+#include <netipx/ipx.h>
+#include <netrom/netrom.h>
+#if HAVE_RPC_XDR_H
+# include <rpc/xdr.h>
+#endif
+#include <scsi/scsi.h>
+#include <sys/mtio.h>
+#include <sys/kd.h>
+#include <sys/shm.h>
+#include <sys/statvfs.h>
+#include <sys/timex.h>
+#if defined(__mips64)
+# include <sys/procfs.h>
+#endif
+#include <sys/user.h>
+#include <linux/cyclades.h>
+#include <linux/if_eql.h>
+#include <linux/if_plip.h>
+#include <linux/lp.h>
+#include <linux/mroute.h>
+#include <linux/mroute6.h>
+#include <linux/scc.h>
+#include <linux/serial.h>
+#include <sys/msg.h>
+#include <sys/ipc.h>
+#include <crypt.h>
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+#if SANITIZER_ANDROID
+#include <linux/kd.h>
+#include <linux/mtio.h>
+#include <linux/ppp_defs.h>
+#include <linux/if_ppp.h>
+#endif
+
+#if SANITIZER_LINUX
+#include <link.h>
+#include <sys/vfs.h>
+#include <sys/epoll.h>
+#include <linux/capability.h>
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_MAC
+#include <net/ethernet.h>
+#include <sys/filio.h>
+#include <sys/sockio.h>
+#endif
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_posix.h"
+
+namespace __sanitizer {
+  unsigned struct_utsname_sz = sizeof(struct utsname);
+  unsigned struct_stat_sz = sizeof(struct stat);
+#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+  unsigned struct_stat64_sz = sizeof(struct stat64);
+#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+  unsigned struct_rusage_sz = sizeof(struct rusage);
+  unsigned struct_tm_sz = sizeof(struct tm);
+  unsigned struct_passwd_sz = sizeof(struct passwd);
+  unsigned struct_group_sz = sizeof(struct group);
+  unsigned siginfo_t_sz = sizeof(siginfo_t);
+  unsigned struct_sigaction_sz = sizeof(struct sigaction);
+  unsigned struct_stack_t_sz = sizeof(stack_t);
+  unsigned struct_itimerval_sz = sizeof(struct itimerval);
+  unsigned pthread_t_sz = sizeof(pthread_t);
+  unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+  unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+  unsigned pid_t_sz = sizeof(pid_t);
+  unsigned timeval_sz = sizeof(timeval);
+  unsigned uid_t_sz = sizeof(uid_t);
+  unsigned gid_t_sz = sizeof(gid_t);
+  unsigned mbstate_t_sz = sizeof(mbstate_t);
+  unsigned sigset_t_sz = sizeof(sigset_t);
+  unsigned struct_timezone_sz = sizeof(struct timezone);
+  unsigned struct_tms_sz = sizeof(struct tms);
+  unsigned struct_sigevent_sz = sizeof(struct sigevent);
+  unsigned struct_sched_param_sz = sizeof(struct sched_param);
+  unsigned struct_regex_sz = sizeof(regex_t);
+  unsigned struct_regmatch_sz = sizeof(regmatch_t);
+
+#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+  unsigned struct_statfs64_sz = sizeof(struct statfs64);
+#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+
+#if !SANITIZER_ANDROID
+  unsigned struct_fstab_sz = sizeof(struct fstab);
+  unsigned struct_statfs_sz = sizeof(struct statfs);
+  unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+  unsigned ucontext_t_sz = sizeof(ucontext_t);
+#endif // !SANITIZER_ANDROID
+
+#if SANITIZER_LINUX
+  unsigned struct_epoll_event_sz = sizeof(struct epoll_event);
+  unsigned struct_sysinfo_sz = sizeof(struct sysinfo);
+  unsigned __user_cap_header_struct_sz =
+      sizeof(struct __user_cap_header_struct);
+  unsigned __user_cap_data_struct_sz = sizeof(struct __user_cap_data_struct);
+  unsigned struct_new_utsname_sz = sizeof(struct new_utsname);
+  unsigned struct_old_utsname_sz = sizeof(struct old_utsname);
+  unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname);
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX
+  unsigned struct_rlimit_sz = sizeof(struct rlimit);
+  unsigned struct_timespec_sz = sizeof(struct timespec);
+  unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+  unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  // Use pre-computed size of struct ustat to avoid <sys/ustat.h> which
+  // has been removed from glibc 2.28.
+#if defined(__aarch64__) || defined(__s390x__) || defined (__mips64) \
+  || defined(__powerpc64__) || defined(__arch64__) || defined(__sparcv9) \
+  || defined(__x86_64__) || (defined(__riscv) && __riscv_xlen == 64)
+#define SIZEOF_STRUCT_USTAT 32
+#elif defined(__arm__) || defined(__i386__) || defined(__mips__) \
+  || defined(__powerpc__) || defined(__s390__) || defined(__sparc__)
+#define SIZEOF_STRUCT_USTAT 20
+#else
+#error Unknown size of struct ustat
+#endif
+  unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT;
+  unsigned struct_rlimit64_sz = sizeof(struct rlimit64);
+  unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
+  unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned struct_timex_sz = sizeof(struct timex);
+  unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+  unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
+  unsigned struct_statvfs_sz = sizeof(struct statvfs);
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+  const uptr sig_ign = (uptr)SIG_IGN;
+  const uptr sig_dfl = (uptr)SIG_DFL;
+  const uptr sig_err = (uptr)SIG_ERR;
+  const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+#if SANITIZER_LINUX
+  int e_tabsz = (int)E_TABSZ;
+#endif
+
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned struct_shminfo_sz = sizeof(struct shminfo);
+  unsigned struct_shm_info_sz = sizeof(struct shm_info);
+  int shmctl_ipc_stat = (int)IPC_STAT;
+  int shmctl_ipc_info = (int)IPC_INFO;
+  int shmctl_shm_info = (int)SHM_INFO;
+  int shmctl_shm_stat = (int)SHM_STAT;
+#endif
+
+#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+  unsigned struct_utmp_sz = sizeof(struct utmp);
+#endif
+#if !SANITIZER_ANDROID
+  unsigned struct_utmpx_sz = sizeof(struct utmpx);
+#endif
+
+  int map_fixed = MAP_FIXED;
+
+  int af_inet = (int)AF_INET;
+  int af_inet6 = (int)AF_INET6;
+
+  uptr __sanitizer_in_addr_sz(int af) {
+    if (af == AF_INET)
+      return sizeof(struct in_addr);
+    else if (af == AF_INET6)
+      return sizeof(struct in6_addr);
+    else
+      return 0;
+  }
+
+#if SANITIZER_LINUX
+unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
+#elif SANITIZER_FREEBSD
+unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  int glob_nomatch = GLOB_NOMATCH;
+  int glob_altdirfunc = GLOB_ALTDIRFUNC;
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
+    (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+      defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+      defined(__s390__))
+#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
+  unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
+  unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
+#elif defined(__aarch64__)
+  unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
+  unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state);
+#elif defined(__s390__)
+  unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
+  unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
+#else
+  unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
+  unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
+#endif // __mips64 || __powerpc64__ || __aarch64__
+#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
+    defined(__aarch64__) || defined(__arm__) || defined(__s390__)
+  unsigned struct_user_fpxregs_struct_sz = 0;
+#else
+  unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
+#endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__
+// || __s390__
+#ifdef __arm__
+  unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
+#else
+  unsigned struct_user_vfpregs_struct_sz = 0;
+#endif
+
+  int ptrace_peektext = PTRACE_PEEKTEXT;
+  int ptrace_peekdata = PTRACE_PEEKDATA;
+  int ptrace_peekuser = PTRACE_PEEKUSER;
+#if (defined(PTRACE_GETREGS) && defined(PTRACE_SETREGS)) || \
+    (defined(PT_GETREGS) && defined(PT_SETREGS))
+  int ptrace_getregs = PTRACE_GETREGS;
+  int ptrace_setregs = PTRACE_SETREGS;
+#else
+  int ptrace_getregs = -1;
+  int ptrace_setregs = -1;
+#endif
+#if (defined(PTRACE_GETFPREGS) && defined(PTRACE_SETFPREGS)) || \
+    (defined(PT_GETFPREGS) && defined(PT_SETFPREGS))
+  int ptrace_getfpregs = PTRACE_GETFPREGS;
+  int ptrace_setfpregs = PTRACE_SETFPREGS;
+#else
+  int ptrace_getfpregs = -1;
+  int ptrace_setfpregs = -1;
+#endif
+#if (defined(PTRACE_GETFPXREGS) && defined(PTRACE_SETFPXREGS)) || \
+    (defined(PT_GETFPXREGS) && defined(PT_SETFPXREGS))
+  int ptrace_getfpxregs = PTRACE_GETFPXREGS;
+  int ptrace_setfpxregs = PTRACE_SETFPXREGS;
+#else
+  int ptrace_getfpxregs = -1;
+  int ptrace_setfpxregs = -1;
+#endif // PTRACE_GETFPXREGS/PTRACE_SETFPXREGS
+#if defined(PTRACE_GETVFPREGS) && defined(PTRACE_SETVFPREGS)
+  int ptrace_getvfpregs = PTRACE_GETVFPREGS;
+  int ptrace_setvfpregs = PTRACE_SETVFPREGS;
+#else
+  int ptrace_getvfpregs = -1;
+  int ptrace_setvfpregs = -1;
+#endif
+  int ptrace_geteventmsg = PTRACE_GETEVENTMSG;
+#if (defined(PTRACE_GETSIGINFO) && defined(PTRACE_SETSIGINFO)) ||              \
+    (defined(PT_GETSIGINFO) && defined(PT_SETSIGINFO))
+  int ptrace_getsiginfo = PTRACE_GETSIGINFO;
+  int ptrace_setsiginfo = PTRACE_SETSIGINFO;
+#else
+  int ptrace_getsiginfo = -1;
+  int ptrace_setsiginfo = -1;
+#endif // PTRACE_GETSIGINFO/PTRACE_SETSIGINFO
+#if defined(PTRACE_GETREGSET) && defined(PTRACE_SETREGSET)
+  int ptrace_getregset = PTRACE_GETREGSET;
+  int ptrace_setregset = PTRACE_SETREGSET;
+#else
+  int ptrace_getregset = -1;
+  int ptrace_setregset = -1;
+#endif // PTRACE_GETREGSET/PTRACE_SETREGSET
+#endif
+
+  unsigned path_max = PATH_MAX;
+
+  // ioctl arguments
+  unsigned struct_ifreq_sz = sizeof(struct ifreq);
+  unsigned struct_termios_sz = sizeof(struct termios);
+  unsigned struct_winsize_sz = sizeof(struct winsize);
+
+#if SANITIZER_LINUX
+  unsigned struct_arpreq_sz = sizeof(struct arpreq);
+  unsigned struct_cdrom_msf_sz = sizeof(struct cdrom_msf);
+  unsigned struct_cdrom_multisession_sz = sizeof(struct cdrom_multisession);
+  unsigned struct_cdrom_read_audio_sz = sizeof(struct cdrom_read_audio);
+  unsigned struct_cdrom_subchnl_sz = sizeof(struct cdrom_subchnl);
+  unsigned struct_cdrom_ti_sz = sizeof(struct cdrom_ti);
+  unsigned struct_cdrom_tocentry_sz = sizeof(struct cdrom_tocentry);
+  unsigned struct_cdrom_tochdr_sz = sizeof(struct cdrom_tochdr);
+  unsigned struct_cdrom_volctrl_sz = sizeof(struct cdrom_volctrl);
+  unsigned struct_ff_effect_sz = sizeof(struct ff_effect);
+  unsigned struct_floppy_drive_params_sz = sizeof(struct floppy_drive_params);
+  unsigned struct_floppy_drive_struct_sz = sizeof(struct floppy_drive_struct);
+  unsigned struct_floppy_fdc_state_sz = sizeof(struct floppy_fdc_state);
+  unsigned struct_floppy_max_errors_sz = sizeof(struct floppy_max_errors);
+  unsigned struct_floppy_raw_cmd_sz = sizeof(struct floppy_raw_cmd);
+  unsigned struct_floppy_struct_sz = sizeof(struct floppy_struct);
+  unsigned struct_floppy_write_errors_sz = sizeof(struct floppy_write_errors);
+  unsigned struct_format_descr_sz = sizeof(struct format_descr);
+  unsigned struct_hd_driveid_sz = sizeof(struct hd_driveid);
+  unsigned struct_hd_geometry_sz = sizeof(struct hd_geometry);
+  unsigned struct_input_absinfo_sz = sizeof(struct input_absinfo);
+  unsigned struct_input_id_sz = sizeof(struct input_id);
+  unsigned struct_mtpos_sz = sizeof(struct mtpos);
+  unsigned struct_rtentry_sz = sizeof(struct rtentry);
+  unsigned struct_termio_sz = sizeof(struct termio);
+  unsigned struct_vt_consize_sz = sizeof(struct vt_consize);
+  unsigned struct_vt_sizes_sz = sizeof(struct vt_sizes);
+  unsigned struct_vt_stat_sz = sizeof(struct vt_stat);
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX
+#if SOUND_VERSION >= 0x040000
+  unsigned struct_copr_buffer_sz = 0;
+  unsigned struct_copr_debug_buf_sz = 0;
+  unsigned struct_copr_msg_sz = 0;
+#else
+  unsigned struct_copr_buffer_sz = sizeof(struct copr_buffer);
+  unsigned struct_copr_debug_buf_sz = sizeof(struct copr_debug_buf);
+  unsigned struct_copr_msg_sz = sizeof(struct copr_msg);
+#endif
+  unsigned struct_midi_info_sz = sizeof(struct midi_info);
+  unsigned struct_mtget_sz = sizeof(struct mtget);
+  unsigned struct_mtop_sz = sizeof(struct mtop);
+  unsigned struct_sbi_instrument_sz = sizeof(struct sbi_instrument);
+  unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec);
+  unsigned struct_synth_info_sz = sizeof(struct synth_info);
+  unsigned struct_vt_mode_sz = sizeof(struct vt_mode);
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct);
+  unsigned struct_cyclades_monitor_sz = sizeof(struct cyclades_monitor);
+#if EV_VERSION > (0x010000)
+  unsigned struct_input_keymap_entry_sz = sizeof(struct input_keymap_entry);
+#else
+  unsigned struct_input_keymap_entry_sz = 0;
+#endif
+  unsigned struct_ipx_config_data_sz = sizeof(struct ipx_config_data);
+  unsigned struct_kbdiacrs_sz = sizeof(struct kbdiacrs);
+  unsigned struct_kbentry_sz = sizeof(struct kbentry);
+  unsigned struct_kbkeycode_sz = sizeof(struct kbkeycode);
+  unsigned struct_kbsentry_sz = sizeof(struct kbsentry);
+  unsigned struct_mtconfiginfo_sz = sizeof(struct mtconfiginfo);
+  unsigned struct_nr_parms_struct_sz = sizeof(struct nr_parms_struct);
+  unsigned struct_scc_modem_sz = sizeof(struct scc_modem);
+  unsigned struct_scc_stat_sz = sizeof(struct scc_stat);
+  unsigned struct_serial_multiport_struct_sz
+      = sizeof(struct serial_multiport_struct);
+  unsigned struct_serial_struct_sz = sizeof(struct serial_struct);
+  unsigned struct_sockaddr_ax25_sz = sizeof(struct sockaddr_ax25);
+  unsigned struct_unimapdesc_sz = sizeof(struct unimapdesc);
+  unsigned struct_unimapinit_sz = sizeof(struct unimapinit);
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
+  unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
+#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+
+#if !SANITIZER_ANDROID && !SANITIZER_MAC
+  unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+  unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+#endif
+
+  const unsigned long __sanitizer_bufsiz = BUFSIZ;
+
+  const unsigned IOCTL_NOT_PRESENT = 0;
+
+  unsigned IOCTL_FIOASYNC = FIOASYNC;
+  unsigned IOCTL_FIOCLEX = FIOCLEX;
+  unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+  unsigned IOCTL_FIONBIO = FIONBIO;
+  unsigned IOCTL_FIONCLEX = FIONCLEX;
+  unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+  unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+  unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+  unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+  unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+  unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+  unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+  unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+  unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+  unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+  unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+  unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+  unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+  unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+  unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+  unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+  unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+  unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+  unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+  unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+  unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+  unsigned IOCTL_TIOCCONS = TIOCCONS;
+  unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+  unsigned IOCTL_TIOCGETD = TIOCGETD;
+  unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+  unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+  unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+  unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+  unsigned IOCTL_TIOCMGET = TIOCMGET;
+  unsigned IOCTL_TIOCMSET = TIOCMSET;
+  unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+  unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+  unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+  unsigned IOCTL_TIOCPKT = TIOCPKT;
+  unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+  unsigned IOCTL_TIOCSETD = TIOCSETD;
+  unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+  unsigned IOCTL_TIOCSTI = TIOCSTI;
+  unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
+  unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
+#endif
+
+#if SANITIZER_LINUX
+  unsigned IOCTL_EVIOCGABS = EVIOCGABS(0);
+  unsigned IOCTL_EVIOCGBIT = EVIOCGBIT(0, 0);
+  unsigned IOCTL_EVIOCGEFFECTS = EVIOCGEFFECTS;
+  unsigned IOCTL_EVIOCGID = EVIOCGID;
+  unsigned IOCTL_EVIOCGKEY = EVIOCGKEY(0);
+  unsigned IOCTL_EVIOCGKEYCODE = EVIOCGKEYCODE;
+  unsigned IOCTL_EVIOCGLED = EVIOCGLED(0);
+  unsigned IOCTL_EVIOCGNAME = EVIOCGNAME(0);
+  unsigned IOCTL_EVIOCGPHYS = EVIOCGPHYS(0);
+  unsigned IOCTL_EVIOCGRAB = EVIOCGRAB;
+  unsigned IOCTL_EVIOCGREP = EVIOCGREP;
+  unsigned IOCTL_EVIOCGSND = EVIOCGSND(0);
+  unsigned IOCTL_EVIOCGSW = EVIOCGSW(0);
+  unsigned IOCTL_EVIOCGUNIQ = EVIOCGUNIQ(0);
+  unsigned IOCTL_EVIOCGVERSION = EVIOCGVERSION;
+  unsigned IOCTL_EVIOCRMFF = EVIOCRMFF;
+  unsigned IOCTL_EVIOCSABS = EVIOCSABS(0);
+  unsigned IOCTL_EVIOCSFF = EVIOCSFF;
+  unsigned IOCTL_EVIOCSKEYCODE = EVIOCSKEYCODE;
+  unsigned IOCTL_EVIOCSREP = EVIOCSREP;
+  unsigned IOCTL_BLKFLSBUF = BLKFLSBUF;
+  unsigned IOCTL_BLKGETSIZE = BLKGETSIZE;
+  unsigned IOCTL_BLKRAGET = BLKRAGET;
+  unsigned IOCTL_BLKRASET = BLKRASET;
+  unsigned IOCTL_BLKROGET = BLKROGET;
+  unsigned IOCTL_BLKROSET = BLKROSET;
+  unsigned IOCTL_BLKRRPART = BLKRRPART;
+  unsigned IOCTL_CDROMAUDIOBUFSIZ = CDROMAUDIOBUFSIZ;
+  unsigned IOCTL_CDROMEJECT = CDROMEJECT;
+  unsigned IOCTL_CDROMEJECT_SW = CDROMEJECT_SW;
+  unsigned IOCTL_CDROMMULTISESSION = CDROMMULTISESSION;
+  unsigned IOCTL_CDROMPAUSE = CDROMPAUSE;
+  unsigned IOCTL_CDROMPLAYMSF = CDROMPLAYMSF;
+  unsigned IOCTL_CDROMPLAYTRKIND = CDROMPLAYTRKIND;
+  unsigned IOCTL_CDROMREADAUDIO = CDROMREADAUDIO;
+  unsigned IOCTL_CDROMREADCOOKED = CDROMREADCOOKED;
+  unsigned IOCTL_CDROMREADMODE1 = CDROMREADMODE1;
+  unsigned IOCTL_CDROMREADMODE2 = CDROMREADMODE2;
+  unsigned IOCTL_CDROMREADRAW = CDROMREADRAW;
+  unsigned IOCTL_CDROMREADTOCENTRY = CDROMREADTOCENTRY;
+  unsigned IOCTL_CDROMREADTOCHDR = CDROMREADTOCHDR;
+  unsigned IOCTL_CDROMRESET = CDROMRESET;
+  unsigned IOCTL_CDROMRESUME = CDROMRESUME;
+  unsigned IOCTL_CDROMSEEK = CDROMSEEK;
+  unsigned IOCTL_CDROMSTART = CDROMSTART;
+  unsigned IOCTL_CDROMSTOP = CDROMSTOP;
+  unsigned IOCTL_CDROMSUBCHNL = CDROMSUBCHNL;
+  unsigned IOCTL_CDROMVOLCTRL = CDROMVOLCTRL;
+  unsigned IOCTL_CDROMVOLREAD = CDROMVOLREAD;
+  unsigned IOCTL_CDROM_GET_UPC = CDROM_GET_UPC;
+  unsigned IOCTL_FDCLRPRM = FDCLRPRM;
+  unsigned IOCTL_FDDEFPRM = FDDEFPRM;
+  unsigned IOCTL_FDFLUSH = FDFLUSH;
+  unsigned IOCTL_FDFMTBEG = FDFMTBEG;
+  unsigned IOCTL_FDFMTEND = FDFMTEND;
+  unsigned IOCTL_FDFMTTRK = FDFMTTRK;
+  unsigned IOCTL_FDGETDRVPRM = FDGETDRVPRM;
+  unsigned IOCTL_FDGETDRVSTAT = FDGETDRVSTAT;
+  unsigned IOCTL_FDGETDRVTYP = FDGETDRVTYP;
+  unsigned IOCTL_FDGETFDCSTAT = FDGETFDCSTAT;
+  unsigned IOCTL_FDGETMAXERRS = FDGETMAXERRS;
+  unsigned IOCTL_FDGETPRM = FDGETPRM;
+  unsigned IOCTL_FDMSGOFF = FDMSGOFF;
+  unsigned IOCTL_FDMSGON = FDMSGON;
+  unsigned IOCTL_FDPOLLDRVSTAT = FDPOLLDRVSTAT;
+  unsigned IOCTL_FDRAWCMD = FDRAWCMD;
+  unsigned IOCTL_FDRESET = FDRESET;
+  unsigned IOCTL_FDSETDRVPRM = FDSETDRVPRM;
+  unsigned IOCTL_FDSETEMSGTRESH = FDSETEMSGTRESH;
+  unsigned IOCTL_FDSETMAXERRS = FDSETMAXERRS;
+  unsigned IOCTL_FDSETPRM = FDSETPRM;
+  unsigned IOCTL_FDTWADDLE = FDTWADDLE;
+  unsigned IOCTL_FDWERRORCLR = FDWERRORCLR;
+  unsigned IOCTL_FDWERRORGET = FDWERRORGET;
+  unsigned IOCTL_HDIO_DRIVE_CMD = HDIO_DRIVE_CMD;
+  unsigned IOCTL_HDIO_GETGEO = HDIO_GETGEO;
+  unsigned IOCTL_HDIO_GET_32BIT = HDIO_GET_32BIT;
+  unsigned IOCTL_HDIO_GET_DMA = HDIO_GET_DMA;
+  unsigned IOCTL_HDIO_GET_IDENTITY = HDIO_GET_IDENTITY;
+  unsigned IOCTL_HDIO_GET_KEEPSETTINGS = HDIO_GET_KEEPSETTINGS;
+  unsigned IOCTL_HDIO_GET_MULTCOUNT = HDIO_GET_MULTCOUNT;
+  unsigned IOCTL_HDIO_GET_NOWERR = HDIO_GET_NOWERR;
+  unsigned IOCTL_HDIO_GET_UNMASKINTR = HDIO_GET_UNMASKINTR;
+  unsigned IOCTL_HDIO_SET_32BIT = HDIO_SET_32BIT;
+  unsigned IOCTL_HDIO_SET_DMA = HDIO_SET_DMA;
+  unsigned IOCTL_HDIO_SET_KEEPSETTINGS = HDIO_SET_KEEPSETTINGS;
+  unsigned IOCTL_HDIO_SET_MULTCOUNT = HDIO_SET_MULTCOUNT;
+  unsigned IOCTL_HDIO_SET_NOWERR = HDIO_SET_NOWERR;
+  unsigned IOCTL_HDIO_SET_UNMASKINTR = HDIO_SET_UNMASKINTR;
+  unsigned IOCTL_MTIOCPOS = MTIOCPOS;
+  unsigned IOCTL_PPPIOCGASYNCMAP = PPPIOCGASYNCMAP;
+  unsigned IOCTL_PPPIOCGDEBUG = PPPIOCGDEBUG;
+  unsigned IOCTL_PPPIOCGFLAGS = PPPIOCGFLAGS;
+  unsigned IOCTL_PPPIOCGUNIT = PPPIOCGUNIT;
+  unsigned IOCTL_PPPIOCGXASYNCMAP = PPPIOCGXASYNCMAP;
+  unsigned IOCTL_PPPIOCSASYNCMAP = PPPIOCSASYNCMAP;
+  unsigned IOCTL_PPPIOCSDEBUG = PPPIOCSDEBUG;
+  unsigned IOCTL_PPPIOCSFLAGS = PPPIOCSFLAGS;
+  unsigned IOCTL_PPPIOCSMAXCID = PPPIOCSMAXCID;
+  unsigned IOCTL_PPPIOCSMRU = PPPIOCSMRU;
+  unsigned IOCTL_PPPIOCSXASYNCMAP = PPPIOCSXASYNCMAP;
+  unsigned IOCTL_SIOCADDRT = SIOCADDRT;
+  unsigned IOCTL_SIOCDARP = SIOCDARP;
+  unsigned IOCTL_SIOCDELRT = SIOCDELRT;
+  unsigned IOCTL_SIOCDRARP = SIOCDRARP;
+  unsigned IOCTL_SIOCGARP = SIOCGARP;
+  unsigned IOCTL_SIOCGIFENCAP = SIOCGIFENCAP;
+  unsigned IOCTL_SIOCGIFHWADDR = SIOCGIFHWADDR;
+  unsigned IOCTL_SIOCGIFMAP = SIOCGIFMAP;
+  unsigned IOCTL_SIOCGIFMEM = SIOCGIFMEM;
+  unsigned IOCTL_SIOCGIFNAME = SIOCGIFNAME;
+  unsigned IOCTL_SIOCGIFSLAVE = SIOCGIFSLAVE;
+  unsigned IOCTL_SIOCGRARP = SIOCGRARP;
+  unsigned IOCTL_SIOCGSTAMP = SIOCGSTAMP;
+  unsigned IOCTL_SIOCSARP = SIOCSARP;
+  unsigned IOCTL_SIOCSIFENCAP = SIOCSIFENCAP;
+  unsigned IOCTL_SIOCSIFHWADDR = SIOCSIFHWADDR;
+  unsigned IOCTL_SIOCSIFLINK = SIOCSIFLINK;
+  unsigned IOCTL_SIOCSIFMAP = SIOCSIFMAP;
+  unsigned IOCTL_SIOCSIFMEM = SIOCSIFMEM;
+  unsigned IOCTL_SIOCSIFSLAVE = SIOCSIFSLAVE;
+  unsigned IOCTL_SIOCSRARP = SIOCSRARP;
+# if SOUND_VERSION >= 0x040000
+  unsigned IOCTL_SNDCTL_COPR_HALT = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_LOAD = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_RCODE = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_RCVMSG = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_RDATA = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_RESET = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_RUN = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_SENDMSG = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_WCODE = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SNDCTL_COPR_WDATA = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_READ_BITS = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_READ_CHANNELS = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_READ_FILTER = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_READ_RATE = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_SOUND_PCM_WRITE_FILTER = IOCTL_NOT_PRESENT;
+# else  // SOUND_VERSION
+  unsigned IOCTL_SNDCTL_COPR_HALT = SNDCTL_COPR_HALT;
+  unsigned IOCTL_SNDCTL_COPR_LOAD = SNDCTL_COPR_LOAD;
+  unsigned IOCTL_SNDCTL_COPR_RCODE = SNDCTL_COPR_RCODE;
+  unsigned IOCTL_SNDCTL_COPR_RCVMSG = SNDCTL_COPR_RCVMSG;
+  unsigned IOCTL_SNDCTL_COPR_RDATA = SNDCTL_COPR_RDATA;
+  unsigned IOCTL_SNDCTL_COPR_RESET = SNDCTL_COPR_RESET;
+  unsigned IOCTL_SNDCTL_COPR_RUN = SNDCTL_COPR_RUN;
+  unsigned IOCTL_SNDCTL_COPR_SENDMSG = SNDCTL_COPR_SENDMSG;
+  unsigned IOCTL_SNDCTL_COPR_WCODE = SNDCTL_COPR_WCODE;
+  unsigned IOCTL_SNDCTL_COPR_WDATA = SNDCTL_COPR_WDATA;
+  unsigned IOCTL_SOUND_PCM_READ_BITS = SOUND_PCM_READ_BITS;
+  unsigned IOCTL_SOUND_PCM_READ_CHANNELS = SOUND_PCM_READ_CHANNELS;
+  unsigned IOCTL_SOUND_PCM_READ_FILTER = SOUND_PCM_READ_FILTER;
+  unsigned IOCTL_SOUND_PCM_READ_RATE = SOUND_PCM_READ_RATE;
+  unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS = SOUND_PCM_WRITE_CHANNELS;
+  unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER;
+#endif // SOUND_VERSION
+  unsigned IOCTL_TCFLSH = TCFLSH;
+  unsigned IOCTL_TCGETA = TCGETA;
+  unsigned IOCTL_TCGETS = TCGETS;
+  unsigned IOCTL_TCSBRK = TCSBRK;
+  unsigned IOCTL_TCSBRKP = TCSBRKP;
+  unsigned IOCTL_TCSETA = TCSETA;
+  unsigned IOCTL_TCSETAF = TCSETAF;
+  unsigned IOCTL_TCSETAW = TCSETAW;
+  unsigned IOCTL_TCSETS = TCSETS;
+  unsigned IOCTL_TCSETSF = TCSETSF;
+  unsigned IOCTL_TCSETSW = TCSETSW;
+  unsigned IOCTL_TCXONC = TCXONC;
+  unsigned IOCTL_TIOCGLCKTRMIOS = TIOCGLCKTRMIOS;
+  unsigned IOCTL_TIOCGSOFTCAR = TIOCGSOFTCAR;
+  unsigned IOCTL_TIOCINQ = TIOCINQ;
+  unsigned IOCTL_TIOCLINUX = TIOCLINUX;
+  unsigned IOCTL_TIOCSERCONFIG = TIOCSERCONFIG;
+  unsigned IOCTL_TIOCSERGETLSR = TIOCSERGETLSR;
+  unsigned IOCTL_TIOCSERGWILD = TIOCSERGWILD;
+  unsigned IOCTL_TIOCSERSWILD = TIOCSERSWILD;
+  unsigned IOCTL_TIOCSLCKTRMIOS = TIOCSLCKTRMIOS;
+  unsigned IOCTL_TIOCSSOFTCAR = TIOCSSOFTCAR;
+  unsigned IOCTL_VT_DISALLOCATE = VT_DISALLOCATE;
+  unsigned IOCTL_VT_GETSTATE = VT_GETSTATE;
+  unsigned IOCTL_VT_RESIZE = VT_RESIZE;
+  unsigned IOCTL_VT_RESIZEX = VT_RESIZEX;
+  unsigned IOCTL_VT_SENDSIG = VT_SENDSIG;
+  unsigned IOCTL_MTIOCGET = MTIOCGET;
+  unsigned IOCTL_MTIOCTOP = MTIOCTOP;
+  unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE = SNDCTL_DSP_GETBLKSIZE;
+  unsigned IOCTL_SNDCTL_DSP_GETFMTS = SNDCTL_DSP_GETFMTS;
+  unsigned IOCTL_SNDCTL_DSP_NONBLOCK = SNDCTL_DSP_NONBLOCK;
+  unsigned IOCTL_SNDCTL_DSP_POST = SNDCTL_DSP_POST;
+  unsigned IOCTL_SNDCTL_DSP_RESET = SNDCTL_DSP_RESET;
+  unsigned IOCTL_SNDCTL_DSP_SETFMT = SNDCTL_DSP_SETFMT;
+  unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT = SNDCTL_DSP_SETFRAGMENT;
+  unsigned IOCTL_SNDCTL_DSP_SPEED = SNDCTL_DSP_SPEED;
+  unsigned IOCTL_SNDCTL_DSP_STEREO = SNDCTL_DSP_STEREO;
+  unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE = SNDCTL_DSP_SUBDIVIDE;
+  unsigned IOCTL_SNDCTL_DSP_SYNC = SNDCTL_DSP_SYNC;
+  unsigned IOCTL_SNDCTL_FM_4OP_ENABLE = SNDCTL_FM_4OP_ENABLE;
+  unsigned IOCTL_SNDCTL_FM_LOAD_INSTR = SNDCTL_FM_LOAD_INSTR;
+  unsigned IOCTL_SNDCTL_MIDI_INFO = SNDCTL_MIDI_INFO;
+  unsigned IOCTL_SNDCTL_MIDI_PRETIME = SNDCTL_MIDI_PRETIME;
+  unsigned IOCTL_SNDCTL_SEQ_CTRLRATE = SNDCTL_SEQ_CTRLRATE;
+  unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT = SNDCTL_SEQ_GETINCOUNT;
+  unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT = SNDCTL_SEQ_GETOUTCOUNT;
+  unsigned IOCTL_SNDCTL_SEQ_NRMIDIS = SNDCTL_SEQ_NRMIDIS;
+  unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS = SNDCTL_SEQ_NRSYNTHS;
+  unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND = SNDCTL_SEQ_OUTOFBAND;
+  unsigned IOCTL_SNDCTL_SEQ_PANIC = SNDCTL_SEQ_PANIC;
+  unsigned IOCTL_SNDCTL_SEQ_PERCMODE = SNDCTL_SEQ_PERCMODE;
+  unsigned IOCTL_SNDCTL_SEQ_RESET = SNDCTL_SEQ_RESET;
+  unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES = SNDCTL_SEQ_RESETSAMPLES;
+  unsigned IOCTL_SNDCTL_SEQ_SYNC = SNDCTL_SEQ_SYNC;
+  unsigned IOCTL_SNDCTL_SEQ_TESTMIDI = SNDCTL_SEQ_TESTMIDI;
+  unsigned IOCTL_SNDCTL_SEQ_THRESHOLD = SNDCTL_SEQ_THRESHOLD;
+  unsigned IOCTL_SNDCTL_SYNTH_INFO = SNDCTL_SYNTH_INFO;
+  unsigned IOCTL_SNDCTL_SYNTH_MEMAVL = SNDCTL_SYNTH_MEMAVL;
+  unsigned IOCTL_SNDCTL_TMR_CONTINUE = SNDCTL_TMR_CONTINUE;
+  unsigned IOCTL_SNDCTL_TMR_METRONOME = SNDCTL_TMR_METRONOME;
+  unsigned IOCTL_SNDCTL_TMR_SELECT = SNDCTL_TMR_SELECT;
+  unsigned IOCTL_SNDCTL_TMR_SOURCE = SNDCTL_TMR_SOURCE;
+  unsigned IOCTL_SNDCTL_TMR_START = SNDCTL_TMR_START;
+  unsigned IOCTL_SNDCTL_TMR_STOP = SNDCTL_TMR_STOP;
+  unsigned IOCTL_SNDCTL_TMR_TEMPO = SNDCTL_TMR_TEMPO;
+  unsigned IOCTL_SNDCTL_TMR_TIMEBASE = SNDCTL_TMR_TIMEBASE;
+  unsigned IOCTL_SOUND_MIXER_READ_ALTPCM = SOUND_MIXER_READ_ALTPCM;
+  unsigned IOCTL_SOUND_MIXER_READ_BASS = SOUND_MIXER_READ_BASS;
+  unsigned IOCTL_SOUND_MIXER_READ_CAPS = SOUND_MIXER_READ_CAPS;
+  unsigned IOCTL_SOUND_MIXER_READ_CD = SOUND_MIXER_READ_CD;
+  unsigned IOCTL_SOUND_MIXER_READ_DEVMASK = SOUND_MIXER_READ_DEVMASK;
+  unsigned IOCTL_SOUND_MIXER_READ_ENHANCE = SOUND_MIXER_READ_ENHANCE;
+  unsigned IOCTL_SOUND_MIXER_READ_IGAIN = SOUND_MIXER_READ_IGAIN;
+  unsigned IOCTL_SOUND_MIXER_READ_IMIX = SOUND_MIXER_READ_IMIX;
+  unsigned IOCTL_SOUND_MIXER_READ_LINE = SOUND_MIXER_READ_LINE;
+  unsigned IOCTL_SOUND_MIXER_READ_LINE1 = SOUND_MIXER_READ_LINE1;
+  unsigned IOCTL_SOUND_MIXER_READ_LINE2 = SOUND_MIXER_READ_LINE2;
+  unsigned IOCTL_SOUND_MIXER_READ_LINE3 = SOUND_MIXER_READ_LINE3;
+  unsigned IOCTL_SOUND_MIXER_READ_LOUD = SOUND_MIXER_READ_LOUD;
+  unsigned IOCTL_SOUND_MIXER_READ_MIC = SOUND_MIXER_READ_MIC;
+  unsigned IOCTL_SOUND_MIXER_READ_MUTE = SOUND_MIXER_READ_MUTE;
+  unsigned IOCTL_SOUND_MIXER_READ_OGAIN = SOUND_MIXER_READ_OGAIN;
+  unsigned IOCTL_SOUND_MIXER_READ_PCM = SOUND_MIXER_READ_PCM;
+  unsigned IOCTL_SOUND_MIXER_READ_RECLEV = SOUND_MIXER_READ_RECLEV;
+  unsigned IOCTL_SOUND_MIXER_READ_RECMASK = SOUND_MIXER_READ_RECMASK;
+  unsigned IOCTL_SOUND_MIXER_READ_RECSRC = SOUND_MIXER_READ_RECSRC;
+  unsigned IOCTL_SOUND_MIXER_READ_SPEAKER = SOUND_MIXER_READ_SPEAKER;
+  unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS = SOUND_MIXER_READ_STEREODEVS;
+  unsigned IOCTL_SOUND_MIXER_READ_SYNTH = SOUND_MIXER_READ_SYNTH;
+  unsigned IOCTL_SOUND_MIXER_READ_TREBLE = SOUND_MIXER_READ_TREBLE;
+  unsigned IOCTL_SOUND_MIXER_READ_VOLUME = SOUND_MIXER_READ_VOLUME;
+  unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM = SOUND_MIXER_WRITE_ALTPCM;
+  unsigned IOCTL_SOUND_MIXER_WRITE_BASS = SOUND_MIXER_WRITE_BASS;
+  unsigned IOCTL_SOUND_MIXER_WRITE_CD = SOUND_MIXER_WRITE_CD;
+  unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE = SOUND_MIXER_WRITE_ENHANCE;
+  unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN = SOUND_MIXER_WRITE_IGAIN;
+  unsigned IOCTL_SOUND_MIXER_WRITE_IMIX = SOUND_MIXER_WRITE_IMIX;
+  unsigned IOCTL_SOUND_MIXER_WRITE_LINE = SOUND_MIXER_WRITE_LINE;
+  unsigned IOCTL_SOUND_MIXER_WRITE_LINE1 = SOUND_MIXER_WRITE_LINE1;
+  unsigned IOCTL_SOUND_MIXER_WRITE_LINE2 = SOUND_MIXER_WRITE_LINE2;
+  unsigned IOCTL_SOUND_MIXER_WRITE_LINE3 = SOUND_MIXER_WRITE_LINE3;
+  unsigned IOCTL_SOUND_MIXER_WRITE_LOUD = SOUND_MIXER_WRITE_LOUD;
+  unsigned IOCTL_SOUND_MIXER_WRITE_MIC = SOUND_MIXER_WRITE_MIC;
+  unsigned IOCTL_SOUND_MIXER_WRITE_MUTE = SOUND_MIXER_WRITE_MUTE;
+  unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN = SOUND_MIXER_WRITE_OGAIN;
+  unsigned IOCTL_SOUND_MIXER_WRITE_PCM = SOUND_MIXER_WRITE_PCM;
+  unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV = SOUND_MIXER_WRITE_RECLEV;
+  unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC = SOUND_MIXER_WRITE_RECSRC;
+  unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER = SOUND_MIXER_WRITE_SPEAKER;
+  unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH = SOUND_MIXER_WRITE_SYNTH;
+  unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE = SOUND_MIXER_WRITE_TREBLE;
+  unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME = SOUND_MIXER_WRITE_VOLUME;
+  unsigned IOCTL_VT_ACTIVATE = VT_ACTIVATE;
+  unsigned IOCTL_VT_GETMODE = VT_GETMODE;
+  unsigned IOCTL_VT_OPENQRY = VT_OPENQRY;
+  unsigned IOCTL_VT_RELDISP = VT_RELDISP;
+  unsigned IOCTL_VT_SETMODE = VT_SETMODE;
+  unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE;
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH;
+  unsigned IOCTL_CYGETDEFTIMEOUT = CYGETDEFTIMEOUT;
+  unsigned IOCTL_CYGETMON = CYGETMON;
+  unsigned IOCTL_CYGETTHRESH = CYGETTHRESH;
+  unsigned IOCTL_CYGETTIMEOUT = CYGETTIMEOUT;
+  unsigned IOCTL_CYSETDEFTHRESH = CYSETDEFTHRESH;
+  unsigned IOCTL_CYSETDEFTIMEOUT = CYSETDEFTIMEOUT;
+  unsigned IOCTL_CYSETTHRESH = CYSETTHRESH;
+  unsigned IOCTL_CYSETTIMEOUT = CYSETTIMEOUT;
+  unsigned IOCTL_EQL_EMANCIPATE = EQL_EMANCIPATE;
+  unsigned IOCTL_EQL_ENSLAVE = EQL_ENSLAVE;
+  unsigned IOCTL_EQL_GETMASTRCFG = EQL_GETMASTRCFG;
+  unsigned IOCTL_EQL_GETSLAVECFG = EQL_GETSLAVECFG;
+  unsigned IOCTL_EQL_SETMASTRCFG = EQL_SETMASTRCFG;
+  unsigned IOCTL_EQL_SETSLAVECFG = EQL_SETSLAVECFG;
+#if EV_VERSION > (0x010000)
+  unsigned IOCTL_EVIOCGKEYCODE_V2 = EVIOCGKEYCODE_V2;
+  unsigned IOCTL_EVIOCGPROP = EVIOCGPROP(0);
+  unsigned IOCTL_EVIOCSKEYCODE_V2 = EVIOCSKEYCODE_V2;
+#else
+  unsigned IOCTL_EVIOCGKEYCODE_V2 = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_EVIOCGPROP = IOCTL_NOT_PRESENT;
+  unsigned IOCTL_EVIOCSKEYCODE_V2 = IOCTL_NOT_PRESENT;
+#endif
+  unsigned IOCTL_FS_IOC_GETFLAGS = FS_IOC_GETFLAGS;
+  unsigned IOCTL_FS_IOC_GETVERSION = FS_IOC_GETVERSION;
+  unsigned IOCTL_FS_IOC_SETFLAGS = FS_IOC_SETFLAGS;
+  unsigned IOCTL_FS_IOC_SETVERSION = FS_IOC_SETVERSION;
+  unsigned IOCTL_GIO_CMAP = GIO_CMAP;
+  unsigned IOCTL_GIO_FONT = GIO_FONT;
+  unsigned IOCTL_GIO_UNIMAP = GIO_UNIMAP;
+  unsigned IOCTL_GIO_UNISCRNMAP = GIO_UNISCRNMAP;
+  unsigned IOCTL_KDADDIO = KDADDIO;
+  unsigned IOCTL_KDDELIO = KDDELIO;
+  unsigned IOCTL_KDGETKEYCODE = KDGETKEYCODE;
+  unsigned IOCTL_KDGKBDIACR = KDGKBDIACR;
+  unsigned IOCTL_KDGKBENT = KDGKBENT;
+  unsigned IOCTL_KDGKBLED = KDGKBLED;
+  unsigned IOCTL_KDGKBMETA = KDGKBMETA;
+  unsigned IOCTL_KDGKBSENT = KDGKBSENT;
+  unsigned IOCTL_KDMAPDISP = KDMAPDISP;
+  unsigned IOCTL_KDSETKEYCODE = KDSETKEYCODE;
+  unsigned IOCTL_KDSIGACCEPT = KDSIGACCEPT;
+  unsigned IOCTL_KDSKBDIACR = KDSKBDIACR;
+  unsigned IOCTL_KDSKBENT = KDSKBENT;
+  unsigned IOCTL_KDSKBLED = KDSKBLED;
+  unsigned IOCTL_KDSKBMETA = KDSKBMETA;
+  unsigned IOCTL_KDSKBSENT = KDSKBSENT;
+  unsigned IOCTL_KDUNMAPDISP = KDUNMAPDISP;
+  unsigned IOCTL_LPABORT = LPABORT;
+  unsigned IOCTL_LPABORTOPEN = LPABORTOPEN;
+  unsigned IOCTL_LPCAREFUL = LPCAREFUL;
+  unsigned IOCTL_LPCHAR = LPCHAR;
+  unsigned IOCTL_LPGETIRQ = LPGETIRQ;
+  unsigned IOCTL_LPGETSTATUS = LPGETSTATUS;
+  unsigned IOCTL_LPRESET = LPRESET;
+  unsigned IOCTL_LPSETIRQ = LPSETIRQ;
+  unsigned IOCTL_LPTIME = LPTIME;
+  unsigned IOCTL_LPWAIT = LPWAIT;
+  unsigned IOCTL_MTIOCGETCONFIG = MTIOCGETCONFIG;
+  unsigned IOCTL_MTIOCSETCONFIG = MTIOCSETCONFIG;
+  unsigned IOCTL_PIO_CMAP = PIO_CMAP;
+  unsigned IOCTL_PIO_FONT = PIO_FONT;
+  unsigned IOCTL_PIO_UNIMAP = PIO_UNIMAP;
+  unsigned IOCTL_PIO_UNIMAPCLR = PIO_UNIMAPCLR;
+  unsigned IOCTL_PIO_UNISCRNMAP = PIO_UNISCRNMAP;
+  unsigned IOCTL_SCSI_IOCTL_GET_IDLUN = SCSI_IOCTL_GET_IDLUN;
+  unsigned IOCTL_SCSI_IOCTL_PROBE_HOST = SCSI_IOCTL_PROBE_HOST;
+  unsigned IOCTL_SCSI_IOCTL_TAGGED_DISABLE = SCSI_IOCTL_TAGGED_DISABLE;
+  unsigned IOCTL_SCSI_IOCTL_TAGGED_ENABLE = SCSI_IOCTL_TAGGED_ENABLE;
+  unsigned IOCTL_SIOCAIPXITFCRT = SIOCAIPXITFCRT;
+  unsigned IOCTL_SIOCAIPXPRISLT = SIOCAIPXPRISLT;
+  unsigned IOCTL_SIOCAX25ADDUID = SIOCAX25ADDUID;
+  unsigned IOCTL_SIOCAX25DELUID = SIOCAX25DELUID;
+  unsigned IOCTL_SIOCAX25GETPARMS = SIOCAX25GETPARMS;
+  unsigned IOCTL_SIOCAX25GETUID = SIOCAX25GETUID;
+  unsigned IOCTL_SIOCAX25NOUID = SIOCAX25NOUID;
+  unsigned IOCTL_SIOCAX25SETPARMS = SIOCAX25SETPARMS;
+  unsigned IOCTL_SIOCDEVPLIP = SIOCDEVPLIP;
+  unsigned IOCTL_SIOCIPXCFGDATA = SIOCIPXCFGDATA;
+  unsigned IOCTL_SIOCNRDECOBS = SIOCNRDECOBS;
+  unsigned IOCTL_SIOCNRGETPARMS = SIOCNRGETPARMS;
+  unsigned IOCTL_SIOCNRRTCTL = SIOCNRRTCTL;
+  unsigned IOCTL_SIOCNRSETPARMS = SIOCNRSETPARMS;
+  unsigned IOCTL_TIOCGSERIAL = TIOCGSERIAL;
+  unsigned IOCTL_TIOCSERGETMULTI = TIOCSERGETMULTI;
+  unsigned IOCTL_TIOCSERSETMULTI = TIOCSERSETMULTI;
+  unsigned IOCTL_TIOCSSERIAL = TIOCSSERIAL;
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP;
+  unsigned IOCTL_KDDISABIO = KDDISABIO;
+  unsigned IOCTL_KDENABIO = KDENABIO;
+  unsigned IOCTL_KDGETLED = KDGETLED;
+  unsigned IOCTL_KDGETMODE = KDGETMODE;
+  unsigned IOCTL_KDGKBMODE = KDGKBMODE;
+  unsigned IOCTL_KDGKBTYPE = KDGKBTYPE;
+  unsigned IOCTL_KDMKTONE = KDMKTONE;
+  unsigned IOCTL_KDSETLED = KDSETLED;
+  unsigned IOCTL_KDSETMODE = KDSETMODE;
+  unsigned IOCTL_KDSKBMODE = KDSKBMODE;
+  unsigned IOCTL_KIOCSOUND = KIOCSOUND;
+  unsigned IOCTL_PIO_SCRNMAP = PIO_SCRNMAP;
+  unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE;
+  unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE;
+#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+
+  const int si_SEGV_MAPERR = SEGV_MAPERR;
+  const int si_SEGV_ACCERR = SEGV_ACCERR;
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+#if SANITIZER_LINUX
+// FIXME: We define those on Linux and Mac, but only check on Linux.
+COMPILER_CHECK(IOC_NRBITS == _IOC_NRBITS);
+COMPILER_CHECK(IOC_TYPEBITS == _IOC_TYPEBITS);
+COMPILER_CHECK(IOC_SIZEBITS == _IOC_SIZEBITS);
+COMPILER_CHECK(IOC_DIRBITS == _IOC_DIRBITS);
+COMPILER_CHECK(IOC_NRMASK == _IOC_NRMASK);
+COMPILER_CHECK(IOC_TYPEMASK == _IOC_TYPEMASK);
+COMPILER_CHECK(IOC_SIZEMASK == _IOC_SIZEMASK);
+COMPILER_CHECK(IOC_DIRMASK == _IOC_DIRMASK);
+COMPILER_CHECK(IOC_NRSHIFT == _IOC_NRSHIFT);
+COMPILER_CHECK(IOC_TYPESHIFT == _IOC_TYPESHIFT);
+COMPILER_CHECK(IOC_SIZESHIFT == _IOC_SIZESHIFT);
+COMPILER_CHECK(IOC_DIRSHIFT == _IOC_DIRSHIFT);
+COMPILER_CHECK(IOC_NONE == _IOC_NONE);
+COMPILER_CHECK(IOC_WRITE == _IOC_WRITE);
+COMPILER_CHECK(IOC_READ == _IOC_READ);
+COMPILER_CHECK(EVIOC_ABS_MAX == ABS_MAX);
+COMPILER_CHECK(EVIOC_EV_MAX == EV_MAX);
+COMPILER_CHECK(IOC_SIZE(0x12345678) == _IOC_SIZE(0x12345678));
+COMPILER_CHECK(IOC_DIR(0x12345678) == _IOC_DIR(0x12345678));
+COMPILER_CHECK(IOC_NR(0x12345678) == _IOC_NR(0x12345678));
+COMPILER_CHECK(IOC_TYPE(0x12345678) == _IOC_TYPE(0x12345678));
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD
+
+#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+#endif
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+#if SANITIZER_LINUX && (__ANDROID_API__ >= 21 || __GLIBC_PREREQ (2, 14))
+CHECK_TYPE_SIZE(mmsghdr);
+CHECK_SIZE_AND_OFFSET(mmsghdr, msg_hdr);
+CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len);
+#endif
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_ino);
+#if SANITIZER_MAC
+CHECK_SIZE_AND_OFFSET(dirent, d_seekoff);
+#elif SANITIZER_FREEBSD
+// There is no 'd_off' field on FreeBSD.
+#else
+CHECK_SIZE_AND_OFFSET(dirent, d_off);
+#endif
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
+CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent64, d_off);
+CHECK_SIZE_AND_OFFSET(dirent64, d_reclen);
+#endif
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+#if !defined(__s390x__) || __GLIBC_PREREQ (2, 20)
+// On s390x glibc 2.19 and earlier sa_flags was unsigned long, and sa_resv
+// didn't exist.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
+#endif
+#if SANITIZER_LINUX && (!SANITIZER_ANDROID || !SANITIZER_MIPS32)
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer);
+#endif
+
+#if SANITIZER_LINUX
+CHECK_TYPE_SIZE(__sysctl_args);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, name);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, nlen);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, oldval);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, oldlenp);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, newval);
+CHECK_SIZE_AND_OFFSET(__sysctl_args, newlen);
+
+CHECK_TYPE_SIZE(__kernel_uid_t);
+CHECK_TYPE_SIZE(__kernel_gid_t);
+
+#if SANITIZER_USES_UID16_SYSCALLS
+CHECK_TYPE_SIZE(__kernel_old_uid_t);
+CHECK_TYPE_SIZE(__kernel_old_gid_t);
+#endif
+
+CHECK_TYPE_SIZE(__kernel_off_t);
+CHECK_TYPE_SIZE(__kernel_loff_t);
+CHECK_TYPE_SIZE(__kernel_fd_set);
+#endif
+
+#if !SANITIZER_ANDROID
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+#endif
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+#if SANITIZER_LINUX
+CHECK_TYPE_SIZE(mntent);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_fsname);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_dir);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_type);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_opts);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_freq);
+CHECK_SIZE_AND_OFFSET(mntent, mnt_passno);
+#endif
+
+CHECK_TYPE_SIZE(ether_addr);
+
+#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+CHECK_TYPE_SIZE(ipc_perm);
+# if SANITIZER_FREEBSD
+CHECK_SIZE_AND_OFFSET(ipc_perm, key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
+# else
+CHECK_SIZE_AND_OFFSET(ipc_perm, __key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, __seq);
+# endif
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+#if !SANITIZER_LINUX || __GLIBC_PREREQ (2, 31)
+/* glibc 2.30 and earlier provided 16-bit mode field instead of 32-bit
+   on many architectures.  */
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+#endif
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+#endif
+
+CHECK_TYPE_SIZE(clock_t);
+
+#if SANITIZER_LINUX
+CHECK_TYPE_SIZE(clockid_t);
+#endif
+
+#if !SANITIZER_ANDROID
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+#if SANITIZER_LINUX || SANITIZER_FREEBSD
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+# if SANITIZER_FREEBSD
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+# else
+COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) ==
+               sizeof(((ifaddrs *)nullptr)->ifa_ifu));
+COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) ==
+               offsetof(ifaddrs, ifa_ifu));
+# endif // SANITIZER_FREEBSD
+#else
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+#endif // SANITIZER_LINUX
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+#endif
+
+#if SANITIZER_LINUX
+COMPILER_CHECK(sizeof(__sanitizer_struct_mallinfo) == sizeof(struct mallinfo));
+#endif
+
+#if !SANITIZER_ANDROID
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+#endif
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+#if !SANITIZER_ANDROID
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+#endif
+
+#if SANITIZER_MAC
+CHECK_SIZE_AND_OFFSET(passwd, pw_change);
+CHECK_SIZE_AND_OFFSET(passwd, pw_expire);
+CHECK_SIZE_AND_OFFSET(passwd, pw_class);
+#endif
+
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+#if HAVE_RPC_XDR_H
+CHECK_TYPE_SIZE(XDR);
+CHECK_SIZE_AND_OFFSET(XDR, x_op);
+CHECK_SIZE_AND_OFFSET(XDR, x_ops);
+CHECK_SIZE_AND_OFFSET(XDR, x_public);
+CHECK_SIZE_AND_OFFSET(XDR, x_private);
+CHECK_SIZE_AND_OFFSET(XDR, x_base);
+CHECK_SIZE_AND_OFFSET(XDR, x_handy);
+COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE);
+COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE);
+COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+COMPILER_CHECK(sizeof(__sanitizer_FILE) <= sizeof(FILE));
+CHECK_SIZE_AND_OFFSET(FILE, _flags);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_read_ptr);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_read_end);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_read_base);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_write_ptr);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_write_end);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_write_base);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_buf_base);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_buf_end);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_save_base);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_backup_base);
+CHECK_SIZE_AND_OFFSET(FILE, _IO_save_end);
+CHECK_SIZE_AND_OFFSET(FILE, _markers);
+CHECK_SIZE_AND_OFFSET(FILE, _chain);
+CHECK_SIZE_AND_OFFSET(FILE, _fileno);
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+COMPILER_CHECK(sizeof(__sanitizer__obstack_chunk) <= sizeof(_obstack_chunk));
+CHECK_SIZE_AND_OFFSET(_obstack_chunk, limit);
+CHECK_SIZE_AND_OFFSET(_obstack_chunk, prev);
+CHECK_TYPE_SIZE(obstack);
+CHECK_SIZE_AND_OFFSET(obstack, chunk_size);
+CHECK_SIZE_AND_OFFSET(obstack, chunk);
+CHECK_SIZE_AND_OFFSET(obstack, object_base);
+CHECK_SIZE_AND_OFFSET(obstack, next_free);
+
+CHECK_TYPE_SIZE(cookie_io_functions_t);
+CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, read);
+CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, write);
+CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, seek);
+CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, close);
+#endif
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD
+CHECK_TYPE_SIZE(sem_t);
+#endif
+
+#if SANITIZER_LINUX && defined(__arm__)
+COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN);
+#endif
+
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h
new file mode 100644
index 0000000000..658b0abaec
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -0,0 +1,1453 @@
+//===-- sanitizer_platform_limits_posix.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific POSIX data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
+#define SANITIZER_PLATFORM_LIMITS_POSIX_H
+
+#if SANITIZER_LINUX || SANITIZER_MAC
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+#if defined(__sparc__)
+// FIXME: This can't be included from tsan which does not support sparc yet.
+#include "sanitizer_glibc_version.h"
+#endif
+
+# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle))
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+#if !SANITIZER_IOS
+extern unsigned struct_stat64_sz;
+#endif
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_stack_t_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs64_sz;
+extern unsigned struct_regex_sz;
+extern unsigned struct_regmatch_sz;
+
+#if !SANITIZER_ANDROID
+extern unsigned struct_fstab_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+#endif // !SANITIZER_ANDROID
+
+#if SANITIZER_LINUX
+
+#if defined(__x86_64__)
+const unsigned struct_kernel_stat_sz = 144;
+const unsigned struct_kernel_stat64_sz = 0;
+#elif defined(__i386__)
+const unsigned struct_kernel_stat_sz = 64;
+const unsigned struct_kernel_stat64_sz = 96;
+#elif defined(__arm__)
+const unsigned struct_kernel_stat_sz = 64;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__aarch64__)
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__powerpc__) && !defined(__powerpc64__)
+const unsigned struct_kernel_stat_sz = 72;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__powerpc64__)
+const unsigned struct_kernel_stat_sz = 144;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__mips__)
+const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
+                                           ? FIRST_32_SECOND_64(104, 128)
+                                           : FIRST_32_SECOND_64(160, 216);
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__s390__) && !defined(__s390x__)
+const unsigned struct_kernel_stat_sz = 64;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__s390x__)
+const unsigned struct_kernel_stat_sz = 144;
+const unsigned struct_kernel_stat64_sz = 0;
+#elif defined(__sparc__) && defined(__arch64__)
+const unsigned struct___old_kernel_stat_sz = 0;
+const unsigned struct_kernel_stat_sz = 104;
+const unsigned struct_kernel_stat64_sz = 144;
+#elif defined(__sparc__) && !defined(__arch64__)
+const unsigned struct___old_kernel_stat_sz = 0;
+const unsigned struct_kernel_stat_sz = 64;
+const unsigned struct_kernel_stat64_sz = 104;
+#elif defined(__riscv) && __riscv_xlen == 64
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 104;
+#endif
+struct __sanitizer_perf_event_attr {
+  unsigned type;
+  unsigned size;
+  // More fields that vary with the kernel version.
+};
+
+extern unsigned struct_epoll_event_sz;
+extern unsigned struct_sysinfo_sz;
+extern unsigned __user_cap_header_struct_sz;
+extern unsigned __user_cap_data_struct_sz;
+extern unsigned struct_new_utsname_sz;
+extern unsigned struct_old_utsname_sz;
+extern unsigned struct_oldold_utsname_sz;
+
+const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long);
+#endif  // SANITIZER_LINUX
+
+#if SANITIZER_LINUX
+
+#if defined(__powerpc64__) || defined(__s390__)
+const unsigned struct___old_kernel_stat_sz = 0;
+#elif !defined(__sparc__)
+const unsigned struct___old_kernel_stat_sz = 32;
+#endif
+
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+
+struct __sanitizer_iocb {
+  u64 aio_data;
+  u32 aio_key_or_aio_reserved1;  // Simply crazy.
+  u32 aio_reserved1_or_aio_key;  // Luckily, we don't need these.
+  u16 aio_lio_opcode;
+  s16 aio_reqprio;
+  u32 aio_fildes;
+  u64 aio_buf;
+  u64 aio_nbytes;
+  s64 aio_offset;
+  u64 aio_reserved2;
+  u64 aio_reserved3;
+};
+
+struct __sanitizer_io_event {
+  u64 data;
+  u64 obj;
+  u64 res;
+  u64 res2;
+};
+
+const unsigned iocb_cmd_pread = 0;
+const unsigned iocb_cmd_pwrite = 1;
+const unsigned iocb_cmd_preadv = 7;
+const unsigned iocb_cmd_pwritev = 8;
+
+struct __sanitizer___sysctl_args {
+  int *name;
+  int nlen;
+  void *oldval;
+  uptr *oldlenp;
+  void *newval;
+  uptr newlen;
+  unsigned long ___unused[4];
+};
+
+const unsigned old_sigset_t_sz = sizeof(unsigned long);
+
+struct __sanitizer_sem_t {
+#if SANITIZER_ANDROID && defined(_LP64)
+  int data[4];
+#elif SANITIZER_ANDROID && !defined(_LP64)
+  int data;
+#elif SANITIZER_LINUX
+  uptr data[4];
+#endif
+};
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_ANDROID
+struct __sanitizer_struct_mallinfo {
+  uptr v[10];
+};
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+struct __sanitizer_struct_mallinfo {
+  int v[10];
+};
+
+extern unsigned struct_ustat_sz;
+extern unsigned struct_rlimit64_sz;
+extern unsigned struct_statvfs64_sz;
+
+struct __sanitizer_ipc_perm {
+  int __key;
+  int uid;
+  int gid;
+  int cuid;
+  int cgid;
+#ifdef __powerpc__
+  unsigned mode;
+  unsigned __seq;
+  u64 __unused1;
+  u64 __unused2;
+#elif defined(__sparc__)
+  unsigned mode;
+  unsigned short __pad2;
+  unsigned short __seq;
+  unsigned long long __unused1;
+  unsigned long long __unused2;
+#else
+  unsigned int mode;
+  unsigned short __seq;
+  unsigned short __pad2;
+#if defined(__x86_64__) && !defined(_LP64)
+  u64 __unused1;
+  u64 __unused2;
+#else
+  unsigned long __unused1;
+  unsigned long __unused2;
+#endif
+#endif
+};
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+#if defined(__sparc__)
+#if !defined(__arch64__)
+  u32 __pad1;
+#endif
+  long shm_atime;
+#if !defined(__arch64__)
+  u32 __pad2;
+#endif
+  long shm_dtime;
+#if !defined(__arch64__)
+  u32 __pad3;
+#endif
+  long shm_ctime;
+  uptr shm_segsz;
+  int shm_cpid;
+  int shm_lpid;
+  unsigned long shm_nattch;
+  unsigned long __glibc_reserved1;
+  unsigned long __glibc_reserved2;
+#else
+#ifndef __powerpc__
+  uptr shm_segsz;
+#elif !defined(__powerpc64__)
+  uptr __unused0;
+#endif
+#if defined(__x86_64__) && !defined(_LP64)
+  u64 shm_atime;
+  u64 shm_dtime;
+  u64 shm_ctime;
+#else
+  uptr shm_atime;
+#if !defined(_LP64) && !defined(__mips__)
+  uptr __unused1;
+#endif
+  uptr shm_dtime;
+#if !defined(_LP64) && !defined(__mips__)
+  uptr __unused2;
+#endif
+  uptr shm_ctime;
+#if !defined(_LP64) && !defined(__mips__)
+  uptr __unused3;
+#endif
+#endif
+#ifdef __powerpc__
+  uptr shm_segsz;
+#endif
+  int shm_cpid;
+  int shm_lpid;
+#if defined(__x86_64__) && !defined(_LP64)
+  u64 shm_nattch;
+  u64 __unused4;
+  u64 __unused5;
+#else
+  uptr shm_nattch;
+  uptr __unused4;
+  uptr __unused5;
+#endif
+#endif
+};
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timex_sz;
+extern unsigned struct_statvfs_sz;
+extern unsigned struct_crypt_data_sz;
+#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+#if !SANITIZER_ANDROID
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  unsigned int ifa_flags;
+  void *ifa_addr;     // (struct sockaddr *)
+  void *ifa_netmask;  // (struct sockaddr *)
+  // This is a union on Linux.
+# ifdef ifa_dstaddr
+# undef ifa_dstaddr
+# endif
+  void *ifa_dstaddr;  // (struct sockaddr *)
+  void *ifa_data;
+};
+#endif  // !SANITIZER_ANDROID
+
+#if SANITIZER_MAC
+typedef unsigned long __sanitizer_pthread_key_t;
+#else
+typedef unsigned __sanitizer_pthread_key_t;
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+
+struct __sanitizer_XDR {
+  int x_op;
+  void *x_ops;
+  uptr x_public;
+  uptr x_private;
+  uptr x_base;
+  unsigned x_handy;
+};
+
+const int __sanitizer_XDR_ENCODE = 0;
+const int __sanitizer_XDR_DECODE = 1;
+const int __sanitizer_XDR_FREE = 2;
+#endif
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  int pw_uid;
+  int pw_gid;
+#if SANITIZER_MAC
+  long pw_change;
+  char *pw_class;
+#endif
+#if !(SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32))
+  char *pw_gecos;
+#endif
+  char *pw_dir;
+  char *pw_shell;
+#if SANITIZER_MAC
+  long pw_expire;
+#endif
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+#if defined(__x86_64__) && !defined(_LP64)
+typedef long long __sanitizer_time_t;
+#else
+typedef long __sanitizer_time_t;
+#endif
+
+typedef long __sanitizer_suseconds_t;
+
+struct __sanitizer_timeval {
+  __sanitizer_time_t tv_sec;
+  __sanitizer_suseconds_t tv_usec;
+};
+
+struct __sanitizer_itimerval {
+  struct __sanitizer_timeval it_interval;
+  struct __sanitizer_timeval it_value;
+};
+
+struct __sanitizer_timeb {
+  __sanitizer_time_t time;
+  unsigned short millitm;
+  short timezone;
+  short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+  long int tm_gmtoff;
+  const char *tm_zone;
+};
+
+#if SANITIZER_LINUX
+struct __sanitizer_mntent {
+  char *mnt_fsname;
+  char *mnt_dir;
+  char *mnt_type;
+  char *mnt_opts;
+  int mnt_freq;
+  int mnt_passno;
+};
+
+struct __sanitizer_file_handle {
+  unsigned int handle_bytes;
+  int handle_type;
+  unsigned char f_handle[1];  // variable sized
+};
+#endif
+
+#if SANITIZER_MAC
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+#else
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  uptr msg_iovlen;
+  void *msg_control;
+  uptr msg_controllen;
+  int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+  uptr cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+#endif
+
+#if SANITIZER_LINUX
+struct __sanitizer_mmsghdr {
+  __sanitizer_msghdr msg_hdr;
+  unsigned int msg_len;
+};
+#endif
+
+#if SANITIZER_MAC
+struct __sanitizer_dirent {
+  unsigned long long d_ino;
+  unsigned long long d_seekoff;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#elif SANITIZER_ANDROID || defined(__x86_64__)
+struct __sanitizer_dirent {
+  unsigned long long d_ino;
+  unsigned long long d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#else
+struct __sanitizer_dirent {
+  uptr d_ino;
+  uptr d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+struct __sanitizer_dirent64 {
+  unsigned long long d_ino;
+  unsigned long long d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#endif
+
+#if defined(__x86_64__) && !defined(_LP64)
+typedef long long __sanitizer_clock_t;
+#else
+typedef long __sanitizer_clock_t;
+#endif
+
+#if SANITIZER_LINUX
+typedef int __sanitizer_clockid_t;
+#endif
+
+#if SANITIZER_LINUX
+#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
+    defined(__mips__)
+typedef unsigned __sanitizer___kernel_uid_t;
+typedef unsigned __sanitizer___kernel_gid_t;
+#else
+typedef unsigned short __sanitizer___kernel_uid_t;
+typedef unsigned short __sanitizer___kernel_gid_t;
+#endif
+#if defined(__x86_64__) && !defined(_LP64)
+typedef long long __sanitizer___kernel_off_t;
+#else
+typedef long __sanitizer___kernel_off_t;
+#endif
+
+#if defined(__powerpc__) || defined(__mips__)
+typedef unsigned int __sanitizer___kernel_old_uid_t;
+typedef unsigned int __sanitizer___kernel_old_gid_t;
+#else
+typedef unsigned short __sanitizer___kernel_old_uid_t;
+typedef unsigned short __sanitizer___kernel_old_gid_t;
+#endif
+
+typedef long long __sanitizer___kernel_loff_t;
+typedef struct {
+  unsigned long fds_bits[1024 / (8 * sizeof(long))];
+} __sanitizer___kernel_fd_set;
+#endif
+
+// This thing depends on the platform. We are only interested in the upper
+// limit. Verified with a compiler assert in .cpp.
+union __sanitizer_pthread_attr_t {
+  char size[128];
+  void *align;
+};
+
+#if SANITIZER_ANDROID
+# if SANITIZER_MIPS
+typedef unsigned long __sanitizer_sigset_t[16 / sizeof(unsigned long)];
+# else
+typedef unsigned long __sanitizer_sigset_t;
+# endif
+#elif SANITIZER_MAC
+typedef unsigned __sanitizer_sigset_t;
+#elif SANITIZER_LINUX
+struct __sanitizer_sigset_t {
+  // The size is determined by looking at sizeof of real sigset_t on linux.
+  uptr val[128 / sizeof(uptr)];
+};
+#endif
+
+struct __sanitizer_siginfo {
+  // The size is determined by looking at sizeof of real siginfo_t on linux.
+  u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
+                                                  __sanitizer_siginfo *siginfo,
+                                                  void *uctx);
+
+// Linux system headers define the 'sa_handler' and 'sa_sigaction' macros.
+#if SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 64)
+struct __sanitizer_sigaction {
+  unsigned sa_flags;
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+  __sanitizer_sigset_t sa_mask;
+  void (*sa_restorer)();
+};
+#elif SANITIZER_ANDROID && SANITIZER_MIPS32  // check this before WORDSIZE == 32
+struct __sanitizer_sigaction {
+  unsigned sa_flags;
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+  __sanitizer_sigset_t sa_mask;
+};
+#elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32)
+struct __sanitizer_sigaction {
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+  __sanitizer_sigset_t sa_mask;
+  uptr sa_flags;
+  void (*sa_restorer)();
+};
+#else // !SANITIZER_ANDROID
+struct __sanitizer_sigaction {
+#if defined(__mips__) && !SANITIZER_FREEBSD
+  unsigned int sa_flags;
+#endif
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+#if SANITIZER_FREEBSD
+  int sa_flags;
+  __sanitizer_sigset_t sa_mask;
+#else
+#if defined(__s390x__)
+  int sa_resv;
+#else
+  __sanitizer_sigset_t sa_mask;
+#endif
+#ifndef __mips__
+#if defined(__sparc__)
+#if __GLIBC_PREREQ (2, 20)
+  // On sparc glibc 2.19 and earlier sa_flags was unsigned long.
+#if defined(__arch64__)
+  // To maintain ABI compatibility on sparc64 when switching to an int,
+  // __glibc_reserved0 was added.
+  int __glibc_reserved0;
+#endif
+  int sa_flags;
+#else
+  unsigned long sa_flags;
+#endif
+#else
+  int sa_flags;
+#endif
+#endif
+#endif
+#if SANITIZER_LINUX
+  void (*sa_restorer)();
+#endif
+#if defined(__mips__) && (SANITIZER_WORDSIZE == 32)
+  int sa_resv[1];
+#endif
+#if defined(__s390x__)
+  __sanitizer_sigset_t sa_mask;
+#endif
+};
+#endif // !SANITIZER_ANDROID
+
+#if defined(__mips__)
+struct __sanitizer_kernel_sigset_t {
+  uptr sig[2];
+};
+#else
+struct __sanitizer_kernel_sigset_t {
+  u8 sig[8];
+};
+#endif
+
+// Linux system headers define the 'sa_handler' and 'sa_sigaction' macros.
+#if SANITIZER_MIPS
+struct __sanitizer_kernel_sigaction_t {
+  unsigned int sa_flags;
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
+  };
+  __sanitizer_kernel_sigset_t sa_mask;
+  void (*sa_restorer)(void);
+};
+#else
+struct __sanitizer_kernel_sigaction_t {
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
+  };
+  unsigned long sa_flags;
+  void (*sa_restorer)(void);
+  __sanitizer_kernel_sigset_t sa_mask;
+};
+#endif
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+#if SANITIZER_LINUX
+extern int e_tabsz;
+#endif
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+#if SANITIZER_LINUX
+struct __sanitizer_dl_phdr_info {
+  uptr dlpi_addr;
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+  short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+#endif
+
+struct __sanitizer_protoent {
+  char *p_name;
+  char **p_aliases;
+  int p_proto;
+};
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+#if SANITIZER_ANDROID || SANITIZER_MAC
+  unsigned ai_addrlen;
+  char *ai_canonname;
+  void *ai_addr;
+#else // LINUX
+  unsigned ai_addrlen;
+  void *ai_addr;
+  char *ai_canonname;
+#endif
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+#if SANITIZER_ANDROID || SANITIZER_MAC
+typedef unsigned __sanitizer_nfds_t;
+#else
+typedef unsigned long __sanitizer_nfds_t;
+#endif
+
+#if !SANITIZER_ANDROID
+# if SANITIZER_LINUX
+struct __sanitizer_glob_t {
+  uptr gl_pathc;
+  char **gl_pathv;
+  uptr gl_offs;
+  int gl_flags;
+
+  void (*gl_closedir)(void *dirp);
+  void *(*gl_readdir)(void *dirp);
+  void *(*gl_opendir)(const char *);
+  int (*gl_lstat)(const char *, void *);
+  int (*gl_stat)(const char *, void *);
+};
+# endif  // SANITIZER_LINUX
+
+# if SANITIZER_LINUX
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+# endif
+#endif  // !SANITIZER_ANDROID
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+  uptr we_wordc;
+  char **we_wordv;
+  uptr we_offs;
+};
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+struct __sanitizer_FILE {
+  int _flags;
+  char *_IO_read_ptr;
+  char *_IO_read_end;
+  char *_IO_read_base;
+  char *_IO_write_base;
+  char *_IO_write_ptr;
+  char *_IO_write_end;
+  char *_IO_buf_base;
+  char *_IO_buf_end;
+  char *_IO_save_base;
+  char *_IO_backup_base;
+  char *_IO_save_end;
+  void *_markers;
+  __sanitizer_FILE *_chain;
+  int _fileno;
+};
+# define SANITIZER_HAS_STRUCT_FILE 1
+#else
+typedef void __sanitizer_FILE;
+# define SANITIZER_HAS_STRUCT_FILE 0
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
+    (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
+     defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+     defined(__s390__))
+extern unsigned struct_user_regs_struct_sz;
+extern unsigned struct_user_fpregs_struct_sz;
+extern unsigned struct_user_fpxregs_struct_sz;
+extern unsigned struct_user_vfpregs_struct_sz;
+
+extern int ptrace_peektext;
+extern int ptrace_peekdata;
+extern int ptrace_peekuser;
+extern int ptrace_getregs;
+extern int ptrace_setregs;
+extern int ptrace_getfpregs;
+extern int ptrace_setfpregs;
+extern int ptrace_getfpxregs;
+extern int ptrace_setfpxregs;
+extern int ptrace_getvfpregs;
+extern int ptrace_setvfpregs;
+extern int ptrace_getsiginfo;
+extern int ptrace_setsiginfo;
+extern int ptrace_getregset;
+extern int ptrace_setregset;
+extern int ptrace_geteventmsg;
+#endif
+
+#if SANITIZER_LINUX  && !SANITIZER_ANDROID
+extern unsigned struct_shminfo_sz;
+extern unsigned struct_shm_info_sz;
+extern int shmctl_ipc_stat;
+extern int shmctl_ipc_info;
+extern int shmctl_shm_info;
+extern int shmctl_shm_stat;
+#endif
+
+#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+extern unsigned struct_utmp_sz;
+#endif
+#if !SANITIZER_ANDROID
+extern unsigned struct_utmpx_sz;
+#endif
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+#if SANITIZER_MAC
+} __attribute__((packed));
+#else
+};
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+struct __sanitizer__obstack_chunk {
+  char *limit;
+  struct __sanitizer__obstack_chunk *prev;
+};
+
+struct __sanitizer_obstack {
+  long chunk_size;
+  struct __sanitizer__obstack_chunk *chunk;
+  char *object_base;
+  char *next_free;
+  uptr more_fields[7];
+};
+
+typedef uptr (*__sanitizer_cookie_io_read)(void *cookie, char *buf, uptr size);
+typedef uptr (*__sanitizer_cookie_io_write)(void *cookie, const char *buf,
+                                            uptr size);
+typedef int (*__sanitizer_cookie_io_seek)(void *cookie, u64 *offset,
+                                          int whence);
+typedef int (*__sanitizer_cookie_io_close)(void *cookie);
+
+struct __sanitizer_cookie_io_functions_t {
+  __sanitizer_cookie_io_read read;
+  __sanitizer_cookie_io_write write;
+  __sanitizer_cookie_io_seek seek;
+  __sanitizer_cookie_io_close close;
+};
+#endif
+
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || \
+    defined(__sparc__)
+#define IOC_SIZEBITS 13
+#define IOC_DIRBITS 3
+#define IOC_NONE 1U
+#define IOC_WRITE 4U
+#define IOC_READ 2U
+#else
+#define IOC_SIZEBITS 14
+#define IOC_DIRBITS 2
+#define IOC_NONE 0U
+#define IOC_WRITE 1U
+#define IOC_READ 2U
+#endif
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#if defined(IOC_DIRMASK)
+#undef IOC_DIRMASK
+#endif
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+#define EVIOC_EV_MAX 0x1f
+#define EVIOC_ABS_MAX 0x3f
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+
+#if defined(__sparc__)
+// In sparc the 14 bits SIZE field overlaps with the
+// least significant bit of DIR, so either IOC_READ or
+// IOC_WRITE shall be 1 in order to get a non-zero SIZE.
+#define IOC_SIZE(nr) \
+  ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff))
+#else
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+#endif
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+#if SANITIZER_LINUX
+extern unsigned struct_arpreq_sz;
+extern unsigned struct_cdrom_msf_sz;
+extern unsigned struct_cdrom_multisession_sz;
+extern unsigned struct_cdrom_read_audio_sz;
+extern unsigned struct_cdrom_subchnl_sz;
+extern unsigned struct_cdrom_ti_sz;
+extern unsigned struct_cdrom_tocentry_sz;
+extern unsigned struct_cdrom_tochdr_sz;
+extern unsigned struct_cdrom_volctrl_sz;
+extern unsigned struct_ff_effect_sz;
+extern unsigned struct_floppy_drive_params_sz;
+extern unsigned struct_floppy_drive_struct_sz;
+extern unsigned struct_floppy_fdc_state_sz;
+extern unsigned struct_floppy_max_errors_sz;
+extern unsigned struct_floppy_raw_cmd_sz;
+extern unsigned struct_floppy_struct_sz;
+extern unsigned struct_floppy_write_errors_sz;
+extern unsigned struct_format_descr_sz;
+extern unsigned struct_hd_driveid_sz;
+extern unsigned struct_hd_geometry_sz;
+extern unsigned struct_input_absinfo_sz;
+extern unsigned struct_input_id_sz;
+extern unsigned struct_mtpos_sz;
+extern unsigned struct_termio_sz;
+extern unsigned struct_vt_consize_sz;
+extern unsigned struct_vt_sizes_sz;
+extern unsigned struct_vt_stat_sz;
+#endif  // SANITIZER_LINUX
+
+#if SANITIZER_LINUX
+extern unsigned struct_copr_buffer_sz;
+extern unsigned struct_copr_debug_buf_sz;
+extern unsigned struct_copr_msg_sz;
+extern unsigned struct_midi_info_sz;
+extern unsigned struct_mtget_sz;
+extern unsigned struct_mtop_sz;
+extern unsigned struct_rtentry_sz;
+extern unsigned struct_sbi_instrument_sz;
+extern unsigned struct_seq_event_rec_sz;
+extern unsigned struct_synth_info_sz;
+extern unsigned struct_vt_mode_sz;
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern unsigned struct_ax25_parms_struct_sz;
+extern unsigned struct_cyclades_monitor_sz;
+extern unsigned struct_input_keymap_entry_sz;
+extern unsigned struct_ipx_config_data_sz;
+extern unsigned struct_kbdiacrs_sz;
+extern unsigned struct_kbentry_sz;
+extern unsigned struct_kbkeycode_sz;
+extern unsigned struct_kbsentry_sz;
+extern unsigned struct_mtconfiginfo_sz;
+extern unsigned struct_nr_parms_struct_sz;
+extern unsigned struct_scc_modem_sz;
+extern unsigned struct_scc_stat_sz;
+extern unsigned struct_serial_multiport_struct_sz;
+extern unsigned struct_serial_struct_sz;
+extern unsigned struct_sockaddr_ax25_sz;
+extern unsigned struct_unimapdesc_sz;
+extern unsigned struct_unimapinit_sz;
+#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+extern const unsigned long __sanitizer_bufsiz;
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern unsigned struct_audio_buf_info_sz;
+extern unsigned struct_ppp_stats_sz;
+#endif  // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+
+#if !SANITIZER_ANDROID && !SANITIZER_MAC
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+#endif
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCCONS;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern unsigned IOCTL_SIOCGETSGCNT;
+extern unsigned IOCTL_SIOCGETVIFCNT;
+#endif
+#if SANITIZER_LINUX
+extern unsigned IOCTL_EVIOCGABS;
+extern unsigned IOCTL_EVIOCGBIT;
+extern unsigned IOCTL_EVIOCGEFFECTS;
+extern unsigned IOCTL_EVIOCGID;
+extern unsigned IOCTL_EVIOCGKEY;
+extern unsigned IOCTL_EVIOCGKEYCODE;
+extern unsigned IOCTL_EVIOCGLED;
+extern unsigned IOCTL_EVIOCGNAME;
+extern unsigned IOCTL_EVIOCGPHYS;
+extern unsigned IOCTL_EVIOCGRAB;
+extern unsigned IOCTL_EVIOCGREP;
+extern unsigned IOCTL_EVIOCGSND;
+extern unsigned IOCTL_EVIOCGSW;
+extern unsigned IOCTL_EVIOCGUNIQ;
+extern unsigned IOCTL_EVIOCGVERSION;
+extern unsigned IOCTL_EVIOCRMFF;
+extern unsigned IOCTL_EVIOCSABS;
+extern unsigned IOCTL_EVIOCSFF;
+extern unsigned IOCTL_EVIOCSKEYCODE;
+extern unsigned IOCTL_EVIOCSREP;
+extern unsigned IOCTL_BLKFLSBUF;
+extern unsigned IOCTL_BLKGETSIZE;
+extern unsigned IOCTL_BLKRAGET;
+extern unsigned IOCTL_BLKRASET;
+extern unsigned IOCTL_BLKROGET;
+extern unsigned IOCTL_BLKROSET;
+extern unsigned IOCTL_BLKRRPART;
+extern unsigned IOCTL_CDROMAUDIOBUFSIZ;
+extern unsigned IOCTL_CDROMEJECT;
+extern unsigned IOCTL_CDROMEJECT_SW;
+extern unsigned IOCTL_CDROMMULTISESSION;
+extern unsigned IOCTL_CDROMPAUSE;
+extern unsigned IOCTL_CDROMPLAYMSF;
+extern unsigned IOCTL_CDROMPLAYTRKIND;
+extern unsigned IOCTL_CDROMREADAUDIO;
+extern unsigned IOCTL_CDROMREADCOOKED;
+extern unsigned IOCTL_CDROMREADMODE1;
+extern unsigned IOCTL_CDROMREADMODE2;
+extern unsigned IOCTL_CDROMREADRAW;
+extern unsigned IOCTL_CDROMREADTOCENTRY;
+extern unsigned IOCTL_CDROMREADTOCHDR;
+extern unsigned IOCTL_CDROMRESET;
+extern unsigned IOCTL_CDROMRESUME;
+extern unsigned IOCTL_CDROMSEEK;
+extern unsigned IOCTL_CDROMSTART;
+extern unsigned IOCTL_CDROMSTOP;
+extern unsigned IOCTL_CDROMSUBCHNL;
+extern unsigned IOCTL_CDROMVOLCTRL;
+extern unsigned IOCTL_CDROMVOLREAD;
+extern unsigned IOCTL_CDROM_GET_UPC;
+extern unsigned IOCTL_FDCLRPRM;
+extern unsigned IOCTL_FDDEFPRM;
+extern unsigned IOCTL_FDFLUSH;
+extern unsigned IOCTL_FDFMTBEG;
+extern unsigned IOCTL_FDFMTEND;
+extern unsigned IOCTL_FDFMTTRK;
+extern unsigned IOCTL_FDGETDRVPRM;
+extern unsigned IOCTL_FDGETDRVSTAT;
+extern unsigned IOCTL_FDGETDRVTYP;
+extern unsigned IOCTL_FDGETFDCSTAT;
+extern unsigned IOCTL_FDGETMAXERRS;
+extern unsigned IOCTL_FDGETPRM;
+extern unsigned IOCTL_FDMSGOFF;
+extern unsigned IOCTL_FDMSGON;
+extern unsigned IOCTL_FDPOLLDRVSTAT;
+extern unsigned IOCTL_FDRAWCMD;
+extern unsigned IOCTL_FDRESET;
+extern unsigned IOCTL_FDSETDRVPRM;
+extern unsigned IOCTL_FDSETEMSGTRESH;
+extern unsigned IOCTL_FDSETMAXERRS;
+extern unsigned IOCTL_FDSETPRM;
+extern unsigned IOCTL_FDTWADDLE;
+extern unsigned IOCTL_FDWERRORCLR;
+extern unsigned IOCTL_FDWERRORGET;
+extern unsigned IOCTL_HDIO_DRIVE_CMD;
+extern unsigned IOCTL_HDIO_GETGEO;
+extern unsigned IOCTL_HDIO_GET_32BIT;
+extern unsigned IOCTL_HDIO_GET_DMA;
+extern unsigned IOCTL_HDIO_GET_IDENTITY;
+extern unsigned IOCTL_HDIO_GET_KEEPSETTINGS;
+extern unsigned IOCTL_HDIO_GET_MULTCOUNT;
+extern unsigned IOCTL_HDIO_GET_NOWERR;
+extern unsigned IOCTL_HDIO_GET_UNMASKINTR;
+extern unsigned IOCTL_HDIO_SET_32BIT;
+extern unsigned IOCTL_HDIO_SET_DMA;
+extern unsigned IOCTL_HDIO_SET_KEEPSETTINGS;
+extern unsigned IOCTL_HDIO_SET_MULTCOUNT;
+extern unsigned IOCTL_HDIO_SET_NOWERR;
+extern unsigned IOCTL_HDIO_SET_UNMASKINTR;
+extern unsigned IOCTL_MTIOCPOS;
+extern unsigned IOCTL_PPPIOCGASYNCMAP;
+extern unsigned IOCTL_PPPIOCGDEBUG;
+extern unsigned IOCTL_PPPIOCGFLAGS;
+extern unsigned IOCTL_PPPIOCGUNIT;
+extern unsigned IOCTL_PPPIOCGXASYNCMAP;
+extern unsigned IOCTL_PPPIOCSASYNCMAP;
+extern unsigned IOCTL_PPPIOCSDEBUG;
+extern unsigned IOCTL_PPPIOCSFLAGS;
+extern unsigned IOCTL_PPPIOCSMAXCID;
+extern unsigned IOCTL_PPPIOCSMRU;
+extern unsigned IOCTL_PPPIOCSXASYNCMAP;
+extern unsigned IOCTL_SIOCDARP;
+extern unsigned IOCTL_SIOCDRARP;
+extern unsigned IOCTL_SIOCGARP;
+extern unsigned IOCTL_SIOCGIFENCAP;
+extern unsigned IOCTL_SIOCGIFHWADDR;
+extern unsigned IOCTL_SIOCGIFMAP;
+extern unsigned IOCTL_SIOCGIFMEM;
+extern unsigned IOCTL_SIOCGIFNAME;
+extern unsigned IOCTL_SIOCGIFSLAVE;
+extern unsigned IOCTL_SIOCGRARP;
+extern unsigned IOCTL_SIOCGSTAMP;
+extern unsigned IOCTL_SIOCSARP;
+extern unsigned IOCTL_SIOCSIFENCAP;
+extern unsigned IOCTL_SIOCSIFHWADDR;
+extern unsigned IOCTL_SIOCSIFLINK;
+extern unsigned IOCTL_SIOCSIFMAP;
+extern unsigned IOCTL_SIOCSIFMEM;
+extern unsigned IOCTL_SIOCSIFSLAVE;
+extern unsigned IOCTL_SIOCSRARP;
+extern unsigned IOCTL_SNDCTL_COPR_HALT;
+extern unsigned IOCTL_SNDCTL_COPR_LOAD;
+extern unsigned IOCTL_SNDCTL_COPR_RCODE;
+extern unsigned IOCTL_SNDCTL_COPR_RCVMSG;
+extern unsigned IOCTL_SNDCTL_COPR_RDATA;
+extern unsigned IOCTL_SNDCTL_COPR_RESET;
+extern unsigned IOCTL_SNDCTL_COPR_RUN;
+extern unsigned IOCTL_SNDCTL_COPR_SENDMSG;
+extern unsigned IOCTL_SNDCTL_COPR_WCODE;
+extern unsigned IOCTL_SNDCTL_COPR_WDATA;
+extern unsigned IOCTL_TCFLSH;
+extern unsigned IOCTL_TCGETA;
+extern unsigned IOCTL_TCGETS;
+extern unsigned IOCTL_TCSBRK;
+extern unsigned IOCTL_TCSBRKP;
+extern unsigned IOCTL_TCSETA;
+extern unsigned IOCTL_TCSETAF;
+extern unsigned IOCTL_TCSETAW;
+extern unsigned IOCTL_TCSETS;
+extern unsigned IOCTL_TCSETSF;
+extern unsigned IOCTL_TCSETSW;
+extern unsigned IOCTL_TCXONC;
+extern unsigned IOCTL_TIOCGLCKTRMIOS;
+extern unsigned IOCTL_TIOCGSOFTCAR;
+extern unsigned IOCTL_TIOCINQ;
+extern unsigned IOCTL_TIOCLINUX;
+extern unsigned IOCTL_TIOCSERCONFIG;
+extern unsigned IOCTL_TIOCSERGETLSR;
+extern unsigned IOCTL_TIOCSERGWILD;
+extern unsigned IOCTL_TIOCSERSWILD;
+extern unsigned IOCTL_TIOCSLCKTRMIOS;
+extern unsigned IOCTL_TIOCSSOFTCAR;
+extern unsigned IOCTL_VT_DISALLOCATE;
+extern unsigned IOCTL_VT_GETSTATE;
+extern unsigned IOCTL_VT_RESIZE;
+extern unsigned IOCTL_VT_RESIZEX;
+extern unsigned IOCTL_VT_SENDSIG;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+extern unsigned IOCTL_SIOCADDRT;
+extern unsigned IOCTL_SIOCDELRT;
+extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE;
+extern unsigned IOCTL_SNDCTL_DSP_GETFMTS;
+extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK;
+extern unsigned IOCTL_SNDCTL_DSP_POST;
+extern unsigned IOCTL_SNDCTL_DSP_RESET;
+extern unsigned IOCTL_SNDCTL_DSP_SETFMT;
+extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT;
+extern unsigned IOCTL_SNDCTL_DSP_SPEED;
+extern unsigned IOCTL_SNDCTL_DSP_STEREO;
+extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE;
+extern unsigned IOCTL_SNDCTL_DSP_SYNC;
+extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE;
+extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR;
+extern unsigned IOCTL_SNDCTL_MIDI_INFO;
+extern unsigned IOCTL_SNDCTL_MIDI_PRETIME;
+extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE;
+extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS;
+extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS;
+extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND;
+extern unsigned IOCTL_SNDCTL_SEQ_PANIC;
+extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE;
+extern unsigned IOCTL_SNDCTL_SEQ_RESET;
+extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES;
+extern unsigned IOCTL_SNDCTL_SEQ_SYNC;
+extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI;
+extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD;
+extern unsigned IOCTL_SNDCTL_SYNTH_INFO;
+extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL;
+extern unsigned IOCTL_SNDCTL_TMR_CONTINUE;
+extern unsigned IOCTL_SNDCTL_TMR_METRONOME;
+extern unsigned IOCTL_SNDCTL_TMR_SELECT;
+extern unsigned IOCTL_SNDCTL_TMR_SOURCE;
+extern unsigned IOCTL_SNDCTL_TMR_START;
+extern unsigned IOCTL_SNDCTL_TMR_STOP;
+extern unsigned IOCTL_SNDCTL_TMR_TEMPO;
+extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE;
+extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_BASS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CAPS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CD;
+extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE;
+extern unsigned IOCTL_SOUND_MIXER_READ_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_READ_MIC;
+extern unsigned IOCTL_SOUND_MIXER_READ_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_PCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS;
+extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_CD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME;
+extern unsigned IOCTL_SOUND_PCM_READ_BITS;
+extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_READ_FILTER;
+extern unsigned IOCTL_SOUND_PCM_READ_RATE;
+extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER;
+extern unsigned IOCTL_VT_ACTIVATE;
+extern unsigned IOCTL_VT_GETMODE;
+extern unsigned IOCTL_VT_OPENQRY;
+extern unsigned IOCTL_VT_RELDISP;
+extern unsigned IOCTL_VT_SETMODE;
+extern unsigned IOCTL_VT_WAITACTIVE;
+#endif  // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+extern unsigned IOCTL_CYGETDEFTHRESH;
+extern unsigned IOCTL_CYGETDEFTIMEOUT;
+extern unsigned IOCTL_CYGETMON;
+extern unsigned IOCTL_CYGETTHRESH;
+extern unsigned IOCTL_CYGETTIMEOUT;
+extern unsigned IOCTL_CYSETDEFTHRESH;
+extern unsigned IOCTL_CYSETDEFTIMEOUT;
+extern unsigned IOCTL_CYSETTHRESH;
+extern unsigned IOCTL_CYSETTIMEOUT;
+extern unsigned IOCTL_EQL_EMANCIPATE;
+extern unsigned IOCTL_EQL_ENSLAVE;
+extern unsigned IOCTL_EQL_GETMASTRCFG;
+extern unsigned IOCTL_EQL_GETSLAVECFG;
+extern unsigned IOCTL_EQL_SETMASTRCFG;
+extern unsigned IOCTL_EQL_SETSLAVECFG;
+extern unsigned IOCTL_EVIOCGKEYCODE_V2;
+extern unsigned IOCTL_EVIOCGPROP;
+extern unsigned IOCTL_EVIOCSKEYCODE_V2;
+extern unsigned IOCTL_FS_IOC_GETFLAGS;
+extern unsigned IOCTL_FS_IOC_GETVERSION;
+extern unsigned IOCTL_FS_IOC_SETFLAGS;
+extern unsigned IOCTL_FS_IOC_SETVERSION;
+extern unsigned IOCTL_GIO_CMAP;
+extern unsigned IOCTL_GIO_FONT;
+extern unsigned IOCTL_GIO_UNIMAP;
+extern unsigned IOCTL_GIO_UNISCRNMAP;
+extern unsigned IOCTL_KDADDIO;
+extern unsigned IOCTL_KDDELIO;
+extern unsigned IOCTL_KDGETKEYCODE;
+extern unsigned IOCTL_KDGKBDIACR;
+extern unsigned IOCTL_KDGKBENT;
+extern unsigned IOCTL_KDGKBLED;
+extern unsigned IOCTL_KDGKBMETA;
+extern unsigned IOCTL_KDGKBSENT;
+extern unsigned IOCTL_KDMAPDISP;
+extern unsigned IOCTL_KDSETKEYCODE;
+extern unsigned IOCTL_KDSIGACCEPT;
+extern unsigned IOCTL_KDSKBDIACR;
+extern unsigned IOCTL_KDSKBENT;
+extern unsigned IOCTL_KDSKBLED;
+extern unsigned IOCTL_KDSKBMETA;
+extern unsigned IOCTL_KDSKBSENT;
+extern unsigned IOCTL_KDUNMAPDISP;
+extern unsigned IOCTL_LPABORT;
+extern unsigned IOCTL_LPABORTOPEN;
+extern unsigned IOCTL_LPCAREFUL;
+extern unsigned IOCTL_LPCHAR;
+extern unsigned IOCTL_LPGETIRQ;
+extern unsigned IOCTL_LPGETSTATUS;
+extern unsigned IOCTL_LPRESET;
+extern unsigned IOCTL_LPSETIRQ;
+extern unsigned IOCTL_LPTIME;
+extern unsigned IOCTL_LPWAIT;
+extern unsigned IOCTL_MTIOCGETCONFIG;
+extern unsigned IOCTL_MTIOCSETCONFIG;
+extern unsigned IOCTL_PIO_CMAP;
+extern unsigned IOCTL_PIO_FONT;
+extern unsigned IOCTL_PIO_UNIMAP;
+extern unsigned IOCTL_PIO_UNIMAPCLR;
+extern unsigned IOCTL_PIO_UNISCRNMAP;
+extern unsigned IOCTL_SCSI_IOCTL_GET_IDLUN;
+extern unsigned IOCTL_SCSI_IOCTL_PROBE_HOST;
+extern unsigned IOCTL_SCSI_IOCTL_TAGGED_DISABLE;
+extern unsigned IOCTL_SCSI_IOCTL_TAGGED_ENABLE;
+extern unsigned IOCTL_SIOCAIPXITFCRT;
+extern unsigned IOCTL_SIOCAIPXPRISLT;
+extern unsigned IOCTL_SIOCAX25ADDUID;
+extern unsigned IOCTL_SIOCAX25DELUID;
+extern unsigned IOCTL_SIOCAX25GETPARMS;
+extern unsigned IOCTL_SIOCAX25GETUID;
+extern unsigned IOCTL_SIOCAX25NOUID;
+extern unsigned IOCTL_SIOCAX25SETPARMS;
+extern unsigned IOCTL_SIOCDEVPLIP;
+extern unsigned IOCTL_SIOCIPXCFGDATA;
+extern unsigned IOCTL_SIOCNRDECOBS;
+extern unsigned IOCTL_SIOCNRGETPARMS;
+extern unsigned IOCTL_SIOCNRRTCTL;
+extern unsigned IOCTL_SIOCNRSETPARMS;
+extern unsigned IOCTL_SNDCTL_DSP_GETISPACE;
+extern unsigned IOCTL_SNDCTL_DSP_GETOSPACE;
+extern unsigned IOCTL_TIOCGSERIAL;
+extern unsigned IOCTL_TIOCSERGETMULTI;
+extern unsigned IOCTL_TIOCSERSETMULTI;
+extern unsigned IOCTL_TIOCSSERIAL;
+extern unsigned IOCTL_GIO_SCRNMAP;
+extern unsigned IOCTL_KDDISABIO;
+extern unsigned IOCTL_KDENABIO;
+extern unsigned IOCTL_KDGETLED;
+extern unsigned IOCTL_KDGETMODE;
+extern unsigned IOCTL_KDGKBMODE;
+extern unsigned IOCTL_KDGKBTYPE;
+extern unsigned IOCTL_KDMKTONE;
+extern unsigned IOCTL_KDSETLED;
+extern unsigned IOCTL_KDSETMODE;
+extern unsigned IOCTL_KDSKBMODE;
+extern unsigned IOCTL_KIOCSOUND;
+extern unsigned IOCTL_PIO_SCRNMAP;
+#endif
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                      \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *)NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==         \
+                 offsetof(struct CLASS, MEMBER))
+
+#define SIGACTION_SYMNAME sigaction
+
+#endif  // SANITIZER_LINUX || SANITIZER_MAC
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp
new file mode 100644
index 0000000000..6ec1a1bdd1
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp
@@ -0,0 +1,366 @@
+//===-- sanitizer_platform_limits_solaris.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific Solaris data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_SOLARIS
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link.h>
+#include <net/if.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/ip_mroute.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <rpc/xdr.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/ethernet.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/mtio.h>
+#include <sys/ptyvar.h>
+#include <sys/resource.h>
+#include <sys/shm.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/stat.h>
+#include <sys/statfs.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <termios.h>
+#include <time.h>
+#include <utmp.h>
+#include <utmpx.h>
+#include <wchar.h>
+#include <wordexp.h>
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_solaris.h"
+
+namespace __sanitizer {
+  unsigned struct_utsname_sz = sizeof(struct utsname);
+  unsigned struct_stat_sz = sizeof(struct stat);
+  unsigned struct_stat64_sz = sizeof(struct stat64);
+  unsigned struct_rusage_sz = sizeof(struct rusage);
+  unsigned struct_tm_sz = sizeof(struct tm);
+  unsigned struct_passwd_sz = sizeof(struct passwd);
+  unsigned struct_group_sz = sizeof(struct group);
+  unsigned siginfo_t_sz = sizeof(siginfo_t);
+  unsigned struct_sigaction_sz = sizeof(struct sigaction);
+  unsigned struct_stack_t_sz = sizeof(stack_t);
+  unsigned struct_itimerval_sz = sizeof(struct itimerval);
+  unsigned pthread_t_sz = sizeof(pthread_t);
+  unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+  unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+  unsigned pid_t_sz = sizeof(pid_t);
+  unsigned timeval_sz = sizeof(timeval);
+  unsigned uid_t_sz = sizeof(uid_t);
+  unsigned gid_t_sz = sizeof(gid_t);
+  unsigned mbstate_t_sz = sizeof(mbstate_t);
+  unsigned sigset_t_sz = sizeof(sigset_t);
+  unsigned struct_timezone_sz = sizeof(struct timezone);
+  unsigned struct_tms_sz = sizeof(struct tms);
+  unsigned struct_sigevent_sz = sizeof(struct sigevent);
+  unsigned struct_sched_param_sz = sizeof(struct sched_param);
+  unsigned struct_statfs_sz = sizeof(struct statfs);
+  unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+  unsigned ucontext_t_sz = sizeof(ucontext_t);
+  unsigned struct_timespec_sz = sizeof(struct timespec);
+#if SANITIZER_SOLARIS32
+  unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
+#endif
+  unsigned struct_statvfs_sz = sizeof(struct statvfs);
+
+  const uptr sig_ign = (uptr)SIG_IGN;
+  const uptr sig_dfl = (uptr)SIG_DFL;
+  const uptr sig_err = (uptr)SIG_ERR;
+  const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+  int shmctl_ipc_stat = (int)IPC_STAT;
+
+  unsigned struct_utmp_sz = sizeof(struct utmp);
+  unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+  int map_fixed = MAP_FIXED;
+
+  int af_inet = (int)AF_INET;
+  int af_inet6 = (int)AF_INET6;
+
+  uptr __sanitizer_in_addr_sz(int af) {
+    if (af == AF_INET)
+      return sizeof(struct in_addr);
+    else if (af == AF_INET6)
+      return sizeof(struct in6_addr);
+    else
+      return 0;
+  }
+
+  unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
+
+  int glob_nomatch = GLOB_NOMATCH;
+
+  unsigned path_max = PATH_MAX;
+
+  // ioctl arguments
+  unsigned struct_ifreq_sz = sizeof(struct ifreq);
+  unsigned struct_termios_sz = sizeof(struct termios);
+  unsigned struct_winsize_sz = sizeof(struct winsize);
+
+  unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+  unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+
+  const unsigned IOCTL_NOT_PRESENT = 0;
+
+  unsigned IOCTL_FIOASYNC = FIOASYNC;
+  unsigned IOCTL_FIOCLEX = FIOCLEX;
+  unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+  unsigned IOCTL_FIONBIO = FIONBIO;
+  unsigned IOCTL_FIONCLEX = FIONCLEX;
+  unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+  unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+  unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+  unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+  unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+  unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+  unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+  unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+  unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+  unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+  unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+  unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+  unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+  unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+  unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+  unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+  unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+  unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+  unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+  unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+  unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+  unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+  unsigned IOCTL_TIOCGETD = TIOCGETD;
+  unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+  unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+  unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+  unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+  unsigned IOCTL_TIOCMGET = TIOCMGET;
+  unsigned IOCTL_TIOCMSET = TIOCMSET;
+  unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+  unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+  unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+  unsigned IOCTL_TIOCPKT = TIOCPKT;
+  unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+  unsigned IOCTL_TIOCSETD = TIOCSETD;
+  unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+  unsigned IOCTL_TIOCSTI = TIOCSTI;
+  unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+
+  unsigned IOCTL_MTIOCGET = MTIOCGET;
+  unsigned IOCTL_MTIOCTOP = MTIOCTOP;
+
+  const int si_SEGV_MAPERR = SEGV_MAPERR;
+  const int si_SEGV_ACCERR = SEGV_ACCERR;
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent, d_off);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+#if SANITIZER_SOLARIS32
+COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
+CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent64, d_off);
+CHECK_SIZE_AND_OFFSET(dirent64, d_reclen);
+#endif
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) ==
+               sizeof(((ifaddrs *)nullptr)->ifa_ifu));
+COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) ==
+               offsetof(ifaddrs, ifa_ifu));
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+CHECK_TYPE_SIZE(XDR);
+CHECK_SIZE_AND_OFFSET(XDR, x_op);
+CHECK_SIZE_AND_OFFSET(XDR, x_ops);
+CHECK_SIZE_AND_OFFSET(XDR, x_public);
+CHECK_SIZE_AND_OFFSET(XDR, x_private);
+CHECK_SIZE_AND_OFFSET(XDR, x_base);
+CHECK_SIZE_AND_OFFSET(XDR, x_handy);
+COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE);
+COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE);
+COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
+
+CHECK_TYPE_SIZE(sem_t);
+
+#endif  // SANITIZER_SOLARIS
diff --git a/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h
new file mode 100644
index 0000000000..85995e7979
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h
@@ -0,0 +1,495 @@
+//===-- sanitizer_platform_limits_solaris.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific Solaris data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_SOLARIS_H
+#define SANITIZER_PLATFORM_LIMITS_SOLARIS_H
+
+#if SANITIZER_SOLARIS
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_stat64_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_stack_t_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs64_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+
+extern unsigned struct_timespec_sz;
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+
+struct __sanitizer_sem_t {
+  //u64 data[6];
+  u32 sem_count;
+  u16 sem_type;
+  u16 sem_magic;
+  u64 sem_pad1[3];
+  u64 sem_pad2[2];
+};
+
+struct __sanitizer_ipc_perm {
+  unsigned int uid;           // uid_t
+  unsigned int gid;           // gid_t
+  unsigned int cuid;          // uid_t
+  unsigned int cgid;          // gid_t
+  unsigned int mode;          // mode_t
+  unsigned int seq;           // uint_t
+  int key;                    // key_t
+#if !defined(_LP64)
+  int pad[4];
+#endif
+};
+
+struct __sanitizer_shmid_ds {
+  __sanitizer_ipc_perm shm_perm;
+  unsigned long shm_segsz;    // size_t
+  unsigned long shm_flags;    // uintptr_t
+  unsigned short shm_lkcnt;   // ushort_t
+  int shm_lpid;               // pid_t
+  int shm_cpid;               // pid_t
+  unsigned long shm_nattch;   // shmatt_t
+  unsigned long shm_cnattch;  // ulong_t
+#if defined(_LP64)
+  long shm_atime;             // time_t
+  long shm_dtime;
+  long shm_ctime;
+  void *shm_amp;
+  u64 shm_gransize;           // uint64_t
+  u64 shm_allocated;          // uint64_t
+  u64 shm_pad4[1];            // int64_t
+#else
+  long shm_atime;             // time_t
+  int shm_pad1;               // int32_t
+  long shm_dtime;             // time_t
+  int shm_pad2;               // int32_t
+  long shm_ctime;             // time_t
+  void *shm_amp;
+  u64 shm_gransize;           // uint64_t
+  u64 shm_allocated;          // uint64_t
+#endif
+};
+
+extern unsigned struct_statvfs_sz;
+#if SANITIZER_SOLARIS32
+extern unsigned struct_statvfs64_sz;
+#endif
+
+struct __sanitizer_iovec {
+  void *iov_base;
+  uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+  struct __sanitizer_ifaddrs *ifa_next;
+  char *ifa_name;
+  u64 ifa_flags;     // uint64_t
+  void *ifa_addr;    // (struct sockaddr *)
+  void *ifa_netmask; // (struct sockaddr *)
+  // This is a union on Linux.
+# ifdef ifa_dstaddr
+# undef ifa_dstaddr
+# endif
+  void *ifa_dstaddr; // (struct sockaddr *)
+  void *ifa_data;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+struct __sanitizer_XDR {
+  int x_op;
+  void *x_ops;
+  uptr x_public;
+  uptr x_private;
+  uptr x_base;
+  unsigned x_handy;
+};
+
+const int __sanitizer_XDR_ENCODE = 0;
+const int __sanitizer_XDR_DECODE = 1;
+const int __sanitizer_XDR_FREE = 2;
+
+struct __sanitizer_passwd {
+  char *pw_name;
+  char *pw_passwd;
+  unsigned int pw_uid;    // uid_t
+  unsigned int pw_gid;    // gid_t
+  char *pw_age;
+  char *pw_comment;
+  char *pw_gecos;
+  char *pw_dir;
+  char *pw_shell;
+};
+
+struct __sanitizer_group {
+  char *gr_name;
+  char *gr_passwd;
+  int gr_gid;
+  char **gr_mem;
+};
+
+typedef long __sanitizer_time_t;
+
+typedef long __sanitizer_suseconds_t;
+
+struct __sanitizer_timeval {
+  __sanitizer_time_t tv_sec;
+  __sanitizer_suseconds_t tv_usec;
+};
+
+struct __sanitizer_itimerval {
+  struct __sanitizer_timeval it_interval;
+  struct __sanitizer_timeval it_value;
+};
+
+struct __sanitizer_timeb {
+  __sanitizer_time_t time;
+  unsigned short millitm;
+  short timezone;
+  short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+  u8 octet[6];
+};
+
+struct __sanitizer_tm {
+  int tm_sec;
+  int tm_min;
+  int tm_hour;
+  int tm_mday;
+  int tm_mon;
+  int tm_year;
+  int tm_wday;
+  int tm_yday;
+  int tm_isdst;
+};
+
+struct __sanitizer_msghdr {
+  void *msg_name;
+  unsigned msg_namelen;
+  struct __sanitizer_iovec *msg_iov;
+  unsigned msg_iovlen;
+  void *msg_control;
+  unsigned msg_controllen;
+  int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+  unsigned cmsg_len;
+  int cmsg_level;
+  int cmsg_type;
+};
+
+#if SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)
+struct __sanitizer_dirent {
+  unsigned long long d_ino;
+  long long d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#else
+struct __sanitizer_dirent {
+  unsigned long d_ino;
+  long d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+#endif
+
+struct __sanitizer_dirent64 {
+  unsigned long long d_ino;
+  unsigned long long d_off;
+  unsigned short d_reclen;
+  // more fields that we don't care about
+};
+
+typedef long __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+// This thing depends on the platform. We are only interested in the upper
+// limit. Verified with a compiler assert in .cpp.
+union __sanitizer_pthread_attr_t {
+  char size[128];
+  void *align;
+};
+
+struct __sanitizer_sigset_t {
+  // uint32_t * 4
+  unsigned int __bits[4];
+};
+
+struct __sanitizer_siginfo {
+  // The size is determined by looking at sizeof of real siginfo_t on linux.
+  u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr =
+    void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx);
+
+struct __sanitizer_sigaction {
+  int sa_flags;
+  union {
+    __sanitizer_sigactionhandler_ptr sigaction;
+    __sanitizer_sighandler_ptr handler;
+  };
+  __sanitizer_sigset_t sa_mask;
+#if !defined(_LP64)
+  int sa_resv[2];
+#endif
+};
+
+struct __sanitizer_kernel_sigset_t {
+  u8 sig[8];
+};
+
+struct __sanitizer_kernel_sigaction_t {
+  union {
+    void (*handler)(int signo);
+    void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
+  };
+  unsigned long sa_flags;
+  void (*sa_restorer)(void);
+  __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+  uptr dlpi_addr;
+  const char *dlpi_name;
+  const void *dlpi_phdr;
+  short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+  int ai_flags;
+  int ai_family;
+  int ai_socktype;
+  int ai_protocol;
+#if defined(__sparcv9)
+  int _ai_pad;
+#endif
+  unsigned ai_addrlen;
+  char *ai_canonname;
+  void *ai_addr;
+  struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+  char *h_name;
+  char **h_aliases;
+  int h_addrtype;
+  int h_length;
+  char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+  int fd;
+  short events;
+  short revents;
+};
+
+typedef unsigned long __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+  uptr gl_pathc;
+  char **gl_pathv;
+  uptr gl_offs;
+  char **gl_pathp;
+  int gl_pathn;
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+  uptr we_wordc;
+  char **we_wordv;
+  uptr we_offs;
+  char **we_wordp;
+  int we_wordn;
+};
+
+typedef void __sanitizer_FILE;
+#define SANITIZER_HAS_STRUCT_FILE 0
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern int shmctl_ipc_stat;
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+  int ifc_len;
+  union {
+    void *ifcu_req;
+  } ifc_ifcu;
+};
+
+// <sys/ioccom.h>
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#define IOC_SIZEBITS 12
+#define IOC_DIRBITS 4
+#undef IOC_NONE
+#define IOC_NONE 2U     // IOC_VOID
+#define IOC_READ 4U     // IOC_OUT
+#define IOC_WRITE 8U    // IOC_IN
+
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+
+#if defined(__sparc__)
+// In sparc the 14 bits SIZE field overlaps with the
+// least significant bit of DIR, so either IOC_READ or
+// IOC_WRITE shall be 1 in order to get a non-zero SIZE.
+#define IOC_SIZE(nr) \
+  ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff))
+#else
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+#endif
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+}  // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+  COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER)                       \
+  COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \
+                 sizeof(((CLASS *) NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) ==          \
+                 offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER)                       \
+  COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \
+                 sizeof(((struct CLASS *) NULL)->MEMBER));                \
+  COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) ==          \
+                 offsetof(struct CLASS, MEMBER))
+
+#endif  // SANITIZER_SOLARIS
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_posix.cpp b/lib/tsan/sanitizer_common/sanitizer_posix.cpp
new file mode 100644
index 0000000000..e21661b42f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_posix.cpp
@@ -0,0 +1,398 @@
+//===-- sanitizer_posix.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements POSIX-specific functions from
+// sanitizer_posix.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_POSIX
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_posix.h"
+#include "sanitizer_procmaps.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/mman.h>
+
+#if SANITIZER_FREEBSD
+// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
+// that, it was never implemented.  So just define it to zero.
+#undef  MAP_NORESERVE
+#define MAP_NORESERVE 0
+#endif
+
+namespace __sanitizer {
+
+// ------------- sanitizer_common.h
+uptr GetMmapGranularity() {
+  return GetPageSize();
+}
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANON, mem_type);
+  int reserrno;
+  if (UNLIKELY(internal_iserror(res, &reserrno)))
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report);
+  IncreaseTotalMmap(size);
+  return (void *)res;
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+  if (!addr || !size) return;
+  uptr res = internal_munmap(addr, size);
+  if (UNLIKELY(internal_iserror(res))) {
+    Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+           SanitizerToolName, size, size, addr);
+    CHECK("unable to unmap" && 0);
+  }
+  DecreaseTotalMmap(size);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANON, mem_type);
+  int reserrno;
+  if (UNLIKELY(internal_iserror(res, &reserrno))) {
+    if (reserrno == ENOMEM)
+      return nullptr;
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
+  }
+  IncreaseTotalMmap(size);
+  return (void *)res;
+}
+
+// We want to map a chunk of address space aligned to 'alignment'.
+// We do it by mapping a bit more and then unmapping redundant pieces.
+// We probably can do it with fewer syscalls in some OS-dependent way.
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
+  CHECK(IsPowerOfTwo(size));
+  CHECK(IsPowerOfTwo(alignment));
+  uptr map_size = size + alignment;
+  uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
+  if (UNLIKELY(!map_res))
+    return nullptr;
+  uptr map_end = map_res + map_size;
+  uptr res = map_res;
+  if (!IsAligned(res, alignment)) {
+    res = (map_res + alignment - 1) & ~(alignment - 1);
+    UnmapOrDie((void*)map_res, res - map_res);
+  }
+  uptr end = res + size;
+  if (end != map_end)
+    UnmapOrDie((void*)end, map_end - end);
+  return (void*)res;
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr p = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, mem_type);
+  int reserrno;
+  if (UNLIKELY(internal_iserror(p, &reserrno)))
+    ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno);
+  IncreaseTotalMmap(size);
+  return (void *)p;
+}
+
+static void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem,
+                           const char *name) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached());
+  uptr p = MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANON | MAP_FIXED, name);
+  int reserrno;
+  if (UNLIKELY(internal_iserror(p, &reserrno))) {
+    if (tolerate_enomem && reserrno == ENOMEM)
+      return nullptr;
+    char mem_type[40];
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+                      fixed_addr);
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
+  }
+  IncreaseTotalMmap(size);
+  return (void *)p;
+}
+
+void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) {
+  return MmapFixedImpl(fixed_addr, size, false /*tolerate_enomem*/, name);
+}
+
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) {
+  return MmapFixedImpl(fixed_addr, size, true /*tolerate_enomem*/, name);
+}
+
+bool MprotectNoAccess(uptr addr, uptr size) {
+  return 0 == internal_mprotect((void*)addr, size, PROT_NONE);
+}
+
+bool MprotectReadOnly(uptr addr, uptr size) {
+  return 0 == internal_mprotect((void *)addr, size, PROT_READ);
+}
+
+#if !SANITIZER_MAC
+void MprotectMallocZones(void *addr, int prot) {}
+#endif
+
+fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) {
+  if (ShouldMockFailureToOpen(filename))
+    return kInvalidFd;
+  int flags;
+  switch (mode) {
+    case RdOnly: flags = O_RDONLY; break;
+    case WrOnly: flags = O_WRONLY | O_CREAT | O_TRUNC; break;
+    case RdWr: flags = O_RDWR | O_CREAT; break;
+  }
+  fd_t res = internal_open(filename, flags, 0660);
+  if (internal_iserror(res, errno_p))
+    return kInvalidFd;
+  return ReserveStandardFds(res);
+}
+
+void CloseFile(fd_t fd) {
+  internal_close(fd);
+}
+
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read,
+                  error_t *error_p) {
+  uptr res = internal_read(fd, buff, buff_size);
+  if (internal_iserror(res, error_p))
+    return false;
+  if (bytes_read)
+    *bytes_read = res;
+  return true;
+}
+
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written,
+                 error_t *error_p) {
+  uptr res = internal_write(fd, buff, buff_size);
+  if (internal_iserror(res, error_p))
+    return false;
+  if (bytes_written)
+    *bytes_written = res;
+  return true;
+}
+
+void *MapFileToMemory(const char *file_name, uptr *buff_size) {
+  fd_t fd = OpenFile(file_name, RdOnly);
+  CHECK(fd != kInvalidFd);
+  uptr fsize = internal_filesize(fd);
+  CHECK_NE(fsize, (uptr)-1);
+  CHECK_GT(fsize, 0);
+  *buff_size = RoundUpTo(fsize, GetPageSizeCached());
+  uptr map = internal_mmap(nullptr, *buff_size, PROT_READ, MAP_PRIVATE, fd, 0);
+  return internal_iserror(map) ? nullptr : (void *)map;
+}
+
+void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset) {
+  uptr flags = MAP_SHARED;
+  if (addr) flags |= MAP_FIXED;
+  uptr p = internal_mmap(addr, size, PROT_READ | PROT_WRITE, flags, fd, offset);
+  int mmap_errno = 0;
+  if (internal_iserror(p, &mmap_errno)) {
+    Printf("could not map writable file (%d, %lld, %zu): %zd, errno: %d\n",
+           fd, (long long)offset, size, p, mmap_errno);
+    return nullptr;
+  }
+  return (void *)p;
+}
+
+static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
+                                        uptr start2, uptr end2) {
+  CHECK(start1 <= end1);
+  CHECK(start2 <= end2);
+  return (end1 < start2) || (end2 < start1);
+}
+
+// FIXME: this is thread-unsafe, but should not cause problems most of the time.
+// When the shadow is mapped only a single thread usually exists (plus maybe
+// several worker threads on Mac, which aren't expected to map big chunks of
+// memory).
+bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  if (proc_maps.Error())
+    return true; // and hope for the best
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (segment.start == segment.end) continue;  // Empty range.
+    CHECK_NE(0, segment.end);
+    if (!IntervalsAreSeparate(segment.start, segment.end - 1, range_start,
+                              range_end))
+      return false;
+  }
+  return true;
+}
+
+void DumpProcessMap() {
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  const sptr kBufSize = 4095;
+  char *filename = (char*)MmapOrDie(kBufSize, __func__);
+  MemoryMappedSegment segment(filename, kBufSize);
+  Report("Process memory map follows:\n");
+  while (proc_maps.Next(&segment)) {
+    Printf("\t%p-%p\t%s\n", (void *)segment.start, (void *)segment.end,
+           segment.filename);
+  }
+  Report("End of process memory map.\n");
+  UnmapOrDie(filename, kBufSize);
+}
+
+const char *GetPwd() {
+  return GetEnv("PWD");
+}
+
+bool IsPathSeparator(const char c) {
+  return c == '/';
+}
+
+bool IsAbsolutePath(const char *path) {
+  return path != nullptr && IsPathSeparator(path[0]);
+}
+
+void ReportFile::Write(const char *buffer, uptr length) {
+  SpinMutexLock l(mu);
+  ReopenIfNecessary();
+  internal_write(fd, buffer, length);
+}
+
+bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
+  MemoryMappingLayout proc_maps(/*cache_enabled*/false);
+  InternalScopedString buff(kMaxPathLength);
+  MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+  while (proc_maps.Next(&segment)) {
+    if (segment.IsExecutable() &&
+        internal_strcmp(module, segment.filename) == 0) {
+      *start = segment.start;
+      *end = segment.end;
+      return true;
+    }
+  }
+  return false;
+}
+
+uptr SignalContext::GetAddress() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return (uptr)si->si_addr;
+}
+
+bool SignalContext::IsMemoryAccess() const {
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return si->si_signo == SIGSEGV;
+}
+
+int SignalContext::GetType() const {
+  return static_cast<const siginfo_t *>(siginfo)->si_signo;
+}
+
+const char *SignalContext::Describe() const {
+  switch (GetType()) {
+    case SIGFPE:
+      return "FPE";
+    case SIGILL:
+      return "ILL";
+    case SIGABRT:
+      return "ABRT";
+    case SIGSEGV:
+      return "SEGV";
+    case SIGBUS:
+      return "BUS";
+    case SIGTRAP:
+      return "TRAP";
+  }
+  return "UNKNOWN SIGNAL";
+}
+
+fd_t ReserveStandardFds(fd_t fd) {
+  CHECK_GE(fd, 0);
+  if (fd > 2)
+    return fd;
+  bool used[3];
+  internal_memset(used, 0, sizeof(used));
+  while (fd <= 2) {
+    used[fd] = true;
+    fd = internal_dup(fd);
+  }
+  for (int i = 0; i <= 2; ++i)
+    if (used[i])
+      internal_close(i);
+  return fd;
+}
+
+bool ShouldMockFailureToOpen(const char *path) {
+  return common_flags()->test_only_emulate_no_memorymap &&
+         internal_strncmp(path, "/proc/", 6) == 0;
+}
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO
+int GetNamedMappingFd(const char *name, uptr size, int *flags) {
+  if (!common_flags()->decorate_proc_maps || !name)
+    return -1;
+  char shmname[200];
+  CHECK(internal_strlen(name) < sizeof(shmname) - 10);
+  internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu [%s]",
+                    internal_getpid(), name);
+  int o_cloexec = 0;
+#if defined(O_CLOEXEC)
+  o_cloexec = O_CLOEXEC;
+#endif
+  int fd = ReserveStandardFds(
+      internal_open(shmname, O_RDWR | O_CREAT | O_TRUNC | o_cloexec, S_IRWXU));
+  CHECK_GE(fd, 0);
+  if (!o_cloexec) {
+    int res = fcntl(fd, F_SETFD, FD_CLOEXEC);
+    CHECK_EQ(0, res);
+  }
+  int res = internal_ftruncate(fd, size);
+  CHECK_EQ(0, res);
+  res = internal_unlink(shmname);
+  CHECK_EQ(0, res);
+  *flags &= ~(MAP_ANON | MAP_ANONYMOUS);
+  return fd;
+}
+#else
+int GetNamedMappingFd(const char *name, uptr size, int *flags) {
+  return -1;
+}
+#endif
+
+#if SANITIZER_ANDROID
+#define PR_SET_VMA 0x53564d41
+#define PR_SET_VMA_ANON_NAME 0
+void DecorateMapping(uptr addr, uptr size, const char *name) {
+  if (!common_flags()->decorate_proc_maps || !name)
+    return;
+  internal_prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, addr, size, (uptr)name);
+}
+#else
+void DecorateMapping(uptr addr, uptr size, const char *name) {
+}
+#endif
+
+uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name) {
+  int fd = GetNamedMappingFd(name, length, &flags);
+  uptr res = internal_mmap(addr, length, prot, flags, fd, 0);
+  if (!internal_iserror(res))
+    DecorateMapping(res, length, name);
+  return res;
+}
+
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_POSIX
diff --git a/lib/tsan/sanitizer_common/sanitizer_posix.h b/lib/tsan/sanitizer_common/sanitizer_posix.h
new file mode 100644
index 0000000000..a1b49702da
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_posix.h
@@ -0,0 +1,125 @@
+//===-- sanitizer_posix.h -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and declares some useful POSIX-specific functions.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_POSIX_H
+#define SANITIZER_POSIX_H
+
+// ----------- ATTENTION -------------
+// This header should NOT include any other headers from sanitizer runtime.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_freebsd.h"
+#include "sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_platform_limits_openbsd.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
+
+#if !SANITIZER_POSIX
+// Make it hard to accidentally use any of functions declared in this file:
+#error This file should only be included on POSIX
+#endif
+
+namespace __sanitizer {
+
+// I/O
+// Don't use directly, use __sanitizer::OpenFile() instead.
+uptr internal_open(const char *filename, int flags);
+uptr internal_open(const char *filename, int flags, u32 mode);
+uptr internal_close(fd_t fd);
+
+uptr internal_read(fd_t fd, void *buf, uptr count);
+uptr internal_write(fd_t fd, const void *buf, uptr count);
+
+// Memory
+uptr internal_mmap(void *addr, uptr length, int prot, int flags,
+                   int fd, u64 offset);
+uptr internal_munmap(void *addr, uptr length);
+int internal_mprotect(void *addr, uptr length, int prot);
+
+// OS
+uptr internal_filesize(fd_t fd);  // -1 on error.
+uptr internal_stat(const char *path, void *buf);
+uptr internal_lstat(const char *path, void *buf);
+uptr internal_fstat(fd_t fd, void *buf);
+uptr internal_dup(int oldfd);
+uptr internal_dup2(int oldfd, int newfd);
+uptr internal_readlink(const char *path, char *buf, uptr bufsize);
+uptr internal_unlink(const char *path);
+uptr internal_rename(const char *oldpath, const char *newpath);
+uptr internal_lseek(fd_t fd, OFF_T offset, int whence);
+
+#if SANITIZER_NETBSD
+uptr internal_ptrace(int request, int pid, void *addr, int data);
+#else
+uptr internal_ptrace(int request, int pid, void *addr, void *data);
+#endif
+uptr internal_waitpid(int pid, int *status, int options);
+
+int internal_fork();
+fd_t internal_spawn(const char *argv[], const char *envp[], pid_t *pid);
+
+int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
+                    uptr *oldlenp, const void *newp, uptr newlen);
+int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
+                          const void *newp, uptr newlen);
+
+// These functions call appropriate pthread_ functions directly, bypassing
+// the interceptor. They are weak and may not be present in some tools.
+SANITIZER_WEAK_ATTRIBUTE
+int real_pthread_create(void *th, void *attr, void *(*callback)(void *),
+                        void *param);
+SANITIZER_WEAK_ATTRIBUTE
+int real_pthread_join(void *th, void **ret);
+
+#define DEFINE_REAL_PTHREAD_FUNCTIONS                                          \
+  namespace __sanitizer {                                                      \
+  int real_pthread_create(void *th, void *attr, void *(*callback)(void *),     \
+                          void *param) {                                       \
+    return REAL(pthread_create)(th, attr, callback, param);                    \
+  }                                                                            \
+  int real_pthread_join(void *th, void **ret) {                                \
+    return REAL(pthread_join(th, ret));                                        \
+  }                                                                            \
+  }  // namespace __sanitizer
+
+int my_pthread_attr_getstack(void *attr, void **addr, uptr *size);
+
+// A routine named real_sigaction() must be implemented by each sanitizer in
+// order for internal_sigaction() to bypass interceptors.
+int internal_sigaction(int signum, const void *act, void *oldact);
+void internal_sigfillset(__sanitizer_sigset_t *set);
+void internal_sigemptyset(__sanitizer_sigset_t *set);
+bool internal_sigismember(__sanitizer_sigset_t *set, int signum);
+
+uptr internal_execve(const char *filename, char *const argv[],
+                     char *const envp[]);
+
+bool IsStateDetached(int state);
+
+// Move the fd out of {0, 1, 2} range.
+fd_t ReserveStandardFds(fd_t fd);
+
+bool ShouldMockFailureToOpen(const char *path);
+
+// Create a non-file mapping with a given /proc/self/maps name.
+uptr MmapNamed(void *addr, uptr length, int prot, int flags, const char *name);
+
+// Platforms should implement at most one of these.
+// 1. Provide a pre-decorated file descriptor to use instead of an anonymous
+// mapping.
+int GetNamedMappingFd(const char *name, uptr size, int *flags);
+// 2. Add name to an existing anonymous mapping. The caller must keep *name
+// alive at least as long as the mapping exists.
+void DecorateMapping(uptr addr, uptr size, const char *name);
+
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_POSIX_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_printf.cpp b/lib/tsan/sanitizer_common/sanitizer_printf.cpp
new file mode 100644
index 0000000000..a032787114
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_printf.cpp
@@ -0,0 +1,358 @@
+//===-- sanitizer_printf.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer.
+//
+// Internal printf function, used inside run-time libraries.
+// We can't use libc printf because we intercept some of the functions used
+// inside it.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+
+#if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 &&               \
+      !defined(va_copy)
+# define va_copy(dst, src) ((dst) = (src))
+#endif
+
+namespace __sanitizer {
+
+static int AppendChar(char **buff, const char *buff_end, char c) {
+  if (*buff < buff_end) {
+    **buff = c;
+    (*buff)++;
+  }
+  return 1;
+}
+
+// Appends number in a given base to buffer. If its length is less than
+// |minimal_num_length|, it is padded with leading zeroes or spaces, depending
+// on the value of |pad_with_zero|.
+static int AppendNumber(char **buff, const char *buff_end, u64 absolute_value,
+                        u8 base, u8 minimal_num_length, bool pad_with_zero,
+                        bool negative, bool uppercase) {
+  uptr const kMaxLen = 30;
+  RAW_CHECK(base == 10 || base == 16);
+  RAW_CHECK(base == 10 || !negative);
+  RAW_CHECK(absolute_value || !negative);
+  RAW_CHECK(minimal_num_length < kMaxLen);
+  int result = 0;
+  if (negative && minimal_num_length)
+    --minimal_num_length;
+  if (negative && pad_with_zero)
+    result += AppendChar(buff, buff_end, '-');
+  uptr num_buffer[kMaxLen];
+  int pos = 0;
+  do {
+    RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow");
+    num_buffer[pos++] = absolute_value % base;
+    absolute_value /= base;
+  } while (absolute_value > 0);
+  if (pos < minimal_num_length) {
+    // Make sure compiler doesn't insert call to memset here.
+    internal_memset(&num_buffer[pos], 0,
+                    sizeof(num_buffer[0]) * (minimal_num_length - pos));
+    pos = minimal_num_length;
+  }
+  RAW_CHECK(pos > 0);
+  pos--;
+  for (; pos >= 0 && num_buffer[pos] == 0; pos--) {
+    char c = (pad_with_zero || pos == 0) ? '0' : ' ';
+    result += AppendChar(buff, buff_end, c);
+  }
+  if (negative && !pad_with_zero) result += AppendChar(buff, buff_end, '-');
+  for (; pos >= 0; pos--) {
+    char digit = static_cast<char>(num_buffer[pos]);
+    digit = (digit < 10) ? '0' + digit : (uppercase ? 'A' : 'a') + digit - 10;
+    result += AppendChar(buff, buff_end, digit);
+  }
+  return result;
+}
+
+static int AppendUnsigned(char **buff, const char *buff_end, u64 num, u8 base,
+                          u8 minimal_num_length, bool pad_with_zero,
+                          bool uppercase) {
+  return AppendNumber(buff, buff_end, num, base, minimal_num_length,
+                      pad_with_zero, false /* negative */, uppercase);
+}
+
+static int AppendSignedDecimal(char **buff, const char *buff_end, s64 num,
+                               u8 minimal_num_length, bool pad_with_zero) {
+  bool negative = (num < 0);
+  return AppendNumber(buff, buff_end, (u64)(negative ? -num : num), 10,
+                      minimal_num_length, pad_with_zero, negative,
+                      false /* uppercase */);
+}
+
+
+// Use the fact that explicitly requesting 0 width (%0s) results in UB and
+// interpret width == 0 as "no width requested":
+// width == 0 - no width requested
+// width  < 0 - left-justify s within and pad it to -width chars, if necessary
+// width  > 0 - right-justify s, not implemented yet
+static int AppendString(char **buff, const char *buff_end, int width,
+                        int max_chars, const char *s) {
+  if (!s)
+    s = "<null>";
+  int result = 0;
+  for (; *s; s++) {
+    if (max_chars >= 0 && result >= max_chars)
+      break;
+    result += AppendChar(buff, buff_end, *s);
+  }
+  // Only the left justified strings are supported.
+  while (width < -result)
+    result += AppendChar(buff, buff_end, ' ');
+  return result;
+}
+
+static int AppendPointer(char **buff, const char *buff_end, u64 ptr_value) {
+  int result = 0;
+  result += AppendString(buff, buff_end, 0, -1, "0x");
+  result += AppendUnsigned(buff, buff_end, ptr_value, 16,
+                           SANITIZER_POINTER_FORMAT_LENGTH,
+                           true /* pad_with_zero */, false /* uppercase */);
+  return result;
+}
+
+int VSNPrintf(char *buff, int buff_length,
+              const char *format, va_list args) {
+  static const char *kPrintfFormatsHelp =
+      "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
+      "%[-]([0-9]*)?(\\.\\*)?s; %c\n";
+  RAW_CHECK(format);
+  RAW_CHECK(buff_length > 0);
+  const char *buff_end = &buff[buff_length - 1];
+  const char *cur = format;
+  int result = 0;
+  for (; *cur; cur++) {
+    if (*cur != '%') {
+      result += AppendChar(&buff, buff_end, *cur);
+      continue;
+    }
+    cur++;
+    bool left_justified = *cur == '-';
+    if (left_justified)
+      cur++;
+    bool have_width = (*cur >= '0' && *cur <= '9');
+    bool pad_with_zero = (*cur == '0');
+    int width = 0;
+    if (have_width) {
+      while (*cur >= '0' && *cur <= '9') {
+        width = width * 10 + *cur++ - '0';
+      }
+    }
+    bool have_precision = (cur[0] == '.' && cur[1] == '*');
+    int precision = -1;
+    if (have_precision) {
+      cur += 2;
+      precision = va_arg(args, int);
+    }
+    bool have_z = (*cur == 'z');
+    cur += have_z;
+    bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
+    cur += have_ll * 2;
+    s64 dval;
+    u64 uval;
+    const bool have_length = have_z || have_ll;
+    const bool have_flags = have_width || have_length;
+    // At the moment only %s supports precision and left-justification.
+    CHECK(!((precision >= 0 || left_justified) && *cur != 's'));
+    switch (*cur) {
+      case 'd': {
+        dval = have_ll ? va_arg(args, s64)
+             : have_z ? va_arg(args, sptr)
+             : va_arg(args, int);
+        result += AppendSignedDecimal(&buff, buff_end, dval, width,
+                                      pad_with_zero);
+        break;
+      }
+      case 'u':
+      case 'x':
+      case 'X': {
+        uval = have_ll ? va_arg(args, u64)
+             : have_z ? va_arg(args, uptr)
+             : va_arg(args, unsigned);
+        bool uppercase = (*cur == 'X');
+        result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
+                                 width, pad_with_zero, uppercase);
+        break;
+      }
+      case 'p': {
+        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
+        break;
+      }
+      case 's': {
+        RAW_CHECK_MSG(!have_length, kPrintfFormatsHelp);
+        // Only left-justified width is supported.
+        CHECK(!have_width || left_justified);
+        result += AppendString(&buff, buff_end, left_justified ? -width : width,
+                               precision, va_arg(args, char*));
+        break;
+      }
+      case 'c': {
+        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        result += AppendChar(&buff, buff_end, va_arg(args, int));
+        break;
+      }
+      case '%' : {
+        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        result += AppendChar(&buff, buff_end, '%');
+        break;
+      }
+      default: {
+        RAW_CHECK_MSG(false, kPrintfFormatsHelp);
+      }
+    }
+  }
+  RAW_CHECK(buff <= buff_end);
+  AppendChar(&buff, buff_end + 1, '\0');
+  return result;
+}
+
+static void (*PrintfAndReportCallback)(const char *);
+void SetPrintfAndReportCallback(void (*callback)(const char *)) {
+  PrintfAndReportCallback = callback;
+}
+
+// Can be overriden in frontend.
+#if SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS)
+// Implementation must be defined in frontend.
+extern "C" void __sanitizer_on_print(const char *str);
+#else
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_on_print, const char *str) {
+  (void)str;
+}
+#endif
+
+static void CallPrintfAndReportCallback(const char *str) {
+  __sanitizer_on_print(str);
+  if (PrintfAndReportCallback)
+    PrintfAndReportCallback(str);
+}
+
+static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
+                                              char *local_buffer,
+                                              int buffer_size,
+                                              const char *format,
+                                              va_list args) {
+  va_list args2;
+  va_copy(args2, args);
+  const int kLen = 16 * 1024;
+  int needed_length;
+  char *buffer = local_buffer;
+  // First try to print a message using a local buffer, and then fall back to
+  // mmaped buffer.
+  for (int use_mmap = 0; use_mmap < 2; use_mmap++) {
+    if (use_mmap) {
+      va_end(args);
+      va_copy(args, args2);
+      buffer = (char*)MmapOrDie(kLen, "Report");
+      buffer_size = kLen;
+    }
+    needed_length = 0;
+    // Check that data fits into the current buffer.
+#   define CHECK_NEEDED_LENGTH \
+      if (needed_length >= buffer_size) { \
+        if (!use_mmap) continue; \
+        RAW_CHECK_MSG(needed_length < kLen, \
+                      "Buffer in Report is too short!\n"); \
+      }
+    // Fuchsia's logging infrastructure always keeps track of the logging
+    // process, thread, and timestamp, so never prepend such information.
+    if (!SANITIZER_FUCHSIA && append_pid) {
+      int pid = internal_getpid();
+      const char *exe_name = GetProcessName();
+      if (common_flags()->log_exe_name && exe_name) {
+        needed_length += internal_snprintf(buffer, buffer_size,
+                                           "==%s", exe_name);
+        CHECK_NEEDED_LENGTH
+      }
+      needed_length += internal_snprintf(
+          buffer + needed_length, buffer_size - needed_length, "==%d==", pid);
+      CHECK_NEEDED_LENGTH
+    }
+    needed_length += VSNPrintf(buffer + needed_length,
+                               buffer_size - needed_length, format, args);
+    CHECK_NEEDED_LENGTH
+    // If the message fit into the buffer, print it and exit.
+    break;
+#   undef CHECK_NEEDED_LENGTH
+  }
+  RawWrite(buffer);
+
+  // Remove color sequences from the message.
+  RemoveANSIEscapeSequencesFromString(buffer);
+  CallPrintfAndReportCallback(buffer);
+  LogMessageOnPrintf(buffer);
+
+  // If we had mapped any memory, clean up.
+  if (buffer != local_buffer)
+    UnmapOrDie((void *)buffer, buffer_size);
+  va_end(args2);
+}
+
+static void NOINLINE SharedPrintfCode(bool append_pid, const char *format,
+                                      va_list args) {
+  // |local_buffer| is small enough not to overflow the stack and/or violate
+  // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other
+  // hand, the bigger the buffer is, the more the chance the error report will
+  // fit into it.
+  char local_buffer[400];
+  SharedPrintfCodeNoBuffer(append_pid, local_buffer, ARRAY_SIZE(local_buffer),
+                           format, args);
+}
+
+FORMAT(1, 2)
+void Printf(const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  SharedPrintfCode(false, format, args);
+  va_end(args);
+}
+
+// Like Printf, but prints the current PID before the output string.
+FORMAT(1, 2)
+void Report(const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  SharedPrintfCode(true, format, args);
+  va_end(args);
+}
+
+// Writes at most "length" symbols to "buffer" (including trailing '\0').
+// Returns the number of symbols that should have been written to buffer
+// (not including trailing '\0'). Thus, the string is truncated
+// iff return value is not less than "length".
+FORMAT(3, 4)
+int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  int needed_length = VSNPrintf(buffer, length, format, args);
+  va_end(args);
+  return needed_length;
+}
+
+FORMAT(2, 3)
+void InternalScopedString::append(const char *format, ...) {
+  CHECK_LT(length_, size());
+  va_list args;
+  va_start(args, format);
+  VSNPrintf(data() + length_, size() - length_, format, args);
+  va_end(args);
+  length_ += internal_strlen(data() + length_);
+  CHECK_LT(length_, size());
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps.h b/lib/tsan/sanitizer_common/sanitizer_procmaps.h
new file mode 100644
index 0000000000..665ed45fa9
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps.h
@@ -0,0 +1,100 @@
+//===-- sanitizer_procmaps.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer.
+//
+// Information about the process mappings.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_PROCMAPS_H
+#define SANITIZER_PROCMAPS_H
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_MAC || SANITIZER_SOLARIS ||  \
+    SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_fuchsia.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mac.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+// Memory protection masks.
+static const uptr kProtectionRead = 1;
+static const uptr kProtectionWrite = 2;
+static const uptr kProtectionExecute = 4;
+static const uptr kProtectionShared = 8;
+
+struct MemoryMappedSegmentData;
+
+class MemoryMappedSegment {
+ public:
+  explicit MemoryMappedSegment(char *buff = nullptr, uptr size = 0)
+      : filename(buff), filename_size(size), data_(nullptr) {}
+  ~MemoryMappedSegment() {}
+
+  bool IsReadable() const { return protection & kProtectionRead; }
+  bool IsWritable() const { return protection & kProtectionWrite; }
+  bool IsExecutable() const { return protection & kProtectionExecute; }
+  bool IsShared() const { return protection & kProtectionShared; }
+
+  void AddAddressRanges(LoadedModule *module);
+
+  uptr start;
+  uptr end;
+  uptr offset;
+  char *filename;  // owned by caller
+  uptr filename_size;
+  uptr protection;
+  ModuleArch arch;
+  u8 uuid[kModuleUUIDSize];
+
+ private:
+  friend class MemoryMappingLayout;
+
+  // This field is assigned and owned by MemoryMappingLayout if needed
+  MemoryMappedSegmentData *data_;
+};
+
+class MemoryMappingLayout {
+ public:
+  explicit MemoryMappingLayout(bool cache_enabled);
+  ~MemoryMappingLayout();
+  bool Next(MemoryMappedSegment *segment);
+  bool Error() const;
+  void Reset();
+  // In some cases, e.g. when running under a sandbox on Linux, ASan is unable
+  // to obtain the memory mappings. It should fall back to pre-cached data
+  // instead of aborting.
+  static void CacheMemoryMappings();
+
+  // Adds all mapped objects into a vector.
+  void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);
+
+ private:
+  void LoadFromCache();
+
+  MemoryMappingLayoutData data_;
+};
+
+// Returns code range for the specified module.
+bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end);
+
+bool IsDecimal(char c);
+uptr ParseDecimal(const char **p);
+bool IsHex(char c);
+uptr ParseHex(const char **p);
+
+}  // namespace __sanitizer
+
+#endif
+#endif  // SANITIZER_PROCMAPS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp
new file mode 100644
index 0000000000..02ff7c0e91
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp
@@ -0,0 +1,139 @@
+//===-- sanitizer_procmaps_bsd.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings
+// (FreeBSD, OpenBSD and NetBSD-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#include "sanitizer_common.h"
+#if SANITIZER_FREEBSD
+#include "sanitizer_freebsd.h"
+#endif
+#include "sanitizer_procmaps.h"
+
+// clang-format off
+#include <sys/types.h>
+#include <sys/sysctl.h>
+// clang-format on
+#include <unistd.h>
+#if SANITIZER_FREEBSD
+#include <sys/user.h>
+#endif
+
+#include <limits.h>
+#if SANITIZER_OPENBSD
+#define KVME_PROT_READ KVE_PROT_READ
+#define KVME_PROT_WRITE KVE_PROT_WRITE
+#define KVME_PROT_EXEC KVE_PROT_EXEC
+#endif
+
+// Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
+#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
+#include <osreldate.h>
+#if __FreeBSD_version <= 902001 // v9.2
+#define kinfo_vmentry xkinfo_vmentry
+#endif
+#endif
+
+namespace __sanitizer {
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
+  const int Mib[] = {
+#if SANITIZER_FREEBSD
+    CTL_KERN,
+    KERN_PROC,
+    KERN_PROC_VMMAP,
+    getpid()
+#elif SANITIZER_OPENBSD
+    CTL_KERN,
+    KERN_PROC_VMMAP,
+    getpid()
+#elif SANITIZER_NETBSD
+    CTL_VM,
+    VM_PROC,
+    VM_PROC_MAP,
+    getpid(),
+    sizeof(struct kinfo_vmentry)
+#else
+#error "not supported"
+#endif
+  };
+
+  uptr Size = 0;
+  int Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0);
+  CHECK_EQ(Err, 0);
+  CHECK_GT(Size, 0);
+
+#if !SANITIZER_OPENBSD
+  size_t MmapedSize = Size * 4 / 3;
+  void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
+  Size = MmapedSize;
+  Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0);
+  CHECK_EQ(Err, 0);
+  proc_maps->data = (char *)VmMap;
+#else
+  size_t PageSize = GetPageSize();
+  size_t MmapedSize = Size;
+  MmapedSize = ((MmapedSize - 1) / PageSize + 1) * PageSize;
+  char *Mem = (char *)MmapOrDie(MmapedSize, "ReadProcMaps()");
+  Size = 2 * Size + 10 * sizeof(struct kinfo_vmentry);
+  if (Size > 0x10000)
+    Size = 0x10000;
+  Size = (Size / sizeof(struct kinfo_vmentry)) * sizeof(struct kinfo_vmentry);
+  Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0);
+  CHECK_EQ(Err, 0);
+  MmapedSize = Size;
+  proc_maps->data = Mem;
+#endif
+
+  proc_maps->mmaped_size = MmapedSize;
+  proc_maps->len = Size;
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  CHECK(!Error()); // can not fail
+  char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+  if (data_.current >= last)
+    return false;
+  const struct kinfo_vmentry *VmEntry =
+      (const struct kinfo_vmentry *)data_.current;
+
+  segment->start = (uptr)VmEntry->kve_start;
+  segment->end = (uptr)VmEntry->kve_end;
+  segment->offset = (uptr)VmEntry->kve_offset;
+
+  segment->protection = 0;
+  if ((VmEntry->kve_protection & KVME_PROT_READ) != 0)
+    segment->protection |= kProtectionRead;
+  if ((VmEntry->kve_protection & KVME_PROT_WRITE) != 0)
+    segment->protection |= kProtectionWrite;
+  if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0)
+    segment->protection |= kProtectionExecute;
+
+#if !SANITIZER_OPENBSD
+  if (segment->filename != NULL && segment->filename_size > 0) {
+    internal_snprintf(segment->filename,
+                      Min(segment->filename_size, (uptr)PATH_MAX), "%s",
+                      VmEntry->kve_path);
+  }
+#endif
+
+#if SANITIZER_FREEBSD
+  data_.current += VmEntry->kve_structsize;
+#else
+  data_.current += sizeof(*VmEntry);
+#endif
+
+  return true;
+}
+
+} // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp
new file mode 100644
index 0000000000..e0cb47f8ca
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp
@@ -0,0 +1,174 @@
+//===-- sanitizer_procmaps_common.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (common parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||                \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+#include "sanitizer_common.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_procmaps.h"
+
+namespace __sanitizer {
+
+static ProcSelfMapsBuff cached_proc_self_maps;
+static StaticSpinMutex cache_lock;
+
+static int TranslateDigit(char c) {
+  if (c >= '0' && c <= '9')
+    return c - '0';
+  if (c >= 'a' && c <= 'f')
+    return c - 'a' + 10;
+  if (c >= 'A' && c <= 'F')
+    return c - 'A' + 10;
+  return -1;
+}
+
+// Parse a number and promote 'p' up to the first non-digit character.
+static uptr ParseNumber(const char **p, int base) {
+  uptr n = 0;
+  int d;
+  CHECK(base >= 2 && base <= 16);
+  while ((d = TranslateDigit(**p)) >= 0 && d < base) {
+    n = n * base + d;
+    (*p)++;
+  }
+  return n;
+}
+
+bool IsDecimal(char c) {
+  int d = TranslateDigit(c);
+  return d >= 0 && d < 10;
+}
+
+uptr ParseDecimal(const char **p) {
+  return ParseNumber(p, 10);
+}
+
+bool IsHex(char c) {
+  int d = TranslateDigit(c);
+  return d >= 0 && d < 16;
+}
+
+uptr ParseHex(const char **p) {
+  return ParseNumber(p, 16);
+}
+
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+  // data_ should be unused on this platform
+  CHECK(!data_);
+  module->addAddressRange(start, end, IsExecutable(), IsWritable());
+}
+
+MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
+  // FIXME: in the future we may want to cache the mappings on demand only.
+  if (cache_enabled)
+    CacheMemoryMappings();
+
+  // Read maps after the cache update to capture the maps/unmaps happening in
+  // the process of updating.
+  ReadProcMaps(&data_.proc_self_maps);
+  if (cache_enabled && data_.proc_self_maps.mmaped_size == 0)
+    LoadFromCache();
+
+  Reset();
+}
+
+bool MemoryMappingLayout::Error() const {
+  return data_.current == nullptr;
+}
+
+MemoryMappingLayout::~MemoryMappingLayout() {
+  // Only unmap the buffer if it is different from the cached one. Otherwise
+  // it will be unmapped when the cache is refreshed.
+  if (data_.proc_self_maps.data != cached_proc_self_maps.data)
+    UnmapOrDie(data_.proc_self_maps.data, data_.proc_self_maps.mmaped_size);
+}
+
+void MemoryMappingLayout::Reset() {
+  data_.current = data_.proc_self_maps.data;
+}
+
+// static
+void MemoryMappingLayout::CacheMemoryMappings() {
+  ProcSelfMapsBuff new_proc_self_maps;
+  ReadProcMaps(&new_proc_self_maps);
+  // Don't invalidate the cache if the mappings are unavailable.
+  if (new_proc_self_maps.mmaped_size == 0)
+    return;
+  SpinMutexLock l(&cache_lock);
+  if (cached_proc_self_maps.mmaped_size)
+    UnmapOrDie(cached_proc_self_maps.data, cached_proc_self_maps.mmaped_size);
+  cached_proc_self_maps = new_proc_self_maps;
+}
+
+void MemoryMappingLayout::LoadFromCache() {
+  SpinMutexLock l(&cache_lock);
+  if (cached_proc_self_maps.data)
+    data_.proc_self_maps = cached_proc_self_maps;
+}
+
+void MemoryMappingLayout::DumpListOfModules(
+    InternalMmapVectorNoCtor<LoadedModule> *modules) {
+  Reset();
+  InternalScopedString module_name(kMaxPathLength);
+  MemoryMappedSegment segment(module_name.data(), module_name.size());
+  for (uptr i = 0; Next(&segment); i++) {
+    const char *cur_name = segment.filename;
+    if (cur_name[0] == '\0')
+      continue;
+    // Don't subtract 'cur_beg' from the first entry:
+    // * If a binary is compiled w/o -pie, then the first entry in
+    //   process maps is likely the binary itself (all dynamic libs
+    //   are mapped higher in address space). For such a binary,
+    //   instruction offset in binary coincides with the actual
+    //   instruction address in virtual memory (as code section
+    //   is mapped to a fixed memory range).
+    // * If a binary is compiled with -pie, all the modules are
+    //   mapped high at address space (in particular, higher than
+    //   shadow memory of the tool), so the module can't be the
+    //   first entry.
+    uptr base_address = (i ? segment.start : 0) - segment.offset;
+    LoadedModule cur_module;
+    cur_module.set(cur_name, base_address);
+    segment.AddAddressRanges(&cur_module);
+    modules->push_back(cur_module);
+  }
+}
+
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
+  char *smaps = nullptr;
+  uptr smaps_cap = 0;
+  uptr smaps_len = 0;
+  if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len))
+    return;
+  uptr start = 0;
+  bool file = false;
+  const char *pos = smaps;
+  while (pos < smaps + smaps_len) {
+    if (IsHex(pos[0])) {
+      start = ParseHex(&pos);
+      for (; *pos != '/' && *pos > '\n'; pos++) {}
+      file = *pos == '/';
+    } else if (internal_strncmp(pos, "Rss:", 4) == 0) {
+      while (!IsDecimal(*pos)) pos++;
+      uptr rss = ParseDecimal(&pos) * 1024;
+      cb(start, rss, file, stats, stats_size);
+    }
+    while (*pos++ != '\n') {}
+  }
+  UnmapOrDie(smaps, smaps_cap);
+}
+
+} // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_fuchsia.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_fuchsia.cpp
new file mode 100644
index 0000000000..cc3e9be064
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_fuchsia.cpp
@@ -0,0 +1,80 @@
+//===-- sanitizer_procmaps_fuchsia.cpp
+//----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (Fuchsia-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_procmaps.h"
+
+namespace __sanitizer {
+
+// The cache flag is ignored on Fuchsia because a process can always get this
+// information via its process-self handle.
+MemoryMappingLayout::MemoryMappingLayout(bool) { Reset(); }
+
+void MemoryMappingLayout::Reset() {
+  data_.data.clear();
+  data_.current = 0;
+
+  size_t count;
+  zx_status_t status = _zx_object_get_info(
+      _zx_process_self(), ZX_INFO_PROCESS_MAPS, nullptr, 0, nullptr, &count);
+  if (status != ZX_OK) {
+    return;
+  }
+
+  size_t filled;
+  do {
+    data_.data.resize(count);
+    status = _zx_object_get_info(
+        _zx_process_self(), ZX_INFO_PROCESS_MAPS, data_.data.data(),
+        count * sizeof(zx_info_maps_t), &filled, &count);
+    if (status != ZX_OK) {
+      data_.data.clear();
+      return;
+    }
+  } while (filled < count);
+}
+
+MemoryMappingLayout::~MemoryMappingLayout() {}
+
+bool MemoryMappingLayout::Error() const { return data_.data.empty(); }
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  while (data_.current < data_.data.size()) {
+    const auto &entry = data_.data[data_.current++];
+    if (entry.type == ZX_INFO_MAPS_TYPE_MAPPING) {
+      segment->start = entry.base;
+      segment->end = entry.base + entry.size;
+      segment->offset = entry.u.mapping.vmo_offset;
+      const auto flags = entry.u.mapping.mmu_flags;
+      segment->protection =
+          ((flags & ZX_VM_PERM_READ) ? kProtectionRead : 0) |
+          ((flags & ZX_VM_PERM_WRITE) ? kProtectionWrite : 0) |
+          ((flags & ZX_VM_PERM_EXECUTE) ? kProtectionExecute : 0);
+      if (segment->filename && segment->filename_size > 0) {
+        uptr len = Min(sizeof(entry.name), segment->filename_size) - 1;
+        internal_strncpy(segment->filename, entry.name, len);
+        segment->filename[len] = 0;
+      }
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_linux.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_linux.cpp
new file mode 100644
index 0000000000..c7af57355b
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_linux.cpp
@@ -0,0 +1,81 @@
+//===-- sanitizer_procmaps_linux.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (Linux-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_LINUX
+#include "sanitizer_common.h"
+#include "sanitizer_procmaps.h"
+
+namespace __sanitizer {
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
+  if (!ReadFileToBuffer("/proc/self/maps", &proc_maps->data,
+                        &proc_maps->mmaped_size, &proc_maps->len)) {
+    proc_maps->data = nullptr;
+    proc_maps->mmaped_size = 0;
+    proc_maps->len = 0;
+  }
+}
+
+static bool IsOneOf(char c, char c1, char c2) {
+  return c == c1 || c == c2;
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  if (Error()) return false; // simulate empty maps
+  char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+  if (data_.current >= last) return false;
+  char *next_line =
+      (char *)internal_memchr(data_.current, '\n', last - data_.current);
+  if (next_line == 0)
+    next_line = last;
+  // Example: 08048000-08056000 r-xp 00000000 03:0c 64593   /foo/bar
+  segment->start = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, '-');
+  segment->end = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  CHECK(IsOneOf(*data_.current, '-', 'r'));
+  segment->protection = 0;
+  if (*data_.current++ == 'r') segment->protection |= kProtectionRead;
+  CHECK(IsOneOf(*data_.current, '-', 'w'));
+  if (*data_.current++ == 'w') segment->protection |= kProtectionWrite;
+  CHECK(IsOneOf(*data_.current, '-', 'x'));
+  if (*data_.current++ == 'x') segment->protection |= kProtectionExecute;
+  CHECK(IsOneOf(*data_.current, 's', 'p'));
+  if (*data_.current++ == 's') segment->protection |= kProtectionShared;
+  CHECK_EQ(*data_.current++, ' ');
+  segment->offset = ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ':');
+  ParseHex(&data_.current);
+  CHECK_EQ(*data_.current++, ' ');
+  while (IsDecimal(*data_.current)) data_.current++;
+  // Qemu may lack the trailing space.
+  // https://github.com/google/sanitizers/issues/160
+  // CHECK_EQ(*data_.current++, ' ');
+  // Skip spaces.
+  while (data_.current < next_line && *data_.current == ' ') data_.current++;
+  // Fill in the filename.
+  if (segment->filename) {
+    uptr len =
+        Min((uptr)(next_line - data_.current), segment->filename_size - 1);
+    internal_strncpy(segment->filename, data_.current, len);
+    segment->filename[len] = 0;
+  }
+
+  data_.current = next_line + 1;
+  return true;
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_LINUX
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp
new file mode 100644
index 0000000000..d02afcfe87
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp
@@ -0,0 +1,379 @@
+//===-- sanitizer_procmaps_mac.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (Mac-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_common.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_procmaps.h"
+
+#include <mach-o/dyld.h>
+#include <mach-o/loader.h>
+#include <mach/mach.h>
+
+// These are not available in older macOS SDKs.
+#ifndef CPU_SUBTYPE_X86_64_H
+#define CPU_SUBTYPE_X86_64_H  ((cpu_subtype_t)8)   /* Haswell */
+#endif
+#ifndef CPU_SUBTYPE_ARM_V7S
+#define CPU_SUBTYPE_ARM_V7S   ((cpu_subtype_t)11)  /* Swift */
+#endif
+#ifndef CPU_SUBTYPE_ARM_V7K
+#define CPU_SUBTYPE_ARM_V7K   ((cpu_subtype_t)12)
+#endif
+#ifndef CPU_TYPE_ARM64
+#define CPU_TYPE_ARM64        (CPU_TYPE_ARM | CPU_ARCH_ABI64)
+#endif
+
+namespace __sanitizer {
+
+// Contains information used to iterate through sections.
+struct MemoryMappedSegmentData {
+  char name[kMaxSegName];
+  uptr nsects;
+  const char *current_load_cmd_addr;
+  u32 lc_type;
+  uptr base_virt_addr;
+  uptr addr_mask;
+};
+
+template <typename Section>
+static void NextSectionLoad(LoadedModule *module, MemoryMappedSegmentData *data,
+                            bool isWritable) {
+  const Section *sc = (const Section *)data->current_load_cmd_addr;
+  data->current_load_cmd_addr += sizeof(Section);
+
+  uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr;
+  uptr sec_end = sec_start + sc->size;
+  module->addAddressRange(sec_start, sec_end, /*executable=*/false, isWritable,
+                          sc->sectname);
+}
+
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+  // Don't iterate over sections when the caller hasn't set up the
+  // data pointer, when there are no sections, or when the segment
+  // is executable. Avoid iterating over executable sections because
+  // it will confuse libignore, and because the extra granularity
+  // of information is not needed by any sanitizers.
+  if (!data_ || !data_->nsects || IsExecutable()) {
+    module->addAddressRange(start, end, IsExecutable(), IsWritable(),
+                            data_ ? data_->name : nullptr);
+    return;
+  }
+
+  do {
+    if (data_->lc_type == LC_SEGMENT) {
+      NextSectionLoad<struct section>(module, data_, IsWritable());
+#ifdef MH_MAGIC_64
+    } else if (data_->lc_type == LC_SEGMENT_64) {
+      NextSectionLoad<struct section_64>(module, data_, IsWritable());
+#endif
+    }
+  } while (--data_->nsects);
+}
+
+MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
+  Reset();
+}
+
+MemoryMappingLayout::~MemoryMappingLayout() {
+}
+
+bool MemoryMappingLayout::Error() const {
+  return false;
+}
+
+// More information about Mach-O headers can be found in mach-o/loader.h
+// Each Mach-O image has a header (mach_header or mach_header_64) starting with
+// a magic number, and a list of linker load commands directly following the
+// header.
+// A load command is at least two 32-bit words: the command type and the
+// command size in bytes. We're interested only in segment load commands
+// (LC_SEGMENT and LC_SEGMENT_64), which tell that a part of the file is mapped
+// into the task's address space.
+// The |vmaddr|, |vmsize| and |fileoff| fields of segment_command or
+// segment_command_64 correspond to the memory address, memory size and the
+// file offset of the current memory segment.
+// Because these fields are taken from the images as is, one needs to add
+// _dyld_get_image_vmaddr_slide() to get the actual addresses at runtime.
+
+void MemoryMappingLayout::Reset() {
+  // Count down from the top.
+  // TODO(glider): as per man 3 dyld, iterating over the headers with
+  // _dyld_image_count is thread-unsafe. We need to register callbacks for
+  // adding and removing images which will invalidate the MemoryMappingLayout
+  // state.
+  data_.current_image = _dyld_image_count();
+  data_.current_load_cmd_count = -1;
+  data_.current_load_cmd_addr = 0;
+  data_.current_magic = 0;
+  data_.current_filetype = 0;
+  data_.current_arch = kModuleArchUnknown;
+  internal_memset(data_.current_uuid, 0, kModuleUUIDSize);
+}
+
+// The dyld load address should be unchanged throughout process execution,
+// and it is expensive to compute once many libraries have been loaded,
+// so cache it here and do not reset.
+static mach_header *dyld_hdr = 0;
+static const char kDyldPath[] = "/usr/lib/dyld";
+static const int kDyldImageIdx = -1;
+
+// static
+void MemoryMappingLayout::CacheMemoryMappings() {
+  // No-op on Mac for now.
+}
+
+void MemoryMappingLayout::LoadFromCache() {
+  // No-op on Mac for now.
+}
+
+// _dyld_get_image_header() and related APIs don't report dyld itself.
+// We work around this by manually recursing through the memory map
+// until we hit a Mach header matching dyld instead. These recurse
+// calls are expensive, but the first memory map generation occurs
+// early in the process, when dyld is one of the only images loaded,
+// so it will be hit after only a few iterations.
+static mach_header *get_dyld_image_header() {
+  unsigned depth = 1;
+  vm_size_t size = 0;
+  vm_address_t address = 0;
+  kern_return_t err = KERN_SUCCESS;
+  mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+
+  while (true) {
+    struct vm_region_submap_info_64 info;
+    err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
+                               (vm_region_info_t)&info, &count);
+    if (err != KERN_SUCCESS) return nullptr;
+
+    if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
+      mach_header *hdr = (mach_header *)address;
+      if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
+          hdr->filetype == MH_DYLINKER) {
+        return hdr;
+      }
+    }
+    address += size;
+  }
+}
+
+const mach_header *get_dyld_hdr() {
+  if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
+
+  return dyld_hdr;
+}
+
+// Next and NextSegmentLoad were inspired by base/sysinfo.cc in
+// Google Perftools, https://github.com/gperftools/gperftools.
+
+// NextSegmentLoad scans the current image for the next segment load command
+// and returns the start and end addresses and file offset of the corresponding
+// segment.
+// Note that the segment addresses are not necessarily sorted.
+template <u32 kLCSegment, typename SegmentCommand>
+static bool NextSegmentLoad(MemoryMappedSegment *segment,
+                            MemoryMappedSegmentData *seg_data,
+                            MemoryMappingLayoutData *layout_data) {
+  const char *lc = layout_data->current_load_cmd_addr;
+  layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
+  if (((const load_command *)lc)->cmd == kLCSegment) {
+    const SegmentCommand* sc = (const SegmentCommand *)lc;
+    uptr base_virt_addr, addr_mask;
+    if (layout_data->current_image == kDyldImageIdx) {
+      base_virt_addr = (uptr)get_dyld_hdr();
+      // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
+      // it contains an absolute address rather than an offset for dyld.
+      // To make matters even more complicated, this absolute address
+      // isn't actually the absolute segment address, but the offset portion
+      // of the address is accurate when combined with the dyld base address,
+      // and the mask will give just this offset.
+      addr_mask = 0xfffff;
+    } else {
+      base_virt_addr =
+          (uptr)_dyld_get_image_vmaddr_slide(layout_data->current_image);
+      addr_mask = ~0;
+    }
+
+    segment->start = (sc->vmaddr & addr_mask) + base_virt_addr;
+    segment->end = segment->start + sc->vmsize;
+    // Most callers don't need section information, so only fill this struct
+    // when required.
+    if (seg_data) {
+      seg_data->nsects = sc->nsects;
+      seg_data->current_load_cmd_addr =
+          (const char *)lc + sizeof(SegmentCommand);
+      seg_data->lc_type = kLCSegment;
+      seg_data->base_virt_addr = base_virt_addr;
+      seg_data->addr_mask = addr_mask;
+      internal_strncpy(seg_data->name, sc->segname,
+                       ARRAY_SIZE(seg_data->name));
+    }
+
+    // Return the initial protection.
+    segment->protection = sc->initprot;
+    segment->offset = (layout_data->current_filetype ==
+                       /*MH_EXECUTE*/ 0x2)
+                          ? sc->vmaddr
+                          : sc->fileoff;
+    if (segment->filename) {
+      const char *src = (layout_data->current_image == kDyldImageIdx)
+                            ? kDyldPath
+                            : _dyld_get_image_name(layout_data->current_image);
+      internal_strncpy(segment->filename, src, segment->filename_size);
+    }
+    segment->arch = layout_data->current_arch;
+    internal_memcpy(segment->uuid, layout_data->current_uuid, kModuleUUIDSize);
+    return true;
+  }
+  return false;
+}
+
+ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) {
+  cpusubtype = cpusubtype & ~CPU_SUBTYPE_MASK;
+  switch (cputype) {
+    case CPU_TYPE_I386:
+      return kModuleArchI386;
+    case CPU_TYPE_X86_64:
+      if (cpusubtype == CPU_SUBTYPE_X86_64_ALL) return kModuleArchX86_64;
+      if (cpusubtype == CPU_SUBTYPE_X86_64_H) return kModuleArchX86_64H;
+      CHECK(0 && "Invalid subtype of x86_64");
+      return kModuleArchUnknown;
+    case CPU_TYPE_ARM:
+      if (cpusubtype == CPU_SUBTYPE_ARM_V6) return kModuleArchARMV6;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7) return kModuleArchARMV7;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7S) return kModuleArchARMV7S;
+      if (cpusubtype == CPU_SUBTYPE_ARM_V7K) return kModuleArchARMV7K;
+      CHECK(0 && "Invalid subtype of ARM");
+      return kModuleArchUnknown;
+    case CPU_TYPE_ARM64:
+      return kModuleArchARM64;
+    default:
+      CHECK(0 && "Invalid CPU type");
+      return kModuleArchUnknown;
+  }
+}
+
+static const load_command *NextCommand(const load_command *lc) {
+  return (const load_command *)((const char *)lc + lc->cmdsize);
+}
+
+static void FindUUID(const load_command *first_lc, u8 *uuid_output) {
+  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
+    if (lc->cmd != LC_UUID) continue;
+
+    const uuid_command *uuid_lc = (const uuid_command *)lc;
+    const uint8_t *uuid = &uuid_lc->uuid[0];
+    internal_memcpy(uuid_output, uuid, kModuleUUIDSize);
+    return;
+  }
+}
+
+static bool IsModuleInstrumented(const load_command *first_lc) {
+  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
+    if (lc->cmd != LC_LOAD_DYLIB) continue;
+
+    const dylib_command *dylib_lc = (const dylib_command *)lc;
+    uint32_t dylib_name_offset = dylib_lc->dylib.name.offset;
+    const char *dylib_name = ((const char *)dylib_lc) + dylib_name_offset;
+    dylib_name = StripModuleName(dylib_name);
+    if (dylib_name != 0 && (internal_strstr(dylib_name, "libclang_rt."))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
+    const mach_header *hdr = (data_.current_image == kDyldImageIdx)
+                                 ? get_dyld_hdr()
+                                 : _dyld_get_image_header(data_.current_image);
+    if (!hdr) continue;
+    if (data_.current_load_cmd_count < 0) {
+      // Set up for this image;
+      data_.current_load_cmd_count = hdr->ncmds;
+      data_.current_magic = hdr->magic;
+      data_.current_filetype = hdr->filetype;
+      data_.current_arch = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype);
+      switch (data_.current_magic) {
+#ifdef MH_MAGIC_64
+        case MH_MAGIC_64: {
+          data_.current_load_cmd_addr =
+              (const char *)hdr + sizeof(mach_header_64);
+          break;
+        }
+#endif
+        case MH_MAGIC: {
+          data_.current_load_cmd_addr = (const char *)hdr + sizeof(mach_header);
+          break;
+        }
+        default: {
+          continue;
+        }
+      }
+      FindUUID((const load_command *)data_.current_load_cmd_addr,
+               data_.current_uuid);
+      data_.current_instrumented = IsModuleInstrumented(
+          (const load_command *)data_.current_load_cmd_addr);
+    }
+
+    for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {
+      switch (data_.current_magic) {
+        // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
+#ifdef MH_MAGIC_64
+        case MH_MAGIC_64: {
+          if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>(
+                  segment, segment->data_, &data_))
+            return true;
+          break;
+        }
+#endif
+        case MH_MAGIC: {
+          if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(
+                  segment, segment->data_, &data_))
+            return true;
+          break;
+        }
+      }
+    }
+    // If we get here, no more load_cmd's in this image talk about
+    // segments.  Go on to the next image.
+  }
+  return false;
+}
+
+void MemoryMappingLayout::DumpListOfModules(
+    InternalMmapVectorNoCtor<LoadedModule> *modules) {
+  Reset();
+  InternalScopedString module_name(kMaxPathLength);
+  MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+  MemoryMappedSegmentData data;
+  segment.data_ = &data;
+  while (Next(&segment)) {
+    if (segment.filename[0] == '\0') continue;
+    LoadedModule *cur_module = nullptr;
+    if (!modules->empty() &&
+        0 == internal_strcmp(segment.filename, modules->back().full_name())) {
+      cur_module = &modules->back();
+    } else {
+      modules->push_back(LoadedModule());
+      cur_module = &modules->back();
+      cur_module->set(segment.filename, segment.start, segment.arch,
+                      segment.uuid, data_.current_instrumented);
+    }
+    segment.AddAddressRanges(cur_module);
+  }
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp b/lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp
new file mode 100644
index 0000000000..8793423a60
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp
@@ -0,0 +1,67 @@
+//===-- sanitizer_procmaps_solaris.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (Solaris-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_SOLARIS
+#include "sanitizer_common.h"
+#include "sanitizer_procmaps.h"
+
+// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
+#undef _FILE_OFFSET_BITS
+#include <procfs.h>
+#include <limits.h>
+
+namespace __sanitizer {
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
+  if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data,
+                        &proc_maps->mmaped_size, &proc_maps->len)) {
+    proc_maps->data = nullptr;
+    proc_maps->mmaped_size = 0;
+    proc_maps->len = 0;
+  }
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+  if (Error()) return false; // simulate empty maps
+  char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+  if (data_.current >= last) return false;
+
+  prxmap_t *xmapentry = (prxmap_t*)data_.current;
+
+  segment->start = (uptr)xmapentry->pr_vaddr;
+  segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size);
+  segment->offset = (uptr)xmapentry->pr_offset;
+
+  segment->protection = 0;
+  if ((xmapentry->pr_mflags & MA_READ) != 0)
+    segment->protection |= kProtectionRead;
+  if ((xmapentry->pr_mflags & MA_WRITE) != 0)
+    segment->protection |= kProtectionWrite;
+  if ((xmapentry->pr_mflags & MA_EXEC) != 0)
+    segment->protection |= kProtectionExecute;
+
+  if (segment->filename != NULL && segment->filename_size > 0) {
+    char proc_path[PATH_MAX + 1];
+
+    internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
+                      xmapentry->pr_mapname);
+    internal_readlink(proc_path, segment->filename, segment->filename_size);
+  }
+
+  data_.current += sizeof(prxmap_t);
+
+  return true;
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SOLARIS
diff --git a/lib/tsan/sanitizer_common/sanitizer_ptrauth.h b/lib/tsan/sanitizer_common/sanitizer_ptrauth.h
new file mode 100644
index 0000000000..4d0d96a64f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_ptrauth.h
@@ -0,0 +1,21 @@
+//===-- sanitizer_ptrauth.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PTRAUTH_H
+#define SANITIZER_PTRAUTH_H
+
+#if __has_feature(ptrauth_calls)
+#include <ptrauth.h>
+#else
+// Copied from <ptrauth.h>
+#define ptrauth_strip(__value, __key) __value
+#define ptrauth_auth_data(__value, __old_key, __old_data) __value
+#define ptrauth_string_discriminator(__string) ((int)0)
+#endif
+
+#endif // SANITIZER_PTRAUTH_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_quarantine.h b/lib/tsan/sanitizer_common/sanitizer_quarantine.h
new file mode 100644
index 0000000000..992f23152c
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_quarantine.h
@@ -0,0 +1,317 @@
+//===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Memory quarantine for AddressSanitizer and potentially other tools.
+// Quarantine caches some specified amount of memory in per-thread caches,
+// then evicts to global FIFO queue. When the queue reaches specified threshold,
+// oldest memory is recycled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_QUARANTINE_H
+#define SANITIZER_QUARANTINE_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_list.h"
+
+namespace __sanitizer {
+
+template<typename Node> class QuarantineCache;
+
+struct QuarantineBatch {
+  static const uptr kSize = 1021;
+  QuarantineBatch *next;
+  uptr size;
+  uptr count;
+  void *batch[kSize];
+
+  void init(void *ptr, uptr size) {
+    count = 1;
+    batch[0] = ptr;
+    this->size = size + sizeof(QuarantineBatch);  // Account for the batch size.
+  }
+
+  // The total size of quarantined nodes recorded in this batch.
+  uptr quarantined_size() const {
+    return size - sizeof(QuarantineBatch);
+  }
+
+  void push_back(void *ptr, uptr size) {
+    CHECK_LT(count, kSize);
+    batch[count++] = ptr;
+    this->size += size;
+  }
+
+  bool can_merge(const QuarantineBatch* const from) const {
+    return count + from->count <= kSize;
+  }
+
+  void merge(QuarantineBatch* const from) {
+    CHECK_LE(count + from->count, kSize);
+    CHECK_GE(size, sizeof(QuarantineBatch));
+
+    for (uptr i = 0; i < from->count; ++i)
+      batch[count + i] = from->batch[i];
+    count += from->count;
+    size += from->quarantined_size();
+
+    from->count = 0;
+    from->size = sizeof(QuarantineBatch);
+  }
+};
+
+COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13));  // 8Kb.
+
+// The callback interface is:
+// void Callback::Recycle(Node *ptr);
+// void *cb.Allocate(uptr size);
+// void cb.Deallocate(void *ptr);
+template<typename Callback, typename Node>
+class Quarantine {
+ public:
+  typedef QuarantineCache<Callback> Cache;
+
+  explicit Quarantine(LinkerInitialized)
+      : cache_(LINKER_INITIALIZED) {
+  }
+
+  void Init(uptr size, uptr cache_size) {
+    // Thread local quarantine size can be zero only when global quarantine size
+    // is zero (it allows us to perform just one atomic read per Put() call).
+    CHECK((size == 0 && cache_size == 0) || cache_size != 0);
+
+    atomic_store_relaxed(&max_size_, size);
+    atomic_store_relaxed(&min_size_, size / 10 * 9);  // 90% of max size.
+    atomic_store_relaxed(&max_cache_size_, cache_size);
+
+    cache_mutex_.Init();
+    recycle_mutex_.Init();
+  }
+
+  uptr GetSize() const { return atomic_load_relaxed(&max_size_); }
+  uptr GetCacheSize() const {
+    return atomic_load_relaxed(&max_cache_size_);
+  }
+
+  void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
+    uptr cache_size = GetCacheSize();
+    if (cache_size) {
+      c->Enqueue(cb, ptr, size);
+    } else {
+      // GetCacheSize() == 0 only when GetSize() == 0 (see Init).
+      cb.Recycle(ptr);
+    }
+    // Check cache size anyway to accommodate for runtime cache_size change.
+    if (c->Size() > cache_size)
+      Drain(c, cb);
+  }
+
+  void NOINLINE Drain(Cache *c, Callback cb) {
+    {
+      SpinMutexLock l(&cache_mutex_);
+      cache_.Transfer(c);
+    }
+    if (cache_.Size() > GetSize() && recycle_mutex_.TryLock())
+      Recycle(atomic_load_relaxed(&min_size_), cb);
+  }
+
+  void NOINLINE DrainAndRecycle(Cache *c, Callback cb) {
+    {
+      SpinMutexLock l(&cache_mutex_);
+      cache_.Transfer(c);
+    }
+    recycle_mutex_.Lock();
+    Recycle(0, cb);
+  }
+
+  void PrintStats() const {
+    // It assumes that the world is stopped, just as the allocator's PrintStats.
+    Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n",
+           GetSize() >> 20, GetCacheSize() >> 10);
+    cache_.PrintStats();
+  }
+
+ private:
+  // Read-only data.
+  char pad0_[kCacheLineSize];
+  atomic_uintptr_t max_size_;
+  atomic_uintptr_t min_size_;
+  atomic_uintptr_t max_cache_size_;
+  char pad1_[kCacheLineSize];
+  StaticSpinMutex cache_mutex_;
+  StaticSpinMutex recycle_mutex_;
+  Cache cache_;
+  char pad2_[kCacheLineSize];
+
+  void NOINLINE Recycle(uptr min_size, Callback cb) {
+    Cache tmp;
+    {
+      SpinMutexLock l(&cache_mutex_);
+      // Go over the batches and merge partially filled ones to
+      // save some memory, otherwise batches themselves (since the memory used
+      // by them is counted against quarantine limit) can overcome the actual
+      // user's quarantined chunks, which diminishes the purpose of the
+      // quarantine.
+      uptr cache_size = cache_.Size();
+      uptr overhead_size = cache_.OverheadSize();
+      CHECK_GE(cache_size, overhead_size);
+      // Do the merge only when overhead exceeds this predefined limit (might
+      // require some tuning). It saves us merge attempt when the batch list
+      // quarantine is unlikely to contain batches suitable for merge.
+      const uptr kOverheadThresholdPercents = 100;
+      if (cache_size > overhead_size &&
+          overhead_size * (100 + kOverheadThresholdPercents) >
+              cache_size * kOverheadThresholdPercents) {
+        cache_.MergeBatches(&tmp);
+      }
+      // Extract enough chunks from the quarantine to get below the max
+      // quarantine size and leave some leeway for the newly quarantined chunks.
+      while (cache_.Size() > min_size) {
+        tmp.EnqueueBatch(cache_.DequeueBatch());
+      }
+    }
+    recycle_mutex_.Unlock();
+    DoRecycle(&tmp, cb);
+  }
+
+  void NOINLINE DoRecycle(Cache *c, Callback cb) {
+    while (QuarantineBatch *b = c->DequeueBatch()) {
+      const uptr kPrefetch = 16;
+      CHECK(kPrefetch <= ARRAY_SIZE(b->batch));
+      for (uptr i = 0; i < kPrefetch; i++)
+        PREFETCH(b->batch[i]);
+      for (uptr i = 0, count = b->count; i < count; i++) {
+        if (i + kPrefetch < count)
+          PREFETCH(b->batch[i + kPrefetch]);
+        cb.Recycle((Node*)b->batch[i]);
+      }
+      cb.Deallocate(b);
+    }
+  }
+};
+
+// Per-thread cache of memory blocks.
+template<typename Callback>
+class QuarantineCache {
+ public:
+  explicit QuarantineCache(LinkerInitialized) {
+  }
+
+  QuarantineCache()
+      : size_() {
+    list_.clear();
+  }
+
+  // Total memory used, including internal accounting.
+  uptr Size() const {
+    return atomic_load_relaxed(&size_);
+  }
+
+  // Memory used for internal accounting.
+  uptr OverheadSize() const {
+    return list_.size() * sizeof(QuarantineBatch);
+  }
+
+  void Enqueue(Callback cb, void *ptr, uptr size) {
+    if (list_.empty() || list_.back()->count == QuarantineBatch::kSize) {
+      QuarantineBatch *b = (QuarantineBatch *)cb.Allocate(sizeof(*b));
+      CHECK(b);
+      b->init(ptr, size);
+      EnqueueBatch(b);
+    } else {
+      list_.back()->push_back(ptr, size);
+      SizeAdd(size);
+    }
+  }
+
+  void Transfer(QuarantineCache *from_cache) {
+    list_.append_back(&from_cache->list_);
+    SizeAdd(from_cache->Size());
+
+    atomic_store_relaxed(&from_cache->size_, 0);
+  }
+
+  void EnqueueBatch(QuarantineBatch *b) {
+    list_.push_back(b);
+    SizeAdd(b->size);
+  }
+
+  QuarantineBatch *DequeueBatch() {
+    if (list_.empty())
+      return nullptr;
+    QuarantineBatch *b = list_.front();
+    list_.pop_front();
+    SizeSub(b->size);
+    return b;
+  }
+
+  void MergeBatches(QuarantineCache *to_deallocate) {
+    uptr extracted_size = 0;
+    QuarantineBatch *current = list_.front();
+    while (current && current->next) {
+      if (current->can_merge(current->next)) {
+        QuarantineBatch *extracted = current->next;
+        // Move all the chunks into the current batch.
+        current->merge(extracted);
+        CHECK_EQ(extracted->count, 0);
+        CHECK_EQ(extracted->size, sizeof(QuarantineBatch));
+        // Remove the next batch from the list and account for its size.
+        list_.extract(current, extracted);
+        extracted_size += extracted->size;
+        // Add it to deallocation list.
+        to_deallocate->EnqueueBatch(extracted);
+      } else {
+        current = current->next;
+      }
+    }
+    SizeSub(extracted_size);
+  }
+
+  void PrintStats() const {
+    uptr batch_count = 0;
+    uptr total_overhead_bytes = 0;
+    uptr total_bytes = 0;
+    uptr total_quarantine_chunks = 0;
+    for (List::ConstIterator it = list_.begin(); it != list_.end(); ++it) {
+      batch_count++;
+      total_bytes += (*it).size;
+      total_overhead_bytes += (*it).size - (*it).quarantined_size();
+      total_quarantine_chunks += (*it).count;
+    }
+    uptr quarantine_chunks_capacity = batch_count * QuarantineBatch::kSize;
+    int chunks_usage_percent = quarantine_chunks_capacity == 0 ?
+        0 : total_quarantine_chunks * 100 / quarantine_chunks_capacity;
+    uptr total_quarantined_bytes = total_bytes - total_overhead_bytes;
+    int memory_overhead_percent = total_quarantined_bytes == 0 ?
+        0 : total_overhead_bytes * 100 / total_quarantined_bytes;
+    Printf("Global quarantine stats: batches: %zd; bytes: %zd (user: %zd); "
+           "chunks: %zd (capacity: %zd); %d%% chunks used; %d%% memory overhead"
+           "\n",
+           batch_count, total_bytes, total_quarantined_bytes,
+           total_quarantine_chunks, quarantine_chunks_capacity,
+           chunks_usage_percent, memory_overhead_percent);
+  }
+
+ private:
+  typedef IntrusiveList<QuarantineBatch> List;
+
+  List list_;
+  atomic_uintptr_t size_;
+
+  void SizeAdd(uptr add) {
+    atomic_store_relaxed(&size_, Size() + add);
+  }
+  void SizeSub(uptr sub) {
+    atomic_store_relaxed(&size_, Size() - sub);
+  }
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_QUARANTINE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_report_decorator.h b/lib/tsan/sanitizer_common/sanitizer_report_decorator.h
new file mode 100644
index 0000000000..d276c2cdd8
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_report_decorator.h
@@ -0,0 +1,48 @@
+//===-- sanitizer_report_decorator.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tags to decorate the sanitizer reports.
+// Currently supported tags:
+//   * None.
+//   * ANSI color sequences.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_REPORT_DECORATOR_H
+#define SANITIZER_REPORT_DECORATOR_H
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+class SanitizerCommonDecorator {
+  // FIXME: This is not portable. It assumes the special strings are printed to
+  // stdout, which is not the case on Windows (see SetConsoleTextAttribute()).
+ public:
+  SanitizerCommonDecorator() : ansi_(ColorizeReports()) {}
+  const char *Bold() const { return ansi_ ? "\033[1m" : ""; }
+  const char *Default() const { return ansi_ ? "\033[1m\033[0m"  : ""; }
+  const char *Warning() const { return Red(); }
+  const char *Error() const { return Red(); }
+  const char *MemoryByte() const { return Magenta(); }
+
+ protected:
+  const char *Black()   const { return ansi_ ? "\033[1m\033[30m" : ""; }
+  const char *Red()     const { return ansi_ ? "\033[1m\033[31m" : ""; }
+  const char *Green()   const { return ansi_ ? "\033[1m\033[32m" : ""; }
+  const char *Yellow()  const { return ansi_ ? "\033[1m\033[33m" : ""; }
+  const char *Blue()    const { return ansi_ ? "\033[1m\033[34m" : ""; }
+  const char *Magenta() const { return ansi_ ? "\033[1m\033[35m" : ""; }
+  const char *Cyan()    const { return ansi_ ? "\033[1m\033[36m" : ""; }
+  const char *White()   const { return ansi_ ? "\033[1m\033[37m" : ""; }
+ private:
+  bool ansi_;
+};
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_REPORT_DECORATOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_ring_buffer.h b/lib/tsan/sanitizer_common/sanitizer_ring_buffer.h
new file mode 100644
index 0000000000..2a46e933b7
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_ring_buffer.h
@@ -0,0 +1,161 @@
+//===-- sanitizer_ring_buffer.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Simple ring buffer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_RING_BUFFER_H
+#define SANITIZER_RING_BUFFER_H
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+// RingBuffer<T>: fixed-size ring buffer optimized for speed of push().
+// T should be a POD type and sizeof(T) should be divisible by sizeof(void*).
+// At creation, all elements are zero.
+template<class T>
+class RingBuffer {
+ public:
+  COMPILER_CHECK(sizeof(T) % sizeof(void *) == 0);
+  static RingBuffer *New(uptr Size) {
+    void *Ptr = MmapOrDie(SizeInBytes(Size), "RingBuffer");
+    RingBuffer *RB = reinterpret_cast<RingBuffer*>(Ptr);
+    uptr End = reinterpret_cast<uptr>(Ptr) + SizeInBytes(Size);
+    RB->last_ = RB->next_ = reinterpret_cast<T*>(End - sizeof(T));
+    return RB;
+  }
+  void Delete() {
+    UnmapOrDie(this, SizeInBytes(size()));
+  }
+  uptr size() const {
+    return last_ + 1 -
+           reinterpret_cast<T *>(reinterpret_cast<uptr>(this) +
+                                 2 * sizeof(T *));
+  }
+
+  static uptr SizeInBytes(uptr Size) {
+    return Size * sizeof(T) + 2 * sizeof(T*);
+  }
+
+  uptr SizeInBytes() { return SizeInBytes(size()); }
+
+  void push(T t) {
+    *next_ = t;
+    next_--;
+    // The condition below works only if sizeof(T) is divisible by sizeof(T*).
+    if (next_ <= reinterpret_cast<T*>(&next_))
+      next_ = last_;
+  }
+
+  T operator[](uptr Idx) const {
+    CHECK_LT(Idx, size());
+    sptr IdxNext = Idx + 1;
+    if (IdxNext > last_ - next_)
+      IdxNext -= size();
+    return next_[IdxNext];
+  }
+
+ private:
+  RingBuffer() {}
+  ~RingBuffer() {}
+  RingBuffer(const RingBuffer&) = delete;
+
+  // Data layout:
+  // LNDDDDDDDD
+  // D: data elements.
+  // L: last_, always points to the last data element.
+  // N: next_, initially equals to last_, is decremented on every push,
+  //    wraps around if it's less or equal than its own address.
+  T *last_;
+  T *next_;
+  T data_[1];  // flexible array.
+};
+
+// A ring buffer with externally provided storage that encodes its state in 8
+// bytes. Has significant constraints on size and alignment of storage.
+// See a comment in hwasan/hwasan_thread_list.h for the motivation behind this.
+#if SANITIZER_WORDSIZE == 64
+template <class T>
+class CompactRingBuffer {
+  // Top byte of long_ stores the buffer size in pages.
+  // Lower bytes store the address of the next buffer element.
+  static constexpr int kPageSizeBits = 12;
+  static constexpr int kSizeShift = 56;
+  static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1;
+
+  uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; }
+
+  void Init(void *storage, uptr size) {
+    CHECK_EQ(sizeof(CompactRingBuffer<T>), sizeof(void *));
+    CHECK(IsPowerOfTwo(size));
+    CHECK_GE(size, 1 << kPageSizeBits);
+    CHECK_LE(size, 128 << kPageSizeBits);
+    CHECK_EQ(size % 4096, 0);
+    CHECK_EQ(size % sizeof(T), 0);
+    CHECK_EQ((uptr)storage % (size * 2), 0);
+    long_ = (uptr)storage | ((size >> kPageSizeBits) << kSizeShift);
+  }
+
+  void SetNext(const T *next) {
+    long_ = (long_ & ~kNextMask) | (uptr)next;
+  }
+
+ public:
+  CompactRingBuffer(void *storage, uptr size) {
+    Init(storage, size);
+  }
+
+  // A copy constructor of sorts.
+  CompactRingBuffer(const CompactRingBuffer &other, void *storage) {
+    uptr size = other.GetStorageSize();
+    internal_memcpy(storage, other.StartOfStorage(), size);
+    Init(storage, size);
+    uptr Idx = other.Next() - (const T *)other.StartOfStorage();
+    SetNext((const T *)storage + Idx);
+  }
+
+  T *Next() const { return (T *)(long_ & kNextMask); }
+
+  void *StartOfStorage() const {
+    return (void *)((uptr)Next() & ~(GetStorageSize() - 1));
+  }
+
+  void *EndOfStorage() const {
+    return (void *)((uptr)StartOfStorage() + GetStorageSize());
+  }
+
+  uptr size() const { return GetStorageSize() / sizeof(T); }
+
+  void push(T t) {
+    T *next = Next();
+    *next = t;
+    next++;
+    next = (T *)((uptr)next & ~GetStorageSize());
+    SetNext(next);
+  }
+
+  const T &operator[](uptr Idx) const {
+    CHECK_LT(Idx, size());
+    const T *Begin = (const T *)StartOfStorage();
+    sptr StorageIdx = Next() - Begin;
+    StorageIdx -= (sptr)(Idx + 1);
+    if (StorageIdx < 0)
+      StorageIdx += size();
+    return Begin[StorageIdx];
+  }
+
+ public:
+  ~CompactRingBuffer() {}
+  CompactRingBuffer(const CompactRingBuffer &) = delete;
+
+  uptr long_;
+};
+#endif
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_RING_BUFFER_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_rtems.cpp b/lib/tsan/sanitizer_common/sanitizer_rtems.cpp
new file mode 100644
index 0000000000..29bcfcfa6f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_rtems.cpp
@@ -0,0 +1,283 @@
+//===-- sanitizer_rtems.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements RTEMS-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_rtems.h"
+#if SANITIZER_RTEMS
+
+#define posix_memalign __real_posix_memalign
+#define free __real_free
+#define memset __real_memset
+
+#include "sanitizer_file.h"
+#include "sanitizer_symbolizer.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// There is no mmap on RTEMS.  Use memalign, etc.
+#define __mmap_alloc_aligned posix_memalign
+#define __mmap_free free
+#define __mmap_memset memset
+
+namespace __sanitizer {
+
+#include "sanitizer_syscall_generic.inc"
+
+void NORETURN internal__exit(int exitcode) {
+  _exit(exitcode);
+}
+
+uptr internal_sched_yield() {
+  return sched_yield();
+}
+
+uptr internal_getpid() {
+  return getpid();
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+  UNIMPLEMENTED();
+}
+
+bool FileExists(const char *filename) {
+  struct stat st;
+  if (stat(filename, &st))
+    return false;
+  // Sanity check: filename is a regular file.
+  return S_ISREG(st.st_mode);
+}
+
+uptr GetThreadSelf() { return static_cast<uptr>(pthread_self()); }
+
+tid_t GetTid() { return GetThreadSelf(); }
+
+void Abort() { abort(); }
+
+int Atexit(void (*function)(void)) { return atexit(function); }
+
+void SleepForSeconds(int seconds) { sleep(seconds); }
+
+void SleepForMillis(int millis) { usleep(millis * 1000); }
+
+bool SupportsColoredOutput(fd_t fd) { return false; }
+
+void GetThreadStackTopAndBottom(bool at_initialization,
+                                uptr *stack_top, uptr *stack_bottom) {
+  pthread_attr_t attr;
+  pthread_attr_init(&attr);
+  CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
+  void *base = nullptr;
+  size_t size = 0;
+  CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0);
+  CHECK_EQ(pthread_attr_destroy(&attr), 0);
+
+  *stack_bottom = reinterpret_cast<uptr>(base);
+  *stack_top = *stack_bottom + size;
+}
+
+void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+                          uptr *tls_addr, uptr *tls_size) {
+  uptr stack_top, stack_bottom;
+  GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
+  *stk_addr = stack_bottom;
+  *stk_size = stack_top - stack_bottom;
+  *tls_addr = *tls_size = 0;
+}
+
+void InitializePlatformEarly() {}
+void MaybeReexec() {}
+void CheckASLR() {}
+void CheckMPROTECT() {}
+void DisableCoreDumperIfNecessary() {}
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
+void SetAlternateSignalStack() {}
+void UnsetAlternateSignalStack() {}
+void InitTlsSize() {}
+
+void PrintModuleMap() {}
+
+void SignalContext::DumpAllRegisters(void *context) {}
+const char *DescribeSignalOrException(int signo) { UNIMPLEMENTED(); }
+
+enum MutexState { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
+
+BlockingMutex::BlockingMutex() {
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  CHECK_EQ(owner_, 0);
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
+    return;
+  while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
+    internal_sched_yield();
+  }
+}
+
+void BlockingMutex::Unlock() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
+  CHECK_NE(v, MtxUnlocked);
+}
+
+void BlockingMutex::CheckLocked() {
+  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+  CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
+}
+
+uptr GetPageSize() { return getpagesize(); }
+
+uptr GetMmapGranularity() { return GetPageSize(); }
+
+uptr GetMaxVirtualAddress() {
+  return (1ULL << 32) - 1;  // 0xffffffff
+}
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+  void* ptr = 0;
+  int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size);
+  if (UNLIKELY(res))
+    ReportMmapFailureAndDie(size, mem_type, "allocate", res, raw_report);
+  __mmap_memset(ptr, 0, size);
+  IncreaseTotalMmap(size);
+  return ptr;
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  void* ptr = 0;
+  int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size);
+  if (UNLIKELY(res)) {
+    if (res == ENOMEM)
+      return nullptr;
+    ReportMmapFailureAndDie(size, mem_type, "allocate", false);
+  }
+  __mmap_memset(ptr, 0, size);
+  IncreaseTotalMmap(size);
+  return ptr;
+}
+
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
+  CHECK(IsPowerOfTwo(size));
+  CHECK(IsPowerOfTwo(alignment));
+  void* ptr = 0;
+  int res = __mmap_alloc_aligned(&ptr, alignment, size);
+  if (res)
+    ReportMmapFailureAndDie(size, mem_type, "align allocate", res, false);
+  __mmap_memset(ptr, 0, size);
+  IncreaseTotalMmap(size);
+  return ptr;
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+  return MmapOrDie(size, mem_type, false);
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+  if (!addr || !size) return;
+  __mmap_free(addr);
+  DecreaseTotalMmap(size);
+}
+
+fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) {
+  int flags;
+  switch (mode) {
+    case RdOnly: flags = O_RDONLY; break;
+    case WrOnly: flags = O_WRONLY | O_CREAT | O_TRUNC; break;
+    case RdWr: flags = O_RDWR | O_CREAT; break;
+  }
+  fd_t res = open(filename, flags, 0660);
+  if (internal_iserror(res, errno_p))
+    return kInvalidFd;
+  return res;
+}
+
+void CloseFile(fd_t fd) {
+  close(fd);
+}
+
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read,
+                  error_t *error_p) {
+  uptr res = read(fd, buff, buff_size);
+  if (internal_iserror(res, error_p))
+    return false;
+  if (bytes_read)
+    *bytes_read = res;
+  return true;
+}
+
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written,
+                 error_t *error_p) {
+  uptr res = write(fd, buff, buff_size);
+  if (internal_iserror(res, error_p))
+    return false;
+  if (bytes_written)
+    *bytes_written = res;
+  return true;
+}
+
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
+void DumpProcessMap() {}
+
+// There is no page protection so everything is "accessible."
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+  return true;
+}
+
+char **GetArgv() { return nullptr; }
+char **GetEnviron() { return nullptr; }
+
+const char *GetEnv(const char *name) {
+  return getenv(name);
+}
+
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+  internal_strncpy(buf, "StubBinaryName", buf_len);
+  return internal_strlen(buf);
+}
+
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
+  internal_strncpy(buf, "StubProcessName", buf_len);
+  return internal_strlen(buf);
+}
+
+bool IsPathSeparator(const char c) {
+  return c == '/';
+}
+
+bool IsAbsolutePath(const char *path) {
+  return path != nullptr && IsPathSeparator(path[0]);
+}
+
+void ReportFile::Write(const char *buffer, uptr length) {
+  SpinMutexLock l(mu);
+  static const char *kWriteError =
+      "ReportFile::Write() can't output requested buffer!\n";
+  ReopenIfNecessary();
+  if (length != write(fd, buffer, length)) {
+    write(fd, kWriteError, internal_strlen(kWriteError));
+    Die();
+  }
+}
+
+uptr MainThreadStackBase, MainThreadStackSize;
+uptr MainThreadTlsBase, MainThreadTlsSize;
+
+} // namespace __sanitizer
+
+#endif  // SANITIZER_RTEMS
diff --git a/lib/tsan/sanitizer_common/sanitizer_rtems.h b/lib/tsan/sanitizer_common/sanitizer_rtems.h
new file mode 100644
index 0000000000..e8adfd500d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_rtems.h
@@ -0,0 +1,20 @@
+//===-- sanitizer_rtems.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// provides definitions for RTEMS-specific functions.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_RTEMS_H
+#define SANITIZER_RTEMS_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_RTEMS
+#include "sanitizer_common.h"
+
+#endif  // SANITIZER_RTEMS
+#endif  // SANITIZER_RTEMS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc b/lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc
new file mode 100644
index 0000000000..68d9eb6596
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc
@@ -0,0 +1,86 @@
+//===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Signal interceptors for sanitizers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_interceptors.h"
+
+using namespace __sanitizer;
+
+#if SANITIZER_NETBSD
+#define sigaction_symname __sigaction14
+#else
+#define sigaction_symname sigaction
+#endif
+
+#ifndef SIGNAL_INTERCEPTOR_SIGNAL_IMPL
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signum, handler) \
+  { return REAL(func)(signum, handler); }
+#endif
+
+#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
+  { return REAL(sigaction_symname)(signum, act, oldact); }
+#endif
+
+#if SANITIZER_INTERCEPT_BSD_SIGNAL
+INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+  SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
+}
+#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal)
+#else  // SANITIZER_INTERCEPT_BSD_SIGNAL
+#define INIT_BSD_SIGNAL
+#endif  // SANITIZER_INTERCEPT_BSD_SIGNAL
+
+#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+INTERCEPTOR(uptr, signal, int signum, uptr handler) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
+    return (uptr) nullptr;
+  SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
+}
+#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal)
+
+INTERCEPTOR(int, sigaction_symname, int signum,
+            const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) {
+  if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+  SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact);
+}
+#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname)
+
+namespace __sanitizer {
+int real_sigaction(int signum, const void *act, void *oldact) {
+  return REAL(sigaction_symname)(signum, (const __sanitizer_sigaction *)act,
+                         (__sanitizer_sigaction *)oldact);
+}
+}  // namespace __sanitizer
+#else  // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+#define INIT_SIGNAL
+#define INIT_SIGACTION
+// We need to have defined REAL(sigaction) on other systems.
+namespace __sanitizer {
+struct __sanitizer_sigaction;
+}
+DEFINE_REAL(int, sigaction, int signum, const __sanitizer_sigaction *act,
+            __sanitizer_sigaction *oldact)
+#endif  // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+
+static void InitializeSignalInterceptors() {
+  static bool was_called_once;
+  CHECK(!was_called_once);
+  was_called_once = true;
+
+  INIT_BSD_SIGNAL;
+  INIT_SIGNAL;
+  INIT_SIGACTION;
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_solaris.cpp b/lib/tsan/sanitizer_common/sanitizer_solaris.cpp
new file mode 100644
index 0000000000..035f2d0ca2
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_solaris.cpp
@@ -0,0 +1,230 @@
+//===-- sanitizer_solaris.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements Solaris-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_SOLARIS
+
+#include <stdio.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_procmaps.h"
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <sched.h>
+#include <thread.h>
+#include <synch.h>
+#include <signal.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+
+namespace __sanitizer {
+
+//#include "sanitizer_syscall_generic.inc"
+
+#define _REAL(func) _ ## func
+#define DECLARE__REAL(ret_type, func, ...) \
+  extern "C" ret_type _REAL(func)(__VA_ARGS__)
+#define DECLARE__REAL_AND_INTERNAL(ret_type, func, ...) \
+  DECLARE__REAL(ret_type, func, __VA_ARGS__); \
+  ret_type internal_ ## func(__VA_ARGS__)
+
+#if !defined(_LP64) && _FILE_OFFSET_BITS == 64
+#define _REAL64(func) _ ## func ## 64
+#else
+#define _REAL64(func) _REAL(func)
+#endif
+#define DECLARE__REAL64(ret_type, func, ...) \
+  extern "C" ret_type _REAL64(func)(__VA_ARGS__)
+#define DECLARE__REAL_AND_INTERNAL64(ret_type, func, ...) \
+  DECLARE__REAL64(ret_type, func, __VA_ARGS__); \
+  ret_type internal_ ## func(__VA_ARGS__)
+
+// ---------------------- sanitizer_libc.h
+DECLARE__REAL_AND_INTERNAL64(uptr, mmap, void *addr, uptr /*size_t*/ length,
+                             int prot, int flags, int fd, OFF_T offset) {
+  return (uptr)_REAL64(mmap)(addr, length, prot, flags, fd, offset);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) {
+  return _REAL(munmap)(addr, length);
+}
+
+DECLARE__REAL_AND_INTERNAL(int, mprotect, void *addr, uptr length, int prot) {
+  return _REAL(mprotect)(addr, length, prot);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) {
+  return _REAL(close)(fd);
+}
+
+extern "C" int _REAL64(open)(const char *, int, ...);
+
+uptr internal_open(const char *filename, int flags) {
+  return _REAL64(open)(filename, flags);
+}
+
+uptr internal_open(const char *filename, int flags, u32 mode) {
+  return _REAL64(open)(filename, flags, mode);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) {
+  return _REAL(read)(fd, buf, count);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, write, fd_t fd, const void *buf, uptr count) {
+  return _REAL(write)(fd, buf, count);
+}
+
+// FIXME: There's only _ftruncate64 beginning with Solaris 11.
+DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) {
+  return ftruncate(fd, size);
+}
+
+DECLARE__REAL_AND_INTERNAL64(uptr, stat, const char *path, void *buf) {
+  return _REAL64(stat)(path, (struct stat *)buf);
+}
+
+DECLARE__REAL_AND_INTERNAL64(uptr, lstat, const char *path, void *buf) {
+  return _REAL64(lstat)(path, (struct stat *)buf);
+}
+
+DECLARE__REAL_AND_INTERNAL64(uptr, fstat, fd_t fd, void *buf) {
+  return _REAL64(fstat)(fd, (struct stat *)buf);
+}
+
+uptr internal_filesize(fd_t fd) {
+  struct stat st;
+  if (internal_fstat(fd, &st))
+    return -1;
+  return (uptr)st.st_size;
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, dup, int oldfd) {
+  return _REAL(dup)(oldfd);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) {
+  return _REAL(dup2)(oldfd, newfd);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, readlink, const char *path, char *buf,
+                           uptr bufsize) {
+  return _REAL(readlink)(path, buf, bufsize);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, unlink, const char *path) {
+  return _REAL(unlink)(path);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, rename, const char *oldpath,
+                           const char *newpath) {
+  return _REAL(rename)(oldpath, newpath);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) {
+  return sched_yield();
+}
+
+DECLARE__REAL_AND_INTERNAL(void, _exit, int exitcode) {
+  _exit(exitcode);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename,
+                           char *const argv[], char *const envp[]) {
+  return _REAL(execve)(filename, argv, envp);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, waitpid, int pid, int *status, int options) {
+  return _REAL(waitpid)(pid, status, options);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) {
+  return _REAL(getpid)();
+}
+
+// FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *.
+DECLARE__REAL_AND_INTERNAL64(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
+                             unsigned int count) {
+  return _REAL64(getdents)(fd, dirp, count);
+}
+
+DECLARE__REAL_AND_INTERNAL64(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
+  return _REAL64(lseek)(fd, offset, whence);
+}
+
+// FIXME: This might be wrong: _sigfillset doesn't take a
+// __sanitizer_sigset_t *.
+DECLARE__REAL_AND_INTERNAL(void, sigfillset, __sanitizer_sigset_t *set) {
+  _REAL(sigfillset)(set);
+}
+
+// FIXME: This might be wrong: _sigprocmask doesn't take __sanitizer_sigset_t *.
+DECLARE__REAL_AND_INTERNAL(uptr, sigprocmask, int how,
+                           __sanitizer_sigset_t *set,
+                           __sanitizer_sigset_t *oldset) {
+  return _REAL(sigprocmask)(how, set, oldset);
+}
+
+DECLARE__REAL_AND_INTERNAL(int, fork, void) {
+  // TODO(glider): this may call user's pthread_atfork() handlers which is bad.
+  return _REAL(fork)();
+}
+
+u64 NanoTime() {
+  return gethrtime();
+}
+
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+  // FIXME: No internal variant.
+  return clock_gettime(clk_id, (timespec *)tp);
+}
+
+// ----------------- sanitizer_common.h
+BlockingMutex::BlockingMutex() {
+  CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
+  internal_memset(this, 0, sizeof(*this));
+  CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0);
+}
+
+void BlockingMutex::Lock() {
+  CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
+  CHECK_NE(owner_, (uptr)thr_self());
+  CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0);
+  CHECK(!owner_);
+  owner_ = (uptr)thr_self();
+}
+
+void BlockingMutex::Unlock() {
+  CHECK(owner_ == (uptr)thr_self());
+  owner_ = 0;
+  CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
+}
+
+void BlockingMutex::CheckLocked() {
+  CHECK_EQ((uptr)thr_self(), owner_);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SOLARIS
diff --git a/lib/tsan/sanitizer_common/sanitizer_stackdepot.h b/lib/tsan/sanitizer_common/sanitizer_stackdepot.h
new file mode 100644
index 0000000000..bf29cb9a00
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stackdepot.h
@@ -0,0 +1,71 @@
+//===-- sanitizer_stackdepot.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_STACKDEPOT_H
+#define SANITIZER_STACKDEPOT_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+// StackDepot efficiently stores huge amounts of stack traces.
+struct StackDepotNode;
+struct StackDepotHandle {
+  StackDepotNode *node_;
+  StackDepotHandle() : node_(nullptr) {}
+  explicit StackDepotHandle(StackDepotNode *node) : node_(node) {}
+  bool valid() { return node_; }
+  u32 id();
+  int use_count();
+  void inc_use_count_unsafe();
+};
+
+const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20);
+
+StackDepotStats *StackDepotGetStats();
+u32 StackDepotPut(StackTrace stack);
+StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
+// Retrieves a stored stack trace by the id.
+StackTrace StackDepotGet(u32 id);
+
+void StackDepotLockAll();
+void StackDepotUnlockAll();
+
+// Instantiating this class creates a snapshot of StackDepot which can be
+// efficiently queried with StackDepotGet(). You can use it concurrently with
+// StackDepot, but the snapshot is only guaranteed to contain those stack traces
+// which were stored before it was instantiated.
+class StackDepotReverseMap {
+ public:
+  StackDepotReverseMap();
+  StackTrace Get(u32 id);
+
+ private:
+  struct IdDescPair {
+    u32 id;
+    StackDepotNode *desc;
+
+    static bool IdComparator(const IdDescPair &a, const IdDescPair &b);
+  };
+
+  InternalMmapVector<IdDescPair> map_;
+
+  // Disallow evil constructors.
+  StackDepotReverseMap(const StackDepotReverseMap&);
+  void operator=(const StackDepotReverseMap&);
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_STACKDEPOT_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h b/lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h
new file mode 100644
index 0000000000..ef1b4f7f70
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h
@@ -0,0 +1,177 @@
+//===-- sanitizer_stackdepotbase.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of a mapping from arbitrary values to unique 32-bit
+// identifiers.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_STACKDEPOTBASE_H
+#define SANITIZER_STACKDEPOTBASE_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_persistent_allocator.h"
+
+namespace __sanitizer {
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+class StackDepotBase {
+ public:
+  typedef typename Node::args_type args_type;
+  typedef typename Node::handle_type handle_type;
+  // Maps stack trace to an unique id.
+  handle_type Put(args_type args, bool *inserted = nullptr);
+  // Retrieves a stored stack trace by the id.
+  args_type Get(u32 id);
+
+  StackDepotStats *GetStats() { return &stats; }
+
+  void LockAll();
+  void UnlockAll();
+
+ private:
+  static Node *find(Node *s, args_type args, u32 hash);
+  static Node *lock(atomic_uintptr_t *p);
+  static void unlock(atomic_uintptr_t *p, Node *s);
+
+  static const int kTabSize = 1 << kTabSizeLog;  // Hash table size.
+  static const int kPartBits = 8;
+  static const int kPartShift = sizeof(u32) * 8 - kPartBits - kReservedBits;
+  static const int kPartCount =
+      1 << kPartBits;  // Number of subparts in the table.
+  static const int kPartSize = kTabSize / kPartCount;
+  static const int kMaxId = 1 << kPartShift;
+
+  atomic_uintptr_t tab[kTabSize];   // Hash table of Node's.
+  atomic_uint32_t seq[kPartCount];  // Unique id generators.
+
+  StackDepotStats stats;
+
+  friend class StackDepotReverseMap;
+};
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node *s,
+                                                             args_type args,
+                                                             u32 hash) {
+  // Searches linked list s for the stack, returns its id.
+  for (; s; s = s->link) {
+    if (s->eq(hash, args)) {
+      return s;
+    }
+  }
+  return nullptr;
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
+    atomic_uintptr_t *p) {
+  // Uses the pointer lsb as mutex.
+  for (int i = 0;; i++) {
+    uptr cmp = atomic_load(p, memory_order_relaxed);
+    if ((cmp & 1) == 0 &&
+        atomic_compare_exchange_weak(p, &cmp, cmp | 1, memory_order_acquire))
+      return (Node *)cmp;
+    if (i < 10)
+      proc_yield(10);
+    else
+      internal_sched_yield();
+  }
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::unlock(
+    atomic_uintptr_t *p, Node *s) {
+  DCHECK_EQ((uptr)s & 1, 0);
+  atomic_store(p, (uptr)s, memory_order_release);
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::handle_type
+StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
+                                                      bool *inserted) {
+  if (inserted) *inserted = false;
+  if (!Node::is_valid(args)) return handle_type();
+  uptr h = Node::hash(args);
+  atomic_uintptr_t *p = &tab[h % kTabSize];
+  uptr v = atomic_load(p, memory_order_consume);
+  Node *s = (Node *)(v & ~1);
+  // First, try to find the existing stack.
+  Node *node = find(s, args, h);
+  if (node) return node->get_handle();
+  // If failed, lock, retry and insert new.
+  Node *s2 = lock(p);
+  if (s2 != s) {
+    node = find(s2, args, h);
+    if (node) {
+      unlock(p, s2);
+      return node->get_handle();
+    }
+  }
+  uptr part = (h % kTabSize) / kPartSize;
+  u32 id = atomic_fetch_add(&seq[part], 1, memory_order_relaxed) + 1;
+  stats.n_uniq_ids++;
+  CHECK_LT(id, kMaxId);
+  id |= part << kPartShift;
+  CHECK_NE(id, 0);
+  CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
+  uptr memsz = Node::storage_size(args);
+  s = (Node *)PersistentAlloc(memsz);
+  stats.allocated += memsz;
+  s->id = id;
+  s->store(args, h);
+  s->link = s2;
+  unlock(p, s);
+  if (inserted) *inserted = true;
+  return s->get_handle();
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::args_type
+StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
+  if (id == 0) {
+    return args_type();
+  }
+  CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
+  // High kPartBits contain part id, so we need to scan at most kPartSize lists.
+  uptr part = id >> kPartShift;
+  for (int i = 0; i != kPartSize; i++) {
+    uptr idx = part * kPartSize + i;
+    CHECK_LT(idx, kTabSize);
+    atomic_uintptr_t *p = &tab[idx];
+    uptr v = atomic_load(p, memory_order_consume);
+    Node *s = (Node *)(v & ~1);
+    for (; s; s = s->link) {
+      if (s->id == id) {
+        return s->load();
+      }
+    }
+  }
+  return args_type();
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
+  for (int i = 0; i < kTabSize; ++i) {
+    lock(&tab[i]);
+  }
+}
+
+template <class Node, int kReservedBits, int kTabSizeLog>
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
+  for (int i = 0; i < kTabSize; ++i) {
+    atomic_uintptr_t *p = &tab[i];
+    uptr s = atomic_load(p, memory_order_relaxed);
+    unlock(p, (Node *)(s & ~1UL));
+  }
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_STACKDEPOTBASE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace.h b/lib/tsan/sanitizer_common/sanitizer_stacktrace.h
new file mode 100644
index 0000000000..f1f29e9f32
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace.h
@@ -0,0 +1,176 @@
+//===-- sanitizer_stacktrace.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_STACKTRACE_H
+#define SANITIZER_STACKTRACE_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+struct BufferedStackTrace;
+
+static const u32 kStackTraceMax = 256;
+
+#if SANITIZER_LINUX && defined(__mips__)
+# define SANITIZER_CAN_FAST_UNWIND 0
+#elif SANITIZER_WINDOWS
+# define SANITIZER_CAN_FAST_UNWIND 0
+#elif SANITIZER_OPENBSD
+# define SANITIZER_CAN_FAST_UNWIND 0
+#else
+# define SANITIZER_CAN_FAST_UNWIND 1
+#endif
+
+// Fast unwind is the only option on Mac for now; we will need to
+// revisit this macro when slow unwind works on Mac, see
+// https://github.com/google/sanitizers/issues/137
+#if SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_RTEMS
+# define SANITIZER_CAN_SLOW_UNWIND 0
+#else
+# define SANITIZER_CAN_SLOW_UNWIND 1
+#endif
+
+struct StackTrace {
+  const uptr *trace;
+  u32 size;
+  u32 tag;
+
+  static const int TAG_UNKNOWN = 0;
+  static const int TAG_ALLOC = 1;
+  static const int TAG_DEALLOC = 2;
+  static const int TAG_CUSTOM = 100; // Tool specific tags start here.
+
+  StackTrace() : trace(nullptr), size(0), tag(0) {}
+  StackTrace(const uptr *trace, u32 size) : trace(trace), size(size), tag(0) {}
+  StackTrace(const uptr *trace, u32 size, u32 tag)
+      : trace(trace), size(size), tag(tag) {}
+
+  // Prints a symbolized stacktrace, followed by an empty line.
+  void Print() const;
+
+  static bool WillUseFastUnwind(bool request_fast_unwind) {
+    if (!SANITIZER_CAN_FAST_UNWIND)
+      return false;
+    if (!SANITIZER_CAN_SLOW_UNWIND)
+      return true;
+    return request_fast_unwind;
+  }
+
+  static uptr GetCurrentPc();
+  static inline uptr GetPreviousInstructionPc(uptr pc);
+  static uptr GetNextInstructionPc(uptr pc);
+  typedef bool (*SymbolizeCallback)(const void *pc, char *out_buffer,
+                                    int out_size);
+};
+
+// Performance-critical, must be in the header.
+ALWAYS_INLINE
+uptr StackTrace::GetPreviousInstructionPc(uptr pc) {
+#if defined(__arm__)
+  // T32 (Thumb) branch instructions might be 16 or 32 bit long,
+  // so we return (pc-2) in that case in order to be safe.
+  // For A32 mode we return (pc-4) because all instructions are 32 bit long.
+  return (pc - 3) & (~1);
+#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__)
+  // PCs are always 4 byte aligned.
+  return pc - 4;
+#elif defined(__sparc__) || defined(__mips__)
+  return pc - 8;
+#else
+  return pc - 1;
+#endif
+}
+
+// StackTrace that owns the buffer used to store the addresses.
+struct BufferedStackTrace : public StackTrace {
+  uptr trace_buffer[kStackTraceMax];
+  uptr top_frame_bp;  // Optional bp of a top frame.
+
+  BufferedStackTrace() : StackTrace(trace_buffer, 0), top_frame_bp(0) {}
+
+  void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0);
+
+  // Get the stack trace with the given pc and bp.
+  // The pc will be in the position 0 of the resulting stack trace.
+  // The bp may refer to the current frame or to the caller's frame.
+  void Unwind(uptr pc, uptr bp, void *context, bool request_fast,
+              u32 max_depth = kStackTraceMax) {
+    top_frame_bp = (max_depth > 0) ? bp : 0;
+    // Small max_depth optimization
+    if (max_depth <= 1) {
+      if (max_depth == 1)
+        trace_buffer[0] = pc;
+      size = max_depth;
+      return;
+    }
+    UnwindImpl(pc, bp, context, request_fast, max_depth);
+  }
+
+  void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top,
+              uptr stack_bottom, bool request_fast_unwind);
+
+  void Reset() {
+    *static_cast<StackTrace *>(this) = StackTrace(trace_buffer, 0);
+    top_frame_bp = 0;
+  }
+
+ private:
+  // Every runtime defines its own implementation of this method
+  void UnwindImpl(uptr pc, uptr bp, void *context, bool request_fast,
+                  u32 max_depth);
+
+  // UnwindFast/Slow have platform-specific implementations
+  void UnwindFast(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom,
+                  u32 max_depth);
+  void UnwindSlow(uptr pc, u32 max_depth);
+  void UnwindSlow(uptr pc, void *context, u32 max_depth);
+
+  void PopStackFrames(uptr count);
+  uptr LocatePcInTrace(uptr pc);
+
+  BufferedStackTrace(const BufferedStackTrace &) = delete;
+  void operator=(const BufferedStackTrace &) = delete;
+
+  friend class FastUnwindTest;
+};
+
+// Check if given pointer points into allocated stack area.
+static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
+  return frame > stack_bottom && frame < stack_top - 2 * sizeof (uhwptr);
+}
+
+}  // namespace __sanitizer
+
+// Use this macro if you want to print stack trace with the caller
+// of the current function in the top frame.
+#define GET_CALLER_PC_BP \
+  uptr bp = GET_CURRENT_FRAME();              \
+  uptr pc = GET_CALLER_PC();
+
+#define GET_CALLER_PC_BP_SP \
+  GET_CALLER_PC_BP;                           \
+  uptr local_stack;                           \
+  uptr sp = (uptr)&local_stack
+
+// Use this macro if you want to print stack trace with the current
+// function in the top frame.
+#define GET_CURRENT_PC_BP \
+  uptr bp = GET_CURRENT_FRAME();              \
+  uptr pc = StackTrace::GetCurrentPc()
+
+#define GET_CURRENT_PC_BP_SP \
+  GET_CURRENT_PC_BP;                          \
+  uptr local_stack;                           \
+  uptr sp = (uptr)&local_stack
+
+
+#endif  // SANITIZER_STACKTRACE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h b/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h
new file mode 100644
index 0000000000..f7f7629f77
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h
@@ -0,0 +1,71 @@
+//===-- sanitizer_stacktrace_printer.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers' run-time libraries.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_STACKTRACE_PRINTER_H
+#define SANITIZER_STACKTRACE_PRINTER_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_symbolizer.h"
+
+namespace __sanitizer {
+
+// Render the contents of "info" structure, which represents the contents of
+// stack frame "frame_no" and appends it to the "buffer". "format" is a
+// string with placeholders, which is copied to the output with
+// placeholders substituted with the contents of "info". For example,
+// format string
+//   "  frame %n: function %F at %S"
+// will be turned into
+//   "  frame 10: function foo::bar() at my/file.cc:10"
+// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
+// source files and modules, and "strip_func_prefix" to strip prefixes of
+// function names.
+// Here's the full list of available placeholders:
+//   %% - represents a '%' character;
+//   %n - frame number (copy of frame_no);
+//   %p - PC in hex format;
+//   %m - path to module (binary or shared object);
+//   %o - offset in the module in hex format;
+//   %f - function name;
+//   %q - offset in the function in hex format (*if available*);
+//   %s - path to source file;
+//   %l - line in the source file;
+//   %c - column in the source file;
+//   %F - if function is known to be <foo>, prints "in <foo>", possibly
+//        followed by the offset in this function, but only if source file
+//        is unknown;
+//   %S - prints file/line/column information;
+//   %L - prints location information: file/line/column, if it is known, or
+//        module+offset if it is known, or (<unknown module>) string.
+//   %M - prints module basename and offset, if it is known, or PC.
+void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+                 const AddressInfo &info, bool vs_style,
+                 const char *strip_path_prefix = "",
+                 const char *strip_func_prefix = "");
+
+void RenderSourceLocation(InternalScopedString *buffer, const char *file,
+                          int line, int column, bool vs_style,
+                          const char *strip_path_prefix);
+
+void RenderModuleLocation(InternalScopedString *buffer, const char *module,
+                          uptr offset, ModuleArch arch,
+                          const char *strip_path_prefix);
+
+// Same as RenderFrame, but for data section (global variables).
+// Accepts %s, %l from above.
+// Also accepts:
+//   %g - name of the global variable.
+void RenderData(InternalScopedString *buffer, const char *format,
+                const DataInfo *DI, const char *strip_path_prefix = "");
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_STACKTRACE_PRINTER_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld.h b/lib/tsan/sanitizer_common/sanitizer_stoptheworld.h
new file mode 100644
index 0000000000..4e42400571
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stoptheworld.h
@@ -0,0 +1,64 @@
+//===-- sanitizer_stoptheworld.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the StopTheWorld function which suspends the execution of the current
+// process and runs the user-supplied callback in the same address space.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_STOPTHEWORLD_H
+#define SANITIZER_STOPTHEWORLD_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+enum PtraceRegistersStatus {
+  REGISTERS_UNAVAILABLE_FATAL = -1,
+  REGISTERS_UNAVAILABLE = 0,
+  REGISTERS_AVAILABLE = 1
+};
+
+// Holds the list of suspended threads and provides an interface to dump their
+// register contexts.
+class SuspendedThreadsList {
+ public:
+  SuspendedThreadsList() = default;
+
+  // Can't declare pure virtual functions in sanitizer runtimes:
+  // __cxa_pure_virtual might be unavailable. Use UNIMPLEMENTED() instead.
+  virtual PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                                  uptr *sp) const {
+    UNIMPLEMENTED();
+  }
+
+  // The buffer in GetRegistersAndSP should be at least this big.
+  virtual uptr RegisterCount() const { UNIMPLEMENTED(); }
+  virtual uptr ThreadCount() const { UNIMPLEMENTED(); }
+  virtual tid_t GetThreadID(uptr index) const { UNIMPLEMENTED(); }
+
+ private:
+  // Prohibit copy and assign.
+  SuspendedThreadsList(const SuspendedThreadsList&);
+  void operator=(const SuspendedThreadsList&);
+};
+
+typedef void (*StopTheWorldCallback)(
+    const SuspendedThreadsList &suspended_threads_list,
+    void *argument);
+
+// Suspend all threads in the current process and run the callback on the list
+// of suspended threads. This function will resume the threads before returning.
+// The callback should not call any libc functions. The callback must not call
+// exit() nor _exit() and instead return to the caller.
+// This function should NOT be called from multiple threads simultaneously.
+void StopTheWorld(StopTheWorldCallback callback, void *argument);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_STOPTHEWORLD_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.cpp b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.cpp
new file mode 100644
index 0000000000..3a246443ed
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.cpp
@@ -0,0 +1,42 @@
+//===-- sanitizer_stoptheworld_fuchsia.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FUCHSIA
+
+#include <zircon/sanitizer.h>
+
+#include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+
+// The Fuchsia implementation stops the world but doesn't offer a real
+// SuspendedThreadsList argument.  This is enough for ASan's use case,
+// and LSan does not use this API on Fuchsia.
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  struct Params {
+    StopTheWorldCallback callback;
+    void *argument;
+  } params = {callback, argument};
+  __sanitizer_memory_snapshot(
+      nullptr, nullptr, nullptr, nullptr,
+      [](zx_status_t, void *data) {
+        auto params = reinterpret_cast<Params *>(data);
+        params->callback({}, params->argument);
+      },
+      &params);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp
new file mode 100644
index 0000000000..6c577426ad
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp
@@ -0,0 +1,182 @@
+//===-- sanitizer_stoptheworld_mac.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__) || \
+                      defined(__i386))
+
+#include <mach/mach.h>
+#include <mach/thread_info.h>
+#include <pthread.h>
+
+#include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+typedef struct {
+  tid_t tid;
+  thread_t thread;
+} SuspendedThreadInfo;
+
+class SuspendedThreadsListMac : public SuspendedThreadsList {
+ public:
+  SuspendedThreadsListMac() : threads_(1024) {}
+
+  tid_t GetThreadID(uptr index) const;
+  thread_t GetThread(uptr index) const;
+  uptr ThreadCount() const;
+  bool ContainsThread(thread_t thread) const;
+  void Append(thread_t thread);
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                          uptr *sp) const;
+  uptr RegisterCount() const;
+
+ private:
+  InternalMmapVector<SuspendedThreadInfo> threads_;
+};
+
+struct RunThreadArgs {
+  StopTheWorldCallback callback;
+  void *argument;
+};
+
+void *RunThread(void *arg) {
+  struct RunThreadArgs *run_args = (struct RunThreadArgs *)arg;
+  SuspendedThreadsListMac suspended_threads_list;
+
+  thread_array_t threads;
+  mach_msg_type_number_t num_threads;
+  kern_return_t err = task_threads(mach_task_self(), &threads, &num_threads);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Failed to get threads for task (errno %d).\n", err);
+    return nullptr;
+  }
+
+  thread_t thread_self = mach_thread_self();
+  for (unsigned int i = 0; i < num_threads; ++i) {
+    if (threads[i] == thread_self) continue;
+
+    thread_suspend(threads[i]);
+    suspended_threads_list.Append(threads[i]);
+  }
+
+  run_args->callback(suspended_threads_list, run_args->argument);
+
+  uptr num_suspended = suspended_threads_list.ThreadCount();
+  for (unsigned int i = 0; i < num_suspended; ++i) {
+    thread_resume(suspended_threads_list.GetThread(i));
+  }
+  return nullptr;
+}
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  struct RunThreadArgs arg = {callback, argument};
+  pthread_t run_thread = (pthread_t)internal_start_thread(RunThread, &arg);
+  internal_join_thread(run_thread);
+}
+
+#if defined(__x86_64__)
+typedef x86_thread_state64_t regs_struct;
+
+#define SP_REG __rsp
+
+#elif defined(__aarch64__)
+typedef arm_thread_state64_t regs_struct;
+
+# if __DARWIN_UNIX03
+#  define SP_REG __sp
+# else
+#  define SP_REG sp
+# endif
+
+#elif defined(__i386)
+typedef x86_thread_state32_t regs_struct;
+
+#define SP_REG __esp
+
+#else
+#error "Unsupported architecture"
+#endif
+
+tid_t SuspendedThreadsListMac::GetThreadID(uptr index) const {
+  CHECK_LT(index, threads_.size());
+  return threads_[index].tid;
+}
+
+thread_t SuspendedThreadsListMac::GetThread(uptr index) const {
+  CHECK_LT(index, threads_.size());
+  return threads_[index].thread;
+}
+
+uptr SuspendedThreadsListMac::ThreadCount() const {
+  return threads_.size();
+}
+
+bool SuspendedThreadsListMac::ContainsThread(thread_t thread) const {
+  for (uptr i = 0; i < threads_.size(); i++) {
+    if (threads_[i].thread == thread) return true;
+  }
+  return false;
+}
+
+void SuspendedThreadsListMac::Append(thread_t thread) {
+  thread_identifier_info_data_t info;
+  mach_msg_type_number_t info_count = THREAD_IDENTIFIER_INFO_COUNT;
+  kern_return_t err = thread_info(thread, THREAD_IDENTIFIER_INFO,
+                                  (thread_info_t)&info, &info_count);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Error - unable to get thread ident for a thread\n");
+    return;
+  }
+  threads_.push_back({info.thread_id, thread});
+}
+
+PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
+    uptr index, uptr *buffer, uptr *sp) const {
+  thread_t thread = GetThread(index);
+  regs_struct regs;
+  int err;
+  mach_msg_type_number_t reg_count = MACHINE_THREAD_STATE_COUNT;
+  err = thread_get_state(thread, MACHINE_THREAD_STATE, (thread_state_t)&regs,
+                         &reg_count);
+  if (err != KERN_SUCCESS) {
+    VReport(1, "Error - unable to get registers for a thread\n");
+    // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid,
+    // or the thread does not exist. The other possible error case,
+    // MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's
+    // still safe to proceed.
+    return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL
+                                        : REGISTERS_UNAVAILABLE;
+  }
+
+  internal_memcpy(buffer, &regs, sizeof(regs));
+#if defined(__aarch64__) && defined(arm_thread_state64_get_sp)
+  *sp = arm_thread_state64_get_sp(regs);
+#else
+  *sp = regs.SP_REG;
+#endif
+
+  // On x86_64 and aarch64, we must account for the stack redzone, which is 128
+  // bytes.
+  if (SANITIZER_WORDSIZE == 64) *sp -= 128;
+
+  return REGISTERS_AVAILABLE;
+}
+
+uptr SuspendedThreadsListMac::RegisterCount() const {
+  return MACHINE_THREAD_STATE_COUNT;
+}
+} // namespace __sanitizer
+
+#endif  // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) ||
+        //                   defined(__i386))
diff --git a/lib/tsan/sanitizer_common/sanitizer_suppressions.cpp b/lib/tsan/sanitizer_common/sanitizer_suppressions.cpp
new file mode 100644
index 0000000000..44c83a66c5
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_suppressions.cpp
@@ -0,0 +1,181 @@
+//===-- sanitizer_suppressions.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Suppression parsing/matching code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_suppressions.h"
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_file.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+
+namespace __sanitizer {
+
+SuppressionContext::SuppressionContext(const char *suppression_types[],
+                                       int suppression_types_num)
+    : suppression_types_(suppression_types),
+      suppression_types_num_(suppression_types_num),
+      can_parse_(true) {
+  CHECK_LE(suppression_types_num_, kMaxSuppressionTypes);
+  internal_memset(has_suppression_type_, 0, suppression_types_num_);
+}
+
+#if !SANITIZER_FUCHSIA
+static bool GetPathAssumingFileIsRelativeToExec(const char *file_path,
+                                                /*out*/char *new_file_path,
+                                                uptr new_file_path_size) {
+  InternalScopedString exec(kMaxPathLength);
+  if (ReadBinaryNameCached(exec.data(), exec.size())) {
+    const char *file_name_pos = StripModuleName(exec.data());
+    uptr path_to_exec_len = file_name_pos - exec.data();
+    internal_strncat(new_file_path, exec.data(),
+                     Min(path_to_exec_len, new_file_path_size - 1));
+    internal_strncat(new_file_path, file_path,
+                     new_file_path_size - internal_strlen(new_file_path) - 1);
+    return true;
+  }
+  return false;
+}
+
+static const char *FindFile(const char *file_path,
+                            /*out*/char *new_file_path,
+                            uptr new_file_path_size) {
+  // If we cannot find the file, check if its location is relative to
+  // the location of the executable.
+  if (!FileExists(file_path) && !IsAbsolutePath(file_path) &&
+      GetPathAssumingFileIsRelativeToExec(file_path, new_file_path,
+                                          new_file_path_size)) {
+    return new_file_path;
+  }
+  return file_path;
+}
+#else
+static const char *FindFile(const char *file_path, char *, uptr) {
+  return file_path;
+}
+#endif
+
+void SuppressionContext::ParseFromFile(const char *filename) {
+  if (filename[0] == '\0')
+    return;
+
+  InternalScopedString new_file_path(kMaxPathLength);
+  filename = FindFile(filename, new_file_path.data(), new_file_path.size());
+
+  // Read the file.
+  VPrintf(1, "%s: reading suppressions file at %s\n",
+          SanitizerToolName, filename);
+  char *file_contents;
+  uptr buffer_size;
+  uptr contents_size;
+  if (!ReadFileToBuffer(filename, &file_contents, &buffer_size,
+                        &contents_size)) {
+    Printf("%s: failed to read suppressions file '%s'\n", SanitizerToolName,
+           filename);
+    Die();
+  }
+
+  Parse(file_contents);
+}
+
+bool SuppressionContext::Match(const char *str, const char *type,
+                               Suppression **s) {
+  can_parse_ = false;
+  if (!HasSuppressionType(type))
+    return false;
+  for (uptr i = 0; i < suppressions_.size(); i++) {
+    Suppression &cur = suppressions_[i];
+    if (0 == internal_strcmp(cur.type, type) && TemplateMatch(cur.templ, str)) {
+      *s = &cur;
+      return true;
+    }
+  }
+  return false;
+}
+
+static const char *StripPrefix(const char *str, const char *prefix) {
+  while (*str && *str == *prefix) {
+    str++;
+    prefix++;
+  }
+  if (!*prefix)
+    return str;
+  return 0;
+}
+
+void SuppressionContext::Parse(const char *str) {
+  // Context must not mutate once Match has been called.
+  CHECK(can_parse_);
+  const char *line = str;
+  while (line) {
+    while (line[0] == ' ' || line[0] == '\t')
+      line++;
+    const char *end = internal_strchr(line, '\n');
+    if (end == 0)
+      end = line + internal_strlen(line);
+    if (line != end && line[0] != '#') {
+      const char *end2 = end;
+      while (line != end2 &&
+             (end2[-1] == ' ' || end2[-1] == '\t' || end2[-1] == '\r'))
+        end2--;
+      int type;
+      for (type = 0; type < suppression_types_num_; type++) {
+        const char *next_char = StripPrefix(line, suppression_types_[type]);
+        if (next_char && *next_char == ':') {
+          line = ++next_char;
+          break;
+        }
+      }
+      if (type == suppression_types_num_) {
+        Printf("%s: failed to parse suppressions\n", SanitizerToolName);
+        Die();
+      }
+      Suppression s;
+      s.type = suppression_types_[type];
+      s.templ = (char*)InternalAlloc(end2 - line + 1);
+      internal_memcpy(s.templ, line, end2 - line);
+      s.templ[end2 - line] = 0;
+      suppressions_.push_back(s);
+      has_suppression_type_[type] = true;
+    }
+    if (end[0] == 0)
+      break;
+    line = end + 1;
+  }
+}
+
+uptr SuppressionContext::SuppressionCount() const {
+  return suppressions_.size();
+}
+
+bool SuppressionContext::HasSuppressionType(const char *type) const {
+  for (int i = 0; i < suppression_types_num_; i++) {
+    if (0 == internal_strcmp(type, suppression_types_[i]))
+      return has_suppression_type_[i];
+  }
+  return false;
+}
+
+const Suppression *SuppressionContext::SuppressionAt(uptr i) const {
+  CHECK_LT(i, suppressions_.size());
+  return &suppressions_[i];
+}
+
+void SuppressionContext::GetMatched(
+    InternalMmapVector<Suppression *> *matched) {
+  for (uptr i = 0; i < suppressions_.size(); i++)
+    if (atomic_load_relaxed(&suppressions_[i].hit_count))
+      matched->push_back(&suppressions_[i]);
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_suppressions.h b/lib/tsan/sanitizer_common/sanitizer_suppressions.h
new file mode 100644
index 0000000000..2d88b1f72f
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_suppressions.h
@@ -0,0 +1,56 @@
+//===-- sanitizer_suppressions.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Suppression parsing/matching code.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SUPPRESSIONS_H
+#define SANITIZER_SUPPRESSIONS_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+struct Suppression {
+  Suppression() { internal_memset(this, 0, sizeof(*this)); }
+  const char *type;
+  char *templ;
+  atomic_uint32_t hit_count;
+  uptr weight;
+};
+
+class SuppressionContext {
+ public:
+  // Create new SuppressionContext capable of parsing given suppression types.
+  SuppressionContext(const char *supprression_types[],
+                     int suppression_types_num);
+
+  void ParseFromFile(const char *filename);
+  void Parse(const char *str);
+
+  bool Match(const char *str, const char *type, Suppression **s);
+  uptr SuppressionCount() const;
+  bool HasSuppressionType(const char *type) const;
+  const Suppression *SuppressionAt(uptr i) const;
+  void GetMatched(InternalMmapVector<Suppression *> *matched);
+
+ private:
+  static const int kMaxSuppressionTypes = 64;
+  const char **const suppression_types_;
+  const int suppression_types_num_;
+
+  InternalMmapVector<Suppression> suppressions_;
+  bool has_suppression_type_[kMaxSuppressionTypes];
+  bool can_parse_;
+};
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SUPPRESSIONS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer.h
new file mode 100644
index 0000000000..2476b0ea7b
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer.h
@@ -0,0 +1,223 @@
+//===-- sanitizer_symbolizer.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Symbolizer is used by sanitizers to map instruction address to a location in
+// source code at run-time. Symbolizer either uses __sanitizer_symbolize_*
+// defined in the program, or (if they are missing) tries to find and
+// launch "llvm-symbolizer" commandline tool in a separate process and
+// communicate with it.
+//
+// Generally we should try to avoid calling system library functions during
+// symbolization (and use their replacements from sanitizer_libc.h instead).
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SYMBOLIZER_H
+#define SANITIZER_SYMBOLIZER_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_vector.h"
+
+namespace __sanitizer {
+
+struct AddressInfo {
+  // Owns all the string members. Storage for them is
+  // (de)allocated using sanitizer internal allocator.
+  uptr address;
+
+  char *module;
+  uptr module_offset;
+  ModuleArch module_arch;
+
+  static const uptr kUnknown = ~(uptr)0;
+  char *function;
+  uptr function_offset;
+
+  char *file;
+  int line;
+  int column;
+
+  AddressInfo();
+  // Deletes all strings and resets all fields.
+  void Clear();
+  void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
+};
+
+// Linked list of symbolized frames (each frame is described by AddressInfo).
+struct SymbolizedStack {
+  SymbolizedStack *next;
+  AddressInfo info;
+  static SymbolizedStack *New(uptr addr);
+  // Deletes current, and all subsequent frames in the linked list.
+  // The object cannot be accessed after the call to this function.
+  void ClearAll();
+
+ private:
+  SymbolizedStack();
+};
+
+// For now, DataInfo is used to describe global variable.
+struct DataInfo {
+  // Owns all the string members. Storage for them is
+  // (de)allocated using sanitizer internal allocator.
+  char *module;
+  uptr module_offset;
+  ModuleArch module_arch;
+
+  char *file;
+  uptr line;
+  char *name;
+  uptr start;
+  uptr size;
+
+  DataInfo();
+  void Clear();
+};
+
+struct LocalInfo {
+  char *function_name = nullptr;
+  char *name = nullptr;
+  char *decl_file = nullptr;
+  unsigned decl_line = 0;
+
+  bool has_frame_offset = false;
+  bool has_size = false;
+  bool has_tag_offset = false;
+
+  sptr frame_offset;
+  uptr size;
+  uptr tag_offset;
+
+  void Clear();
+};
+
+struct FrameInfo {
+  char *module;
+  uptr module_offset;
+  ModuleArch module_arch;
+
+  InternalMmapVector<LocalInfo> locals;
+  void Clear();
+};
+
+class SymbolizerTool;
+
+class Symbolizer final {
+ public:
+  /// Initialize and return platform-specific implementation of symbolizer
+  /// (if it wasn't already initialized).
+  static Symbolizer *GetOrInit();
+  static void LateInitialize();
+  // Returns a list of symbolized frames for a given address (containing
+  // all inlined functions, if necessary).
+  SymbolizedStack *SymbolizePC(uptr address);
+  bool SymbolizeData(uptr address, DataInfo *info);
+  bool SymbolizeFrame(uptr address, FrameInfo *info);
+
+  // The module names Symbolizer returns are stable and unique for every given
+  // module.  It is safe to store and compare them as pointers.
+  bool GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
+                                   uptr *module_address);
+  const char *GetModuleNameForPc(uptr pc) {
+    const char *module_name = nullptr;
+    uptr unused;
+    if (GetModuleNameAndOffsetForPC(pc, &module_name, &unused))
+      return module_name;
+    return nullptr;
+  }
+
+  // Release internal caches (if any).
+  void Flush();
+  // Attempts to demangle the provided C++ mangled name.
+  const char *Demangle(const char *name);
+
+  // Allow user to install hooks that would be called before/after Symbolizer
+  // does the actual file/line info fetching. Specific sanitizers may need this
+  // to distinguish system library calls made in user code from calls made
+  // during in-process symbolization.
+  typedef void (*StartSymbolizationHook)();
+  typedef void (*EndSymbolizationHook)();
+  // May be called at most once.
+  void AddHooks(StartSymbolizationHook start_hook,
+                EndSymbolizationHook end_hook);
+
+  void RefreshModules();
+  const LoadedModule *FindModuleForAddress(uptr address);
+
+  void InvalidateModuleList();
+
+ private:
+  // GetModuleNameAndOffsetForPC has to return a string to the caller.
+  // Since the corresponding module might get unloaded later, we should create
+  // our owned copies of the strings that we can safely return.
+  // ModuleNameOwner does not provide any synchronization, thus calls to
+  // its method should be protected by |mu_|.
+  class ModuleNameOwner {
+   public:
+    explicit ModuleNameOwner(BlockingMutex *synchronized_by)
+        : last_match_(nullptr), mu_(synchronized_by) {
+      storage_.reserve(kInitialCapacity);
+    }
+    const char *GetOwnedCopy(const char *str);
+
+   private:
+    static const uptr kInitialCapacity = 1000;
+    InternalMmapVector<const char*> storage_;
+    const char *last_match_;
+
+    BlockingMutex *mu_;
+  } module_names_;
+
+  /// Platform-specific function for creating a Symbolizer object.
+  static Symbolizer *PlatformInit();
+
+  bool FindModuleNameAndOffsetForAddress(uptr address, const char **module_name,
+                                         uptr *module_offset,
+                                         ModuleArch *module_arch);
+  ListOfModules modules_;
+  ListOfModules fallback_modules_;
+  // If stale, need to reload the modules before looking up addresses.
+  bool modules_fresh_;
+
+  // Platform-specific default demangler, must not return nullptr.
+  const char *PlatformDemangle(const char *name);
+
+  static Symbolizer *symbolizer_;
+  static StaticSpinMutex init_mu_;
+
+  // Mutex locked from public methods of |Symbolizer|, so that the internals
+  // (including individual symbolizer tools and platform-specific methods) are
+  // always synchronized.
+  BlockingMutex mu_;
+
+  IntrusiveList<SymbolizerTool> tools_;
+
+  explicit Symbolizer(IntrusiveList<SymbolizerTool> tools);
+
+  static LowLevelAllocator symbolizer_allocator_;
+
+  StartSymbolizationHook start_hook_;
+  EndSymbolizationHook end_hook_;
+  class SymbolizerScope {
+   public:
+    explicit SymbolizerScope(const Symbolizer *sym);
+    ~SymbolizerScope();
+   private:
+    const Symbolizer *sym_;
+  };
+
+  // Calls `LateInitialize()` on all items in `tools_`.
+  void LateInitializeTools();
+};
+
+#ifdef SANITIZER_WINDOWS
+void InitializeDbgHelpIfNeeded();
+#endif
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h
new file mode 100644
index 0000000000..c4061e38c6
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_fuchsia.h
@@ -0,0 +1,42 @@
+//===-- sanitizer_symbolizer_fuchsia.h -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Define Fuchsia's string formats and limits for the markup symbolizer.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SYMBOLIZER_FUCHSIA_H
+#define SANITIZER_SYMBOLIZER_FUCHSIA_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+// See the spec at:
+// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
+constexpr uptr kFormatDemangleMax = 1024;  // Arbitrary.
+
+// Function name or equivalent from PC location.
+constexpr const char *kFormatFunction = "{{{pc:%p}}}";
+constexpr uptr kFormatFunctionMax = 64;  // More than big enough for 64-bit hex.
+
+// Global variable name or equivalent from data memory address.
+constexpr const char *kFormatData = "{{{data:%p}}}";
+
+// One frame in a backtrace (printed on a line by itself).
+constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}";
+
+// Dump trigger element.
+#define FORMAT_DUMPFILE "{{{dumpfile:%s:%s}}}"
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_FUCHSIA_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h
new file mode 100644
index 0000000000..e4c351e667
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h
@@ -0,0 +1,165 @@
+//===-- sanitizer_symbolizer_internal.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Header for internal classes and functions to be used by implementations of
+// symbolizers.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SYMBOLIZER_INTERNAL_H
+#define SANITIZER_SYMBOLIZER_INTERNAL_H
+
+#include "sanitizer_symbolizer.h"
+#include "sanitizer_file.h"
+#include "sanitizer_vector.h"
+
+namespace __sanitizer {
+
+// Parsing helpers, 'str' is searched for delimiter(s) and a string or uptr
+// is extracted. When extracting a string, a newly allocated (using
+// InternalAlloc) and null-terminataed buffer is returned. They return a pointer
+// to the next characted after the found delimiter.
+const char *ExtractToken(const char *str, const char *delims, char **result);
+const char *ExtractInt(const char *str, const char *delims, int *result);
+const char *ExtractUptr(const char *str, const char *delims, uptr *result);
+const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter,
+                                      char **result);
+
+const char *DemangleSwiftAndCXX(const char *name);
+
+// SymbolizerTool is an interface that is implemented by individual "tools"
+// that can perform symbolication (external llvm-symbolizer, libbacktrace,
+// Windows DbgHelp symbolizer, etc.).
+class SymbolizerTool {
+ public:
+  // The main |Symbolizer| class implements a "fallback chain" of symbolizer
+  // tools. In a request to symbolize an address, if one tool returns false,
+  // the next tool in the chain will be tried.
+  SymbolizerTool *next;
+
+  SymbolizerTool() : next(nullptr) { }
+
+  // Can't declare pure virtual functions in sanitizer runtimes:
+  // __cxa_pure_virtual might be unavailable.
+
+  // The |stack| parameter is inout. It is pre-filled with the address,
+  // module base and module offset values and is to be used to construct
+  // other stack frames.
+  virtual bool SymbolizePC(uptr addr, SymbolizedStack *stack) {
+    UNIMPLEMENTED();
+  }
+
+  // The |info| parameter is inout. It is pre-filled with the module base
+  // and module offset values.
+  virtual bool SymbolizeData(uptr addr, DataInfo *info) {
+    UNIMPLEMENTED();
+  }
+
+  virtual bool SymbolizeFrame(uptr addr, FrameInfo *info) {
+    return false;
+  }
+
+  virtual void Flush() {}
+
+  // Return nullptr to fallback to the default platform-specific demangler.
+  virtual const char *Demangle(const char *name) {
+    return nullptr;
+  }
+
+  // Called during the LateInitialize phase of Sanitizer initialization.
+  // Usually this is a safe place to call code that might need to use user
+  // memory allocators.
+  virtual void LateInitialize() {}
+};
+
+// SymbolizerProcess encapsulates communication between the tool and
+// external symbolizer program, running in a different subprocess.
+// SymbolizerProcess may not be used from two threads simultaneously.
+class SymbolizerProcess {
+ public:
+  explicit SymbolizerProcess(const char *path, bool use_posix_spawn = false);
+  const char *SendCommand(const char *command);
+
+ protected:
+  /// The maximum number of arguments required to invoke a tool process.
+  static const unsigned kArgVMax = 6;
+
+  // Customizable by subclasses.
+  virtual bool StartSymbolizerSubprocess();
+  virtual bool ReadFromSymbolizer(char *buffer, uptr max_length);
+  // Return the environment to run the symbolizer in.
+  virtual char **GetEnvP() { return GetEnviron(); }
+
+ private:
+  virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const {
+    UNIMPLEMENTED();
+  }
+
+  /// Fill in an argv array to invoke the child process.
+  virtual void GetArgV(const char *path_to_binary,
+                       const char *(&argv)[kArgVMax]) const {
+    UNIMPLEMENTED();
+  }
+
+  bool Restart();
+  const char *SendCommandImpl(const char *command);
+  bool WriteToSymbolizer(const char *buffer, uptr length);
+
+  const char *path_;
+  fd_t input_fd_;
+  fd_t output_fd_;
+
+  static const uptr kBufferSize = 16 * 1024;
+  char buffer_[kBufferSize];
+
+  static const uptr kMaxTimesRestarted = 5;
+  static const int kSymbolizerStartupTimeMillis = 10;
+  uptr times_restarted_;
+  bool failed_to_start_;
+  bool reported_invalid_path_;
+  bool use_posix_spawn_;
+};
+
+class LLVMSymbolizerProcess;
+
+// This tool invokes llvm-symbolizer in a subprocess. It should be as portable
+// as the llvm-symbolizer tool is.
+class LLVMSymbolizer : public SymbolizerTool {
+ public:
+  explicit LLVMSymbolizer(const char *path, LowLevelAllocator *allocator);
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
+  bool SymbolizeData(uptr addr, DataInfo *info) override;
+  bool SymbolizeFrame(uptr addr, FrameInfo *info) override;
+
+ private:
+  const char *FormatAndSendCommand(const char *command_prefix,
+                                   const char *module_name, uptr module_offset,
+                                   ModuleArch arch);
+
+  LLVMSymbolizerProcess *symbolizer_process_;
+  static const uptr kBufferSize = 16 * 1024;
+  char buffer_[kBufferSize];
+};
+
+// Parses one or more two-line strings in the following format:
+//   <function_name>
+//   <file_name>:<line_number>[:<column_number>]
+// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of
+// them use the same output format.  Returns true if any useful debug
+// information was found.
+void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res);
+
+// Parses a two-line string in the following format:
+//   <symbol_name>
+//   <start_address> <size>
+// Used by LLVMSymbolizer and InternalSymbolizer.
+void ParseSymbolizeDataOutput(const char *str, DataInfo *info);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_INTERNAL_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.h
new file mode 100644
index 0000000000..e2a0f71420
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.h
@@ -0,0 +1,49 @@
+//===-- sanitizer_symbolizer_libbacktrace.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// Header for libbacktrace symbolizer.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SYMBOLIZER_LIBBACKTRACE_H
+#define SANITIZER_SYMBOLIZER_LIBBACKTRACE_H
+
+#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_symbolizer_internal.h"
+
+#ifndef SANITIZER_LIBBACKTRACE
+# define SANITIZER_LIBBACKTRACE 0
+#endif
+
+#ifndef SANITIZER_CP_DEMANGLE
+# define SANITIZER_CP_DEMANGLE 0
+#endif
+
+namespace __sanitizer {
+
+class LibbacktraceSymbolizer : public SymbolizerTool {
+ public:
+  static LibbacktraceSymbolizer *get(LowLevelAllocator *alloc);
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
+
+  bool SymbolizeData(uptr addr, DataInfo *info) override;
+
+  // May return NULL if demangling failed.
+  const char *Demangle(const char *name) override;
+
+ private:
+  explicit LibbacktraceSymbolizer(void *state) : state_(state) {}
+
+  void *state_;  // Leaked.
+};
+
+}  // namespace __sanitizer
+#endif  // SANITIZER_SYMBOLIZER_LIBBACKTRACE_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h
new file mode 100644
index 0000000000..8996131fc1
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h
@@ -0,0 +1,48 @@
+//===-- sanitizer_symbolizer_mac.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Header for Mac-specific "atos" symbolizer.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_SYMBOLIZER_MAC_H
+#define SANITIZER_SYMBOLIZER_MAC_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_symbolizer_internal.h"
+
+namespace __sanitizer {
+
+class DlAddrSymbolizer : public SymbolizerTool {
+ public:
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
+  bool SymbolizeData(uptr addr, DataInfo *info) override;
+};
+
+class AtosSymbolizerProcess;
+
+class AtosSymbolizer : public SymbolizerTool {
+ public:
+  explicit AtosSymbolizer(const char *path, LowLevelAllocator *allocator);
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
+  bool SymbolizeData(uptr addr, DataInfo *info) override;
+  void LateInitialize() override;
+
+ private:
+  AtosSymbolizerProcess *process_;
+};
+
+} // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
+
+#endif // SANITIZER_SYMBOLIZER_MAC_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_rtems.h b/lib/tsan/sanitizer_common/sanitizer_symbolizer_rtems.h
new file mode 100644
index 0000000000..3371092e06
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_rtems.h
@@ -0,0 +1,40 @@
+//===-- sanitizer_symbolizer_rtems.h -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Define RTEMS's string formats and limits for the markup symbolizer.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SYMBOLIZER_RTEMS_H
+#define SANITIZER_SYMBOLIZER_RTEMS_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+// The Myriad RTEMS symbolizer currently only parses backtrace lines,
+// so use a format that the symbolizer understands.  For other
+// markups, keep them the same as the Fuchsia's.
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
+constexpr uptr kFormatDemangleMax = 1024;  // Arbitrary.
+
+// Function name or equivalent from PC location.
+constexpr const char *kFormatFunction = "{{{pc:%p}}}";
+constexpr uptr kFormatFunctionMax = 64;  // More than big enough for 64-bit hex.
+
+// Global variable name or equivalent from data memory address.
+constexpr const char *kFormatData = "{{{data:%p}}}";
+
+// One frame in a backtrace (printed on a line by itself).
+constexpr const char *kFormatFrame = "    [%u] IP: %p";
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_RTEMS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc b/lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc
new file mode 100644
index 0000000000..a43ce3efab
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc
@@ -0,0 +1,38 @@
+//===-- sanitizer_syscall_generic.inc ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic implementations of internal_syscall* and internal_iserror.
+//
+//===----------------------------------------------------------------------===//
+
+// NetBSD uses libc calls directly
+#if !SANITIZER_NETBSD
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_SOLARIS
+# define SYSCALL(name) SYS_ ## name
+#else
+# define SYSCALL(name) __NR_ ## name
+#endif
+
+#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC)
+# define internal_syscall __syscall
+# else
+# define internal_syscall syscall
+#endif
+
+#endif
+
+bool internal_iserror(uptr retval, int *rverrno) {
+  if (retval == (uptr)-1) {
+    if (rverrno)
+      *rverrno = errno;
+    return true;
+  } else {
+    return false;
+  }
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
new file mode 100644
index 0000000000..56c5e99220
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
@@ -0,0 +1,137 @@
+//===-- sanitizer_syscall_linux_aarch64.inc --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/aarch64.
+//
+//===----------------------------------------------------------------------===//
+
+#define SYSCALL(name) __NR_ ## name
+
+static uptr __internal_syscall(u64 nr) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0");
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall0(n) \
+  (__internal_syscall)(n)
+
+static uptr __internal_syscall(u64 nr, u64 arg1) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall1(n, a1) \
+  (__internal_syscall)(n, (u64)(a1))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  register u64 x1 asm("x1") = arg2;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0), "r"(x1)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall2(n, a1, a2) \
+  (__internal_syscall)(n, (u64)(a1), (long)(a2))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  register u64 x1 asm("x1") = arg2;
+  register u64 x2 asm("x2") = arg3;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0), "r"(x1), "r"(x2)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall3(n, a1, a2, a3) \
+  (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
+                               u64 arg4) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  register u64 x1 asm("x1") = arg2;
+  register u64 x2 asm("x2") = arg3;
+  register u64 x3 asm("x3") = arg4;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+  (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
+                               u64 arg4, long arg5) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  register u64 x1 asm("x1") = arg2;
+  register u64 x2 asm("x2") = arg3;
+  register u64 x3 asm("x3") = arg4;
+  register u64 x4 asm("x4") = arg5;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+  (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+                       (u64)(a5))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
+                               u64 arg4, long arg5, long arg6) {
+  register u64 x8 asm("x8") = nr;
+  register u64 x0 asm("x0") = arg1;
+  register u64 x1 asm("x1") = arg2;
+  register u64 x2 asm("x2") = arg3;
+  register u64 x3 asm("x3") = arg4;
+  register u64 x4 asm("x4") = arg5;
+  register u64 x5 asm("x5") = arg6;
+  asm volatile("svc 0"
+               : "=r"(x0)
+               : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x5)
+               : "memory", "cc");
+  return x0;
+}
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+  (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+                       (u64)(a5), (long)(a6))
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+  __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+  __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+// Helper function used to avoid cobbler errno.
+bool internal_iserror(uptr retval, int *rverrno) {
+  if (retval >= (uptr)-4095) {
+    if (rverrno)
+      *rverrno = -retval;
+    return true;
+  }
+  return false;
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_syscall_linux_arm.inc b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_arm.inc
new file mode 100644
index 0000000000..121a9445b4
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_arm.inc
@@ -0,0 +1,137 @@
+//===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/arm.
+//
+//===----------------------------------------------------------------------===//
+
+#define SYSCALL(name) __NR_ ## name
+
+static uptr __internal_syscall(u32 nr) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0");
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall0(n) \
+  (__internal_syscall)(n)
+
+static uptr __internal_syscall(u32 nr, u32 arg1) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall1(n, a1) \
+  (__internal_syscall)(n, (u32)(a1))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  register u32 r1 asm("r1") = arg2;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0), "r"(r1)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall2(n, a1, a2) \
+  (__internal_syscall)(n, (u32)(a1), (long)(a2))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  register u32 r1 asm("r1") = arg2;
+  register u32 r2 asm("r2") = arg3;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0), "r"(r1), "r"(r2)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall3(n, a1, a2, a3) \
+  (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+                               u32 arg4) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  register u32 r1 asm("r1") = arg2;
+  register u32 r2 asm("r2") = arg3;
+  register u32 r3 asm("r3") = arg4;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+  (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+                               u32 arg4, long arg5) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  register u32 r1 asm("r1") = arg2;
+  register u32 r2 asm("r2") = arg3;
+  register u32 r3 asm("r3") = arg4;
+  register u32 r4 asm("r4") = arg5;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+  (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \
+                       (u32)(a5))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+                               u32 arg4, long arg5, long arg6) {
+  register u32 r8 asm("r7") = nr;
+  register u32 r0 asm("r0") = arg1;
+  register u32 r1 asm("r1") = arg2;
+  register u32 r2 asm("r2") = arg3;
+  register u32 r3 asm("r3") = arg4;
+  register u32 r4 asm("r4") = arg5;
+  register u32 r5 asm("r5") = arg6;
+  asm volatile("swi #0"
+               : "=r"(r0)
+               : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5)
+               : "memory", "cc");
+  return r0;
+}
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+  (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \
+                       (u32)(a5), (long)(a6))
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+  __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+  __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+// Helper function used to avoid cobbler errno.
+bool internal_iserror(uptr retval, int *rverrno) {
+  if (retval >= (uptr)-4095) {
+    if (rverrno)
+      *rverrno = -retval;
+    return true;
+  }
+  return false;
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
new file mode 100644
index 0000000000..67e8686d12
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
@@ -0,0 +1,90 @@
+//===-- sanitizer_syscall_linux_x86_64.inc ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/x86_64.
+//
+//===----------------------------------------------------------------------===//
+
+#define SYSCALL(name) __NR_ ## name
+
+static uptr internal_syscall(u64 nr) {
+  u64 retval;
+  asm volatile("syscall" : "=a"(retval) : "a"(nr) : "rcx", "r11",
+               "memory", "cc");
+  return retval;
+}
+
+template <typename T1>
+static uptr internal_syscall(u64 nr, T1 arg1) {
+  u64 retval;
+  asm volatile("syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1) :
+               "rcx", "r11", "memory", "cc");
+  return retval;
+}
+
+template <typename T1, typename T2>
+static uptr internal_syscall(u64 nr, T1 arg1, T2 arg2) {
+  u64 retval;
+  asm volatile("syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1),
+               "S"((u64)arg2) : "rcx", "r11", "memory", "cc");
+  return retval;
+}
+
+template <typename T1, typename T2, typename T3>
+static uptr internal_syscall(u64 nr, T1 arg1, T2 arg2, T3 arg3) {
+  u64 retval;
+  asm volatile("syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1),
+               "S"((u64)arg2), "d"((u64)arg3) : "rcx", "r11", "memory", "cc");
+  return retval;
+}
+
+template <typename T1, typename T2, typename T3, typename T4>
+static uptr internal_syscall(u64 nr, T1 arg1, T2 arg2, T3 arg3, T4 arg4) {
+  u64 retval;
+  asm volatile("mov %5, %%r10;"
+               "syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1),
+               "S"((u64)arg2), "d"((u64)arg3), "r"((u64)arg4) :
+               "rcx", "r11", "r10", "memory", "cc");
+  return retval;
+}
+
+template <typename T1, typename T2, typename T3, typename T4, typename T5>
+static uptr internal_syscall(u64 nr, T1 arg1, T2 arg2, T3 arg3, T4 arg4,
+                             T5 arg5) {
+  u64 retval;
+  asm volatile("mov %5, %%r10;"
+               "mov %6, %%r8;"
+               "syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1),
+               "S"((u64)arg2), "d"((u64)arg3), "r"((u64)arg4), "r"((u64)arg5) :
+               "rcx", "r11", "r10", "r8", "memory", "cc");
+  return retval;
+}
+
+template <typename T1, typename T2, typename T3, typename T4, typename T5,
+          typename T6>
+static uptr internal_syscall(u64 nr, T1 arg1, T2 arg2, T3 arg3, T4 arg4,
+                             T5 arg5, T6 arg6) {
+  u64 retval;
+  asm volatile("mov %5, %%r10;"
+               "mov %6, %%r8;"
+               "mov %7, %%r9;"
+               "syscall" : "=a"(retval) : "a"(nr), "D"((u64)arg1),
+               "S"((u64)arg2), "d"((u64)arg3), "r"((u64)arg4), "r"((u64)arg5),
+               "r"((u64)arg6) : "rcx", "r11", "r10", "r8", "r9",
+               "memory", "cc");
+  return retval;
+}
+
+bool internal_iserror(uptr retval, int *rverrno) {
+  if (retval >= (uptr)-4095) {
+    if (rverrno)
+      *rverrno = -retval;
+    return true;
+  }
+  return false;
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc b/lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc
new file mode 100644
index 0000000000..02b7e11b16
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc
@@ -0,0 +1,3837 @@
+//===-- sanitizer_syscalls_netbsd.inc ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common syscalls handlers for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// This file should be included into the tool's interceptor file,
+// which has to define it's own macros:
+//   COMMON_SYSCALL_PRE_READ_RANGE
+//          Called in prehook for regions that will be read by the kernel and
+//          must be initialized.
+//   COMMON_SYSCALL_PRE_WRITE_RANGE
+//          Called in prehook for regions that will be written to by the kernel
+//          and must be addressable. The actual write range may be smaller than
+//          reported in the prehook. See POST_WRITE_RANGE.
+//   COMMON_SYSCALL_POST_READ_RANGE
+//          Called in posthook for regions that were read by the kernel. Does
+//          not make much sense.
+//   COMMON_SYSCALL_POST_WRITE_RANGE
+//          Called in posthook for regions that were written to by the kernel
+//          and are now initialized.
+//   COMMON_SYSCALL_ACQUIRE(addr)
+//          Acquire memory visibility from addr.
+//   COMMON_SYSCALL_RELEASE(addr)
+//          Release memory visibility to addr.
+//   COMMON_SYSCALL_FD_CLOSE(fd)
+//          Called before closing file descriptor fd.
+//   COMMON_SYSCALL_FD_ACQUIRE(fd)
+//          Acquire memory visibility from fd.
+//   COMMON_SYSCALL_FD_RELEASE(fd)
+//          Release memory visibility to fd.
+//   COMMON_SYSCALL_PRE_FORK()
+//          Called before fork syscall.
+//   COMMON_SYSCALL_POST_FORK(long long res)
+//          Called after fork syscall.
+//
+// DO NOT EDIT! THIS FILE HAS BEEN GENERATED!
+//
+// Generated with: generate_netbsd_syscalls.awk
+// Generated date: 2019-12-24
+// Generated from: syscalls.master,v 1.296 2019/09/22 22:59:39 christos Exp
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_NETBSD
+
+#include "sanitizer_libc.h"
+
+#define PRE_SYSCALL(name)                                                      \
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
+#define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
+#define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
+
+#define POST_SYSCALL(name)                                                     \
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
+#define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
+#define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
+
+#ifndef COMMON_SYSCALL_ACQUIRE
+#define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
+#endif
+
+#ifndef COMMON_SYSCALL_RELEASE
+#define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_CLOSE
+#define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_ACQUIRE
+#define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_FD_RELEASE
+#define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
+#endif
+
+#ifndef COMMON_SYSCALL_PRE_FORK
+#define COMMON_SYSCALL_PRE_FORK()                                              \
+  {}
+#endif
+
+#ifndef COMMON_SYSCALL_POST_FORK
+#define COMMON_SYSCALL_POST_FORK(res)                                          \
+  {}
+#endif
+
+// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
+
+extern "C" {
+#define SYS_MAXSYSARGS 8
+PRE_SYSCALL(syscall)(long long code_, long long args_[SYS_MAXSYSARGS]) {
+  /* Nothing to do */
+}
+POST_SYSCALL(syscall)
+(long long res, long long code_, long long args_[SYS_MAXSYSARGS]) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(exit)(long long rval_) { /* Nothing to do */ }
+POST_SYSCALL(exit)(long long res, long long rval_) { /* Nothing to do */ }
+PRE_SYSCALL(fork)(void) { COMMON_SYSCALL_PRE_FORK(); }
+POST_SYSCALL(fork)(long long res) { COMMON_SYSCALL_POST_FORK(res); }
+PRE_SYSCALL(read)(long long fd_, void *buf_, long long nbyte_) {
+  if (buf_) {
+    PRE_WRITE(buf_, nbyte_);
+  }
+}
+POST_SYSCALL(read)(long long res, long long fd_, void *buf_, long long nbyte_) {
+  if (res > 0) {
+    POST_WRITE(buf_, res);
+  }
+}
+PRE_SYSCALL(write)(long long fd_, void *buf_, long long nbyte_) {
+  if (buf_) {
+    PRE_READ(buf_, nbyte_);
+  }
+}
+POST_SYSCALL(write)
+(long long res, long long fd_, void *buf_, long long nbyte_) {
+  if (res > 0) {
+    POST_READ(buf_, res);
+  }
+}
+PRE_SYSCALL(open)(void *path_, long long flags_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(open)
+(long long res, void *path_, long long flags_, long long mode_) {
+  if (res > 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(close)(long long fd_) { COMMON_SYSCALL_FD_CLOSE((int)fd_); }
+POST_SYSCALL(close)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(compat_50_wait4)
+(long long pid_, void *status_, long long options_, void *rusage_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_wait4)
+(long long res, long long pid_, void *status_, long long options_,
+  void *rusage_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ocreat)(void *path_, long long mode_) { /* TODO */ }
+POST_SYSCALL(compat_43_ocreat)(long long res, void *path_, long long mode_) {
+  /* TODO */
+}
+PRE_SYSCALL(link)(void *path_, void *link_) {
+  const char *path = (const char *)path_;
+  const char *link = (const char *)link_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (link) {
+    PRE_READ(path, __sanitizer::internal_strlen(link) + 1);
+  }
+}
+POST_SYSCALL(link)(long long res, void *path_, void *link_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    const char *link = (const char *)link_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+    if (link) {
+      POST_READ(path, __sanitizer::internal_strlen(link) + 1);
+    }
+  }
+}
+PRE_SYSCALL(unlink)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(unlink)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+/* syscall 11 has been skipped */
+PRE_SYSCALL(chdir)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(chdir)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fchdir)(long long fd_) { /* Nothing to do */ }
+POST_SYSCALL(fchdir)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(compat_50_mknod)(void *path_, long long mode_, long long dev_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_mknod)
+(long long res, void *path_, long long mode_, long long dev_) {
+  /* TODO */
+}
+PRE_SYSCALL(chmod)(void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(chmod)(long long res, void *path_, long long mode_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(chown)(void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(chown)
+(long long res, void *path_, long long uid_, long long gid_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(break)(void *nsize_) { /* Nothing to do */ }
+POST_SYSCALL(break)(long long res, void *nsize_) { /* Nothing to do */ }
+PRE_SYSCALL(compat_20_getfsstat)
+(void *buf_, long long bufsize_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_20_getfsstat)
+(long long res, void *buf_, long long bufsize_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_olseek)
+(long long fd_, long long offset_, long long whence_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_olseek)
+(long long res, long long fd_, long long offset_, long long whence_) {
+  /* TODO */
+}
+PRE_SYSCALL(getpid)(void) { /* Nothing to do */ }
+POST_SYSCALL(getpid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_40_mount)
+(void *type_, void *path_, long long flags_, void *data_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_40_mount)
+(long long res, void *type_, void *path_, long long flags_, void *data_) {
+  /* TODO */
+}
+PRE_SYSCALL(unmount)(void *path_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(unmount)(long long res, void *path_, long long flags_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(setuid)(long long uid_) { /* Nothing to do */ }
+POST_SYSCALL(setuid)(long long res, long long uid_) { /* Nothing to do */ }
+PRE_SYSCALL(getuid)(void) { /* Nothing to do */ }
+POST_SYSCALL(getuid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(geteuid)(void) { /* Nothing to do */ }
+POST_SYSCALL(geteuid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(ptrace)
+(long long req_, long long pid_, void *addr_, long long data_) {
+  if (req_ == ptrace_pt_io) {
+    struct __sanitizer_ptrace_io_desc *addr =
+        (struct __sanitizer_ptrace_io_desc *)addr_;
+    PRE_READ(addr, struct_ptrace_ptrace_io_desc_struct_sz);
+    if (addr->piod_op == ptrace_piod_write_d ||
+        addr->piod_op == ptrace_piod_write_i) {
+      PRE_READ(addr->piod_addr, addr->piod_len);
+    }
+    if (addr->piod_op == ptrace_piod_read_d ||
+        addr->piod_op == ptrace_piod_read_i ||
+        addr->piod_op == ptrace_piod_read_auxv) {
+      PRE_WRITE(addr->piod_addr, addr->piod_len);
+    }
+  } else if (req_ == ptrace_pt_lwpinfo) {
+    struct __sanitizer_ptrace_lwpinfo *addr =
+        (struct __sanitizer_ptrace_lwpinfo *)addr_;
+    PRE_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+    PRE_WRITE(addr, struct_ptrace_ptrace_lwpinfo_struct_sz);
+  } else if (req_ == ptrace_pt_set_event_mask) {
+    PRE_READ(addr_, struct_ptrace_ptrace_event_struct_sz);
+  } else if (req_ == ptrace_pt_get_event_mask) {
+    PRE_WRITE(addr_, struct_ptrace_ptrace_event_struct_sz);
+  } else if (req_ == ptrace_pt_set_siginfo) {
+    PRE_READ(addr_, struct_ptrace_ptrace_siginfo_struct_sz);
+  } else if (req_ == ptrace_pt_get_siginfo) {
+    PRE_WRITE(addr_, struct_ptrace_ptrace_siginfo_struct_sz);
+  } else if (req_ == ptrace_pt_lwpstatus) {
+    struct __sanitizer_ptrace_lwpstatus *addr =
+        (struct __sanitizer_ptrace_lwpstatus *)addr_;
+    PRE_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+    PRE_WRITE(addr, struct_ptrace_ptrace_lwpstatus_struct_sz);
+  } else if (req_ == ptrace_pt_lwpnext) {
+    struct __sanitizer_ptrace_lwpstatus *addr =
+        (struct __sanitizer_ptrace_lwpstatus *)addr_;
+    PRE_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+    PRE_WRITE(addr, struct_ptrace_ptrace_lwpstatus_struct_sz);
+  } else if (req_ == ptrace_pt_setregs) {
+    PRE_READ(addr_, struct_ptrace_reg_struct_sz);
+  } else if (req_ == ptrace_pt_getregs) {
+    PRE_WRITE(addr_, struct_ptrace_reg_struct_sz);
+  } else if (req_ == ptrace_pt_setfpregs) {
+    PRE_READ(addr_, struct_ptrace_fpreg_struct_sz);
+  } else if (req_ == ptrace_pt_getfpregs) {
+    PRE_WRITE(addr_, struct_ptrace_fpreg_struct_sz);
+  } else if (req_ == ptrace_pt_setdbregs) {
+    PRE_READ(addr_, struct_ptrace_dbreg_struct_sz);
+  } else if (req_ == ptrace_pt_getdbregs) {
+    PRE_WRITE(addr_, struct_ptrace_dbreg_struct_sz);
+  }
+}
+POST_SYSCALL(ptrace)
+(long long res, long long req_, long long pid_, void *addr_, long long data_) {
+  if (res == 0) {
+    if (req_ == ptrace_pt_io) {
+      struct __sanitizer_ptrace_io_desc *addr =
+          (struct __sanitizer_ptrace_io_desc *)addr_;
+      POST_READ(addr, struct_ptrace_ptrace_io_desc_struct_sz);
+      if (addr->piod_op == ptrace_piod_write_d ||
+          addr->piod_op == ptrace_piod_write_i) {
+        POST_READ(addr->piod_addr, addr->piod_len);
+      }
+      if (addr->piod_op == ptrace_piod_read_d ||
+          addr->piod_op == ptrace_piod_read_i ||
+          addr->piod_op == ptrace_piod_read_auxv) {
+        POST_WRITE(addr->piod_addr, addr->piod_len);
+      }
+    } else if (req_ == ptrace_pt_lwpinfo) {
+      struct __sanitizer_ptrace_lwpinfo *addr =
+          (struct __sanitizer_ptrace_lwpinfo *)addr_;
+      POST_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+      POST_WRITE(addr, struct_ptrace_ptrace_lwpinfo_struct_sz);
+    } else if (req_ == ptrace_pt_set_event_mask) {
+      POST_READ(addr_, struct_ptrace_ptrace_event_struct_sz);
+    } else if (req_ == ptrace_pt_get_event_mask) {
+      POST_WRITE(addr_, struct_ptrace_ptrace_event_struct_sz);
+    } else if (req_ == ptrace_pt_set_siginfo) {
+      POST_READ(addr_, struct_ptrace_ptrace_siginfo_struct_sz);
+    } else if (req_ == ptrace_pt_get_siginfo) {
+      POST_WRITE(addr_, struct_ptrace_ptrace_siginfo_struct_sz);
+    } else if (req_ == ptrace_pt_lwpstatus) {
+      struct __sanitizer_ptrace_lwpstatus *addr =
+          (struct __sanitizer_ptrace_lwpstatus *)addr_;
+      POST_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+      POST_WRITE(addr, struct_ptrace_ptrace_lwpstatus_struct_sz);
+    } else if (req_ == ptrace_pt_lwpnext) {
+      struct __sanitizer_ptrace_lwpstatus *addr =
+          (struct __sanitizer_ptrace_lwpstatus *)addr_;
+      POST_READ(&addr->pl_lwpid, sizeof(__sanitizer_lwpid_t));
+      POST_WRITE(addr, struct_ptrace_ptrace_lwpstatus_struct_sz);
+    } else if (req_ == ptrace_pt_setregs) {
+      POST_READ(addr_, struct_ptrace_reg_struct_sz);
+    } else if (req_ == ptrace_pt_getregs) {
+      POST_WRITE(addr_, struct_ptrace_reg_struct_sz);
+    } else if (req_ == ptrace_pt_setfpregs) {
+      POST_READ(addr_, struct_ptrace_fpreg_struct_sz);
+    } else if (req_ == ptrace_pt_getfpregs) {
+      POST_WRITE(addr_, struct_ptrace_fpreg_struct_sz);
+    } else if (req_ == ptrace_pt_setdbregs) {
+      POST_READ(addr_, struct_ptrace_dbreg_struct_sz);
+    } else if (req_ == ptrace_pt_getdbregs) {
+      POST_WRITE(addr_, struct_ptrace_dbreg_struct_sz);
+    }
+  }
+}
+PRE_SYSCALL(recvmsg)(long long s_, void *msg_, long long flags_) {
+  PRE_WRITE(msg_, sizeof(__sanitizer_msghdr));
+}
+POST_SYSCALL(recvmsg)
+(long long res, long long s_, void *msg_, long long flags_) {
+  if (res > 0) {
+    POST_WRITE(msg_, sizeof(__sanitizer_msghdr));
+  }
+}
+PRE_SYSCALL(sendmsg)(long long s_, void *msg_, long long flags_) {
+  PRE_READ(msg_, sizeof(__sanitizer_msghdr));
+}
+POST_SYSCALL(sendmsg)
+(long long res, long long s_, void *msg_, long long flags_) {
+  if (res > 0) {
+    POST_READ(msg_, sizeof(__sanitizer_msghdr));
+  }
+}
+PRE_SYSCALL(recvfrom)
+(long long s_, void *buf_, long long len_, long long flags_, void *from_,
+  void *fromlenaddr_) {
+  PRE_WRITE(buf_, len_);
+  PRE_WRITE(from_, struct_sockaddr_sz);
+  PRE_WRITE(fromlenaddr_, sizeof(__sanitizer_socklen_t));
+}
+POST_SYSCALL(recvfrom)
+(long long res, long long s_, void *buf_, long long len_, long long flags_,
+  void *from_, void *fromlenaddr_) {
+  if (res >= 0) {
+    POST_WRITE(buf_, res);
+    POST_WRITE(from_, struct_sockaddr_sz);
+    POST_WRITE(fromlenaddr_, sizeof(__sanitizer_socklen_t));
+  }
+}
+PRE_SYSCALL(accept)(long long s_, void *name_, void *anamelen_) {
+  PRE_WRITE(name_, struct_sockaddr_sz);
+  PRE_WRITE(anamelen_, sizeof(__sanitizer_socklen_t));
+}
+POST_SYSCALL(accept)
+(long long res, long long s_, void *name_, void *anamelen_) {
+  if (res == 0) {
+    POST_WRITE(name_, struct_sockaddr_sz);
+    POST_WRITE(anamelen_, sizeof(__sanitizer_socklen_t));
+  }
+}
+PRE_SYSCALL(getpeername)(long long fdes_, void *asa_, void *alen_) {
+  PRE_WRITE(asa_, struct_sockaddr_sz);
+  PRE_WRITE(alen_, sizeof(__sanitizer_socklen_t));
+}
+POST_SYSCALL(getpeername)
+(long long res, long long fdes_, void *asa_, void *alen_) {
+  if (res == 0) {
+    POST_WRITE(asa_, struct_sockaddr_sz);
+    POST_WRITE(alen_, sizeof(__sanitizer_socklen_t));
+  }
+}
+PRE_SYSCALL(getsockname)(long long fdes_, void *asa_, void *alen_) {
+  PRE_WRITE(asa_, struct_sockaddr_sz);
+  PRE_WRITE(alen_, sizeof(__sanitizer_socklen_t));
+}
+POST_SYSCALL(getsockname)
+(long long res, long long fdes_, void *asa_, void *alen_) {
+  if (res == 0) {
+    POST_WRITE(asa_, struct_sockaddr_sz);
+    POST_WRITE(alen_, sizeof(__sanitizer_socklen_t));
+  }
+}
+PRE_SYSCALL(access)(void *path_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(access)(long long res, void *path_, long long flags_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(chflags)(void *path_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(chflags)(long long res, void *path_, long long flags_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fchflags)(long long fd_, long long flags_) { /* Nothing to do */ }
+POST_SYSCALL(fchflags)(long long res, long long fd_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(sync)(void) { /* Nothing to do */ }
+POST_SYSCALL(sync)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(kill)(long long pid_, long long signum_) { /* Nothing to do */ }
+POST_SYSCALL(kill)(long long res, long long pid_, long long signum_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_43_stat43)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_43_stat43)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(getppid)(void) { /* Nothing to do */ }
+POST_SYSCALL(getppid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_43_lstat43)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_43_lstat43)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(dup)(long long fd_) { /* Nothing to do */ }
+POST_SYSCALL(dup)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(pipe)(void) {
+  /* pipe returns two descriptors through two returned values */
+}
+POST_SYSCALL(pipe)(long long res) {
+  /* pipe returns two descriptors through two returned values */
+}
+PRE_SYSCALL(getegid)(void) { /* Nothing to do */ }
+POST_SYSCALL(getegid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(profil)
+(void *samples_, long long size_, long long offset_, long long scale_) {
+  if (samples_) {
+    PRE_WRITE(samples_, size_);
+  }
+}
+POST_SYSCALL(profil)
+(long long res, void *samples_, long long size_, long long offset_,
+  long long scale_) {
+  if (res == 0) {
+    if (samples_) {
+      POST_WRITE(samples_, size_);
+    }
+  }
+}
+PRE_SYSCALL(ktrace)
+(void *fname_, long long ops_, long long facs_, long long pid_) {
+  const char *fname = (const char *)fname_;
+  if (fname) {
+    PRE_READ(fname, __sanitizer::internal_strlen(fname) + 1);
+  }
+}
+POST_SYSCALL(ktrace)
+(long long res, void *fname_, long long ops_, long long facs_, long long pid_) {
+  const char *fname = (const char *)fname_;
+  if (res == 0) {
+    if (fname) {
+      POST_READ(fname, __sanitizer::internal_strlen(fname) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_13_sigaction13)(long long signum_, void *nsa_, void *osa_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_13_sigaction13)
+(long long res, long long signum_, void *nsa_, void *osa_) {
+  /* TODO */
+}
+PRE_SYSCALL(getgid)(void) { /* Nothing to do */ }
+POST_SYSCALL(getgid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_13_sigprocmask13)(long long how_, long long mask_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_13_sigprocmask13)
+(long long res, long long how_, long long mask_) {
+  /* TODO */
+}
+PRE_SYSCALL(__getlogin)(void *namebuf_, long long namelen_) {
+  if (namebuf_) {
+    PRE_WRITE(namebuf_, namelen_);
+  }
+}
+POST_SYSCALL(__getlogin)(long long res, void *namebuf_, long long namelen_) {
+  if (res == 0) {
+    if (namebuf_) {
+      POST_WRITE(namebuf_, namelen_);
+    }
+  }
+}
+PRE_SYSCALL(__setlogin)(void *namebuf_) {
+  const char *namebuf = (const char *)namebuf_;
+  if (namebuf) {
+    PRE_READ(namebuf, __sanitizer::internal_strlen(namebuf) + 1);
+  }
+}
+POST_SYSCALL(__setlogin)(long long res, void *namebuf_) {
+  if (res == 0) {
+    const char *namebuf = (const char *)namebuf_;
+    if (namebuf) {
+      POST_READ(namebuf, __sanitizer::internal_strlen(namebuf) + 1);
+    }
+  }
+}
+PRE_SYSCALL(acct)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(acct)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_13_sigpending13)(void) { /* TODO */ }
+POST_SYSCALL(compat_13_sigpending13)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_13_sigaltstack13)(void *nss_, void *oss_) { /* TODO */ }
+POST_SYSCALL(compat_13_sigaltstack13)(long long res, void *nss_, void *oss_) {
+  /* TODO */
+}
+PRE_SYSCALL(ioctl)(long long fd_, long long com_, void *data_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(ioctl)(long long res, long long fd_, long long com_, void *data_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_12_oreboot)(long long opt_) { /* TODO */ }
+POST_SYSCALL(compat_12_oreboot)(long long res, long long opt_) { /* TODO */ }
+PRE_SYSCALL(revoke)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(revoke)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(symlink)(void *path_, void *link_) {
+  const char *path = (const char *)path_;
+  const char *link = (const char *)link_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (link) {
+    PRE_READ(link, __sanitizer::internal_strlen(link) + 1);
+  }
+}
+POST_SYSCALL(symlink)(long long res, void *path_, void *link_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    const char *link = (const char *)link_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+    if (link) {
+      POST_READ(link, __sanitizer::internal_strlen(link) + 1);
+    }
+  }
+}
+PRE_SYSCALL(readlink)(void *path_, void *buf_, long long count_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (buf_) {
+    PRE_WRITE(buf_, count_);
+  }
+}
+POST_SYSCALL(readlink)
+(long long res, void *path_, void *buf_, long long count_) {
+  if (res > 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+    if (buf_) {
+      PRE_WRITE(buf_, res);
+    }
+  }
+}
+PRE_SYSCALL(execve)(void *path_, void *argp_, void *envp_) {
+  const char *path = (const char *)path_;
+  char **argp = (char **)argp_;
+  char **envp = (char **)envp_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (argp && argp[0]) {
+    char *a = argp[0];
+    while (a++) {
+      PRE_READ(a, __sanitizer::internal_strlen(a) + 1);
+    }
+  }
+  if (envp && envp[0]) {
+    char *e = envp[0];
+    while (e++) {
+      PRE_READ(e, __sanitizer::internal_strlen(e) + 1);
+    }
+  }
+}
+POST_SYSCALL(execve)(long long res, void *path_, void *argp_, void *envp_) {
+  /* If we are here, something went wrong */
+  const char *path = (const char *)path_;
+  char **argp = (char **)argp_;
+  char **envp = (char **)envp_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (argp && argp[0]) {
+    char *a = argp[0];
+    while (a++) {
+      POST_READ(a, __sanitizer::internal_strlen(a) + 1);
+    }
+  }
+  if (envp && envp[0]) {
+    char *e = envp[0];
+    while (e++) {
+      POST_READ(e, __sanitizer::internal_strlen(e) + 1);
+    }
+  }
+}
+PRE_SYSCALL(umask)(long long newmask_) { /* Nothing to do */ }
+POST_SYSCALL(umask)(long long res, long long newmask_) { /* Nothing to do */ }
+PRE_SYSCALL(chroot)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(chroot)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_43_fstat43)(long long fd_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_43_fstat43)(long long res, long long fd_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogetkerninfo)
+(long long op_, void *where_, void *size_, long long arg_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ogetkerninfo)
+(long long res, long long op_, void *where_, void *size_, long long arg_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogetpagesize)(void) { /* TODO */ }
+POST_SYSCALL(compat_43_ogetpagesize)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_12_msync)(void *addr_, long long len_) { /* TODO */ }
+POST_SYSCALL(compat_12_msync)(long long res, void *addr_, long long len_) {
+  /* TODO */
+}
+PRE_SYSCALL(vfork)(void) { /* Nothing to do */ }
+POST_SYSCALL(vfork)(long long res) { /* Nothing to do */ }
+/* syscall 67 has been skipped */
+/* syscall 68 has been skipped */
+/* syscall 69 has been skipped */
+/* syscall 70 has been skipped */
+PRE_SYSCALL(compat_43_ommap)
+(void *addr_, long long len_, long long prot_, long long flags_, long long fd_,
+  long long pos_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ommap)
+(long long res, void *addr_, long long len_, long long prot_, long long flags_,
+  long long fd_, long long pos_) {
+  /* TODO */
+}
+PRE_SYSCALL(vadvise)(long long anom_) { /* Nothing to do */ }
+POST_SYSCALL(vadvise)(long long res, long long anom_) { /* Nothing to do */ }
+PRE_SYSCALL(munmap)(void *addr_, long long len_) { /* Nothing to do */ }
+POST_SYSCALL(munmap)(long long res, void *addr_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(mprotect)(void *addr_, long long len_, long long prot_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(mprotect)
+(long long res, void *addr_, long long len_, long long prot_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(madvise)(void *addr_, long long len_, long long behav_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(madvise)
+(long long res, void *addr_, long long len_, long long behav_) {
+  /* Nothing to do */
+}
+/* syscall 76 has been skipped */
+/* syscall 77 has been skipped */
+PRE_SYSCALL(mincore)(void *addr_, long long len_, void *vec_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(mincore)(long long res, void *addr_, long long len_, void *vec_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(getgroups)(long long gidsetsize_, void *gidset_) {
+  unsigned int *gidset = (unsigned int *)gidset_;
+  if (gidset) {
+    PRE_WRITE(gidset, sizeof(*gidset) * gidsetsize_);
+  }
+}
+POST_SYSCALL(getgroups)(long long res, long long gidsetsize_, void *gidset_) {
+  if (res == 0) {
+    unsigned int *gidset = (unsigned int *)gidset_;
+    if (gidset) {
+      POST_WRITE(gidset, sizeof(*gidset) * gidsetsize_);
+    }
+  }
+}
+PRE_SYSCALL(setgroups)(long long gidsetsize_, void *gidset_) {
+  unsigned int *gidset = (unsigned int *)gidset_;
+  if (gidset) {
+    PRE_READ(gidset, sizeof(*gidset) * gidsetsize_);
+  }
+}
+POST_SYSCALL(setgroups)(long long res, long long gidsetsize_, void *gidset_) {
+  if (res == 0) {
+    unsigned int *gidset = (unsigned int *)gidset_;
+    if (gidset) {
+      POST_READ(gidset, sizeof(*gidset) * gidsetsize_);
+    }
+  }
+}
+PRE_SYSCALL(getpgrp)(void) { /* Nothing to do */ }
+POST_SYSCALL(getpgrp)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(setpgid)(long long pid_, long long pgid_) { /* Nothing to do */ }
+POST_SYSCALL(setpgid)(long long res, long long pid_, long long pgid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_setitimer)(long long which_, void *itv_, void *oitv_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_setitimer)
+(long long res, long long which_, void *itv_, void *oitv_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_owait)(void) { /* TODO */ }
+POST_SYSCALL(compat_43_owait)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_12_oswapon)(void *name_) { /* TODO */ }
+POST_SYSCALL(compat_12_oswapon)(long long res, void *name_) { /* TODO */ }
+PRE_SYSCALL(compat_50_getitimer)(long long which_, void *itv_) { /* TODO */ }
+POST_SYSCALL(compat_50_getitimer)(long long res, long long which_, void *itv_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogethostname)(void *hostname_, long long len_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ogethostname)
+(long long res, void *hostname_, long long len_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_osethostname)(void *hostname_, long long len_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_osethostname)
+(long long res, void *hostname_, long long len_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogetdtablesize)(void) { /* TODO */ }
+POST_SYSCALL(compat_43_ogetdtablesize)(long long res) { /* TODO */ }
+PRE_SYSCALL(dup2)(long long from_, long long to_) { /* Nothing to do */ }
+POST_SYSCALL(dup2)(long long res, long long from_, long long to_) {
+  /* Nothing to do */
+}
+/* syscall 91 has been skipped */
+PRE_SYSCALL(fcntl)(long long fd_, long long cmd_, void *arg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fcntl)(long long res, long long fd_, long long cmd_, void *arg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_select)
+(long long nd_, void *in_, void *ou_, void *ex_, void *tv_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_select)
+(long long res, long long nd_, void *in_, void *ou_, void *ex_, void *tv_) {
+  /* TODO */
+}
+/* syscall 94 has been skipped */
+PRE_SYSCALL(fsync)(long long fd_) { /* Nothing to do */ }
+POST_SYSCALL(fsync)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(setpriority)(long long which_, long long who_, long long prio_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(setpriority)
+(long long res, long long which_, long long who_, long long prio_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_30_socket)
+(long long domain_, long long type_, long long protocol_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_30_socket)
+(long long res, long long domain_, long long type_, long long protocol_) {
+  /* TODO */
+}
+PRE_SYSCALL(connect)(long long s_, void *name_, long long namelen_) {
+  PRE_READ(name_, namelen_);
+}
+POST_SYSCALL(connect)
+(long long res, long long s_, void *name_, long long namelen_) {
+  if (res == 0) {
+    POST_READ(name_, namelen_);
+  }
+}
+PRE_SYSCALL(compat_43_oaccept)(long long s_, void *name_, void *anamelen_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_oaccept)
+(long long res, long long s_, void *name_, void *anamelen_) {
+  /* TODO */
+}
+PRE_SYSCALL(getpriority)(long long which_, long long who_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(getpriority)(long long res, long long which_, long long who_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_43_osend)
+(long long s_, void *buf_, long long len_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_osend)
+(long long res, long long s_, void *buf_, long long len_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_orecv)
+(long long s_, void *buf_, long long len_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_orecv)
+(long long res, long long s_, void *buf_, long long len_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_13_sigreturn13)(void *sigcntxp_) { /* TODO */ }
+POST_SYSCALL(compat_13_sigreturn13)(long long res, void *sigcntxp_) {
+  /* TODO */
+}
+PRE_SYSCALL(bind)(long long s_, void *name_, long long namelen_) {
+  PRE_READ(name_, namelen_);
+}
+POST_SYSCALL(bind)
+(long long res, long long s_, void *name_, long long namelen_) {
+  if (res == 0) {
+    PRE_READ(name_, namelen_);
+  }
+}
+PRE_SYSCALL(setsockopt)
+(long long s_, long long level_, long long name_, void *val_,
+  long long valsize_) {
+  if (val_) {
+    PRE_READ(val_, valsize_);
+  }
+}
+POST_SYSCALL(setsockopt)
+(long long res, long long s_, long long level_, long long name_, void *val_,
+  long long valsize_) {
+  if (res == 0) {
+    if (val_) {
+      POST_READ(val_, valsize_);
+    }
+  }
+}
+PRE_SYSCALL(listen)(long long s_, long long backlog_) { /* Nothing to do */ }
+POST_SYSCALL(listen)(long long res, long long s_, long long backlog_) {
+  /* Nothing to do */
+}
+/* syscall 107 has been skipped */
+PRE_SYSCALL(compat_43_osigvec)(long long signum_, void *nsv_, void *osv_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_osigvec)
+(long long res, long long signum_, void *nsv_, void *osv_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_osigblock)(long long mask_) { /* TODO */ }
+POST_SYSCALL(compat_43_osigblock)(long long res, long long mask_) { /* TODO */ }
+PRE_SYSCALL(compat_43_osigsetmask)(long long mask_) { /* TODO */ }
+POST_SYSCALL(compat_43_osigsetmask)(long long res, long long mask_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_13_sigsuspend13)(long long mask_) { /* TODO */ }
+POST_SYSCALL(compat_13_sigsuspend13)(long long res, long long mask_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_osigstack)(void *nss_, void *oss_) { /* TODO */ }
+POST_SYSCALL(compat_43_osigstack)(long long res, void *nss_, void *oss_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_orecvmsg)(long long s_, void *msg_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_orecvmsg)
+(long long res, long long s_, void *msg_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_osendmsg)(long long s_, void *msg_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_osendmsg)
+(long long res, long long s_, void *msg_, long long flags_) {
+  /* TODO */
+}
+/* syscall 115 has been skipped */
+PRE_SYSCALL(compat_50_gettimeofday)(void *tp_, void *tzp_) { /* TODO */ }
+POST_SYSCALL(compat_50_gettimeofday)(long long res, void *tp_, void *tzp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_getrusage)(long long who_, void *rusage_) { /* TODO */ }
+POST_SYSCALL(compat_50_getrusage)
+(long long res, long long who_, void *rusage_) {
+  /* TODO */
+}
+PRE_SYSCALL(getsockopt)
+(long long s_, long long level_, long long name_, void *val_, void *avalsize_) {
+  /* TODO */
+}
+POST_SYSCALL(getsockopt)
+(long long res, long long s_, long long level_, long long name_, void *val_,
+  void *avalsize_) {
+  /* TODO */
+}
+/* syscall 119 has been skipped */
+PRE_SYSCALL(readv)(long long fd_, void *iovp_, long long iovcnt_) {
+  struct __sanitizer_iovec *iovp = (struct __sanitizer_iovec *)iovp_;
+  int i;
+  if (iovp) {
+    PRE_READ(iovp, sizeof(struct __sanitizer_iovec) * iovcnt_);
+    for (i = 0; i < iovcnt_; i++) {
+      PRE_WRITE(iovp[i].iov_base, iovp[i].iov_len);
+    }
+  }
+}
+POST_SYSCALL(readv)
+(long long res, long long fd_, void *iovp_, long long iovcnt_) {
+  struct __sanitizer_iovec *iovp = (struct __sanitizer_iovec *)iovp_;
+  int i;
+  uptr m, n = res;
+  if (res > 0) {
+    if (iovp) {
+      POST_READ(iovp, sizeof(struct __sanitizer_iovec) * iovcnt_);
+      for (i = 0; i < iovcnt_ && n > 0; i++) {
+        m = n > iovp[i].iov_len ? iovp[i].iov_len : n;
+        POST_WRITE(iovp[i].iov_base, m);
+        n -= m;
+      }
+    }
+  }
+}
+PRE_SYSCALL(writev)(long long fd_, void *iovp_, long long iovcnt_) {
+  struct __sanitizer_iovec *iovp = (struct __sanitizer_iovec *)iovp_;
+  int i;
+  if (iovp) {
+    PRE_READ(iovp, sizeof(struct __sanitizer_iovec) * iovcnt_);
+    for (i = 0; i < iovcnt_; i++) {
+      PRE_READ(iovp[i].iov_base, iovp[i].iov_len);
+    }
+  }
+}
+POST_SYSCALL(writev)
+(long long res, long long fd_, void *iovp_, long long iovcnt_) {
+  struct __sanitizer_iovec *iovp = (struct __sanitizer_iovec *)iovp_;
+  int i;
+  uptr m, n = res;
+  if (res > 0) {
+    if (iovp) {
+      POST_READ(iovp, sizeof(struct __sanitizer_iovec) * iovcnt_);
+      for (i = 0; i < iovcnt_ && n > 0; i++) {
+        m = n > iovp[i].iov_len ? iovp[i].iov_len : n;
+        POST_READ(iovp[i].iov_base, m);
+        n -= m;
+      }
+    }
+  }
+}
+PRE_SYSCALL(compat_50_settimeofday)(void *tv_, void *tzp_) { /* TODO */ }
+POST_SYSCALL(compat_50_settimeofday)(long long res, void *tv_, void *tzp_) {
+  /* TODO */
+}
+PRE_SYSCALL(fchown)(long long fd_, long long uid_, long long gid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fchown)
+(long long res, long long fd_, long long uid_, long long gid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(fchmod)(long long fd_, long long mode_) { /* Nothing to do */ }
+POST_SYSCALL(fchmod)(long long res, long long fd_, long long mode_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_43_orecvfrom)
+(long long s_, void *buf_, long long len_, long long flags_, void *from_,
+  void *fromlenaddr_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_orecvfrom)
+(long long res, long long s_, void *buf_, long long len_, long long flags_,
+  void *from_, void *fromlenaddr_) {
+  /* TODO */
+}
+PRE_SYSCALL(setreuid)(long long ruid_, long long euid_) { /* Nothing to do */ }
+POST_SYSCALL(setreuid)(long long res, long long ruid_, long long euid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(setregid)(long long rgid_, long long egid_) { /* Nothing to do */ }
+POST_SYSCALL(setregid)(long long res, long long rgid_, long long egid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(rename)(void *from_, void *to_) {
+  const char *from = (const char *)from_;
+  const char *to = (const char *)to_;
+  if (from) {
+    PRE_READ(from, __sanitizer::internal_strlen(from) + 1);
+  }
+  if (to) {
+    PRE_READ(to, __sanitizer::internal_strlen(to) + 1);
+  }
+}
+POST_SYSCALL(rename)(long long res, void *from_, void *to_) {
+  if (res == 0) {
+    const char *from = (const char *)from_;
+    const char *to = (const char *)to_;
+    if (from) {
+      POST_READ(from, __sanitizer::internal_strlen(from) + 1);
+    }
+    if (to) {
+      POST_READ(to, __sanitizer::internal_strlen(to) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_43_otruncate)(void *path_, long long length_) { /* TODO */ }
+POST_SYSCALL(compat_43_otruncate)
+(long long res, void *path_, long long length_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_oftruncate)(long long fd_, long long length_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_oftruncate)
+(long long res, long long fd_, long long length_) {
+  /* TODO */
+}
+PRE_SYSCALL(flock)(long long fd_, long long how_) { /* Nothing to do */ }
+POST_SYSCALL(flock)(long long res, long long fd_, long long how_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(mkfifo)(void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(mkfifo)(long long res, void *path_, long long mode_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(sendto)
+(long long s_, void *buf_, long long len_, long long flags_, void *to_,
+  long long tolen_) {
+  PRE_READ(buf_, len_);
+  PRE_READ(to_, tolen_);
+}
+POST_SYSCALL(sendto)
+(long long res, long long s_, void *buf_, long long len_, long long flags_,
+  void *to_, long long tolen_) {
+  if (res >= 0) {
+    POST_READ(buf_, len_);
+    POST_READ(to_, tolen_);
+  }
+}
+PRE_SYSCALL(shutdown)(long long s_, long long how_) { /* Nothing to do */ }
+POST_SYSCALL(shutdown)(long long res, long long s_, long long how_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(socketpair)
+(long long domain_, long long type_, long long protocol_, void *rsv_) {
+  PRE_WRITE(rsv_, 2 * sizeof(int));
+}
+POST_SYSCALL(socketpair)
+(long long res, long long domain_, long long type_, long long protocol_,
+  void *rsv_) {
+  if (res == 0) {
+    POST_WRITE(rsv_, 2 * sizeof(int));
+  }
+}
+PRE_SYSCALL(mkdir)(void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(mkdir)(long long res, void *path_, long long mode_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(rmdir)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(rmdir)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_50_utimes)(void *path_, void *tptr_) { /* TODO */ }
+POST_SYSCALL(compat_50_utimes)(long long res, void *path_, void *tptr_) {
+  /* TODO */
+}
+/* syscall 139 has been skipped */
+PRE_SYSCALL(compat_50_adjtime)(void *delta_, void *olddelta_) { /* TODO */ }
+POST_SYSCALL(compat_50_adjtime)(long long res, void *delta_, void *olddelta_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogetpeername)(long long fdes_, void *asa_, void *alen_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ogetpeername)
+(long long res, long long fdes_, void *asa_, void *alen_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogethostid)(void) { /* TODO */ }
+POST_SYSCALL(compat_43_ogethostid)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_43_osethostid)(long long hostid_) { /* TODO */ }
+POST_SYSCALL(compat_43_osethostid)(long long res, long long hostid_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_ogetrlimit)(long long which_, void *rlp_) { /* TODO */ }
+POST_SYSCALL(compat_43_ogetrlimit)
+(long long res, long long which_, void *rlp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_osetrlimit)(long long which_, void *rlp_) { /* TODO */ }
+POST_SYSCALL(compat_43_osetrlimit)
+(long long res, long long which_, void *rlp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_okillpg)(long long pgid_, long long signum_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_okillpg)
+(long long res, long long pgid_, long long signum_) {
+  /* TODO */
+}
+PRE_SYSCALL(setsid)(void) { /* Nothing to do */ }
+POST_SYSCALL(setsid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_50_quotactl)
+(void *path_, long long cmd_, long long uid_, void *arg_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_quotactl)
+(long long res, void *path_, long long cmd_, long long uid_, void *arg_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_43_oquota)(void) { /* TODO */ }
+POST_SYSCALL(compat_43_oquota)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_43_ogetsockname)(long long fdec_, void *asa_, void *alen_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ogetsockname)
+(long long res, long long fdec_, void *asa_, void *alen_) {
+  /* TODO */
+}
+/* syscall 151 has been skipped */
+/* syscall 152 has been skipped */
+/* syscall 153 has been skipped */
+/* syscall 154 has been skipped */
+PRE_SYSCALL(nfssvc)(long long flag_, void *argp_) { /* Nothing to do */ }
+POST_SYSCALL(nfssvc)(long long res, long long flag_, void *argp_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_43_ogetdirentries)
+(long long fd_, void *buf_, long long count_, void *basep_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_43_ogetdirentries)
+(long long res, long long fd_, void *buf_, long long count_, void *basep_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_20_statfs)(void *path_, void *buf_) { /* TODO */ }
+POST_SYSCALL(compat_20_statfs)(long long res, void *path_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_20_fstatfs)(long long fd_, void *buf_) { /* TODO */ }
+POST_SYSCALL(compat_20_fstatfs)(long long res, long long fd_, void *buf_) {
+  /* TODO */
+}
+/* syscall 159 has been skipped */
+/* syscall 160 has been skipped */
+PRE_SYSCALL(compat_30_getfh)(void *fname_, void *fhp_) { /* TODO */ }
+POST_SYSCALL(compat_30_getfh)(long long res, void *fname_, void *fhp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_09_ogetdomainname)(void *domainname_, long long len_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_09_ogetdomainname)
+(long long res, void *domainname_, long long len_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_09_osetdomainname)(void *domainname_, long long len_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_09_osetdomainname)
+(long long res, void *domainname_, long long len_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_09_ouname)(void *name_) { /* TODO */ }
+POST_SYSCALL(compat_09_ouname)(long long res, void *name_) { /* TODO */ }
+PRE_SYSCALL(sysarch)(long long op_, void *parms_) { /* TODO */ }
+POST_SYSCALL(sysarch)(long long res, long long op_, void *parms_) { /* TODO */ }
+/* syscall 166 has been skipped */
+/* syscall 167 has been skipped */
+/* syscall 168 has been skipped */
+#if !defined(_LP64)
+PRE_SYSCALL(compat_10_osemsys)
+(long long which_, long long a2_, long long a3_, long long a4_, long long a5_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_10_osemsys)
+(long long res, long long which_, long long a2_, long long a3_, long long a4_,
+  long long a5_) {
+  /* TODO */
+}
+#else
+/* syscall 169 has been skipped */
+#endif
+#if !defined(_LP64)
+PRE_SYSCALL(compat_10_omsgsys)
+(long long which_, long long a2_, long long a3_, long long a4_, long long a5_,
+  long long a6_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_10_omsgsys)
+(long long res, long long which_, long long a2_, long long a3_, long long a4_,
+  long long a5_, long long a6_) {
+  /* TODO */
+}
+#else
+/* syscall 170 has been skipped */
+#endif
+#if !defined(_LP64)
+PRE_SYSCALL(compat_10_oshmsys)
+(long long which_, long long a2_, long long a3_, long long a4_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_10_oshmsys)
+(long long res, long long which_, long long a2_, long long a3_, long long a4_) {
+  /* TODO */
+}
+#else
+/* syscall 171 has been skipped */
+#endif
+/* syscall 172 has been skipped */
+PRE_SYSCALL(pread)
+(long long fd_, void *buf_, long long nbyte_, long long PAD_,
+  long long offset_) {
+  if (buf_) {
+    PRE_WRITE(buf_, nbyte_);
+  }
+}
+POST_SYSCALL(pread)
+(long long res, long long fd_, void *buf_, long long nbyte_, long long PAD_,
+  long long offset_) {
+  if (res > 0) {
+    POST_WRITE(buf_, res);
+  }
+}
+PRE_SYSCALL(pwrite)
+(long long fd_, void *buf_, long long nbyte_, long long PAD_,
+  long long offset_) {
+  if (buf_) {
+    PRE_READ(buf_, nbyte_);
+  }
+}
+POST_SYSCALL(pwrite)
+(long long res, long long fd_, void *buf_, long long nbyte_, long long PAD_,
+  long long offset_) {
+  if (res > 0) {
+    POST_READ(buf_, res);
+  }
+}
+PRE_SYSCALL(compat_30_ntp_gettime)(void *ntvp_) { /* TODO */ }
+POST_SYSCALL(compat_30_ntp_gettime)(long long res, void *ntvp_) { /* TODO */ }
+#if defined(NTP) || !defined(_KERNEL_OPT)
+PRE_SYSCALL(ntp_adjtime)(void *tp_) { /* Nothing to do */ }
+POST_SYSCALL(ntp_adjtime)(long long res, void *tp_) { /* Nothing to do */ }
+#else
+/* syscall 176 has been skipped */
+#endif
+/* syscall 177 has been skipped */
+/* syscall 178 has been skipped */
+/* syscall 179 has been skipped */
+/* syscall 180 has been skipped */
+PRE_SYSCALL(setgid)(long long gid_) { /* Nothing to do */ }
+POST_SYSCALL(setgid)(long long res, long long gid_) { /* Nothing to do */ }
+PRE_SYSCALL(setegid)(long long egid_) { /* Nothing to do */ }
+POST_SYSCALL(setegid)(long long res, long long egid_) { /* Nothing to do */ }
+PRE_SYSCALL(seteuid)(long long euid_) { /* Nothing to do */ }
+POST_SYSCALL(seteuid)(long long res, long long euid_) { /* Nothing to do */ }
+PRE_SYSCALL(lfs_bmapv)(void *fsidp_, void *blkiov_, long long blkcnt_) {
+  /* TODO */
+}
+POST_SYSCALL(lfs_bmapv)
+(long long res, void *fsidp_, void *blkiov_, long long blkcnt_) {
+  /* TODO */
+}
+PRE_SYSCALL(lfs_markv)(void *fsidp_, void *blkiov_, long long blkcnt_) {
+  /* TODO */
+}
+POST_SYSCALL(lfs_markv)
+(long long res, void *fsidp_, void *blkiov_, long long blkcnt_) {
+  /* TODO */
+}
+PRE_SYSCALL(lfs_segclean)(void *fsidp_, long long segment_) { /* TODO */ }
+POST_SYSCALL(lfs_segclean)(long long res, void *fsidp_, long long segment_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_lfs_segwait)(void *fsidp_, void *tv_) { /* TODO */ }
+POST_SYSCALL(compat_50_lfs_segwait)(long long res, void *fsidp_, void *tv_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_12_stat12)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_12_stat12)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_12_fstat12)(long long fd_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_12_fstat12)(long long res, long long fd_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_12_lstat12)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_12_lstat12)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(pathconf)(void *path_, long long name_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(pathconf)(long long res, void *path_, long long name_) {
+  if (res != -1) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fpathconf)(long long fd_, long long name_) { /* Nothing to do */ }
+POST_SYSCALL(fpathconf)(long long res, long long fd_, long long name_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(getsockopt2)
+(long long s_, long long level_, long long name_, void *val_, void *avalsize_) {
+  /* TODO */
+}
+POST_SYSCALL(getsockopt2)
+(long long res, long long s_, long long level_, long long name_, void *val_,
+  void *avalsize_) {
+  /* TODO */
+}
+PRE_SYSCALL(getrlimit)(long long which_, void *rlp_) {
+  PRE_WRITE(rlp_, struct_rlimit_sz);
+}
+POST_SYSCALL(getrlimit)(long long res, long long which_, void *rlp_) {
+  if (res == 0) {
+    POST_WRITE(rlp_, struct_rlimit_sz);
+  }
+}
+PRE_SYSCALL(setrlimit)(long long which_, void *rlp_) {
+  PRE_READ(rlp_, struct_rlimit_sz);
+}
+POST_SYSCALL(setrlimit)(long long res, long long which_, void *rlp_) {
+  if (res == 0) {
+    POST_READ(rlp_, struct_rlimit_sz);
+  }
+}
+PRE_SYSCALL(compat_12_getdirentries)
+(long long fd_, void *buf_, long long count_, void *basep_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_12_getdirentries)
+(long long res, long long fd_, void *buf_, long long count_, void *basep_) {
+  /* TODO */
+}
+PRE_SYSCALL(mmap)
+(void *addr_, long long len_, long long prot_, long long flags_, long long fd_,
+  long long PAD_, long long pos_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(mmap)
+(long long res, void *addr_, long long len_, long long prot_, long long flags_,
+  long long fd_, long long PAD_, long long pos_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__syscall)(long long code_, long long args_[SYS_MAXSYSARGS]) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__syscall)
+(long long res, long long code_, long long args_[SYS_MAXSYSARGS]) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(lseek)
+(long long fd_, long long PAD_, long long offset_, long long whence_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(lseek)
+(long long res, long long fd_, long long PAD_, long long offset_,
+  long long whence_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(truncate)(void *path_, long long PAD_, long long length_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(truncate)
+(long long res, void *path_, long long PAD_, long long length_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(ftruncate)(long long fd_, long long PAD_, long long length_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(ftruncate)
+(long long res, long long fd_, long long PAD_, long long length_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__sysctl)
+(void *name_, long long namelen_, void *oldv_, void *oldlenp_, void *newv_,
+  long long newlen_) {
+  const int *name = (const int *)name_;
+  if (name) {
+    PRE_READ(name, namelen_ * sizeof(*name));
+  }
+  if (newv_) {
+    PRE_READ(name, newlen_);
+  }
+}
+POST_SYSCALL(__sysctl)
+(long long res, void *name_, long long namelen_, void *oldv_, void *oldlenp_,
+  void *newv_, long long newlen_) {
+  if (res == 0) {
+    const int *name = (const int *)name_;
+    if (name) {
+      POST_READ(name, namelen_ * sizeof(*name));
+    }
+    if (newv_) {
+      POST_READ(name, newlen_);
+    }
+  }
+}
+PRE_SYSCALL(mlock)(void *addr_, long long len_) { /* Nothing to do */ }
+POST_SYSCALL(mlock)(long long res, void *addr_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(munlock)(void *addr_, long long len_) { /* Nothing to do */ }
+POST_SYSCALL(munlock)(long long res, void *addr_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(undelete)(void *path_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(undelete)(long long res, void *path_) {
+  if (res == 0) {
+    const char *path = (const char *)path_;
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(compat_50_futimes)(long long fd_, void *tptr_) { /* TODO */ }
+POST_SYSCALL(compat_50_futimes)(long long res, long long fd_, void *tptr_) {
+  /* TODO */
+}
+PRE_SYSCALL(getpgid)(long long pid_) { /* Nothing to do */ }
+POST_SYSCALL(getpgid)(long long res, long long pid_) { /* Nothing to do */ }
+PRE_SYSCALL(reboot)(long long opt_, void *bootstr_) {
+  const char *bootstr = (const char *)bootstr_;
+  if (bootstr) {
+    PRE_READ(bootstr, __sanitizer::internal_strlen(bootstr) + 1);
+  }
+}
+POST_SYSCALL(reboot)(long long res, long long opt_, void *bootstr_) {
+  /* This call should never return */
+  const char *bootstr = (const char *)bootstr_;
+  if (bootstr) {
+    POST_READ(bootstr, __sanitizer::internal_strlen(bootstr) + 1);
+  }
+}
+PRE_SYSCALL(poll)(void *fds_, long long nfds_, long long timeout_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(poll)
+(long long res, void *fds_, long long nfds_, long long timeout_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(afssys)
+(long long id_, long long a1_, long long a2_, long long a3_, long long a4_,
+  long long a5_, long long a6_) {
+  /* TODO */
+}
+POST_SYSCALL(afssys)
+(long long res, long long id_, long long a1_, long long a2_, long long a3_,
+  long long a4_, long long a5_, long long a6_) {
+  /* TODO */
+}
+/* syscall 211 has been skipped */
+/* syscall 212 has been skipped */
+/* syscall 213 has been skipped */
+/* syscall 214 has been skipped */
+/* syscall 215 has been skipped */
+/* syscall 216 has been skipped */
+/* syscall 217 has been skipped */
+/* syscall 218 has been skipped */
+/* syscall 219 has been skipped */
+PRE_SYSCALL(compat_14___semctl)
+(long long semid_, long long semnum_, long long cmd_, void *arg_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_14___semctl)
+(long long res, long long semid_, long long semnum_, long long cmd_,
+  void *arg_) {
+  /* TODO */
+}
+PRE_SYSCALL(semget)(long long key_, long long nsems_, long long semflg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(semget)
+(long long res, long long key_, long long nsems_, long long semflg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(semop)(long long semid_, void *sops_, long long nsops_) {
+  if (sops_) {
+    PRE_READ(sops_, nsops_ * struct_sembuf_sz);
+  }
+}
+POST_SYSCALL(semop)
+(long long res, long long semid_, void *sops_, long long nsops_) {
+  if (res == 0) {
+    if (sops_) {
+      POST_READ(sops_, nsops_ * struct_sembuf_sz);
+    }
+  }
+}
+PRE_SYSCALL(semconfig)(long long flag_) { /* Nothing to do */ }
+POST_SYSCALL(semconfig)(long long res, long long flag_) { /* Nothing to do */ }
+PRE_SYSCALL(compat_14_msgctl)(long long msqid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_14_msgctl)
+(long long res, long long msqid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(msgget)(long long key_, long long msgflg_) { /* Nothing to do */ }
+POST_SYSCALL(msgget)(long long res, long long key_, long long msgflg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(msgsnd)
+(long long msqid_, void *msgp_, long long msgsz_, long long msgflg_) {
+  if (msgp_) {
+    PRE_READ(msgp_, msgsz_);
+  }
+}
+POST_SYSCALL(msgsnd)
+(long long res, long long msqid_, void *msgp_, long long msgsz_,
+  long long msgflg_) {
+  if (res == 0) {
+    if (msgp_) {
+      POST_READ(msgp_, msgsz_);
+    }
+  }
+}
+PRE_SYSCALL(msgrcv)
+(long long msqid_, void *msgp_, long long msgsz_, long long msgtyp_,
+  long long msgflg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(msgrcv)
+(long long res, long long msqid_, void *msgp_, long long msgsz_,
+  long long msgtyp_, long long msgflg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(shmat)(long long shmid_, void *shmaddr_, long long shmflg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(shmat)
+(long long res, long long shmid_, void *shmaddr_, long long shmflg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_14_shmctl)(long long shmid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_14_shmctl)
+(long long res, long long shmid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(shmdt)(void *shmaddr_) { /* Nothing to do */ }
+POST_SYSCALL(shmdt)(long long res, void *shmaddr_) { /* Nothing to do */ }
+PRE_SYSCALL(shmget)(long long key_, long long size_, long long shmflg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(shmget)
+(long long res, long long key_, long long size_, long long shmflg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_clock_gettime)(long long clock_id_, void *tp_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_clock_gettime)
+(long long res, long long clock_id_, void *tp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_clock_settime)(long long clock_id_, void *tp_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_clock_settime)
+(long long res, long long clock_id_, void *tp_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_clock_getres)(long long clock_id_, void *tp_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_clock_getres)
+(long long res, long long clock_id_, void *tp_) {
+  /* TODO */
+}
+PRE_SYSCALL(timer_create)(long long clock_id_, void *evp_, void *timerid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(timer_create)
+(long long res, long long clock_id_, void *evp_, void *timerid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(timer_delete)(long long timerid_) { /* Nothing to do */ }
+POST_SYSCALL(timer_delete)(long long res, long long timerid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_timer_settime)
+(long long timerid_, long long flags_, void *value_, void *ovalue_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_timer_settime)
+(long long res, long long timerid_, long long flags_, void *value_,
+  void *ovalue_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_timer_gettime)(long long timerid_, void *value_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_timer_gettime)
+(long long res, long long timerid_, void *value_) {
+  /* TODO */
+}
+PRE_SYSCALL(timer_getoverrun)(long long timerid_) { /* Nothing to do */ }
+POST_SYSCALL(timer_getoverrun)(long long res, long long timerid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_nanosleep)(void *rqtp_, void *rmtp_) { /* TODO */ }
+POST_SYSCALL(compat_50_nanosleep)(long long res, void *rqtp_, void *rmtp_) {
+  /* TODO */
+}
+PRE_SYSCALL(fdatasync)(long long fd_) { /* Nothing to do */ }
+POST_SYSCALL(fdatasync)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(mlockall)(long long flags_) { /* Nothing to do */ }
+POST_SYSCALL(mlockall)(long long res, long long flags_) { /* Nothing to do */ }
+PRE_SYSCALL(munlockall)(void) { /* Nothing to do */ }
+POST_SYSCALL(munlockall)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_50___sigtimedwait)(void *set_, void *info_, void *timeout_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50___sigtimedwait)
+(long long res, void *set_, void *info_, void *timeout_) {
+  /* TODO */
+}
+PRE_SYSCALL(sigqueueinfo)(long long pid_, void *info_) {
+  if (info_) {
+    PRE_READ(info_, siginfo_t_sz);
+  }
+}
+POST_SYSCALL(sigqueueinfo)(long long res, long long pid_, void *info_) {}
+PRE_SYSCALL(modctl)(long long cmd_, void *arg_) { /* TODO */ }
+POST_SYSCALL(modctl)(long long res, long long cmd_, void *arg_) { /* TODO */ }
+PRE_SYSCALL(_ksem_init)(long long value_, void *idp_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_init)(long long res, long long value_, void *idp_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_ksem_open)
+(void *name_, long long oflag_, long long mode_, long long value_, void *idp_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    PRE_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+POST_SYSCALL(_ksem_open)
+(long long res, void *name_, long long oflag_, long long mode_,
+  long long value_, void *idp_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    POST_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+PRE_SYSCALL(_ksem_unlink)(void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    PRE_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+POST_SYSCALL(_ksem_unlink)(long long res, void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    POST_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+PRE_SYSCALL(_ksem_close)(long long id_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_close)(long long res, long long id_) { /* Nothing to do */ }
+PRE_SYSCALL(_ksem_post)(long long id_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_post)(long long res, long long id_) { /* Nothing to do */ }
+PRE_SYSCALL(_ksem_wait)(long long id_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_wait)(long long res, long long id_) { /* Nothing to do */ }
+PRE_SYSCALL(_ksem_trywait)(long long id_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_trywait)(long long res, long long id_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_ksem_getvalue)(long long id_, void *value_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_getvalue)(long long res, long long id_, void *value_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_ksem_destroy)(long long id_) { /* Nothing to do */ }
+POST_SYSCALL(_ksem_destroy)(long long res, long long id_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_ksem_timedwait)(long long id_, void *abstime_) {
+  if (abstime_) {
+    PRE_READ(abstime_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(_ksem_timedwait)(long long res, long long id_, void *abstime_) {}
+PRE_SYSCALL(mq_open)
+(void *name_, long long oflag_, long long mode_, void *attr_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    PRE_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+POST_SYSCALL(mq_open)
+(long long res, void *name_, long long oflag_, long long mode_, void *attr_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    POST_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+PRE_SYSCALL(mq_close)(long long mqdes_) { /* Nothing to do */ }
+POST_SYSCALL(mq_close)(long long res, long long mqdes_) { /* Nothing to do */ }
+PRE_SYSCALL(mq_unlink)(void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    PRE_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+POST_SYSCALL(mq_unlink)(long long res, void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    POST_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+PRE_SYSCALL(mq_getattr)(long long mqdes_, void *mqstat_) { /* Nothing to do */ }
+POST_SYSCALL(mq_getattr)(long long res, long long mqdes_, void *mqstat_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(mq_setattr)(long long mqdes_, void *mqstat_, void *omqstat_) {
+  if (mqstat_) {
+    PRE_READ(mqstat_, struct_mq_attr_sz);
+  }
+}
+POST_SYSCALL(mq_setattr)
+(long long res, long long mqdes_, void *mqstat_, void *omqstat_) {}
+PRE_SYSCALL(mq_notify)(long long mqdes_, void *notification_) {
+  if (notification_) {
+    PRE_READ(notification_, struct_sigevent_sz);
+  }
+}
+POST_SYSCALL(mq_notify)(long long res, long long mqdes_, void *notification_) {}
+PRE_SYSCALL(mq_send)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, long long msg_prio_) {
+  if (msg_ptr_) {
+    PRE_READ(msg_ptr_, msg_len_);
+  }
+}
+POST_SYSCALL(mq_send)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  long long msg_prio_) {}
+PRE_SYSCALL(mq_receive)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, void *msg_prio_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(mq_receive)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  void *msg_prio_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50_mq_timedsend)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, long long msg_prio_,
+  void *abs_timeout_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_mq_timedsend)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  long long msg_prio_, void *abs_timeout_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_mq_timedreceive)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, void *msg_prio_,
+  void *abs_timeout_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_mq_timedreceive)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  void *msg_prio_, void *abs_timeout_) {
+  /* TODO */
+}
+/* syscall 267 has been skipped */
+/* syscall 268 has been skipped */
+/* syscall 269 has been skipped */
+PRE_SYSCALL(__posix_rename)(void *from_, void *to_) {
+  const char *from = (const char *)from_;
+  const char *to = (const char *)to_;
+  if (from_) {
+    PRE_READ(from, __sanitizer::internal_strlen(from) + 1);
+  }
+  if (to) {
+    PRE_READ(to, __sanitizer::internal_strlen(to) + 1);
+  }
+}
+POST_SYSCALL(__posix_rename)(long long res, void *from_, void *to_) {
+  const char *from = (const char *)from_;
+  const char *to = (const char *)to_;
+  if (from) {
+    POST_READ(from, __sanitizer::internal_strlen(from) + 1);
+  }
+  if (to) {
+    POST_READ(to, __sanitizer::internal_strlen(to) + 1);
+  }
+}
+PRE_SYSCALL(swapctl)(long long cmd_, void *arg_, long long misc_) { /* TODO */ }
+POST_SYSCALL(swapctl)
+(long long res, long long cmd_, void *arg_, long long misc_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_30_getdents)(long long fd_, void *buf_, long long count_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_30_getdents)
+(long long res, long long fd_, void *buf_, long long count_) {
+  /* TODO */
+}
+PRE_SYSCALL(minherit)(void *addr_, long long len_, long long inherit_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(minherit)
+(long long res, void *addr_, long long len_, long long inherit_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(lchmod)(void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lchmod)(long long res, void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(lchown)(void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lchown)
+(long long res, void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(compat_50_lutimes)(void *path_, void *tptr_) { /* TODO */ }
+POST_SYSCALL(compat_50_lutimes)(long long res, void *path_, void *tptr_) {
+  /* TODO */
+}
+PRE_SYSCALL(__msync13)(void *addr_, long long len_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__msync13)
+(long long res, void *addr_, long long len_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_30___stat13)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_30___stat13)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_30___fstat13)(long long fd_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_30___fstat13)(long long res, long long fd_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_30___lstat13)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_30___lstat13)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(__sigaltstack14)(void *nss_, void *oss_) {
+  if (nss_) {
+    PRE_READ(nss_, struct_sigaltstack_sz);
+  }
+  if (oss_) {
+    PRE_READ(oss_, struct_sigaltstack_sz);
+  }
+}
+POST_SYSCALL(__sigaltstack14)(long long res, void *nss_, void *oss_) {}
+PRE_SYSCALL(__vfork14)(void) { /* Nothing to do */ }
+POST_SYSCALL(__vfork14)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(__posix_chown)(void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__posix_chown)
+(long long res, void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(__posix_fchown)(long long fd_, long long uid_, long long gid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__posix_fchown)
+(long long res, long long fd_, long long uid_, long long gid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__posix_lchown)(void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__posix_lchown)
+(long long res, void *path_, long long uid_, long long gid_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(getsid)(long long pid_) { /* Nothing to do */ }
+POST_SYSCALL(getsid)(long long res, long long pid_) { /* Nothing to do */ }
+PRE_SYSCALL(__clone)(long long flags_, void *stack_) { /* Nothing to do */ }
+POST_SYSCALL(__clone)(long long res, long long flags_, void *stack_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(fktrace)
+(long long fd_, long long ops_, long long facs_, long long pid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fktrace)
+(long long res, long long fd_, long long ops_, long long facs_,
+  long long pid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(preadv)
+(long long fd_, void *iovp_, long long iovcnt_, long long PAD_,
+  long long offset_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(preadv)
+(long long res, long long fd_, void *iovp_, long long iovcnt_, long long PAD_,
+  long long offset_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(pwritev)
+(long long fd_, void *iovp_, long long iovcnt_, long long PAD_,
+  long long offset_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(pwritev)
+(long long res, long long fd_, void *iovp_, long long iovcnt_, long long PAD_,
+  long long offset_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_16___sigaction14)
+(long long signum_, void *nsa_, void *osa_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_16___sigaction14)
+(long long res, long long signum_, void *nsa_, void *osa_) {
+  /* TODO */
+}
+PRE_SYSCALL(__sigpending14)(void *set_) { /* Nothing to do */ }
+POST_SYSCALL(__sigpending14)(long long res, void *set_) { /* Nothing to do */ }
+PRE_SYSCALL(__sigprocmask14)(long long how_, void *set_, void *oset_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__sigprocmask14)
+(long long res, long long how_, void *set_, void *oset_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__sigsuspend14)(void *set_) {
+  if (set_) {
+    PRE_READ(set_, sizeof(__sanitizer_sigset_t));
+  }
+}
+POST_SYSCALL(__sigsuspend14)(long long res, void *set_) {
+  if (set_) {
+    PRE_READ(set_, sizeof(__sanitizer_sigset_t));
+  }
+}
+PRE_SYSCALL(compat_16___sigreturn14)(void *sigcntxp_) { /* TODO */ }
+POST_SYSCALL(compat_16___sigreturn14)(long long res, void *sigcntxp_) {
+  /* TODO */
+}
+PRE_SYSCALL(__getcwd)(void *bufp_, long long length_) { /* Nothing to do */ }
+POST_SYSCALL(__getcwd)(long long res, void *bufp_, long long length_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(fchroot)(long long fd_) { /* Nothing to do */ }
+POST_SYSCALL(fchroot)(long long res, long long fd_) { /* Nothing to do */ }
+PRE_SYSCALL(compat_30_fhopen)(void *fhp_, long long flags_) { /* TODO */ }
+POST_SYSCALL(compat_30_fhopen)(long long res, void *fhp_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_30_fhstat)(void *fhp_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_30_fhstat)(long long res, void *fhp_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_20_fhstatfs)(void *fhp_, void *buf_) { /* TODO */ }
+POST_SYSCALL(compat_20_fhstatfs)(long long res, void *fhp_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_____semctl13)
+(long long semid_, long long semnum_, long long cmd_, void *arg_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_____semctl13)
+(long long res, long long semid_, long long semnum_, long long cmd_,
+  void *arg_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___msgctl13)
+(long long msqid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50___msgctl13)
+(long long res, long long msqid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___shmctl13)
+(long long shmid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50___shmctl13)
+(long long res, long long shmid_, long long cmd_, void *buf_) {
+  /* TODO */
+}
+PRE_SYSCALL(lchflags)(void *path_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lchflags)(long long res, void *path_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(issetugid)(void) { /* Nothing to do */ }
+POST_SYSCALL(issetugid)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(utrace)(void *label_, void *addr_, long long len_) {
+  const char *label = (const char *)label_;
+  if (label) {
+    PRE_READ(label, __sanitizer::internal_strlen(label) + 1);
+  }
+  if (addr_) {
+    PRE_READ(addr_, len_);
+  }
+}
+POST_SYSCALL(utrace)(long long res, void *label_, void *addr_, long long len_) {
+  const char *label = (const char *)label_;
+  if (label) {
+    POST_READ(label, __sanitizer::internal_strlen(label) + 1);
+  }
+  if (addr_) {
+    POST_READ(addr_, len_);
+  }
+}
+PRE_SYSCALL(getcontext)(void *ucp_) { /* Nothing to do */ }
+POST_SYSCALL(getcontext)(long long res, void *ucp_) { /* Nothing to do */ }
+PRE_SYSCALL(setcontext)(void *ucp_) {
+  if (ucp_) {
+    PRE_READ(ucp_, ucontext_t_sz);
+  }
+}
+POST_SYSCALL(setcontext)(long long res, void *ucp_) {}
+PRE_SYSCALL(_lwp_create)(void *ucp_, long long flags_, void *new_lwp_) {
+  if (ucp_) {
+    PRE_READ(ucp_, ucontext_t_sz);
+  }
+}
+POST_SYSCALL(_lwp_create)
+(long long res, void *ucp_, long long flags_, void *new_lwp_) {}
+PRE_SYSCALL(_lwp_exit)(void) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_exit)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(_lwp_self)(void) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_self)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(_lwp_wait)(long long wait_for_, void *departed_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_lwp_wait)(long long res, long long wait_for_, void *departed_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_suspend)(long long target_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_suspend)(long long res, long long target_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_continue)(long long target_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_continue)(long long res, long long target_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_wakeup)(long long target_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_wakeup)(long long res, long long target_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_getprivate)(void) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_getprivate)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(_lwp_setprivate)(void *ptr_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_setprivate)(long long res, void *ptr_) { /* Nothing to do */ }
+PRE_SYSCALL(_lwp_kill)(long long target_, long long signo_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_lwp_kill)(long long res, long long target_, long long signo_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_detach)(long long target_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_detach)(long long res, long long target_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_50__lwp_park)
+(void *ts_, long long unpark_, void *hint_, void *unparkhint_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50__lwp_park)
+(long long res, void *ts_, long long unpark_, void *hint_, void *unparkhint_) {
+  /* TODO */
+}
+PRE_SYSCALL(_lwp_unpark)(long long target_, void *hint_) { /* Nothing to do */ }
+POST_SYSCALL(_lwp_unpark)(long long res, long long target_, void *hint_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_unpark_all)(void *targets_, long long ntargets_, void *hint_) {
+  if (targets_) {
+    PRE_READ(targets_, ntargets_ * sizeof(__sanitizer_lwpid_t));
+  }
+}
+POST_SYSCALL(_lwp_unpark_all)
+(long long res, void *targets_, long long ntargets_, void *hint_) {}
+PRE_SYSCALL(_lwp_setname)(long long target_, void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    PRE_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+POST_SYSCALL(_lwp_setname)(long long res, long long target_, void *name_) {
+  const char *name = (const char *)name_;
+  if (name) {
+    POST_READ(name, __sanitizer::internal_strlen(name) + 1);
+  }
+}
+PRE_SYSCALL(_lwp_getname)(long long target_, void *name_, long long len_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_lwp_getname)
+(long long res, long long target_, void *name_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_lwp_ctl)(long long features_, void **address_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_lwp_ctl)(long long res, long long features_, void **address_) {
+  /* Nothing to do */
+}
+/* syscall 326 has been skipped */
+/* syscall 327 has been skipped */
+/* syscall 328 has been skipped */
+/* syscall 329 has been skipped */
+PRE_SYSCALL(compat_60_sa_register)
+(void *newv_, void **oldv_, long long flags_, long long stackinfo_offset_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_60_sa_register)
+(long long res, void *newv_, void **oldv_, long long flags_,
+  long long stackinfo_offset_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_60_sa_stacks)(long long num_, void *stacks_) { /* TODO */ }
+POST_SYSCALL(compat_60_sa_stacks)
+(long long res, long long num_, void *stacks_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_60_sa_enable)(void) { /* TODO */ }
+POST_SYSCALL(compat_60_sa_enable)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_60_sa_setconcurrency)(long long concurrency_) { /* TODO */ }
+POST_SYSCALL(compat_60_sa_setconcurrency)
+(long long res, long long concurrency_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_60_sa_yield)(void) { /* TODO */ }
+POST_SYSCALL(compat_60_sa_yield)(long long res) { /* TODO */ }
+PRE_SYSCALL(compat_60_sa_preempt)(long long sa_id_) { /* TODO */ }
+POST_SYSCALL(compat_60_sa_preempt)(long long res, long long sa_id_) {
+  /* TODO */
+}
+/* syscall 336 has been skipped */
+/* syscall 337 has been skipped */
+/* syscall 338 has been skipped */
+/* syscall 339 has been skipped */
+PRE_SYSCALL(__sigaction_sigtramp)
+(long long signum_, void *nsa_, void *osa_, void *tramp_, long long vers_) {
+  if (nsa_) {
+    PRE_READ(nsa_, sizeof(__sanitizer_sigaction));
+  }
+}
+POST_SYSCALL(__sigaction_sigtramp)
+(long long res, long long signum_, void *nsa_, void *osa_, void *tramp_,
+  long long vers_) {
+  if (nsa_) {
+    PRE_READ(nsa_, sizeof(__sanitizer_sigaction));
+  }
+}
+/* syscall 341 has been skipped */
+/* syscall 342 has been skipped */
+PRE_SYSCALL(rasctl)(void *addr_, long long len_, long long op_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(rasctl)
+(long long res, void *addr_, long long len_, long long op_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(kqueue)(void) { /* Nothing to do */ }
+POST_SYSCALL(kqueue)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(compat_50_kevent)
+(long long fd_, void *changelist_, long long nchanges_, void *eventlist_,
+  long long nevents_, void *timeout_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_kevent)
+(long long res, long long fd_, void *changelist_, long long nchanges_,
+  void *eventlist_, long long nevents_, void *timeout_) {
+  /* TODO */
+}
+PRE_SYSCALL(_sched_setparam)
+(long long pid_, long long lid_, long long policy_, void *params_) {
+  if (params_) {
+    PRE_READ(params_, struct_sched_param_sz);
+  }
+}
+POST_SYSCALL(_sched_setparam)
+(long long res, long long pid_, long long lid_, long long policy_,
+  void *params_) {
+  if (params_) {
+    PRE_READ(params_, struct_sched_param_sz);
+  }
+}
+PRE_SYSCALL(_sched_getparam)
+(long long pid_, long long lid_, void *policy_, void *params_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_sched_getparam)
+(long long res, long long pid_, long long lid_, void *policy_, void *params_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_sched_setaffinity)
+(long long pid_, long long lid_, long long size_, void *cpuset_) {
+  if (cpuset_) {
+    PRE_READ(cpuset_, size_);
+  }
+}
+POST_SYSCALL(_sched_setaffinity)
+(long long res, long long pid_, long long lid_, long long size_,
+  void *cpuset_) {
+  if (cpuset_) {
+    PRE_READ(cpuset_, size_);
+  }
+}
+PRE_SYSCALL(_sched_getaffinity)
+(long long pid_, long long lid_, long long size_, void *cpuset_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_sched_getaffinity)
+(long long res, long long pid_, long long lid_, long long size_,
+  void *cpuset_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(sched_yield)(void) { /* Nothing to do */ }
+POST_SYSCALL(sched_yield)(long long res) { /* Nothing to do */ }
+PRE_SYSCALL(_sched_protect)(long long priority_) { /* Nothing to do */ }
+POST_SYSCALL(_sched_protect)(long long res, long long priority_) {
+  /* Nothing to do */
+}
+/* syscall 352 has been skipped */
+/* syscall 353 has been skipped */
+PRE_SYSCALL(fsync_range)
+(long long fd_, long long flags_, long long start_, long long length_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fsync_range)
+(long long res, long long fd_, long long flags_, long long start_,
+  long long length_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(uuidgen)(void *store_, long long count_) { /* Nothing to do */ }
+POST_SYSCALL(uuidgen)(long long res, void *store_, long long count_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_90_getvfsstat)
+(void *buf_, long long bufsize_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(compat_90_getvfsstat)
+(long long res, void *buf_, long long bufsize_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_90_statvfs1)(void *path_, void *buf_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(compat_90_statvfs1)
+(long long res, void *path_, void *buf_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(compat_90_fstatvfs1)(long long fd_, void *buf_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(compat_90_fstatvfs1)
+(long long res, long long fd_, void *buf_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(compat_30_fhstatvfs1)(void *fhp_, void *buf_, long long flags_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_30_fhstatvfs1)
+(long long res, void *fhp_, void *buf_, long long flags_) {
+  /* TODO */
+}
+PRE_SYSCALL(extattrctl)
+(void *path_, long long cmd_, void *filename_, long long attrnamespace_,
+  void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattrctl)
+(long long res, void *path_, long long cmd_, void *filename_,
+  long long attrnamespace_, void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_set_file)
+(void *path_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_set_file)
+(long long res, void *path_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_get_file)
+(void *path_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_get_file)
+(long long res, void *path_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_delete_file)
+(void *path_, long long attrnamespace_, void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_delete_file)
+(long long res, void *path_, long long attrnamespace_, void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_set_fd)
+(long long fd_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  /* TODO */
+}
+POST_SYSCALL(extattr_set_fd)
+(long long res, long long fd_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  /* TODO */
+}
+PRE_SYSCALL(extattr_get_fd)
+(long long fd_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  /* TODO */
+}
+POST_SYSCALL(extattr_get_fd)
+(long long res, long long fd_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  /* TODO */
+}
+PRE_SYSCALL(extattr_delete_fd)
+(long long fd_, long long attrnamespace_, void *attrname_) {
+  /* TODO */
+}
+POST_SYSCALL(extattr_delete_fd)
+(long long res, long long fd_, long long attrnamespace_, void *attrname_) {
+  /* TODO */
+}
+PRE_SYSCALL(extattr_set_link)
+(void *path_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_set_link)
+(long long res, void *path_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_get_link)
+(void *path_, long long attrnamespace_, void *attrname_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_get_link)
+(long long res, void *path_, long long attrnamespace_, void *attrname_,
+  void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_delete_link)
+(void *path_, long long attrnamespace_, void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_delete_link)
+(long long res, void *path_, long long attrnamespace_, void *attrname_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_list_fd)
+(long long fd_, long long attrnamespace_, void *data_, long long nbytes_) {
+  /* TODO */
+}
+POST_SYSCALL(extattr_list_fd)
+(long long res, long long fd_, long long attrnamespace_, void *data_,
+  long long nbytes_) {
+  /* TODO */
+}
+PRE_SYSCALL(extattr_list_file)
+(void *path_, long long attrnamespace_, void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_list_file)
+(long long res, void *path_, long long attrnamespace_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(extattr_list_link)
+(void *path_, long long attrnamespace_, void *data_, long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(extattr_list_link)
+(long long res, void *path_, long long attrnamespace_, void *data_,
+  long long nbytes_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(compat_50_pselect)
+(long long nd_, void *in_, void *ou_, void *ex_, void *ts_, void *mask_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_pselect)
+(long long res, long long nd_, void *in_, void *ou_, void *ex_, void *ts_,
+  void *mask_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50_pollts)
+(void *fds_, long long nfds_, void *ts_, void *mask_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_pollts)
+(long long res, void *fds_, long long nfds_, void *ts_, void *mask_) {
+  /* TODO */
+}
+PRE_SYSCALL(setxattr)
+(void *path_, void *name_, void *value_, long long size_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(setxattr)
+(long long res, void *path_, void *name_, void *value_, long long size_,
+  long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(lsetxattr)
+(void *path_, void *name_, void *value_, long long size_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lsetxattr)
+(long long res, void *path_, void *name_, void *value_, long long size_,
+  long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(fsetxattr)
+(long long fd_, void *name_, void *value_, long long size_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fsetxattr)
+(long long res, long long fd_, void *name_, void *value_, long long size_,
+  long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(getxattr)(void *path_, void *name_, void *value_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(getxattr)
+(long long res, void *path_, void *name_, void *value_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(lgetxattr)
+(void *path_, void *name_, void *value_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lgetxattr)
+(long long res, void *path_, void *name_, void *value_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(fgetxattr)
+(long long fd_, void *name_, void *value_, long long size_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fgetxattr)
+(long long res, long long fd_, void *name_, void *value_, long long size_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(listxattr)(void *path_, void *list_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(listxattr)
+(long long res, void *path_, void *list_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(llistxattr)(void *path_, void *list_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(llistxattr)
+(long long res, void *path_, void *list_, long long size_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(flistxattr)(long long fd_, void *list_, long long size_) {
+  /* TODO */
+}
+POST_SYSCALL(flistxattr)
+(long long res, long long fd_, void *list_, long long size_) {
+  /* TODO */
+}
+PRE_SYSCALL(removexattr)(void *path_, void *name_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(removexattr)(long long res, void *path_, void *name_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(lremovexattr)(void *path_, void *name_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(lremovexattr)(long long res, void *path_, void *name_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(fremovexattr)(long long fd_, void *name_) { /* TODO */ }
+POST_SYSCALL(fremovexattr)(long long res, long long fd_, void *name_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___stat30)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_50___stat30)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___fstat30)(long long fd_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_50___fstat30)(long long res, long long fd_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___lstat30)(void *path_, void *ub_) { /* TODO */ }
+POST_SYSCALL(compat_50___lstat30)(long long res, void *path_, void *ub_) {
+  /* TODO */
+}
+PRE_SYSCALL(__getdents30)(long long fd_, void *buf_, long long count_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__getdents30)
+(long long res, long long fd_, void *buf_, long long count_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(posix_fadvise)(long long) { /* Nothing to do */ }
+POST_SYSCALL(posix_fadvise)(long long res, long long) { /* Nothing to do */ }
+PRE_SYSCALL(compat_30___fhstat30)(void *fhp_, void *sb_) { /* TODO */ }
+POST_SYSCALL(compat_30___fhstat30)(long long res, void *fhp_, void *sb_) {
+  /* TODO */
+}
+PRE_SYSCALL(compat_50___ntp_gettime30)(void *ntvp_) { /* TODO */ }
+POST_SYSCALL(compat_50___ntp_gettime30)(long long res, void *ntvp_) {
+  /* TODO */
+}
+PRE_SYSCALL(__socket30)
+(long long domain_, long long type_, long long protocol_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__socket30)
+(long long res, long long domain_, long long type_, long long protocol_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__getfh30)(void *fname_, void *fhp_, void *fh_size_) {
+  const char *fname = (const char *)fname_;
+  if (fname) {
+    PRE_READ(fname, __sanitizer::internal_strlen(fname) + 1);
+  }
+}
+POST_SYSCALL(__getfh30)
+(long long res, void *fname_, void *fhp_, void *fh_size_) {
+  const char *fname = (const char *)fname_;
+  if (res == 0) {
+    if (fname) {
+      POST_READ(fname, __sanitizer::internal_strlen(fname) + 1);
+    }
+  }
+}
+PRE_SYSCALL(__fhopen40)(void *fhp_, long long fh_size_, long long flags_) {
+  if (fhp_) {
+    PRE_READ(fhp_, fh_size_);
+  }
+}
+POST_SYSCALL(__fhopen40)
+(long long res, void *fhp_, long long fh_size_, long long flags_) {}
+PRE_SYSCALL(compat_90_fhstatvfs1)
+(void *fhp_, long long fh_size_, void *buf_, long long flags_) {
+  if (fhp_) {
+    PRE_READ(fhp_, fh_size_);
+  }
+}
+POST_SYSCALL(compat_90_fhstatvfs1)
+(long long res, void *fhp_, long long fh_size_, void *buf_, long long flags_) {}
+PRE_SYSCALL(compat_50___fhstat40)(void *fhp_, long long fh_size_, void *sb_) {
+  if (fhp_) {
+    PRE_READ(fhp_, fh_size_);
+  }
+}
+POST_SYSCALL(compat_50___fhstat40)
+(long long res, void *fhp_, long long fh_size_, void *sb_) {}
+PRE_SYSCALL(aio_cancel)(long long fildes_, void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_cancel)(long long res, long long fildes_, void *aiocbp_) {}
+PRE_SYSCALL(aio_error)(void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_error)(long long res, void *aiocbp_) {}
+PRE_SYSCALL(aio_fsync)(long long op_, void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_fsync)(long long res, long long op_, void *aiocbp_) {}
+PRE_SYSCALL(aio_read)(void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_read)(long long res, void *aiocbp_) {}
+PRE_SYSCALL(aio_return)(void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_return)(long long res, void *aiocbp_) {}
+PRE_SYSCALL(compat_50_aio_suspend)
+(void *list_, long long nent_, void *timeout_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_50_aio_suspend)
+(long long res, void *list_, long long nent_, void *timeout_) {
+  /* TODO */
+}
+PRE_SYSCALL(aio_write)(void *aiocbp_) {
+  if (aiocbp_) {
+    PRE_READ(aiocbp_, sizeof(struct __sanitizer_aiocb));
+  }
+}
+POST_SYSCALL(aio_write)(long long res, void *aiocbp_) {}
+PRE_SYSCALL(lio_listio)
+(long long mode_, void *list_, long long nent_, void *sig_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(lio_listio)
+(long long res, long long mode_, void *list_, long long nent_, void *sig_) {
+  /* Nothing to do */
+}
+/* syscall 407 has been skipped */
+/* syscall 408 has been skipped */
+/* syscall 409 has been skipped */
+PRE_SYSCALL(__mount50)
+(void *type_, void *path_, long long flags_, void *data_, long long data_len_) {
+  const char *type = (const char *)type_;
+  const char *path = (const char *)path_;
+  if (type) {
+    PRE_READ(type, __sanitizer::internal_strlen(type) + 1);
+  }
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (data_) {
+    PRE_READ(data_, data_len_);
+  }
+}
+POST_SYSCALL(__mount50)
+(long long res, void *type_, void *path_, long long flags_, void *data_,
+  long long data_len_) {
+  const char *type = (const char *)type_;
+  const char *path = (const char *)path_;
+  if (type) {
+    POST_READ(type, __sanitizer::internal_strlen(type) + 1);
+  }
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (data_) {
+    POST_READ(data_, data_len_);
+  }
+}
+PRE_SYSCALL(mremap)
+(void *old_address_, long long old_size_, void *new_address_,
+  long long new_size_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(mremap)
+(long long res, void *old_address_, long long old_size_, void *new_address_,
+  long long new_size_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(pset_create)(void *psid_) { /* Nothing to do */ }
+POST_SYSCALL(pset_create)(long long res, void *psid_) { /* Nothing to do */ }
+PRE_SYSCALL(pset_destroy)(long long psid_) { /* Nothing to do */ }
+POST_SYSCALL(pset_destroy)(long long res, long long psid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(pset_assign)(long long psid_, long long cpuid_, void *opsid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(pset_assign)
+(long long res, long long psid_, long long cpuid_, void *opsid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(_pset_bind)
+(long long idtype_, long long first_id_, long long second_id_, long long psid_,
+  void *opsid_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(_pset_bind)
+(long long res, long long idtype_, long long first_id_, long long second_id_,
+  long long psid_, void *opsid_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__posix_fadvise50)
+(long long fd_, long long PAD_, long long offset_, long long len_,
+  long long advice_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__posix_fadvise50)
+(long long res, long long fd_, long long PAD_, long long offset_,
+  long long len_, long long advice_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__select50)
+(long long nd_, void *in_, void *ou_, void *ex_, void *tv_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__select50)
+(long long res, long long nd_, void *in_, void *ou_, void *ex_, void *tv_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__gettimeofday50)(void *tp_, void *tzp_) { /* Nothing to do */ }
+POST_SYSCALL(__gettimeofday50)(long long res, void *tp_, void *tzp_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__settimeofday50)(void *tv_, void *tzp_) {
+  if (tv_) {
+    PRE_READ(tv_, timeval_sz);
+  }
+  if (tzp_) {
+    PRE_READ(tzp_, struct_timezone_sz);
+  }
+}
+POST_SYSCALL(__settimeofday50)(long long res, void *tv_, void *tzp_) {}
+PRE_SYSCALL(__utimes50)(void *path_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (tptr) {
+    PRE_READ(tptr[0], struct_timespec_sz);
+    PRE_READ(tptr[1], struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__utimes50)(long long res, void *path_, void *tptr_) {}
+PRE_SYSCALL(__adjtime50)(void *delta_, void *olddelta_) {
+  if (delta_) {
+    PRE_READ(delta_, timeval_sz);
+  }
+}
+POST_SYSCALL(__adjtime50)(long long res, void *delta_, void *olddelta_) {}
+PRE_SYSCALL(__lfs_segwait50)(void *fsidp_, void *tv_) { /* TODO */ }
+POST_SYSCALL(__lfs_segwait50)(long long res, void *fsidp_, void *tv_) {
+  /* TODO */
+}
+PRE_SYSCALL(__futimes50)(long long fd_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  if (tptr) {
+    PRE_READ(tptr[0], struct_timespec_sz);
+    PRE_READ(tptr[1], struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__futimes50)(long long res, long long fd_, void *tptr_) {}
+PRE_SYSCALL(__lutimes50)(void *path_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (tptr) {
+    PRE_READ(tptr[0], struct_timespec_sz);
+    PRE_READ(tptr[1], struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__lutimes50)(long long res, void *path_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (tptr) {
+    POST_READ(tptr[0], struct_timespec_sz);
+    POST_READ(tptr[1], struct_timespec_sz);
+  }
+}
+PRE_SYSCALL(__setitimer50)(long long which_, void *itv_, void *oitv_) {
+  struct __sanitizer_itimerval *itv = (struct __sanitizer_itimerval *)itv_;
+  if (itv) {
+    PRE_READ(&itv->it_interval.tv_sec, sizeof(__sanitizer_time_t));
+    PRE_READ(&itv->it_interval.tv_usec, sizeof(__sanitizer_suseconds_t));
+    PRE_READ(&itv->it_value.tv_sec, sizeof(__sanitizer_time_t));
+    PRE_READ(&itv->it_value.tv_usec, sizeof(__sanitizer_suseconds_t));
+  }
+}
+POST_SYSCALL(__setitimer50)
+(long long res, long long which_, void *itv_, void *oitv_) {}
+PRE_SYSCALL(__getitimer50)(long long which_, void *itv_) { /* Nothing to do */ }
+POST_SYSCALL(__getitimer50)(long long res, long long which_, void *itv_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__clock_gettime50)(long long clock_id_, void *tp_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__clock_gettime50)(long long res, long long clock_id_, void *tp_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__clock_settime50)(long long clock_id_, void *tp_) {
+  if (tp_) {
+    PRE_READ(tp_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__clock_settime50)
+(long long res, long long clock_id_, void *tp_) {}
+PRE_SYSCALL(__clock_getres50)(long long clock_id_, void *tp_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__clock_getres50)(long long res, long long clock_id_, void *tp_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__nanosleep50)(void *rqtp_, void *rmtp_) {
+  if (rqtp_) {
+    PRE_READ(rqtp_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__nanosleep50)(long long res, void *rqtp_, void *rmtp_) {}
+PRE_SYSCALL(____sigtimedwait50)(void *set_, void *info_, void *timeout_) {
+  if (set_) {
+    PRE_READ(set_, sizeof(__sanitizer_sigset_t));
+  }
+  if (timeout_) {
+    PRE_READ(timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(____sigtimedwait50)
+(long long res, void *set_, void *info_, void *timeout_) {}
+PRE_SYSCALL(__mq_timedsend50)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, long long msg_prio_,
+  void *abs_timeout_) {
+  if (msg_ptr_) {
+    PRE_READ(msg_ptr_, msg_len_);
+  }
+  if (abs_timeout_) {
+    PRE_READ(abs_timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__mq_timedsend50)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  long long msg_prio_, void *abs_timeout_) {}
+PRE_SYSCALL(__mq_timedreceive50)
+(long long mqdes_, void *msg_ptr_, long long msg_len_, void *msg_prio_,
+  void *abs_timeout_) {
+  if (msg_ptr_) {
+    PRE_READ(msg_ptr_, msg_len_);
+  }
+  if (abs_timeout_) {
+    PRE_READ(abs_timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__mq_timedreceive50)
+(long long res, long long mqdes_, void *msg_ptr_, long long msg_len_,
+  void *msg_prio_, void *abs_timeout_) {}
+PRE_SYSCALL(compat_60__lwp_park)
+(void *ts_, long long unpark_, void *hint_, void *unparkhint_) {
+  /* TODO */
+}
+POST_SYSCALL(compat_60__lwp_park)
+(long long res, void *ts_, long long unpark_, void *hint_, void *unparkhint_) {
+  /* TODO */
+}
+PRE_SYSCALL(__kevent50)
+(long long fd_, void *changelist_, long long nchanges_, void *eventlist_,
+  long long nevents_, void *timeout_) {
+  if (changelist_) {
+    PRE_READ(changelist_, nchanges_ * struct_kevent_sz);
+  }
+  if (timeout_) {
+    PRE_READ(timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__kevent50)
+(long long res, long long fd_, void *changelist_, long long nchanges_,
+  void *eventlist_, long long nevents_, void *timeout_) {}
+PRE_SYSCALL(__pselect50)
+(long long nd_, void *in_, void *ou_, void *ex_, void *ts_, void *mask_) {
+  if (ts_) {
+    PRE_READ(ts_, struct_timespec_sz);
+  }
+  if (mask_) {
+    PRE_READ(mask_, sizeof(struct __sanitizer_sigset_t));
+  }
+}
+POST_SYSCALL(__pselect50)
+(long long res, long long nd_, void *in_, void *ou_, void *ex_, void *ts_,
+  void *mask_) {}
+PRE_SYSCALL(__pollts50)(void *fds_, long long nfds_, void *ts_, void *mask_) {
+  if (ts_) {
+    PRE_READ(ts_, struct_timespec_sz);
+  }
+  if (mask_) {
+    PRE_READ(mask_, sizeof(struct __sanitizer_sigset_t));
+  }
+}
+POST_SYSCALL(__pollts50)
+(long long res, void *fds_, long long nfds_, void *ts_, void *mask_) {}
+PRE_SYSCALL(__aio_suspend50)(void *list_, long long nent_, void *timeout_) {
+  int i;
+  const struct aiocb *const *list = (const struct aiocb *const *)list_;
+  if (list) {
+    for (i = 0; i < nent_; i++) {
+      if (list[i]) {
+        PRE_READ(list[i], sizeof(struct __sanitizer_aiocb));
+      }
+    }
+  }
+  if (timeout_) {
+    PRE_READ(timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(__aio_suspend50)
+(long long res, void *list_, long long nent_, void *timeout_) {}
+PRE_SYSCALL(__stat50)(void *path_, void *ub_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__stat50)(long long res, void *path_, void *ub_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(__fstat50)(long long fd_, void *sb_) { /* Nothing to do */ }
+POST_SYSCALL(__fstat50)(long long res, long long fd_, void *sb_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__lstat50)(void *path_, void *ub_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__lstat50)(long long res, void *path_, void *ub_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(____semctl50)
+(long long semid_, long long semnum_, long long cmd_, void *arg_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(____semctl50)
+(long long res, long long semid_, long long semnum_, long long cmd_,
+  void *arg_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__shmctl50)(long long shmid_, long long cmd_, void *buf_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__shmctl50)
+(long long res, long long shmid_, long long cmd_, void *buf_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__msgctl50)(long long msqid_, long long cmd_, void *buf_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__msgctl50)
+(long long res, long long msqid_, long long cmd_, void *buf_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__getrusage50)(long long who_, void *rusage_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__getrusage50)(long long res, long long who_, void *rusage_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__timer_settime50)
+(long long timerid_, long long flags_, void *value_, void *ovalue_) {
+  struct __sanitizer_itimerval *value = (struct __sanitizer_itimerval *)value_;
+  if (value) {
+    PRE_READ(&value->it_interval.tv_sec, sizeof(__sanitizer_time_t));
+    PRE_READ(&value->it_interval.tv_usec, sizeof(__sanitizer_suseconds_t));
+    PRE_READ(&value->it_value.tv_sec, sizeof(__sanitizer_time_t));
+    PRE_READ(&value->it_value.tv_usec, sizeof(__sanitizer_suseconds_t));
+  }
+}
+POST_SYSCALL(__timer_settime50)
+(long long res, long long timerid_, long long flags_, void *value_,
+  void *ovalue_) {
+  struct __sanitizer_itimerval *value = (struct __sanitizer_itimerval *)value_;
+  if (res == 0) {
+    if (value) {
+      POST_READ(&value->it_interval.tv_sec, sizeof(__sanitizer_time_t));
+      POST_READ(&value->it_interval.tv_usec, sizeof(__sanitizer_suseconds_t));
+      POST_READ(&value->it_value.tv_sec, sizeof(__sanitizer_time_t));
+      POST_READ(&value->it_value.tv_usec, sizeof(__sanitizer_suseconds_t));
+    }
+  }
+}
+PRE_SYSCALL(__timer_gettime50)(long long timerid_, void *value_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__timer_gettime50)
+(long long res, long long timerid_, void *value_) {
+  /* Nothing to do */
+}
+#if defined(NTP) || !defined(_KERNEL_OPT)
+PRE_SYSCALL(__ntp_gettime50)(void *ntvp_) { /* Nothing to do */ }
+POST_SYSCALL(__ntp_gettime50)(long long res, void *ntvp_) {
+  /* Nothing to do */
+}
+#else
+/* syscall 448 has been skipped */
+#endif
+PRE_SYSCALL(__wait450)
+(long long pid_, void *status_, long long options_, void *rusage_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__wait450)
+(long long res, long long pid_, void *status_, long long options_,
+  void *rusage_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__mknod50)(void *path_, long long mode_, long long dev_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__mknod50)
+(long long res, void *path_, long long mode_, long long dev_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(__fhstat50)(void *fhp_, long long fh_size_, void *sb_) {
+  if (fhp_) {
+    PRE_READ(fhp_, fh_size_);
+  }
+}
+POST_SYSCALL(__fhstat50)
+(long long res, void *fhp_, long long fh_size_, void *sb_) {
+  if (res == 0) {
+    if (fhp_) {
+      POST_READ(fhp_, fh_size_);
+    }
+  }
+}
+/* syscall 452 has been skipped */
+PRE_SYSCALL(pipe2)(void *fildes_, long long flags_) { /* Nothing to do */ }
+POST_SYSCALL(pipe2)(long long res, void *fildes_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(dup3)(long long from_, long long to_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(dup3)
+(long long res, long long from_, long long to_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(kqueue1)(long long flags_) { /* Nothing to do */ }
+POST_SYSCALL(kqueue1)(long long res, long long flags_) { /* Nothing to do */ }
+PRE_SYSCALL(paccept)
+(long long s_, void *name_, void *anamelen_, void *mask_, long long flags_) {
+  if (mask_) {
+    PRE_READ(mask_, sizeof(__sanitizer_sigset_t));
+  }
+}
+POST_SYSCALL(paccept)
+(long long res, long long s_, void *name_, void *anamelen_, void *mask_,
+  long long flags_) {
+  if (res >= 0) {
+    if (mask_) {
+      PRE_READ(mask_, sizeof(__sanitizer_sigset_t));
+    }
+  }
+}
+PRE_SYSCALL(linkat)
+(long long fd1_, void *name1_, long long fd2_, void *name2_, long long flags_) {
+  const char *name1 = (const char *)name1_;
+  const char *name2 = (const char *)name2_;
+  if (name1) {
+    PRE_READ(name1, __sanitizer::internal_strlen(name1) + 1);
+  }
+  if (name2) {
+    PRE_READ(name2, __sanitizer::internal_strlen(name2) + 1);
+  }
+}
+POST_SYSCALL(linkat)
+(long long res, long long fd1_, void *name1_, long long fd2_, void *name2_,
+  long long flags_) {
+  const char *name1 = (const char *)name1_;
+  const char *name2 = (const char *)name2_;
+  if (res == 0) {
+    if (name1) {
+      POST_READ(name1, __sanitizer::internal_strlen(name1) + 1);
+    }
+    if (name2) {
+      POST_READ(name2, __sanitizer::internal_strlen(name2) + 1);
+    }
+  }
+}
+PRE_SYSCALL(renameat)
+(long long fromfd_, void *from_, long long tofd_, void *to_) {
+  const char *from = (const char *)from_;
+  const char *to = (const char *)to_;
+  if (from) {
+    PRE_READ(from, __sanitizer::internal_strlen(from) + 1);
+  }
+  if (to) {
+    PRE_READ(to, __sanitizer::internal_strlen(to) + 1);
+  }
+}
+POST_SYSCALL(renameat)
+(long long res, long long fromfd_, void *from_, long long tofd_, void *to_) {
+  const char *from = (const char *)from_;
+  const char *to = (const char *)to_;
+  if (res == 0) {
+    if (from) {
+      POST_READ(from, __sanitizer::internal_strlen(from) + 1);
+    }
+    if (to) {
+      POST_READ(to, __sanitizer::internal_strlen(to) + 1);
+    }
+  }
+}
+PRE_SYSCALL(mkfifoat)(long long fd_, void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(mkfifoat)
+(long long res, long long fd_, void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(mknodat)
+(long long fd_, void *path_, long long mode_, long long PAD_, long long dev_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(mknodat)
+(long long res, long long fd_, void *path_, long long mode_, long long PAD_,
+  long long dev_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(mkdirat)(long long fd_, void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(mkdirat)
+(long long res, long long fd_, void *path_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(faccessat)
+(long long fd_, void *path_, long long amode_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(faccessat)
+(long long res, long long fd_, void *path_, long long amode_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fchmodat)
+(long long fd_, void *path_, long long mode_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(fchmodat)
+(long long res, long long fd_, void *path_, long long mode_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fchownat)
+(long long fd_, void *path_, long long owner_, long long group_,
+  long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(fchownat)
+(long long res, long long fd_, void *path_, long long owner_, long long group_,
+  long long flag_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(fexecve)(long long fd_, void *argp_, void *envp_) { /* TODO */ }
+POST_SYSCALL(fexecve)(long long res, long long fd_, void *argp_, void *envp_) {
+  /* TODO */
+}
+PRE_SYSCALL(fstatat)(long long fd_, void *path_, void *buf_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(fstatat)
+(long long res, long long fd_, void *path_, void *buf_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(utimensat)
+(long long fd_, void *path_, void *tptr_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+  if (tptr_) {
+    PRE_READ(tptr_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(utimensat)
+(long long res, long long fd_, void *path_, void *tptr_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (res > 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+    if (tptr_) {
+      POST_READ(tptr_, struct_timespec_sz);
+    }
+  }
+}
+PRE_SYSCALL(openat)
+(long long fd_, void *path_, long long oflags_, long long mode_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(openat)
+(long long res, long long fd_, void *path_, long long oflags_,
+  long long mode_) {
+  const char *path = (const char *)path_;
+  if (res > 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(readlinkat)
+(long long fd_, void *path_, void *buf_, long long bufsize_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(readlinkat)
+(long long res, long long fd_, void *path_, void *buf_, long long bufsize_) {
+  const char *path = (const char *)path_;
+  if (res > 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(symlinkat)(void *path1_, long long fd_, void *path2_) {
+  const char *path1 = (const char *)path1_;
+  const char *path2 = (const char *)path2_;
+  if (path1) {
+    PRE_READ(path1, __sanitizer::internal_strlen(path1) + 1);
+  }
+  if (path2) {
+    PRE_READ(path2, __sanitizer::internal_strlen(path2) + 1);
+  }
+}
+POST_SYSCALL(symlinkat)
+(long long res, void *path1_, long long fd_, void *path2_) {
+  const char *path1 = (const char *)path1_;
+  const char *path2 = (const char *)path2_;
+  if (res == 0) {
+    if (path1) {
+      POST_READ(path1, __sanitizer::internal_strlen(path1) + 1);
+    }
+    if (path2) {
+      POST_READ(path2, __sanitizer::internal_strlen(path2) + 1);
+    }
+  }
+}
+PRE_SYSCALL(unlinkat)(long long fd_, void *path_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(unlinkat)
+(long long res, long long fd_, void *path_, long long flag_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(futimens)(long long fd_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  if (tptr) {
+    PRE_READ(tptr[0], struct_timespec_sz);
+    PRE_READ(tptr[1], struct_timespec_sz);
+  }
+}
+POST_SYSCALL(futimens)(long long res, long long fd_, void *tptr_) {
+  struct __sanitizer_timespec **tptr = (struct __sanitizer_timespec **)tptr_;
+  if (res == 0) {
+    if (tptr) {
+      POST_READ(tptr[0], struct_timespec_sz);
+      POST_READ(tptr[1], struct_timespec_sz);
+    }
+  }
+}
+PRE_SYSCALL(__quotactl)(void *path_, void *args_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__quotactl)(long long res, void *path_, void *args_) {
+  const char *path = (const char *)path_;
+  if (res == 0) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(posix_spawn)
+(void *pid_, void *path_, void *file_actions_, void *attrp_, void *argv_,
+  void *envp_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(posix_spawn)
+(long long res, void *pid_, void *path_, void *file_actions_, void *attrp_,
+  void *argv_, void *envp_) {
+  const char *path = (const char *)path_;
+  if (pid_) {
+    if (path) {
+      POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+    }
+  }
+}
+PRE_SYSCALL(recvmmsg)
+(long long s_, void *mmsg_, long long vlen_, long long flags_, void *timeout_) {
+  if (timeout_) {
+    PRE_READ(timeout_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(recvmmsg)
+(long long res, long long s_, void *mmsg_, long long vlen_, long long flags_,
+  void *timeout_) {
+  if (res >= 0) {
+    if (timeout_) {
+      POST_READ(timeout_, struct_timespec_sz);
+    }
+  }
+}
+PRE_SYSCALL(sendmmsg)
+(long long s_, void *mmsg_, long long vlen_, long long flags_) {
+  struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_;
+  if (mmsg) {
+    PRE_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) *
+                       (vlen_ > 1024 ? 1024 : vlen_));
+  }
+}
+POST_SYSCALL(sendmmsg)
+(long long res, long long s_, void *mmsg_, long long vlen_, long long flags_) {
+  struct __sanitizer_mmsghdr *mmsg = (struct __sanitizer_mmsghdr *)mmsg_;
+  if (res >= 0) {
+    if (mmsg) {
+      POST_READ(mmsg, sizeof(struct __sanitizer_mmsghdr) *
+                          (vlen_ > 1024 ? 1024 : vlen_));
+    }
+  }
+}
+PRE_SYSCALL(clock_nanosleep)
+(long long clock_id_, long long flags_, void *rqtp_, void *rmtp_) {
+  if (rqtp_) {
+    PRE_READ(rqtp_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(clock_nanosleep)
+(long long res, long long clock_id_, long long flags_, void *rqtp_,
+  void *rmtp_) {
+  if (rqtp_) {
+    POST_READ(rqtp_, struct_timespec_sz);
+  }
+}
+PRE_SYSCALL(___lwp_park60)
+(long long clock_id_, long long flags_, void *ts_, long long unpark_,
+  void *hint_, void *unparkhint_) {
+  if (ts_) {
+    PRE_READ(ts_, struct_timespec_sz);
+  }
+}
+POST_SYSCALL(___lwp_park60)
+(long long res, long long clock_id_, long long flags_, void *ts_,
+  long long unpark_, void *hint_, void *unparkhint_) {
+  if (res == 0) {
+    if (ts_) {
+      POST_READ(ts_, struct_timespec_sz);
+    }
+  }
+}
+PRE_SYSCALL(posix_fallocate)
+(long long fd_, long long PAD_, long long pos_, long long len_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(posix_fallocate)
+(long long res, long long fd_, long long PAD_, long long pos_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(fdiscard)
+(long long fd_, long long PAD_, long long pos_, long long len_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(fdiscard)
+(long long res, long long fd_, long long PAD_, long long pos_, long long len_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(wait6)
+(long long idtype_, long long id_, void *status_, long long options_,
+  void *wru_, void *info_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(wait6)
+(long long res, long long idtype_, long long id_, void *status_,
+  long long options_, void *wru_, void *info_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(clock_getcpuclockid2)
+(long long idtype_, long long id_, void *clock_id_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(clock_getcpuclockid2)
+(long long res, long long idtype_, long long id_, void *clock_id_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__getvfsstat90)(void *buf_, long long bufsize_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__getvfsstat90)
+(long long res, void *buf_, long long bufsize_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__statvfs190)(void *path_, void *buf_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    PRE_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+POST_SYSCALL(__statvfs190)
+(long long res, void *path_, void *buf_, long long flags_) {
+  const char *path = (const char *)path_;
+  if (path) {
+    POST_READ(path, __sanitizer::internal_strlen(path) + 1);
+  }
+}
+PRE_SYSCALL(__fstatvfs190)(long long fd_, void *buf_, long long flags_) {
+  /* Nothing to do */
+}
+POST_SYSCALL(__fstatvfs190)
+(long long res, long long fd_, void *buf_, long long flags_) {
+  /* Nothing to do */
+}
+PRE_SYSCALL(__fhstatvfs190)
+(void *fhp_, long long fh_size_, void *buf_, long long flags_) {
+  if (fhp_) {
+    PRE_READ(fhp_, fh_size_);
+  }
+}
+POST_SYSCALL(__fhstatvfs190)
+(long long res, void *fhp_, long long fh_size_, void *buf_, long long flags_) {}
+#undef SYS_MAXSYSARGS
+} // extern "C"
+
+#undef PRE_SYSCALL
+#undef PRE_READ
+#undef PRE_WRITE
+#undef POST_SYSCALL
+#undef POST_READ
+#undef POST_WRITE
+
+#endif // SANITIZER_NETBSD
diff --git a/lib/tsan/sanitizer_common/sanitizer_termination.cpp b/lib/tsan/sanitizer_common/sanitizer_termination.cpp
new file mode 100644
index 0000000000..84be6fc323
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_termination.cpp
@@ -0,0 +1,94 @@
+//===-- sanitizer_termination.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file contains the Sanitizer termination functions CheckFailed and Die,
+/// and the callback functionalities associated with them.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+
+namespace __sanitizer {
+
+static const int kMaxNumOfInternalDieCallbacks = 5;
+static DieCallbackType InternalDieCallbacks[kMaxNumOfInternalDieCallbacks];
+
+bool AddDieCallback(DieCallbackType callback) {
+  for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) {
+    if (InternalDieCallbacks[i] == nullptr) {
+      InternalDieCallbacks[i] = callback;
+      return true;
+    }
+  }
+  return false;
+}
+
+bool RemoveDieCallback(DieCallbackType callback) {
+  for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) {
+    if (InternalDieCallbacks[i] == callback) {
+      internal_memmove(&InternalDieCallbacks[i], &InternalDieCallbacks[i + 1],
+                       sizeof(InternalDieCallbacks[0]) *
+                           (kMaxNumOfInternalDieCallbacks - i - 1));
+      InternalDieCallbacks[kMaxNumOfInternalDieCallbacks - 1] = nullptr;
+      return true;
+    }
+  }
+  return false;
+}
+
+static DieCallbackType UserDieCallback;
+void SetUserDieCallback(DieCallbackType callback) {
+  UserDieCallback = callback;
+}
+
+void NORETURN Die() {
+  if (UserDieCallback)
+    UserDieCallback();
+  for (int i = kMaxNumOfInternalDieCallbacks - 1; i >= 0; i--) {
+    if (InternalDieCallbacks[i])
+      InternalDieCallbacks[i]();
+  }
+  if (common_flags()->abort_on_error)
+    Abort();
+  internal__exit(common_flags()->exitcode);
+}
+
+static CheckFailedCallbackType CheckFailedCallback;
+void SetCheckFailedCallback(CheckFailedCallbackType callback) {
+  CheckFailedCallback = callback;
+}
+
+const int kSecondsToSleepWhenRecursiveCheckFailed = 2;
+
+void NORETURN CheckFailed(const char *file, int line, const char *cond,
+                          u64 v1, u64 v2) {
+  static atomic_uint32_t num_calls;
+  if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) > 10) {
+    SleepForSeconds(kSecondsToSleepWhenRecursiveCheckFailed);
+    Trap();
+  }
+
+  if (CheckFailedCallback) {
+    CheckFailedCallback(file, line, cond, v1, v2);
+  }
+  Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond,
+                                                            v1, v2);
+  Die();
+}
+
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_set_death_callback(void (*callback)(void)) {
+  SetUserDieCallback(callback);
+}
+}  // extern "C"
diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp b/lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp
new file mode 100644
index 0000000000..f2c6f27993
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp
@@ -0,0 +1,351 @@
+//===-- sanitizer_thread_registry.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// General thread bookkeeping functionality.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_thread_registry.h"
+
+namespace __sanitizer {
+
+ThreadContextBase::ThreadContextBase(u32 tid)
+    : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0),
+      status(ThreadStatusInvalid), detached(false),
+      thread_type(ThreadType::Regular), parent_tid(0), next(0) {
+  name[0] = '\0';
+  atomic_store(&thread_destroyed, 0, memory_order_release);
+}
+
+ThreadContextBase::~ThreadContextBase() {
+  // ThreadContextBase should never be deleted.
+  CHECK(0);
+}
+
+void ThreadContextBase::SetName(const char *new_name) {
+  name[0] = '\0';
+  if (new_name) {
+    internal_strncpy(name, new_name, sizeof(name));
+    name[sizeof(name) - 1] = '\0';
+  }
+}
+
+void ThreadContextBase::SetDead() {
+  CHECK(status == ThreadStatusRunning ||
+        status == ThreadStatusFinished);
+  status = ThreadStatusDead;
+  user_id = 0;
+  OnDead();
+}
+
+void ThreadContextBase::SetDestroyed() {
+  atomic_store(&thread_destroyed, 1, memory_order_release);
+}
+
+bool ThreadContextBase::GetDestroyed() {
+  return !!atomic_load(&thread_destroyed, memory_order_acquire);
+}
+
+void ThreadContextBase::SetJoined(void *arg) {
+  // FIXME(dvyukov): print message and continue (it's user error).
+  CHECK_EQ(false, detached);
+  CHECK_EQ(ThreadStatusFinished, status);
+  status = ThreadStatusDead;
+  user_id = 0;
+  OnJoined(arg);
+}
+
+void ThreadContextBase::SetFinished() {
+  // ThreadRegistry::FinishThread calls here in ThreadStatusCreated state
+  // for a thread that never actually started.  In that case the thread
+  // should go to ThreadStatusFinished regardless of whether it was created
+  // as detached.
+  if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished;
+  OnFinished();
+}
+
+void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type,
+                                   void *arg) {
+  status = ThreadStatusRunning;
+  os_id = _os_id;
+  thread_type = _thread_type;
+  OnStarted(arg);
+}
+
+void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id,
+                                   bool _detached, u32 _parent_tid, void *arg) {
+  status = ThreadStatusCreated;
+  user_id = _user_id;
+  unique_id = _unique_id;
+  detached = _detached;
+  // Parent tid makes no sense for the main thread.
+  if (tid != 0)
+    parent_tid = _parent_tid;
+  OnCreated(arg);
+}
+
+void ThreadContextBase::Reset() {
+  status = ThreadStatusInvalid;
+  SetName(0);
+  atomic_store(&thread_destroyed, 0, memory_order_release);
+  OnReset();
+}
+
+// ThreadRegistry implementation.
+
+const u32 ThreadRegistry::kUnknownTid = ~0U;
+
+ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
+                               u32 thread_quarantine_size, u32 max_reuse)
+    : context_factory_(factory),
+      max_threads_(max_threads),
+      thread_quarantine_size_(thread_quarantine_size),
+      max_reuse_(max_reuse),
+      mtx_(),
+      n_contexts_(0),
+      total_threads_(0),
+      alive_threads_(0),
+      max_alive_threads_(0),
+      running_threads_(0) {
+  threads_ = (ThreadContextBase **)MmapOrDie(max_threads_ * sizeof(threads_[0]),
+                                             "ThreadRegistry");
+  dead_threads_.clear();
+  invalid_threads_.clear();
+}
+
+void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
+                                        uptr *alive) {
+  BlockingMutexLock l(&mtx_);
+  if (total) *total = n_contexts_;
+  if (running) *running = running_threads_;
+  if (alive) *alive = alive_threads_;
+}
+
+uptr ThreadRegistry::GetMaxAliveThreads() {
+  BlockingMutexLock l(&mtx_);
+  return max_alive_threads_;
+}
+
+u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
+                                 void *arg) {
+  BlockingMutexLock l(&mtx_);
+  u32 tid = kUnknownTid;
+  ThreadContextBase *tctx = QuarantinePop();
+  if (tctx) {
+    tid = tctx->tid;
+  } else if (n_contexts_ < max_threads_) {
+    // Allocate new thread context and tid.
+    tid = n_contexts_++;
+    tctx = context_factory_(tid);
+    threads_[tid] = tctx;
+  } else {
+#if !SANITIZER_GO
+    Report("%s: Thread limit (%u threads) exceeded. Dying.\n",
+           SanitizerToolName, max_threads_);
+#else
+    Printf("race: limit on %u simultaneously alive goroutines is exceeded,"
+        " dying\n", max_threads_);
+#endif
+    Die();
+  }
+  CHECK_NE(tctx, 0);
+  CHECK_NE(tid, kUnknownTid);
+  CHECK_LT(tid, max_threads_);
+  CHECK_EQ(tctx->status, ThreadStatusInvalid);
+  alive_threads_++;
+  if (max_alive_threads_ < alive_threads_) {
+    max_alive_threads_++;
+    CHECK_EQ(alive_threads_, max_alive_threads_);
+  }
+  tctx->SetCreated(user_id, total_threads_++, detached,
+                   parent_tid, arg);
+  return tid;
+}
+
+void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
+                                                    void *arg) {
+  CheckLocked();
+  for (u32 tid = 0; tid < n_contexts_; tid++) {
+    ThreadContextBase *tctx = threads_[tid];
+    if (tctx == 0)
+      continue;
+    cb(tctx, arg);
+  }
+}
+
+u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
+  BlockingMutexLock l(&mtx_);
+  for (u32 tid = 0; tid < n_contexts_; tid++) {
+    ThreadContextBase *tctx = threads_[tid];
+    if (tctx != 0 && cb(tctx, arg))
+      return tctx->tid;
+  }
+  return kUnknownTid;
+}
+
+ThreadContextBase *
+ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
+  CheckLocked();
+  for (u32 tid = 0; tid < n_contexts_; tid++) {
+    ThreadContextBase *tctx = threads_[tid];
+    if (tctx != 0 && cb(tctx, arg))
+      return tctx;
+  }
+  return 0;
+}
+
+static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx,
+                                            void *arg) {
+  return (tctx->os_id == (uptr)arg && tctx->status != ThreadStatusInvalid &&
+      tctx->status != ThreadStatusDead);
+}
+
+ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
+  return FindThreadContextLocked(FindThreadContextByOsIdCallback,
+                                 (void *)os_id);
+}
+
+void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
+  BlockingMutexLock l(&mtx_);
+  CHECK_LT(tid, n_contexts_);
+  ThreadContextBase *tctx = threads_[tid];
+  CHECK_NE(tctx, 0);
+  CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
+           tctx->status);
+  tctx->SetName(name);
+}
+
+void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
+  BlockingMutexLock l(&mtx_);
+  for (u32 tid = 0; tid < n_contexts_; tid++) {
+    ThreadContextBase *tctx = threads_[tid];
+    if (tctx != 0 && tctx->user_id == user_id &&
+        tctx->status != ThreadStatusInvalid) {
+      tctx->SetName(name);
+      return;
+    }
+  }
+}
+
+void ThreadRegistry::DetachThread(u32 tid, void *arg) {
+  BlockingMutexLock l(&mtx_);
+  CHECK_LT(tid, n_contexts_);
+  ThreadContextBase *tctx = threads_[tid];
+  CHECK_NE(tctx, 0);
+  if (tctx->status == ThreadStatusInvalid) {
+    Report("%s: Detach of non-existent thread\n", SanitizerToolName);
+    return;
+  }
+  tctx->OnDetached(arg);
+  if (tctx->status == ThreadStatusFinished) {
+    tctx->SetDead();
+    QuarantinePush(tctx);
+  } else {
+    tctx->detached = true;
+  }
+}
+
+void ThreadRegistry::JoinThread(u32 tid, void *arg) {
+  bool destroyed = false;
+  do {
+    {
+      BlockingMutexLock l(&mtx_);
+      CHECK_LT(tid, n_contexts_);
+      ThreadContextBase *tctx = threads_[tid];
+      CHECK_NE(tctx, 0);
+      if (tctx->status == ThreadStatusInvalid) {
+        Report("%s: Join of non-existent thread\n", SanitizerToolName);
+        return;
+      }
+      if ((destroyed = tctx->GetDestroyed())) {
+        tctx->SetJoined(arg);
+        QuarantinePush(tctx);
+      }
+    }
+    if (!destroyed)
+      internal_sched_yield();
+  } while (!destroyed);
+}
+
+// Normally this is called when the thread is about to exit.  If
+// called in ThreadStatusCreated state, then this thread was never
+// really started.  We just did CreateThread for a prospective new
+// thread before trying to create it, and then failed to actually
+// create it, and so never called StartThread.
+void ThreadRegistry::FinishThread(u32 tid) {
+  BlockingMutexLock l(&mtx_);
+  CHECK_GT(alive_threads_, 0);
+  alive_threads_--;
+  CHECK_LT(tid, n_contexts_);
+  ThreadContextBase *tctx = threads_[tid];
+  CHECK_NE(tctx, 0);
+  bool dead = tctx->detached;
+  if (tctx->status == ThreadStatusRunning) {
+    CHECK_GT(running_threads_, 0);
+    running_threads_--;
+  } else {
+    // The thread never really existed.
+    CHECK_EQ(tctx->status, ThreadStatusCreated);
+    dead = true;
+  }
+  tctx->SetFinished();
+  if (dead) {
+    tctx->SetDead();
+    QuarantinePush(tctx);
+  }
+  tctx->SetDestroyed();
+}
+
+void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
+                                 void *arg) {
+  BlockingMutexLock l(&mtx_);
+  running_threads_++;
+  CHECK_LT(tid, n_contexts_);
+  ThreadContextBase *tctx = threads_[tid];
+  CHECK_NE(tctx, 0);
+  CHECK_EQ(ThreadStatusCreated, tctx->status);
+  tctx->SetStarted(os_id, thread_type, arg);
+}
+
+void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) {
+  if (tctx->tid == 0)
+    return;  // Don't reuse the main thread.  It's a special snowflake.
+  dead_threads_.push_back(tctx);
+  if (dead_threads_.size() <= thread_quarantine_size_)
+    return;
+  tctx = dead_threads_.front();
+  dead_threads_.pop_front();
+  CHECK_EQ(tctx->status, ThreadStatusDead);
+  tctx->Reset();
+  tctx->reuse_count++;
+  if (max_reuse_ > 0 && tctx->reuse_count >= max_reuse_)
+    return;
+  invalid_threads_.push_back(tctx);
+}
+
+ThreadContextBase *ThreadRegistry::QuarantinePop() {
+  if (invalid_threads_.size() == 0)
+    return 0;
+  ThreadContextBase *tctx = invalid_threads_.front();
+  invalid_threads_.pop_front();
+  return tctx;
+}
+
+void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
+  BlockingMutexLock l(&mtx_);
+  CHECK_LT(tid, n_contexts_);
+  ThreadContextBase *tctx = threads_[tid];
+  CHECK_NE(tctx, 0);
+  CHECK_NE(tctx->status, ThreadStatusInvalid);
+  CHECK_NE(tctx->status, ThreadStatusDead);
+  CHECK_EQ(tctx->user_id, 0);
+  tctx->user_id = user_id;
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_thread_registry.h b/lib/tsan/sanitizer_common/sanitizer_thread_registry.h
new file mode 100644
index 0000000000..493aa988f7
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_thread_registry.h
@@ -0,0 +1,160 @@
+//===-- sanitizer_thread_registry.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// General thread bookkeeping functionality.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_THREAD_REGISTRY_H
+#define SANITIZER_THREAD_REGISTRY_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_list.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+enum ThreadStatus {
+  ThreadStatusInvalid,   // Non-existent thread, data is invalid.
+  ThreadStatusCreated,   // Created but not yet running.
+  ThreadStatusRunning,   // The thread is currently running.
+  ThreadStatusFinished,  // Joinable thread is finished but not yet joined.
+  ThreadStatusDead       // Joined, but some info is still available.
+};
+
+enum class ThreadType {
+  Regular, // Normal thread
+  Worker,  // macOS Grand Central Dispatch (GCD) worker thread
+  Fiber,   // Fiber
+};
+
+// Generic thread context. Specific sanitizer tools may inherit from it.
+// If thread is dead, context may optionally be reused for a new thread.
+class ThreadContextBase {
+ public:
+  explicit ThreadContextBase(u32 tid);
+  ~ThreadContextBase();  // Should never be called.
+
+  const u32 tid;  // Thread ID. Main thread should have tid = 0.
+  u64 unique_id;  // Unique thread ID.
+  u32 reuse_count;  // Number of times this tid was reused.
+  tid_t os_id;     // PID (used for reporting).
+  uptr user_id;   // Some opaque user thread id (e.g. pthread_t).
+  char name[64];  // As annotated by user.
+
+  ThreadStatus status;
+  bool detached;
+  ThreadType thread_type;
+
+  u32 parent_tid;
+  ThreadContextBase *next;  // For storing thread contexts in a list.
+
+  atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished
+
+  void SetName(const char *new_name);
+
+  void SetDead();
+  void SetJoined(void *arg);
+  void SetFinished();
+  void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg);
+  void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
+                  u32 _parent_tid, void *arg);
+  void Reset();
+
+  void SetDestroyed();
+  bool GetDestroyed();
+
+  // The following methods may be overriden by subclasses.
+  // Some of them take opaque arg that may be optionally be used
+  // by subclasses.
+  virtual void OnDead() {}
+  virtual void OnJoined(void *arg) {}
+  virtual void OnFinished() {}
+  virtual void OnStarted(void *arg) {}
+  virtual void OnCreated(void *arg) {}
+  virtual void OnReset() {}
+  virtual void OnDetached(void *arg) {}
+};
+
+typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
+
+class ThreadRegistry {
+ public:
+  static const u32 kUnknownTid;
+
+  ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
+                 u32 thread_quarantine_size, u32 max_reuse = 0);
+  void GetNumberOfThreads(uptr *total = nullptr, uptr *running = nullptr,
+                          uptr *alive = nullptr);
+  uptr GetMaxAliveThreads();
+
+  void Lock() { mtx_.Lock(); }
+  void CheckLocked() { mtx_.CheckLocked(); }
+  void Unlock() { mtx_.Unlock(); }
+
+  // Should be guarded by ThreadRegistryLock.
+  ThreadContextBase *GetThreadLocked(u32 tid) {
+    DCHECK_LT(tid, n_contexts_);
+    return threads_[tid];
+  }
+
+  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
+
+  typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
+  // Invokes callback with a specified arg for each thread context.
+  // Should be guarded by ThreadRegistryLock.
+  void RunCallbackForEachThreadLocked(ThreadCallback cb, void *arg);
+
+  typedef bool (*FindThreadCallback)(ThreadContextBase *tctx, void *arg);
+  // Finds a thread using the provided callback. Returns kUnknownTid if no
+  // thread is found.
+  u32 FindThread(FindThreadCallback cb, void *arg);
+  // Should be guarded by ThreadRegistryLock. Return 0 if no thread
+  // is found.
+  ThreadContextBase *FindThreadContextLocked(FindThreadCallback cb,
+                                             void *arg);
+  ThreadContextBase *FindThreadContextByOsIDLocked(tid_t os_id);
+
+  void SetThreadName(u32 tid, const char *name);
+  void SetThreadNameByUserId(uptr user_id, const char *name);
+  void DetachThread(u32 tid, void *arg);
+  void JoinThread(u32 tid, void *arg);
+  void FinishThread(u32 tid);
+  void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
+  void SetThreadUserId(u32 tid, uptr user_id);
+
+ private:
+  const ThreadContextFactory context_factory_;
+  const u32 max_threads_;
+  const u32 thread_quarantine_size_;
+  const u32 max_reuse_;
+
+  BlockingMutex mtx_;
+
+  u32 n_contexts_;      // Number of created thread contexts,
+                        // at most max_threads_.
+  u64 total_threads_;   // Total number of created threads. May be greater than
+                        // max_threads_ if contexts were reused.
+  uptr alive_threads_;  // Created or running.
+  uptr max_alive_threads_;
+  uptr running_threads_;
+
+  ThreadContextBase **threads_;  // Array of thread contexts is leaked.
+  IntrusiveList<ThreadContextBase> dead_threads_;
+  IntrusiveList<ThreadContextBase> invalid_threads_;
+
+  void QuarantinePush(ThreadContextBase *tctx);
+  ThreadContextBase *QuarantinePop();
+};
+
+typedef GenericScopedLock<ThreadRegistry> ThreadRegistryLock;
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_THREAD_REGISTRY_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp
new file mode 100644
index 0000000000..9ca898a306
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -0,0 +1,154 @@
+//===-- sanitizer_tls_get_addr.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Handle the __tls_get_addr call.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_tls_get_addr.h"
+
+#include "sanitizer_flags.h"
+#include "sanitizer_platform_interceptors.h"
+
+namespace __sanitizer {
+#if SANITIZER_INTERCEPT_TLS_GET_ADDR
+
+// The actual parameter that comes to __tls_get_addr
+// is a pointer to a struct with two words in it:
+struct TlsGetAddrParam {
+  uptr dso_id;
+  uptr offset;
+};
+
+// Glibc starting from 2.19 allocates tls using __signal_safe_memalign,
+// which has such header.
+struct Glibc_2_19_tls_header {
+  uptr size;
+  uptr start;
+};
+
+// This must be static TLS
+__attribute__((tls_model("initial-exec")))
+static __thread DTLS dtls;
+
+// Make sure we properly destroy the DTLS objects:
+// this counter should never get too large.
+static atomic_uintptr_t number_of_live_dtls;
+
+static const uptr kDestroyedThread = -1;
+
+static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
+  if (!size) return;
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
+  UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
+  atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
+}
+
+static inline void DTLS_Resize(uptr new_size) {
+  if (dtls.dtv_size >= new_size) return;
+  new_size = RoundUpToPowerOfTwo(new_size);
+  new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV));
+  DTLS::DTV *new_dtv =
+      (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
+  uptr num_live_dtls =
+      atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
+  VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
+  CHECK_LT(num_live_dtls, 1 << 20);
+  uptr old_dtv_size = dtls.dtv_size;
+  DTLS::DTV *old_dtv = dtls.dtv;
+  if (old_dtv_size)
+    internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV));
+  dtls.dtv = new_dtv;
+  dtls.dtv_size = new_size;
+  if (old_dtv_size)
+    DTLS_Deallocate(old_dtv, old_dtv_size);
+}
+
+void DTLS_Destroy() {
+  if (!common_flags()->intercept_tls_get_addr) return;
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
+  uptr s = dtls.dtv_size;
+  dtls.dtv_size = kDestroyedThread;  // Do this before unmap for AS-safety.
+  DTLS_Deallocate(dtls.dtv, s);
+}
+
+#if defined(__powerpc64__) || defined(__mips__)
+// This is glibc's TLS_DTV_OFFSET:
+// "Dynamic thread vector pointers point 0x8000 past the start of each
+//  TLS block."
+static const uptr kDtvOffset = 0x8000;
+#else
+static const uptr kDtvOffset = 0;
+#endif
+
+DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
+                                uptr static_tls_begin, uptr static_tls_end) {
+  if (!common_flags()->intercept_tls_get_addr) return 0;
+  TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
+  uptr dso_id = arg->dso_id;
+  if (dtls.dtv_size == kDestroyedThread) return 0;
+  DTLS_Resize(dso_id + 1);
+  if (dtls.dtv[dso_id].beg) return 0;
+  uptr tls_size = 0;
+  uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
+  VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
+             "num_live_dtls %zd\n",
+          arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
+          atomic_load(&number_of_live_dtls, memory_order_relaxed));
+  if (dtls.last_memalign_ptr == tls_beg) {
+    tls_size = dtls.last_memalign_size;
+    VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
+        tls_beg, tls_size);
+  } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
+    // This is the static TLS block which was initialized / unpoisoned at thread
+    // creation.
+    VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
+    tls_size = 0;
+  } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
+    // We may want to check gnu_get_libc_version().
+    Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
+    tls_size = header->size;
+    tls_beg = header->start;
+    VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
+        tls_beg, tls_size);
+  } else {
+    VReport(2, "__tls_get_addr: Can't guess glibc version\n");
+    // This may happen inside the DTOR of main thread, so just ignore it.
+    tls_size = 0;
+  }
+  dtls.dtv[dso_id].beg = tls_beg;
+  dtls.dtv[dso_id].size = tls_size;
+  return dtls.dtv + dso_id;
+}
+
+void DTLS_on_libc_memalign(void *ptr, uptr size) {
+  if (!common_flags()->intercept_tls_get_addr) return;
+  VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
+  dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
+  dtls.last_memalign_size = size;
+}
+
+DTLS *DTLS_Get() { return &dtls; }
+
+bool DTLSInDestruction(DTLS *dtls) {
+  return dtls->dtv_size == kDestroyedThread;
+}
+
+#else
+void DTLS_on_libc_memalign(void *ptr, uptr size) {}
+DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res,
+  unsigned long, unsigned long) { return 0; }
+DTLS *DTLS_Get() { return 0; }
+void DTLS_Destroy() {}
+bool DTLSInDestruction(DTLS *dtls) {
+  UNREACHABLE("dtls is unsupported on this platform!");
+}
+
+#endif  // SANITIZER_INTERCEPT_TLS_GET_ADDR
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h
new file mode 100644
index 0000000000..c7cd5a8bff
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h
@@ -0,0 +1,62 @@
+//===-- sanitizer_tls_get_addr.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Handle the __tls_get_addr call.
+//
+// All this magic is specific to glibc and is required to workaround
+// the lack of interface that would tell us about the Dynamic TLS (DTLS).
+// https://sourceware.org/bugzilla/show_bug.cgi?id=16291
+//
+// The matters get worse because the glibc implementation changed between
+// 2.18 and 2.19:
+// https://groups.google.com/forum/#!topic/address-sanitizer/BfwYD8HMxTM
+//
+// Before 2.19, every DTLS chunk is allocated with __libc_memalign,
+// which we intercept and thus know where is the DTLS.
+// Since 2.19, DTLS chunks are allocated with __signal_safe_memalign,
+// which is an internal function that wraps a mmap call, neither of which
+// we can intercept. Luckily, __signal_safe_memalign has a simple parseable
+// header which we can use.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_TLS_GET_ADDR_H
+#define SANITIZER_TLS_GET_ADDR_H
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+struct DTLS {
+  // Array of DTLS chunks for the current Thread.
+  // If beg == 0, the chunk is unused.
+  struct DTV {
+    uptr beg, size;
+  };
+
+  uptr dtv_size;
+  DTV *dtv;  // dtv_size elements, allocated by MmapOrDie.
+
+  // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
+  uptr last_memalign_size;
+  uptr last_memalign_ptr;
+};
+
+// Returns pointer and size of a linker-allocated TLS block.
+// Each block is returned exactly once.
+DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res, uptr static_tls_begin,
+                                uptr static_tls_end);
+void DTLS_on_libc_memalign(void *ptr, uptr size);
+DTLS *DTLS_Get();
+void DTLS_Destroy();  // Make sure to call this before the thread is destroyed.
+// Returns true if DTLS of suspended thread is in destruction process.
+bool DTLSInDestruction(DTLS *dtls);
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_TLS_GET_ADDR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_type_traits.cpp b/lib/tsan/sanitizer_common/sanitizer_type_traits.cpp
new file mode 100644
index 0000000000..5ee37d7376
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_type_traits.cpp
@@ -0,0 +1,20 @@
+//===-- sanitizer_type_traits.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a subset of C++ type traits. This is so we can avoid depending
+// on system C++ headers.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_type_traits.h"
+
+namespace __sanitizer {
+
+const bool true_type::value;
+const bool false_type::value;
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_type_traits.h b/lib/tsan/sanitizer_common/sanitizer_type_traits.h
new file mode 100644
index 0000000000..2a58d9874d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_type_traits.h
@@ -0,0 +1,62 @@
+//===-- sanitizer_type_traits.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a subset of C++ type traits. This is so we can avoid depending
+// on system C++ headers.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_TYPE_TRAITS_H
+#define SANITIZER_TYPE_TRAITS_H
+
+namespace __sanitizer {
+
+struct true_type {
+  static const bool value = true;
+};
+
+struct false_type {
+  static const bool value = false;
+};
+
+// is_same<T, U>
+//
+// Type trait to compare if types are the same.
+// E.g.
+//
+// ```
+// is_same<int,int>::value - True
+// is_same<int,char>::value - False
+// ```
+template <typename T, typename U>
+struct is_same : public false_type {};
+
+template <typename T>
+struct is_same<T, T> : public true_type {};
+
+// conditional<B, T, F>
+//
+// Defines type as T if B is true or as F otherwise.
+// E.g. the following is true
+//
+// ```
+// is_same<int, conditional<true, int, double>::type>::value
+// is_same<double, conditional<false, int, double>::type>::value
+// ```
+template <bool B, class T, class F>
+struct conditional {
+  using type = T;
+};
+
+template <class T, class F>
+struct conditional<false, T, F> {
+  using type = F;
+};
+
+}  // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_vector.h b/lib/tsan/sanitizer_common/sanitizer_vector.h
new file mode 100644
index 0000000000..31216f3ec3
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_vector.h
@@ -0,0 +1,124 @@
+//===-- sanitizer_vector.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+//===----------------------------------------------------------------------===//
+
+// Low-fat STL-like vector container.
+
+#ifndef SANITIZER_VECTOR_H
+#define SANITIZER_VECTOR_H
+
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_libc.h"
+
+namespace __sanitizer {
+
+template<typename T>
+class Vector {
+ public:
+  Vector() : begin_(), end_(), last_() {}
+
+  ~Vector() {
+    if (begin_)
+      InternalFree(begin_);
+  }
+
+  void Reset() {
+    if (begin_)
+      InternalFree(begin_);
+    begin_ = 0;
+    end_ = 0;
+    last_ = 0;
+  }
+
+  uptr Size() const {
+    return end_ - begin_;
+  }
+
+  T &operator[](uptr i) {
+    DCHECK_LT(i, end_ - begin_);
+    return begin_[i];
+  }
+
+  const T &operator[](uptr i) const {
+    DCHECK_LT(i, end_ - begin_);
+    return begin_[i];
+  }
+
+  T *PushBack() {
+    EnsureSize(Size() + 1);
+    T *p = &end_[-1];
+    internal_memset(p, 0, sizeof(*p));
+    return p;
+  }
+
+  T *PushBack(const T& v) {
+    EnsureSize(Size() + 1);
+    T *p = &end_[-1];
+    internal_memcpy(p, &v, sizeof(*p));
+    return p;
+  }
+
+  void PopBack() {
+    DCHECK_GT(end_, begin_);
+    end_--;
+  }
+
+  void Resize(uptr size) {
+    if (size == 0) {
+      end_ = begin_;
+      return;
+    }
+    uptr old_size = Size();
+    if (size <= old_size) {
+      end_ = begin_ + size;
+      return;
+    }
+    EnsureSize(size);
+    if (old_size < size) {
+      for (uptr i = old_size; i < size; i++)
+        internal_memset(&begin_[i], 0, sizeof(begin_[i]));
+    }
+  }
+
+ private:
+  T *begin_;
+  T *end_;
+  T *last_;
+
+  void EnsureSize(uptr size) {
+    if (size <= Size())
+      return;
+    if (size <= (uptr)(last_ - begin_)) {
+      end_ = begin_ + size;
+      return;
+    }
+    uptr cap0 = last_ - begin_;
+    uptr cap = cap0 * 5 / 4;  // 25% growth
+    if (cap == 0)
+      cap = 16;
+    if (cap < size)
+      cap = size;
+    T *p = (T*)InternalAlloc(cap * sizeof(T));
+    if (cap0) {
+      internal_memcpy(p, begin_, cap0 * sizeof(T));
+      InternalFree(begin_);
+    }
+    begin_ = p;
+    end_ = begin_ + size;
+    last_ = begin_ + cap;
+  }
+
+  Vector(const Vector&);
+  void operator=(const Vector&);
+};
+}  // namespace __sanitizer
+
+#endif  // #ifndef SANITIZER_VECTOR_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_win.cpp b/lib/tsan/sanitizer_common/sanitizer_win.cpp
new file mode 100644
index 0000000000..fca15beb61
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_win.cpp
@@ -0,0 +1,1129 @@
+//===-- sanitizer_win.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements windows-specific functions from
+// sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#define WIN32_LEAN_AND_MEAN
+#define NOGDI
+#include <windows.h>
+#include <io.h>
+#include <psapi.h>
+#include <stdlib.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_win_defs.h"
+
+#if defined(PSAPI_VERSION) && PSAPI_VERSION == 1
+#pragma comment(lib, "psapi")
+#endif
+#if SANITIZER_WIN_TRACE
+#include <traceloggingprovider.h>
+//  Windows trace logging provider init
+#pragma comment(lib, "advapi32.lib")
+TRACELOGGING_DECLARE_PROVIDER(g_asan_provider);
+// GUID must be the same in utils/AddressSanitizerLoggingProvider.wprp
+TRACELOGGING_DEFINE_PROVIDER(g_asan_provider, "AddressSanitizerLoggingProvider",
+                             (0x6c6c766d, 0x3846, 0x4e6a, 0xa4, 0xfb, 0x5b,
+                              0x53, 0x0b, 0xd0, 0xf3, 0xfa));
+#else
+#define TraceLoggingUnregister(x)
+#endif
+
+// A macro to tell the compiler that this part of the code cannot be reached,
+// if the compiler supports this feature. Since we're using this in
+// code that is called when terminating the process, the expansion of the
+// macro should not terminate the process to avoid infinite recursion.
+#if defined(__clang__)
+# define BUILTIN_UNREACHABLE() __builtin_unreachable()
+#elif defined(__GNUC__) && \
+    (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+# define BUILTIN_UNREACHABLE() __builtin_unreachable()
+#elif defined(_MSC_VER)
+# define BUILTIN_UNREACHABLE() __assume(0)
+#else
+# define BUILTIN_UNREACHABLE()
+#endif
+
+namespace __sanitizer {
+
+#include "sanitizer_syscall_generic.inc"
+
+// --------------------- sanitizer_common.h
+uptr GetPageSize() {
+  SYSTEM_INFO si;
+  GetSystemInfo(&si);
+  return si.dwPageSize;
+}
+
+uptr GetMmapGranularity() {
+  SYSTEM_INFO si;
+  GetSystemInfo(&si);
+  return si.dwAllocationGranularity;
+}
+
+uptr GetMaxUserVirtualAddress() {
+  SYSTEM_INFO si;
+  GetSystemInfo(&si);
+  return (uptr)si.lpMaximumApplicationAddress;
+}
+
+uptr GetMaxVirtualAddress() {
+  return GetMaxUserVirtualAddress();
+}
+
+bool FileExists(const char *filename) {
+  return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES;
+}
+
+uptr internal_getpid() {
+  return GetProcessId(GetCurrentProcess());
+}
+
+int internal_dlinfo(void *handle, int request, void *p) {
+  UNIMPLEMENTED();
+}
+
+// In contrast to POSIX, on Windows GetCurrentThreadId()
+// returns a system-unique identifier.
+tid_t GetTid() {
+  return GetCurrentThreadId();
+}
+
+uptr GetThreadSelf() {
+  return GetTid();
+}
+
+#if !SANITIZER_GO
+void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
+                                uptr *stack_bottom) {
+  CHECK(stack_top);
+  CHECK(stack_bottom);
+  MEMORY_BASIC_INFORMATION mbi;
+  CHECK_NE(VirtualQuery(&mbi /* on stack */, &mbi, sizeof(mbi)), 0);
+  // FIXME: is it possible for the stack to not be a single allocation?
+  // Are these values what ASan expects to get (reserved, not committed;
+  // including stack guard page) ?
+  *stack_top = (uptr)mbi.BaseAddress + mbi.RegionSize;
+  *stack_bottom = (uptr)mbi.AllocationBase;
+}
+#endif  // #if !SANITIZER_GO
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+  void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (rv == 0)
+    ReportMmapFailureAndDie(size, mem_type, "allocate",
+                            GetLastError(), raw_report);
+  return rv;
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+  if (!size || !addr)
+    return;
+
+  MEMORY_BASIC_INFORMATION mbi;
+  CHECK(VirtualQuery(addr, &mbi, sizeof(mbi)));
+
+  // MEM_RELEASE can only be used to unmap whole regions previously mapped with
+  // VirtualAlloc. So we first try MEM_RELEASE since it is better, and if that
+  // fails try MEM_DECOMMIT.
+  if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
+    if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) {
+      Report("ERROR: %s failed to "
+             "deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n",
+             SanitizerToolName, size, size, addr, GetLastError());
+      CHECK("unable to unmap" && 0);
+    }
+  }
+}
+
+static void *ReturnNullptrOnOOMOrDie(uptr size, const char *mem_type,
+                                     const char *mmap_type) {
+  error_t last_error = GetLastError();
+  if (last_error == ERROR_NOT_ENOUGH_MEMORY)
+    return nullptr;
+  ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (rv == 0)
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
+  return rv;
+}
+
+// We want to map a chunk of address space aligned to 'alignment'.
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+                                   const char *mem_type) {
+  CHECK(IsPowerOfTwo(size));
+  CHECK(IsPowerOfTwo(alignment));
+
+  // Windows will align our allocations to at least 64K.
+  alignment = Max(alignment, GetMmapGranularity());
+
+  uptr mapped_addr =
+      (uptr)VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (!mapped_addr)
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
+
+  // If we got it right on the first try, return. Otherwise, unmap it and go to
+  // the slow path.
+  if (IsAligned(mapped_addr, alignment))
+    return (void*)mapped_addr;
+  if (VirtualFree((void *)mapped_addr, 0, MEM_RELEASE) == 0)
+    ReportMmapFailureAndDie(size, mem_type, "deallocate", GetLastError());
+
+  // If we didn't get an aligned address, overallocate, find an aligned address,
+  // unmap, and try to allocate at that aligned address.
+  int retries = 0;
+  const int kMaxRetries = 10;
+  for (; retries < kMaxRetries &&
+         (mapped_addr == 0 || !IsAligned(mapped_addr, alignment));
+       retries++) {
+    // Overallocate size + alignment bytes.
+    mapped_addr =
+        (uptr)VirtualAlloc(0, size + alignment, MEM_RESERVE, PAGE_NOACCESS);
+    if (!mapped_addr)
+      return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
+
+    // Find the aligned address.
+    uptr aligned_addr = RoundUpTo(mapped_addr, alignment);
+
+    // Free the overallocation.
+    if (VirtualFree((void *)mapped_addr, 0, MEM_RELEASE) == 0)
+      ReportMmapFailureAndDie(size, mem_type, "deallocate", GetLastError());
+
+    // Attempt to allocate exactly the number of bytes we need at the aligned
+    // address. This may fail for a number of reasons, in which case we continue
+    // the loop.
+    mapped_addr = (uptr)VirtualAlloc((void *)aligned_addr, size,
+                                     MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  }
+
+  // Fail if we can't make this work quickly.
+  if (retries == kMaxRetries && mapped_addr == 0)
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate aligned");
+
+  return (void *)mapped_addr;
+}
+
+bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
+  // FIXME: is this really "NoReserve"? On Win32 this does not matter much,
+  // but on Win64 it does.
+  (void)name;  // unsupported
+#if !SANITIZER_GO && SANITIZER_WINDOWS64
+  // On asan/Windows64, use MEM_COMMIT would result in error
+  // 1455:ERROR_COMMITMENT_LIMIT.
+  // Asan uses exception handler to commit page on demand.
+  void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_RESERVE, PAGE_READWRITE);
+#else
+  void *p = VirtualAlloc((LPVOID)fixed_addr, size, MEM_RESERVE | MEM_COMMIT,
+                         PAGE_READWRITE);
+#endif
+  if (p == 0) {
+    Report("ERROR: %s failed to "
+           "allocate %p (%zd) bytes at %p (error code: %d)\n",
+           SanitizerToolName, size, size, fixed_addr, GetLastError());
+    return false;
+  }
+  return true;
+}
+
+bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size, const char *name) {
+  // FIXME: Windows support large pages too. Might be worth checking
+  return MmapFixedNoReserve(fixed_addr, size, name);
+}
+
+// Memory space mapped by 'MmapFixedOrDie' must have been reserved by
+// 'MmapFixedNoAccess'.
+void *MmapFixedOrDie(uptr fixed_addr, uptr size, const char *name) {
+  void *p = VirtualAlloc((LPVOID)fixed_addr, size,
+      MEM_COMMIT, PAGE_READWRITE);
+  if (p == 0) {
+    char mem_type[30];
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+                      fixed_addr);
+    ReportMmapFailureAndDie(size, mem_type, "allocate", GetLastError());
+  }
+  return p;
+}
+
+// Uses fixed_addr for now.
+// Will use offset instead once we've implemented this function for real.
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) {
+  return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size));
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size,
+                                    const char *name) {
+  return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size));
+}
+
+void ReservedAddressRange::Unmap(uptr addr, uptr size) {
+  // Only unmap if it covers the entire range.
+  CHECK((addr == reinterpret_cast<uptr>(base_)) && (size == size_));
+  // We unmap the whole range, just null out the base.
+  base_ = nullptr;
+  size_ = 0;
+  UnmapOrDie(reinterpret_cast<void*>(addr), size);
+}
+
+void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size, const char *name) {
+  void *p = VirtualAlloc((LPVOID)fixed_addr, size,
+      MEM_COMMIT, PAGE_READWRITE);
+  if (p == 0) {
+    char mem_type[30];
+    internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
+                      fixed_addr);
+    return ReturnNullptrOnOOMOrDie(size, mem_type, "allocate");
+  }
+  return p;
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+  // FIXME: make this really NoReserve?
+  return MmapOrDie(size, mem_type);
+}
+
+uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
+  base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size) : MmapNoAccess(size);
+  size_ = size;
+  name_ = name;
+  (void)os_handle_;  // unsupported
+  return reinterpret_cast<uptr>(base_);
+}
+
+
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  (void)name; // unsupported
+  void *res = VirtualAlloc((LPVOID)fixed_addr, size,
+                           MEM_RESERVE, PAGE_NOACCESS);
+  if (res == 0)
+    Report("WARNING: %s failed to "
+           "mprotect %p (%zd) bytes at %p (error code: %d)\n",
+           SanitizerToolName, size, size, fixed_addr, GetLastError());
+  return res;
+}
+
+void *MmapNoAccess(uptr size) {
+  void *res = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS);
+  if (res == 0)
+    Report("WARNING: %s failed to "
+           "mprotect %p (%zd) bytes (error code: %d)\n",
+           SanitizerToolName, size, size, GetLastError());
+  return res;
+}
+
+bool MprotectNoAccess(uptr addr, uptr size) {
+  DWORD old_protection;
+  return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
+}
+
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
+  // This is almost useless on 32-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
+}
+
+void SetShadowRegionHugePageMode(uptr addr, uptr size) {
+  // FIXME: probably similar to ReleaseMemoryToOS.
+}
+
+bool DontDumpShadowMemory(uptr addr, uptr length) {
+  // This is almost useless on 32-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
+  return true;
+}
+
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found,
+                              uptr *max_occupied_addr) {
+  uptr address = 0;
+  while (true) {
+    MEMORY_BASIC_INFORMATION info;
+    if (!::VirtualQuery((void*)address, &info, sizeof(info)))
+      return 0;
+
+    if (info.State == MEM_FREE) {
+      uptr shadow_address = RoundUpTo((uptr)info.BaseAddress + left_padding,
+                                      alignment);
+      if (shadow_address + size < (uptr)info.BaseAddress + info.RegionSize)
+        return shadow_address;
+    }
+
+    // Move to the next region.
+    address = (uptr)info.BaseAddress + info.RegionSize;
+  }
+  return 0;
+}
+
+bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
+  MEMORY_BASIC_INFORMATION mbi;
+  CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
+  return mbi.Protect == PAGE_NOACCESS &&
+         (uptr)mbi.BaseAddress + mbi.RegionSize >= range_end;
+}
+
+void *MapFileToMemory(const char *file_name, uptr *buff_size) {
+  UNIMPLEMENTED();
+}
+
+void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset) {
+  UNIMPLEMENTED();
+}
+
+static const int kMaxEnvNameLength = 128;
+static const DWORD kMaxEnvValueLength = 32767;
+
+namespace {
+
+struct EnvVariable {
+  char name[kMaxEnvNameLength];
+  char value[kMaxEnvValueLength];
+};
+
+}  // namespace
+
+static const int kEnvVariables = 5;
+static EnvVariable env_vars[kEnvVariables];
+static int num_env_vars;
+
+const char *GetEnv(const char *name) {
+  // Note: this implementation caches the values of the environment variables
+  // and limits their quantity.
+  for (int i = 0; i < num_env_vars; i++) {
+    if (0 == internal_strcmp(name, env_vars[i].name))
+      return env_vars[i].value;
+  }
+  CHECK_LT(num_env_vars, kEnvVariables);
+  DWORD rv = GetEnvironmentVariableA(name, env_vars[num_env_vars].value,
+                                     kMaxEnvValueLength);
+  if (rv > 0 && rv < kMaxEnvValueLength) {
+    CHECK_LT(internal_strlen(name), kMaxEnvNameLength);
+    internal_strncpy(env_vars[num_env_vars].name, name, kMaxEnvNameLength);
+    num_env_vars++;
+    return env_vars[num_env_vars - 1].value;
+  }
+  return 0;
+}
+
+const char *GetPwd() {
+  UNIMPLEMENTED();
+}
+
+u32 GetUid() {
+  UNIMPLEMENTED();
+}
+
+namespace {
+struct ModuleInfo {
+  const char *filepath;
+  uptr base_address;
+  uptr end_address;
+};
+
+#if !SANITIZER_GO
+int CompareModulesBase(const void *pl, const void *pr) {
+  const ModuleInfo *l = (const ModuleInfo *)pl, *r = (const ModuleInfo *)pr;
+  if (l->base_address < r->base_address)
+    return -1;
+  return l->base_address > r->base_address;
+}
+#endif
+}  // namespace
+
+#if !SANITIZER_GO
+void DumpProcessMap() {
+  Report("Dumping process modules:\n");
+  ListOfModules modules;
+  modules.init();
+  uptr num_modules = modules.size();
+
+  InternalMmapVector<ModuleInfo> module_infos(num_modules);
+  for (size_t i = 0; i < num_modules; ++i) {
+    module_infos[i].filepath = modules[i].full_name();
+    module_infos[i].base_address = modules[i].ranges().front()->beg;
+    module_infos[i].end_address = modules[i].ranges().back()->end;
+  }
+  qsort(module_infos.data(), num_modules, sizeof(ModuleInfo),
+        CompareModulesBase);
+
+  for (size_t i = 0; i < num_modules; ++i) {
+    const ModuleInfo &mi = module_infos[i];
+    if (mi.end_address != 0) {
+      Printf("\t%p-%p %s\n", mi.base_address, mi.end_address,
+             mi.filepath[0] ? mi.filepath : "[no name]");
+    } else if (mi.filepath[0]) {
+      Printf("\t??\?-??? %s\n", mi.filepath);
+    } else {
+      Printf("\t???\n");
+    }
+  }
+}
+#endif
+
+void PrintModuleMap() { }
+
+void DisableCoreDumperIfNecessary() {
+  // Do nothing.
+}
+
+void ReExec() {
+  UNIMPLEMENTED();
+}
+
+void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+
+bool StackSizeIsUnlimited() {
+  UNIMPLEMENTED();
+}
+
+void SetStackSizeLimitInBytes(uptr limit) {
+  UNIMPLEMENTED();
+}
+
+bool AddressSpaceIsUnlimited() {
+  UNIMPLEMENTED();
+}
+
+void SetAddressSpaceUnlimited() {
+  UNIMPLEMENTED();
+}
+
+bool IsPathSeparator(const char c) {
+  return c == '\\' || c == '/';
+}
+
+static bool IsAlpha(char c) {
+  c = ToLower(c);
+  return c >= 'a' && c <= 'z';
+}
+
+bool IsAbsolutePath(const char *path) {
+  return path != nullptr && IsAlpha(path[0]) && path[1] == ':' &&
+         IsPathSeparator(path[2]);
+}
+
+void SleepForSeconds(int seconds) {
+  Sleep(seconds * 1000);
+}
+
+void SleepForMillis(int millis) {
+  Sleep(millis);
+}
+
+u64 NanoTime() {
+  static LARGE_INTEGER frequency = {};
+  LARGE_INTEGER counter;
+  if (UNLIKELY(frequency.QuadPart == 0)) {
+    QueryPerformanceFrequency(&frequency);
+    CHECK_NE(frequency.QuadPart, 0);
+  }
+  QueryPerformanceCounter(&counter);
+  counter.QuadPart *= 1000ULL * 1000000ULL;
+  counter.QuadPart /= frequency.QuadPart;
+  return counter.QuadPart;
+}
+
+u64 MonotonicNanoTime() { return NanoTime(); }
+
+void Abort() {
+  internal__exit(3);
+}
+
+#if !SANITIZER_GO
+// Read the file to extract the ImageBase field from the PE header. If ASLR is
+// disabled and this virtual address is available, the loader will typically
+// load the image at this address. Therefore, we call it the preferred base. Any
+// addresses in the DWARF typically assume that the object has been loaded at
+// this address.
+static uptr GetPreferredBase(const char *modname) {
+  fd_t fd = OpenFile(modname, RdOnly, nullptr);
+  if (fd == kInvalidFd)
+    return 0;
+  FileCloser closer(fd);
+
+  // Read just the DOS header.
+  IMAGE_DOS_HEADER dos_header;
+  uptr bytes_read;
+  if (!ReadFromFile(fd, &dos_header, sizeof(dos_header), &bytes_read) ||
+      bytes_read != sizeof(dos_header))
+    return 0;
+
+  // The file should start with the right signature.
+  if (dos_header.e_magic != IMAGE_DOS_SIGNATURE)
+    return 0;
+
+  // The layout at e_lfanew is:
+  // "PE\0\0"
+  // IMAGE_FILE_HEADER
+  // IMAGE_OPTIONAL_HEADER
+  // Seek to e_lfanew and read all that data.
+  char buf[4 + sizeof(IMAGE_FILE_HEADER) + sizeof(IMAGE_OPTIONAL_HEADER)];
+  if (::SetFilePointer(fd, dos_header.e_lfanew, nullptr, FILE_BEGIN) ==
+      INVALID_SET_FILE_POINTER)
+    return 0;
+  if (!ReadFromFile(fd, &buf[0], sizeof(buf), &bytes_read) ||
+      bytes_read != sizeof(buf))
+    return 0;
+
+  // Check for "PE\0\0" before the PE header.
+  char *pe_sig = &buf[0];
+  if (internal_memcmp(pe_sig, "PE\0\0", 4) != 0)
+    return 0;
+
+  // Skip over IMAGE_FILE_HEADER. We could do more validation here if we wanted.
+  IMAGE_OPTIONAL_HEADER *pe_header =
+      (IMAGE_OPTIONAL_HEADER *)(pe_sig + 4 + sizeof(IMAGE_FILE_HEADER));
+
+  // Check for more magic in the PE header.
+  if (pe_header->Magic != IMAGE_NT_OPTIONAL_HDR_MAGIC)
+    return 0;
+
+  // Finally, return the ImageBase.
+  return (uptr)pe_header->ImageBase;
+}
+
+void ListOfModules::init() {
+  clearOrInit();
+  HANDLE cur_process = GetCurrentProcess();
+
+  // Query the list of modules.  Start by assuming there are no more than 256
+  // modules and retry if that's not sufficient.
+  HMODULE *hmodules = 0;
+  uptr modules_buffer_size = sizeof(HMODULE) * 256;
+  DWORD bytes_required;
+  while (!hmodules) {
+    hmodules = (HMODULE *)MmapOrDie(modules_buffer_size, __FUNCTION__);
+    CHECK(EnumProcessModules(cur_process, hmodules, modules_buffer_size,
+                             &bytes_required));
+    if (bytes_required > modules_buffer_size) {
+      // Either there turned out to be more than 256 hmodules, or new hmodules
+      // could have loaded since the last try.  Retry.
+      UnmapOrDie(hmodules, modules_buffer_size);
+      hmodules = 0;
+      modules_buffer_size = bytes_required;
+    }
+  }
+
+  // |num_modules| is the number of modules actually present,
+  size_t num_modules = bytes_required / sizeof(HMODULE);
+  for (size_t i = 0; i < num_modules; ++i) {
+    HMODULE handle = hmodules[i];
+    MODULEINFO mi;
+    if (!GetModuleInformation(cur_process, handle, &mi, sizeof(mi)))
+      continue;
+
+    // Get the UTF-16 path and convert to UTF-8.
+    wchar_t modname_utf16[kMaxPathLength];
+    int modname_utf16_len =
+        GetModuleFileNameW(handle, modname_utf16, kMaxPathLength);
+    if (modname_utf16_len == 0)
+      modname_utf16[0] = '\0';
+    char module_name[kMaxPathLength];
+    int module_name_len =
+        ::WideCharToMultiByte(CP_UTF8, 0, modname_utf16, modname_utf16_len + 1,
+                              &module_name[0], kMaxPathLength, NULL, NULL);
+    module_name[module_name_len] = '\0';
+
+    uptr base_address = (uptr)mi.lpBaseOfDll;
+    uptr end_address = (uptr)mi.lpBaseOfDll + mi.SizeOfImage;
+
+    // Adjust the base address of the module so that we get a VA instead of an
+    // RVA when computing the module offset. This helps llvm-symbolizer find the
+    // right DWARF CU. In the common case that the image is loaded at it's
+    // preferred address, we will now print normal virtual addresses.
+    uptr preferred_base = GetPreferredBase(&module_name[0]);
+    uptr adjusted_base = base_address - preferred_base;
+
+    LoadedModule cur_module;
+    cur_module.set(module_name, adjusted_base);
+    // We add the whole module as one single address range.
+    cur_module.addAddressRange(base_address, end_address, /*executable*/ true,
+                               /*writable*/ true);
+    modules_.push_back(cur_module);
+  }
+  UnmapOrDie(hmodules, modules_buffer_size);
+}
+
+void ListOfModules::fallbackInit() { clear(); }
+
+// We can't use atexit() directly at __asan_init time as the CRT is not fully
+// initialized at this point.  Place the functions into a vector and use
+// atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers).
+InternalMmapVectorNoCtor<void (*)(void)> atexit_functions;
+
+int Atexit(void (*function)(void)) {
+  atexit_functions.push_back(function);
+  return 0;
+}
+
+static int RunAtexit() {
+  TraceLoggingUnregister(g_asan_provider);
+  int ret = 0;
+  for (uptr i = 0; i < atexit_functions.size(); ++i) {
+    ret |= atexit(atexit_functions[i]);
+  }
+  return ret;
+}
+
+#pragma section(".CRT$XID", long, read)
+__declspec(allocate(".CRT$XID")) int (*__run_atexit)() = RunAtexit;
+#endif
+
+// ------------------ sanitizer_libc.h
+fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *last_error) {
+  // FIXME: Use the wide variants to handle Unicode filenames.
+  fd_t res;
+  if (mode == RdOnly) {
+    res = CreateFileA(filename, GENERIC_READ,
+                      FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                      nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+  } else if (mode == WrOnly) {
+    res = CreateFileA(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS,
+                      FILE_ATTRIBUTE_NORMAL, nullptr);
+  } else {
+    UNIMPLEMENTED();
+  }
+  CHECK(res != kStdoutFd || kStdoutFd == kInvalidFd);
+  CHECK(res != kStderrFd || kStderrFd == kInvalidFd);
+  if (res == kInvalidFd && last_error)
+    *last_error = GetLastError();
+  return res;
+}
+
+void CloseFile(fd_t fd) {
+  CloseHandle(fd);
+}
+
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read,
+                  error_t *error_p) {
+  CHECK(fd != kInvalidFd);
+
+  // bytes_read can't be passed directly to ReadFile:
+  // uptr is unsigned long long on 64-bit Windows.
+  unsigned long num_read_long;
+
+  bool success = ::ReadFile(fd, buff, buff_size, &num_read_long, nullptr);
+  if (!success && error_p)
+    *error_p = GetLastError();
+  if (bytes_read)
+    *bytes_read = num_read_long;
+  return success;
+}
+
+bool SupportsColoredOutput(fd_t fd) {
+  // FIXME: support colored output.
+  return false;
+}
+
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written,
+                 error_t *error_p) {
+  CHECK(fd != kInvalidFd);
+
+  // Handle null optional parameters.
+  error_t dummy_error;
+  error_p = error_p ? error_p : &dummy_error;
+  uptr dummy_bytes_written;
+  bytes_written = bytes_written ? bytes_written : &dummy_bytes_written;
+
+  // Initialize output parameters in case we fail.
+  *error_p = 0;
+  *bytes_written = 0;
+
+  // Map the conventional Unix fds 1 and 2 to Windows handles. They might be
+  // closed, in which case this will fail.
+  if (fd == kStdoutFd || fd == kStderrFd) {
+    fd = GetStdHandle(fd == kStdoutFd ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE);
+    if (fd == 0) {
+      *error_p = ERROR_INVALID_HANDLE;
+      return false;
+    }
+  }
+
+  DWORD bytes_written_32;
+  if (!WriteFile(fd, buff, buff_size, &bytes_written_32, 0)) {
+    *error_p = GetLastError();
+    return false;
+  } else {
+    *bytes_written = bytes_written_32;
+    return true;
+  }
+}
+
+uptr internal_sched_yield() {
+  Sleep(0);
+  return 0;
+}
+
+void internal__exit(int exitcode) {
+  TraceLoggingUnregister(g_asan_provider);
+  // ExitProcess runs some finalizers, so use TerminateProcess to avoid that.
+  // The debugger doesn't stop on TerminateProcess like it does on ExitProcess,
+  // so add our own breakpoint here.
+  if (::IsDebuggerPresent())
+    __debugbreak();
+  TerminateProcess(GetCurrentProcess(), exitcode);
+  BUILTIN_UNREACHABLE();
+}
+
+uptr internal_ftruncate(fd_t fd, uptr size) {
+  UNIMPLEMENTED();
+}
+
+uptr GetRSS() {
+  PROCESS_MEMORY_COUNTERS counters;
+  if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters)))
+    return 0;
+  return counters.WorkingSetSize;
+}
+
+void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
+void internal_join_thread(void *th) { }
+
+// ---------------------- BlockingMutex ---------------- {{{1
+
+BlockingMutex::BlockingMutex() {
+  CHECK(sizeof(SRWLOCK) <= sizeof(opaque_storage_));
+  internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+  AcquireSRWLockExclusive((PSRWLOCK)opaque_storage_);
+  CHECK_EQ(owner_, 0);
+  owner_ = GetThreadSelf();
+}
+
+void BlockingMutex::Unlock() {
+  CheckLocked();
+  owner_ = 0;
+  ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
+}
+
+void BlockingMutex::CheckLocked() {
+  CHECK_EQ(owner_, GetThreadSelf());
+}
+
+uptr GetTlsSize() {
+  return 0;
+}
+
+void InitTlsSize() {
+}
+
+void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+                          uptr *tls_addr, uptr *tls_size) {
+#if SANITIZER_GO
+  *stk_addr = 0;
+  *stk_size = 0;
+  *tls_addr = 0;
+  *tls_size = 0;
+#else
+  uptr stack_top, stack_bottom;
+  GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
+  *stk_addr = stack_bottom;
+  *stk_size = stack_top - stack_bottom;
+  *tls_addr = 0;
+  *tls_size = 0;
+#endif
+}
+
+void ReportFile::Write(const char *buffer, uptr length) {
+  SpinMutexLock l(mu);
+  ReopenIfNecessary();
+  if (!WriteToFile(fd, buffer, length)) {
+    // stderr may be closed, but we may be able to print to the debugger
+    // instead.  This is the case when launching a program from Visual Studio,
+    // and the following routine should write to its console.
+    OutputDebugStringA(buffer);
+  }
+}
+
+void SetAlternateSignalStack() {
+  // FIXME: Decide what to do on Windows.
+}
+
+void UnsetAlternateSignalStack() {
+  // FIXME: Decide what to do on Windows.
+}
+
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {
+  (void)handler;
+  // FIXME: Decide what to do on Windows.
+}
+
+HandleSignalMode GetHandleSignalMode(int signum) {
+  // FIXME: Decide what to do on Windows.
+  return kHandleSignalNo;
+}
+
+// Check based on flags if we should handle this exception.
+bool IsHandledDeadlyException(DWORD exceptionCode) {
+  switch (exceptionCode) {
+    case EXCEPTION_ACCESS_VIOLATION:
+    case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
+    case EXCEPTION_STACK_OVERFLOW:
+    case EXCEPTION_DATATYPE_MISALIGNMENT:
+    case EXCEPTION_IN_PAGE_ERROR:
+      return common_flags()->handle_segv;
+    case EXCEPTION_ILLEGAL_INSTRUCTION:
+    case EXCEPTION_PRIV_INSTRUCTION:
+    case EXCEPTION_BREAKPOINT:
+      return common_flags()->handle_sigill;
+    case EXCEPTION_FLT_DENORMAL_OPERAND:
+    case EXCEPTION_FLT_DIVIDE_BY_ZERO:
+    case EXCEPTION_FLT_INEXACT_RESULT:
+    case EXCEPTION_FLT_INVALID_OPERATION:
+    case EXCEPTION_FLT_OVERFLOW:
+    case EXCEPTION_FLT_STACK_CHECK:
+    case EXCEPTION_FLT_UNDERFLOW:
+    case EXCEPTION_INT_DIVIDE_BY_ZERO:
+    case EXCEPTION_INT_OVERFLOW:
+      return common_flags()->handle_sigfpe;
+  }
+  return false;
+}
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+  SYSTEM_INFO si;
+  GetNativeSystemInfo(&si);
+  uptr page_size = si.dwPageSize;
+  uptr page_mask = ~(page_size - 1);
+
+  for (uptr page = beg & page_mask, end = (beg + size - 1) & page_mask;
+       page <= end;) {
+    MEMORY_BASIC_INFORMATION info;
+    if (VirtualQuery((LPCVOID)page, &info, sizeof(info)) != sizeof(info))
+      return false;
+
+    if (info.Protect == 0 || info.Protect == PAGE_NOACCESS ||
+        info.Protect == PAGE_EXECUTE)
+      return false;
+
+    if (info.RegionSize == 0)
+      return false;
+
+    page += info.RegionSize;
+  }
+
+  return true;
+}
+
+bool SignalContext::IsStackOverflow() const {
+  return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW;
+}
+
+void SignalContext::InitPcSpBp() {
+  EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+  CONTEXT *context_record = (CONTEXT *)context;
+
+  pc = (uptr)exception_record->ExceptionAddress;
+#ifdef _WIN64
+  bp = (uptr)context_record->Rbp;
+  sp = (uptr)context_record->Rsp;
+#else
+  bp = (uptr)context_record->Ebp;
+  sp = (uptr)context_record->Esp;
+#endif
+}
+
+uptr SignalContext::GetAddress() const {
+  EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+  return exception_record->ExceptionInformation[1];
+}
+
+bool SignalContext::IsMemoryAccess() const {
+  return GetWriteFlag() != SignalContext::UNKNOWN;
+}
+
+bool SignalContext::IsTrueFaultingAddress() const {
+  // FIXME: Provide real implementation for this. See Linux and Mac variants.
+  return IsMemoryAccess();
+}
+
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
+  EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+  // The contents of this array are documented at
+  // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx
+  // The first element indicates read as 0, write as 1, or execute as 8.  The
+  // second element is the faulting address.
+  switch (exception_record->ExceptionInformation[0]) {
+    case 0:
+      return SignalContext::READ;
+    case 1:
+      return SignalContext::WRITE;
+    case 8:
+      return SignalContext::UNKNOWN;
+  }
+  return SignalContext::UNKNOWN;
+}
+
+void SignalContext::DumpAllRegisters(void *context) {
+  // FIXME: Implement this.
+}
+
+int SignalContext::GetType() const {
+  return static_cast<const EXCEPTION_RECORD *>(siginfo)->ExceptionCode;
+}
+
+const char *SignalContext::Describe() const {
+  unsigned code = GetType();
+  // Get the string description of the exception if this is a known deadly
+  // exception.
+  switch (code) {
+    case EXCEPTION_ACCESS_VIOLATION:
+      return "access-violation";
+    case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
+      return "array-bounds-exceeded";
+    case EXCEPTION_STACK_OVERFLOW:
+      return "stack-overflow";
+    case EXCEPTION_DATATYPE_MISALIGNMENT:
+      return "datatype-misalignment";
+    case EXCEPTION_IN_PAGE_ERROR:
+      return "in-page-error";
+    case EXCEPTION_ILLEGAL_INSTRUCTION:
+      return "illegal-instruction";
+    case EXCEPTION_PRIV_INSTRUCTION:
+      return "priv-instruction";
+    case EXCEPTION_BREAKPOINT:
+      return "breakpoint";
+    case EXCEPTION_FLT_DENORMAL_OPERAND:
+      return "flt-denormal-operand";
+    case EXCEPTION_FLT_DIVIDE_BY_ZERO:
+      return "flt-divide-by-zero";
+    case EXCEPTION_FLT_INEXACT_RESULT:
+      return "flt-inexact-result";
+    case EXCEPTION_FLT_INVALID_OPERATION:
+      return "flt-invalid-operation";
+    case EXCEPTION_FLT_OVERFLOW:
+      return "flt-overflow";
+    case EXCEPTION_FLT_STACK_CHECK:
+      return "flt-stack-check";
+    case EXCEPTION_FLT_UNDERFLOW:
+      return "flt-underflow";
+    case EXCEPTION_INT_DIVIDE_BY_ZERO:
+      return "int-divide-by-zero";
+    case EXCEPTION_INT_OVERFLOW:
+      return "int-overflow";
+  }
+  return "unknown exception";
+}
+
+uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+  // FIXME: Actually implement this function.
+  CHECK_GT(buf_len, 0);
+  buf[0] = 0;
+  return 0;
+}
+
+uptr ReadLongProcessName(/*out*/char *buf, uptr buf_len) {
+  return ReadBinaryName(buf, buf_len);
+}
+
+void CheckVMASize() {
+  // Do nothing.
+}
+
+void InitializePlatformEarly() {
+  // Do nothing.
+}
+
+void MaybeReexec() {
+  // No need to re-exec on Windows.
+}
+
+void CheckASLR() {
+  // Do nothing
+}
+
+void CheckMPROTECT() {
+  // Do nothing
+}
+
+char **GetArgv() {
+  // FIXME: Actually implement this function.
+  return 0;
+}
+
+char **GetEnviron() {
+  // FIXME: Actually implement this function.
+  return 0;
+}
+
+pid_t StartSubprocess(const char *program, const char *const argv[],
+                      const char *const envp[], fd_t stdin_fd, fd_t stdout_fd,
+                      fd_t stderr_fd) {
+  // FIXME: implement on this platform
+  // Should be implemented based on
+  // SymbolizerProcess::StarAtSymbolizerSubprocess
+  // from lib/sanitizer_common/sanitizer_symbolizer_win.cpp.
+  return -1;
+}
+
+bool IsProcessRunning(pid_t pid) {
+  // FIXME: implement on this platform.
+  return false;
+}
+
+int WaitForProcess(pid_t pid) { return -1; }
+
+// FIXME implement on this platform.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+
+void CheckNoDeepBind(const char *filename, int flag) {
+  // Do nothing.
+}
+
+// FIXME: implement on this platform.
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+  UNIMPLEMENTED();
+}
+
+u32 GetNumberOfCPUs() {
+  SYSTEM_INFO sysinfo = {};
+  GetNativeSystemInfo(&sysinfo);
+  return sysinfo.dwNumberOfProcessors;
+}
+
+#if SANITIZER_WIN_TRACE
+// TODO(mcgov): Rename this project-wide to PlatformLogInit
+void AndroidLogInit(void) {
+  HRESULT hr = TraceLoggingRegister(g_asan_provider);
+  if (!SUCCEEDED(hr))
+    return;
+}
+
+void SetAbortMessage(const char *) {}
+
+void LogFullErrorReport(const char *buffer) {
+  if (common_flags()->log_to_syslog) {
+    InternalMmapVector<wchar_t> filename;
+    DWORD filename_length = 0;
+    do {
+      filename.resize(filename.size() + 0x100);
+      filename_length =
+          GetModuleFileNameW(NULL, filename.begin(), filename.size());
+    } while (filename_length >= filename.size());
+    TraceLoggingWrite(g_asan_provider, "AsanReportEvent",
+                      TraceLoggingValue(filename.begin(), "ExecutableName"),
+                      TraceLoggingValue(buffer, "AsanReportContents"));
+  }
+}
+#endif // SANITIZER_WIN_TRACE
+
+}  // namespace __sanitizer
+
+#endif  // _WIN32
diff --git a/lib/tsan/sanitizer_common/sanitizer_win.h b/lib/tsan/sanitizer_common/sanitizer_win.h
new file mode 100644
index 0000000000..ff8939ca5e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_win.h
@@ -0,0 +1,25 @@
+//===-- sanitizer_win.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Windows-specific declarations.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_H
+#define SANITIZER_WIN_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+// Check based on flags if we should handle the exception.
+bool IsHandledDeadlyException(DWORD exceptionCode);
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_WINDOWS
+#endif  // SANITIZER_WIN_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_win_defs.h b/lib/tsan/sanitizer_common/sanitizer_win_defs.h
new file mode 100644
index 0000000000..bfe38a3323
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_win_defs.h
@@ -0,0 +1,174 @@
+//===-- sanitizer_win_defs.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common definitions for Windows-specific code.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_DEFS_H
+#define SANITIZER_WIN_DEFS_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#ifndef WINAPI
+#if defined(_M_IX86) || defined(__i386__)
+#define WINAPI __stdcall
+#else
+#define WINAPI
+#endif
+#endif
+
+#if defined(_M_IX86) || defined(__i386__)
+#define WIN_SYM_PREFIX "_"
+#else
+#define WIN_SYM_PREFIX
+#endif
+
+// For MinGW, the /export: directives contain undecorated symbols, contrary to
+// link/lld-link. The GNU linker doesn't support /alternatename and /include
+// though, thus lld-link in MinGW mode interprets them in the same way as
+// in the default mode.
+#ifdef __MINGW32__
+#define WIN_EXPORT_PREFIX
+#else
+#define WIN_EXPORT_PREFIX WIN_SYM_PREFIX
+#endif
+
+// Intermediate macro to ensure the parameter is expanded before stringified.
+#define STRINGIFY_(A) #A
+#define STRINGIFY(A) STRINGIFY_(A)
+
+#if !SANITIZER_GO
+
+// ----------------- A workaround for the absence of weak symbols --------------
+// We don't have a direct equivalent of weak symbols when using MSVC, but we can
+// use the /alternatename directive to tell the linker to default a specific
+// symbol to a specific value.
+// Take into account that this is a pragma directive for the linker, so it will
+// be ignored by the compiler and the function will be marked as UNDEF in the
+// symbol table of the resulting object file. The linker won't find the default
+// implementation until it links with that object file.
+// So, suppose we provide a default implementation "fundef" for "fun", and this
+// is compiled into the object file "test.obj" including the pragma directive.
+// If we have some code with references to "fun" and we link that code with
+// "test.obj", it will work because the linker always link object files.
+// But, if "test.obj" is included in a static library, like "test.lib", then the
+// liker will only link to "test.obj" if necessary. If we only included the
+// definition of "fun", it won't link to "test.obj" (from test.lib) because
+// "fun" appears as UNDEF, so it doesn't resolve the symbol "fun", and will
+// result in a link error (the linker doesn't find the pragma directive).
+// So, a workaround is to force linkage with the modules that include weak
+// definitions, with the following macro: WIN_FORCE_LINK()
+
+#define WIN_WEAK_ALIAS(Name, Default)                                          \
+  __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY(Name) "="\
+                                             WIN_SYM_PREFIX STRINGIFY(Default)))
+
+#define WIN_FORCE_LINK(Name)                                                   \
+  __pragma(comment(linker, "/include:" WIN_SYM_PREFIX STRINGIFY(Name)))
+
+#define WIN_EXPORT(ExportedName, Name)                                         \
+  __pragma(comment(linker, "/export:" WIN_EXPORT_PREFIX STRINGIFY(ExportedName)\
+                                  "=" WIN_EXPORT_PREFIX STRINGIFY(Name)))
+
+// We cannot define weak functions on Windows, but we can use WIN_WEAK_ALIAS()
+// which defines an alias to a default implementation, and only works when
+// linking statically.
+// So, to define a weak function "fun", we define a default implementation with
+// a different name "fun__def" and we create a "weak alias" fun = fun__def.
+// Then, users can override it just defining "fun".
+// We impose "extern "C"" because otherwise WIN_WEAK_ALIAS() will fail because
+// of name mangling.
+
+// Dummy name for default implementation of weak function.
+# define WEAK_DEFAULT_NAME(Name) Name##__def
+// Name for exported implementation of weak function.
+# define WEAK_EXPORT_NAME(Name) Name##__dll
+
+// Use this macro when you need to define and export a weak function from a
+// library. For example:
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+# define WIN_WEAK_EXPORT_DEF(ReturnType, Name, ...)                            \
+  WIN_WEAK_ALIAS(Name, WEAK_DEFAULT_NAME(Name))                                \
+  WIN_EXPORT(WEAK_EXPORT_NAME(Name), Name)                                     \
+  extern "C" ReturnType Name(__VA_ARGS__);                                     \
+  extern "C" ReturnType WEAK_DEFAULT_NAME(Name)(__VA_ARGS__)
+
+// Use this macro when you need to import a weak function from a library. It
+// defines a weak alias to the imported function from the dll. For example:
+//   WIN_WEAK_IMPORT_DEF(compare)
+# define WIN_WEAK_IMPORT_DEF(Name)                                             \
+  WIN_WEAK_ALIAS(Name, WEAK_EXPORT_NAME(Name))
+
+// So, for Windows we provide something similar to weak symbols in Linux, with
+// some differences:
+// + A default implementation must always be provided.
+//
+// + When linking statically it works quite similarly. For example:
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//
+//   // client.cc
+//   // We can use the default implementation from the library:
+//   compare(1, 2);
+//   // Or we can override it:
+//   extern "C" bool compare (int a, int b) { return a >= b; }
+//
+//  And it will work fine. If we don't override the function, we need to ensure
+//  that the linker includes the object file with the default implementation.
+//  We can do so with the linker option "-wholearchive:".
+//
+// + When linking dynamically with a library (dll), weak functions are exported
+//  with "__dll" suffix. Clients can use the macro WIN_WEAK_IMPORT_DEF(fun)
+//  which defines a "weak alias" fun = fun__dll.
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//
+//   // client.cc
+//   WIN_WEAK_IMPORT_DEF(compare)
+//   // We can use the default implementation from the library:
+//   compare(1, 2);
+//   // Or we can override it:
+//   extern "C" bool compare (int a, int b) { return a >= b; }
+//
+//  But if we override the function, the dlls don't have access to it (which
+//  is different in linux). If that is desired, the strong definition must be
+//  exported and interception can be used from the rest of the dlls.
+//
+//   // libExample.cc
+//   WIN_WEAK_EXPORT_DEF(bool, compare, int a, int b) { return a > b; }
+//   // When initialized, check if the main executable defined "compare".
+//   int libExample_init() {
+//     uptr fnptr = __interception::InternalGetProcAddress(
+//         (void *)GetModuleHandleA(0), "compare");
+//     if (fnptr && !__interception::OverrideFunction((uptr)compare, fnptr, 0))
+//       abort();
+//     return 0;
+//   }
+//
+//   // client.cc
+//   WIN_WEAK_IMPORT_DEF(compare)
+//   // We override and export compare:
+//   extern "C" __declspec(dllexport) bool compare (int a, int b) {
+//     return a >= b;
+//   }
+//
+
+#else // SANITIZER_GO
+
+// Go neither needs nor wants weak references.
+// The shenanigans above don't work for gcc.
+# define WIN_WEAK_EXPORT_DEF(ReturnType, Name, ...)                            \
+  extern "C" ReturnType Name(__VA_ARGS__)
+
+#endif // SANITIZER_GO
+
+#endif // SANITIZER_WINDOWS
+#endif // SANITIZER_WIN_DEFS_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h b/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h
new file mode 100644
index 0000000000..48c73c4c98
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h
@@ -0,0 +1,181 @@
+//===-- sanitizer_win_dll_thunk.h -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This header provide helper macros to delegate calls to the shared runtime
+// that lives in the main executable. It should be included to dll_thunks that
+// will be linked to the dlls, when the sanitizer is a static library included
+// in the main executable.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_DLL_THUNK_H
+#define SANITIZER_WIN_DLL_THUNK_H
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+uptr dllThunkGetRealAddrOrDie(const char *name);
+
+int dllThunkIntercept(const char* main_function, uptr dll_function);
+
+int dllThunkInterceptWhenPossible(const char* main_function,
+    const char* default_function, uptr dll_function);
+}
+
+extern "C" int __dll_thunk_init();
+
+// ----------------- Function interception helper macros -------------------- //
+// Override dll_function with main_function from main executable.
+#define INTERCEPT_OR_DIE(main_function, dll_function)                          \
+  static int intercept_##dll_function() {                                      \
+    return __sanitizer::dllThunkIntercept(main_function, (__sanitizer::uptr)   \
+        dll_function);                                                         \
+  }                                                                            \
+  __pragma(section(".DLLTH$M", long, read))                                    \
+  __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() =       \
+    intercept_##dll_function;
+
+// Try to override dll_function with main_function from main executable.
+// If main_function is not present, override dll_function with default_function.
+#define INTERCEPT_WHEN_POSSIBLE(main_function, default_function, dll_function) \
+  static int intercept_##dll_function() {                                      \
+    return __sanitizer::dllThunkInterceptWhenPossible(main_function,           \
+        default_function, (__sanitizer::uptr)dll_function);                    \
+  }                                                                            \
+  __pragma(section(".DLLTH$M", long, read))                                    \
+  __declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() =       \
+    intercept_##dll_function;
+
+// -------------------- Function interception macros ------------------------ //
+// Special case of hooks -- ASan own interface functions.  Those are only called
+// after __asan_init, thus an empty implementation is sufficient.
+#define INTERCEPT_SANITIZER_FUNCTION(name)                                     \
+  extern "C" __declspec(noinline) void name() {                                \
+    volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__;                  \
+    static const char function_name[] = #name;                                 \
+    for (const char* ptr = &function_name[0]; *ptr; ++ptr)                     \
+      prevent_icf ^= *ptr;                                                     \
+    (void)prevent_icf;                                                         \
+    __debugbreak();                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name)
+
+// Special case of hooks -- Weak functions, could be redefined in the main
+// executable, but that is not necessary, so we shouldn't die if we can not find
+// a reference. Instead, when the function is not present in the main executable
+// we consider the default impl provided by asan library.
+#define INTERCEPT_SANITIZER_WEAK_FUNCTION(name)                                \
+  extern "C" __declspec(noinline) void name() {                                \
+    volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__;                  \
+    static const char function_name[] = #name;                                 \
+    for (const char* ptr = &function_name[0]; *ptr; ++ptr)                     \
+      prevent_icf ^= *ptr;                                                     \
+    (void)prevent_icf;                                                         \
+    __debugbreak();                                                            \
+  }                                                                            \
+  INTERCEPT_WHEN_POSSIBLE(#name, STRINGIFY(WEAK_EXPORT_NAME(name)), name)
+
+// We can't define our own version of strlen etc. because that would lead to
+// link-time or even type mismatch errors.  Instead, we can declare a function
+// just to be able to get its address.  Me may miss the first few calls to the
+// functions since it can be called before __dll_thunk_init, but that would lead
+// to false negatives in the startup code before user's global initializers,
+// which isn't a big deal.
+#define INTERCEPT_LIBRARY_FUNCTION(name)                                       \
+  extern "C" void name();                                                      \
+  INTERCEPT_OR_DIE(WRAPPER_NAME(name), name)
+
+// Use these macros for functions that could be called before __dll_thunk_init()
+// is executed and don't lead to errors if defined (free, malloc, etc).
+#define INTERCEPT_WRAP_V_V(name)                                               \
+  extern "C" void name() {                                                     \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn();                                                                      \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_W(name)                                               \
+  extern "C" void name(void *arg) {                                            \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg);                                                                   \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_WW(name)                                              \
+  extern "C" void name(void *arg1, void *arg2) {                               \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg1, arg2);                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_V_WWW(name)                                             \
+  extern "C" void name(void *arg1, void *arg2, void *arg3) {                   \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    fn(arg1, arg2, arg3);                                                      \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_V(name)                                               \
+  extern "C" void *name() {                                                    \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn();                                                               \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_W(name)                                               \
+  extern "C" void *name(void *arg) {                                           \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg);                                                            \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WW(name)                                              \
+  extern "C" void *name(void *arg1, void *arg2) {                              \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2);                                                     \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWW(name)                                             \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3) {                  \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3);                                               \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWW(name)                                            \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4) {      \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4);                                         \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWWW(name)                                           \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
+                        void *arg5) {                                          \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4, arg5);                                   \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#define INTERCEPT_WRAP_W_WWWWWW(name)                                          \
+  extern "C" void *name(void *arg1, void *arg2, void *arg3, void *arg4,        \
+                        void *arg5, void *arg6) {                              \
+    typedef decltype(name) *fntype;                                            \
+    static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name);   \
+    return fn(arg1, arg2, arg3, arg4, arg5, arg6);                             \
+  }                                                                            \
+  INTERCEPT_OR_DIE(#name, name);
+
+#endif // SANITIZER_WIN_DLL_THUNK_H
diff --git a/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h b/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h
new file mode 100644
index 0000000000..5e4d8b8def
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h
@@ -0,0 +1,32 @@
+//===-- sanitizer_win_weak_interception.h ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This header provide helper macros to delegate calls of weak functions to the
+// implementation in the main executable when a strong definition is present.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_WIN_WEAK_INTERCEPTION_H
+#define SANITIZER_WIN_WEAK_INTERCEPTION_H
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+int interceptWhenPossible(uptr dll_function, const char *real_function);
+}
+
+// ----------------- Function interception helper macros -------------------- //
+// Weak functions, could be redefined in the main executable, but that is not
+// necessary, so we shouldn't die if we can not find a reference.
+#define INTERCEPT_WEAK(Name) interceptWhenPossible((uptr) Name, #Name);
+
+#define INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)                                \
+  static int intercept_##Name() {                                              \
+    return __sanitizer::interceptWhenPossible((__sanitizer::uptr) Name, #Name);\
+  }                                                                            \
+  __pragma(section(".WEAK$M", long, read))                                     \
+  __declspec(allocate(".WEAK$M")) int (*__weak_intercept_##Name)() =           \
+      intercept_##Name;
+
+#endif // SANITIZER_WIN_WEAK_INTERCEPTION_H
diff --git a/lib/tsan/tsan_clock.cpp b/lib/tsan/tsan_clock.cpp
new file mode 100644
index 0000000000..c91b29cb22
--- /dev/null
+++ b/lib/tsan/tsan_clock.cpp
@@ -0,0 +1,655 @@
+//===-- tsan_clock.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_clock.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+
+// SyncClock and ThreadClock implement vector clocks for sync variables
+// (mutexes, atomic variables, file descriptors, etc) and threads, respectively.
+// ThreadClock contains fixed-size vector clock for maximum number of threads.
+// SyncClock contains growable vector clock for currently necessary number of
+// threads.
+// Together they implement very simple model of operations, namely:
+//
+//   void ThreadClock::acquire(const SyncClock *src) {
+//     for (int i = 0; i < kMaxThreads; i++)
+//       clock[i] = max(clock[i], src->clock[i]);
+//   }
+//
+//   void ThreadClock::release(SyncClock *dst) const {
+//     for (int i = 0; i < kMaxThreads; i++)
+//       dst->clock[i] = max(dst->clock[i], clock[i]);
+//   }
+//
+//   void ThreadClock::releaseStoreAcquire(SyncClock *sc) const {
+//     for (int i = 0; i < kMaxThreads; i++) {
+//       tmp = clock[i];
+//       clock[i] = max(clock[i], sc->clock[i]);
+//       sc->clock[i] = tmp;
+//     }
+//   }
+//
+//   void ThreadClock::ReleaseStore(SyncClock *dst) const {
+//     for (int i = 0; i < kMaxThreads; i++)
+//       dst->clock[i] = clock[i];
+//   }
+//
+//   void ThreadClock::acq_rel(SyncClock *dst) {
+//     acquire(dst);
+//     release(dst);
+//   }
+//
+// Conformance to this model is extensively verified in tsan_clock_test.cpp.
+// However, the implementation is significantly more complex. The complexity
+// allows to implement important classes of use cases in O(1) instead of O(N).
+//
+// The use cases are:
+// 1. Singleton/once atomic that has a single release-store operation followed
+//    by zillions of acquire-loads (the acquire-load is O(1)).
+// 2. Thread-local mutex (both lock and unlock can be O(1)).
+// 3. Leaf mutex (unlock is O(1)).
+// 4. A mutex shared by 2 threads (both lock and unlock can be O(1)).
+// 5. An atomic with a single writer (writes can be O(1)).
+// The implementation dynamically adopts to workload. So if an atomic is in
+// read-only phase, these reads will be O(1); if it later switches to read/write
+// phase, the implementation will correctly handle that by switching to O(N).
+//
+// Thread-safety note: all const operations on SyncClock's are conducted under
+// a shared lock; all non-const operations on SyncClock's are conducted under
+// an exclusive lock; ThreadClock's are private to respective threads and so
+// do not need any protection.
+//
+// Description of SyncClock state:
+// clk_ - variable size vector clock, low kClkBits hold timestamp,
+//   the remaining bits hold "acquired" flag (the actual value is thread's
+//   reused counter);
+//   if acquried == thr->reused_, then the respective thread has already
+//   acquired this clock (except possibly for dirty elements).
+// dirty_ - holds up to two indeces in the vector clock that other threads
+//   need to acquire regardless of "acquired" flag value;
+// release_store_tid_ - denotes that the clock state is a result of
+//   release-store operation by the thread with release_store_tid_ index.
+// release_store_reused_ - reuse count of release_store_tid_.
+
+// We don't have ThreadState in these methods, so this is an ugly hack that
+// works only in C++.
+#if !SANITIZER_GO
+# define CPP_STAT_INC(typ) StatInc(cur_thread(), typ)
+#else
+# define CPP_STAT_INC(typ) (void)0
+#endif
+
+namespace __tsan {
+
+static atomic_uint32_t *ref_ptr(ClockBlock *cb) {
+  return reinterpret_cast<atomic_uint32_t *>(&cb->table[ClockBlock::kRefIdx]);
+}
+
+// Drop reference to the first level block idx.
+static void UnrefClockBlock(ClockCache *c, u32 idx, uptr blocks) {
+  ClockBlock *cb = ctx->clock_alloc.Map(idx);
+  atomic_uint32_t *ref = ref_ptr(cb);
+  u32 v = atomic_load(ref, memory_order_acquire);
+  for (;;) {
+    CHECK_GT(v, 0);
+    if (v == 1)
+      break;
+    if (atomic_compare_exchange_strong(ref, &v, v - 1, memory_order_acq_rel))
+      return;
+  }
+  // First level block owns second level blocks, so them as well.
+  for (uptr i = 0; i < blocks; i++)
+    ctx->clock_alloc.Free(c, cb->table[ClockBlock::kBlockIdx - i]);
+  ctx->clock_alloc.Free(c, idx);
+}
+
+ThreadClock::ThreadClock(unsigned tid, unsigned reused)
+    : tid_(tid)
+    , reused_(reused + 1)  // 0 has special meaning
+    , last_acquire_()
+    , global_acquire_()
+    , cached_idx_()
+    , cached_size_()
+    , cached_blocks_() {
+  CHECK_LT(tid, kMaxTidInClock);
+  CHECK_EQ(reused_, ((u64)reused_ << kClkBits) >> kClkBits);
+  nclk_ = tid_ + 1;
+  internal_memset(clk_, 0, sizeof(clk_));
+}
+
+void ThreadClock::ResetCached(ClockCache *c) {
+  if (cached_idx_) {
+    UnrefClockBlock(c, cached_idx_, cached_blocks_);
+    cached_idx_ = 0;
+    cached_size_ = 0;
+    cached_blocks_ = 0;
+  }
+}
+
+void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(src->size_, kMaxTid);
+  CPP_STAT_INC(StatClockAcquire);
+
+  // Check if it's empty -> no need to do anything.
+  const uptr nclk = src->size_;
+  if (nclk == 0) {
+    CPP_STAT_INC(StatClockAcquireEmpty);
+    return;
+  }
+
+  bool acquired = false;
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    SyncClock::Dirty dirty = src->dirty_[i];
+    unsigned tid = dirty.tid;
+    if (tid != kInvalidTid) {
+      if (clk_[tid] < dirty.epoch) {
+        clk_[tid] = dirty.epoch;
+        acquired = true;
+      }
+    }
+  }
+
+  // Check if we've already acquired src after the last release operation on src
+  if (tid_ >= nclk || src->elem(tid_).reused != reused_) {
+    // O(N) acquire.
+    CPP_STAT_INC(StatClockAcquireFull);
+    nclk_ = max(nclk_, nclk);
+    u64 *dst_pos = &clk_[0];
+    for (ClockElem &src_elem : *src) {
+      u64 epoch = src_elem.epoch;
+      if (*dst_pos < epoch) {
+        *dst_pos = epoch;
+        acquired = true;
+      }
+      dst_pos++;
+    }
+
+    // Remember that this thread has acquired this clock.
+    if (nclk > tid_)
+      src->elem(tid_).reused = reused_;
+  }
+
+  if (acquired) {
+    CPP_STAT_INC(StatClockAcquiredSomething);
+    last_acquire_ = clk_[tid_];
+    ResetCached(c);
+  }
+}
+
+void ThreadClock::releaseStoreAcquire(ClockCache *c, SyncClock *sc) {
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(sc->size_, kMaxTid);
+
+  if (sc->size_ == 0) {
+    // ReleaseStore will correctly set release_store_tid_,
+    // which can be important for future operations.
+    ReleaseStore(c, sc);
+    return;
+  }
+
+  nclk_ = max(nclk_, (uptr) sc->size_);
+
+  // Check if we need to resize sc.
+  if (sc->size_ < nclk_)
+    sc->Resize(c, nclk_);
+
+  bool acquired = false;
+
+  sc->Unshare(c);
+  // Update sc->clk_.
+  sc->FlushDirty();
+  uptr i = 0;
+  for (ClockElem &ce : *sc) {
+    u64 tmp = clk_[i];
+    if (clk_[i] < ce.epoch) {
+      clk_[i] = ce.epoch;
+      acquired = true;
+    }
+    ce.epoch = tmp;
+    ce.reused = 0;
+    i++;
+  }
+  sc->release_store_tid_ = kInvalidTid;
+  sc->release_store_reused_ = 0;
+
+  if (acquired) {
+    CPP_STAT_INC(StatClockAcquiredSomething);
+    last_acquire_ = clk_[tid_];
+    ResetCached(c);
+  }
+}
+
+void ThreadClock::release(ClockCache *c, SyncClock *dst) {
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(dst->size_, kMaxTid);
+
+  if (dst->size_ == 0) {
+    // ReleaseStore will correctly set release_store_tid_,
+    // which can be important for future operations.
+    ReleaseStore(c, dst);
+    return;
+  }
+
+  CPP_STAT_INC(StatClockRelease);
+  // Check if we need to resize dst.
+  if (dst->size_ < nclk_)
+    dst->Resize(c, nclk_);
+
+  // Check if we had not acquired anything from other threads
+  // since the last release on dst. If so, we need to update
+  // only dst->elem(tid_).
+  if (!HasAcquiredAfterRelease(dst)) {
+    UpdateCurrentThread(c, dst);
+    if (dst->release_store_tid_ != tid_ ||
+        dst->release_store_reused_ != reused_)
+      dst->release_store_tid_ = kInvalidTid;
+    return;
+  }
+
+  // O(N) release.
+  CPP_STAT_INC(StatClockReleaseFull);
+  dst->Unshare(c);
+  // First, remember whether we've acquired dst.
+  bool acquired = IsAlreadyAcquired(dst);
+  if (acquired)
+    CPP_STAT_INC(StatClockReleaseAcquired);
+  // Update dst->clk_.
+  dst->FlushDirty();
+  uptr i = 0;
+  for (ClockElem &ce : *dst) {
+    ce.epoch = max(ce.epoch, clk_[i]);
+    ce.reused = 0;
+    i++;
+  }
+  // Clear 'acquired' flag in the remaining elements.
+  if (nclk_ < dst->size_)
+    CPP_STAT_INC(StatClockReleaseClearTail);
+  dst->release_store_tid_ = kInvalidTid;
+  dst->release_store_reused_ = 0;
+  // If we've acquired dst, remember this fact,
+  // so that we don't need to acquire it on next acquire.
+  if (acquired)
+    dst->elem(tid_).reused = reused_;
+}
+
+void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(dst->size_, kMaxTid);
+  CPP_STAT_INC(StatClockStore);
+
+  if (dst->size_ == 0 && cached_idx_ != 0) {
+    // Reuse the cached clock.
+    // Note: we could reuse/cache the cached clock in more cases:
+    // we could update the existing clock and cache it, or replace it with the
+    // currently cached clock and release the old one. And for a shared
+    // existing clock, we could replace it with the currently cached;
+    // or unshare, update and cache. But, for simplicity, we currnetly reuse
+    // cached clock only when the target clock is empty.
+    dst->tab_ = ctx->clock_alloc.Map(cached_idx_);
+    dst->tab_idx_ = cached_idx_;
+    dst->size_ = cached_size_;
+    dst->blocks_ = cached_blocks_;
+    CHECK_EQ(dst->dirty_[0].tid, kInvalidTid);
+    // The cached clock is shared (immutable),
+    // so this is where we store the current clock.
+    dst->dirty_[0].tid = tid_;
+    dst->dirty_[0].epoch = clk_[tid_];
+    dst->release_store_tid_ = tid_;
+    dst->release_store_reused_ = reused_;
+    // Rememeber that we don't need to acquire it in future.
+    dst->elem(tid_).reused = reused_;
+    // Grab a reference.
+    atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
+    return;
+  }
+
+  // Check if we need to resize dst.
+  if (dst->size_ < nclk_)
+    dst->Resize(c, nclk_);
+
+  if (dst->release_store_tid_ == tid_ &&
+      dst->release_store_reused_ == reused_ &&
+      !HasAcquiredAfterRelease(dst)) {
+    CPP_STAT_INC(StatClockStoreFast);
+    UpdateCurrentThread(c, dst);
+    return;
+  }
+
+  // O(N) release-store.
+  CPP_STAT_INC(StatClockStoreFull);
+  dst->Unshare(c);
+  // Note: dst can be larger than this ThreadClock.
+  // This is fine since clk_ beyond size is all zeros.
+  uptr i = 0;
+  for (ClockElem &ce : *dst) {
+    ce.epoch = clk_[i];
+    ce.reused = 0;
+    i++;
+  }
+  for (uptr i = 0; i < kDirtyTids; i++)
+    dst->dirty_[i].tid = kInvalidTid;
+  dst->release_store_tid_ = tid_;
+  dst->release_store_reused_ = reused_;
+  // Rememeber that we don't need to acquire it in future.
+  dst->elem(tid_).reused = reused_;
+
+  // If the resulting clock is cachable, cache it for future release operations.
+  // The clock is always cachable if we released to an empty sync object.
+  if (cached_idx_ == 0 && dst->Cachable()) {
+    // Grab a reference to the ClockBlock.
+    atomic_uint32_t *ref = ref_ptr(dst->tab_);
+    if (atomic_load(ref, memory_order_acquire) == 1)
+      atomic_store_relaxed(ref, 2);
+    else
+      atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
+    cached_idx_ = dst->tab_idx_;
+    cached_size_ = dst->size_;
+    cached_blocks_ = dst->blocks_;
+  }
+}
+
+void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) {
+  CPP_STAT_INC(StatClockAcquireRelease);
+  acquire(c, dst);
+  ReleaseStore(c, dst);
+}
+
+// Updates only single element related to the current thread in dst->clk_.
+void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const {
+  // Update the threads time, but preserve 'acquired' flag.
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    SyncClock::Dirty *dirty = &dst->dirty_[i];
+    const unsigned tid = dirty->tid;
+    if (tid == tid_ || tid == kInvalidTid) {
+      CPP_STAT_INC(StatClockReleaseFast);
+      dirty->tid = tid_;
+      dirty->epoch = clk_[tid_];
+      return;
+    }
+  }
+  // Reset all 'acquired' flags, O(N).
+  // We are going to touch dst elements, so we need to unshare it.
+  dst->Unshare(c);
+  CPP_STAT_INC(StatClockReleaseSlow);
+  dst->elem(tid_).epoch = clk_[tid_];
+  for (uptr i = 0; i < dst->size_; i++)
+    dst->elem(i).reused = 0;
+  dst->FlushDirty();
+}
+
+// Checks whether the current thread has already acquired src.
+bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
+  if (src->elem(tid_).reused != reused_)
+    return false;
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    SyncClock::Dirty dirty = src->dirty_[i];
+    if (dirty.tid != kInvalidTid) {
+      if (clk_[dirty.tid] < dirty.epoch)
+        return false;
+    }
+  }
+  return true;
+}
+
+// Checks whether the current thread has acquired anything
+// from other clocks after releasing to dst (directly or indirectly).
+bool ThreadClock::HasAcquiredAfterRelease(const SyncClock *dst) const {
+  const u64 my_epoch = dst->elem(tid_).epoch;
+  return my_epoch <= last_acquire_ ||
+      my_epoch <= atomic_load_relaxed(&global_acquire_);
+}
+
+// Sets a single element in the vector clock.
+// This function is called only from weird places like AcquireGlobal.
+void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
+  DCHECK_LT(tid, kMaxTid);
+  DCHECK_GE(v, clk_[tid]);
+  clk_[tid] = v;
+  if (nclk_ <= tid)
+    nclk_ = tid + 1;
+  last_acquire_ = clk_[tid_];
+  ResetCached(c);
+}
+
+void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
+  printf("clock=[");
+  for (uptr i = 0; i < nclk_; i++)
+    printf("%s%llu", i == 0 ? "" : ",", clk_[i]);
+  printf("] tid=%u/%u last_acq=%llu", tid_, reused_, last_acquire_);
+}
+
+SyncClock::SyncClock() {
+  ResetImpl();
+}
+
+SyncClock::~SyncClock() {
+  // Reset must be called before dtor.
+  CHECK_EQ(size_, 0);
+  CHECK_EQ(blocks_, 0);
+  CHECK_EQ(tab_, 0);
+  CHECK_EQ(tab_idx_, 0);
+}
+
+void SyncClock::Reset(ClockCache *c) {
+  if (size_)
+    UnrefClockBlock(c, tab_idx_, blocks_);
+  ResetImpl();
+}
+
+void SyncClock::ResetImpl() {
+  tab_ = 0;
+  tab_idx_ = 0;
+  size_ = 0;
+  blocks_ = 0;
+  release_store_tid_ = kInvalidTid;
+  release_store_reused_ = 0;
+  for (uptr i = 0; i < kDirtyTids; i++)
+    dirty_[i].tid = kInvalidTid;
+}
+
+void SyncClock::Resize(ClockCache *c, uptr nclk) {
+  CPP_STAT_INC(StatClockReleaseResize);
+  Unshare(c);
+  if (nclk <= capacity()) {
+    // Memory is already allocated, just increase the size.
+    size_ = nclk;
+    return;
+  }
+  if (size_ == 0) {
+    // Grow from 0 to one-level table.
+    CHECK_EQ(size_, 0);
+    CHECK_EQ(blocks_, 0);
+    CHECK_EQ(tab_, 0);
+    CHECK_EQ(tab_idx_, 0);
+    tab_idx_ = ctx->clock_alloc.Alloc(c);
+    tab_ = ctx->clock_alloc.Map(tab_idx_);
+    internal_memset(tab_, 0, sizeof(*tab_));
+    atomic_store_relaxed(ref_ptr(tab_), 1);
+    size_ = 1;
+  } else if (size_ > blocks_ * ClockBlock::kClockCount) {
+    u32 idx = ctx->clock_alloc.Alloc(c);
+    ClockBlock *new_cb = ctx->clock_alloc.Map(idx);
+    uptr top = size_ - blocks_ * ClockBlock::kClockCount;
+    CHECK_LT(top, ClockBlock::kClockCount);
+    const uptr move = top * sizeof(tab_->clock[0]);
+    internal_memcpy(&new_cb->clock[0], tab_->clock, move);
+    internal_memset(&new_cb->clock[top], 0, sizeof(*new_cb) - move);
+    internal_memset(tab_->clock, 0, move);
+    append_block(idx);
+  }
+  // At this point we have first level table allocated and all clock elements
+  // are evacuated from it to a second level block.
+  // Add second level tables as necessary.
+  while (nclk > capacity()) {
+    u32 idx = ctx->clock_alloc.Alloc(c);
+    ClockBlock *cb = ctx->clock_alloc.Map(idx);
+    internal_memset(cb, 0, sizeof(*cb));
+    append_block(idx);
+  }
+  size_ = nclk;
+}
+
+// Flushes all dirty elements into the main clock array.
+void SyncClock::FlushDirty() {
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    Dirty *dirty = &dirty_[i];
+    if (dirty->tid != kInvalidTid) {
+      CHECK_LT(dirty->tid, size_);
+      elem(dirty->tid).epoch = dirty->epoch;
+      dirty->tid = kInvalidTid;
+    }
+  }
+}
+
+bool SyncClock::IsShared() const {
+  if (size_ == 0)
+    return false;
+  atomic_uint32_t *ref = ref_ptr(tab_);
+  u32 v = atomic_load(ref, memory_order_acquire);
+  CHECK_GT(v, 0);
+  return v > 1;
+}
+
+// Unshares the current clock if it's shared.
+// Shared clocks are immutable, so they need to be unshared before any updates.
+// Note: this does not apply to dirty entries as they are not shared.
+void SyncClock::Unshare(ClockCache *c) {
+  if (!IsShared())
+    return;
+  // First, copy current state into old.
+  SyncClock old;
+  old.tab_ = tab_;
+  old.tab_idx_ = tab_idx_;
+  old.size_ = size_;
+  old.blocks_ = blocks_;
+  old.release_store_tid_ = release_store_tid_;
+  old.release_store_reused_ = release_store_reused_;
+  for (unsigned i = 0; i < kDirtyTids; i++)
+    old.dirty_[i] = dirty_[i];
+  // Then, clear current object.
+  ResetImpl();
+  // Allocate brand new clock in the current object.
+  Resize(c, old.size_);
+  // Now copy state back into this object.
+  Iter old_iter(&old);
+  for (ClockElem &ce : *this) {
+    ce = *old_iter;
+    ++old_iter;
+  }
+  release_store_tid_ = old.release_store_tid_;
+  release_store_reused_ = old.release_store_reused_;
+  for (unsigned i = 0; i < kDirtyTids; i++)
+    dirty_[i] = old.dirty_[i];
+  // Drop reference to old and delete if necessary.
+  old.Reset(c);
+}
+
+// Can we cache this clock for future release operations?
+ALWAYS_INLINE bool SyncClock::Cachable() const {
+  if (size_ == 0)
+    return false;
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    if (dirty_[i].tid != kInvalidTid)
+      return false;
+  }
+  return atomic_load_relaxed(ref_ptr(tab_)) == 1;
+}
+
+// elem linearizes the two-level structure into linear array.
+// Note: this is used only for one time accesses, vector operations use
+// the iterator as it is much faster.
+ALWAYS_INLINE ClockElem &SyncClock::elem(unsigned tid) const {
+  DCHECK_LT(tid, size_);
+  const uptr block = tid / ClockBlock::kClockCount;
+  DCHECK_LE(block, blocks_);
+  tid %= ClockBlock::kClockCount;
+  if (block == blocks_)
+    return tab_->clock[tid];
+  u32 idx = get_block(block);
+  ClockBlock *cb = ctx->clock_alloc.Map(idx);
+  return cb->clock[tid];
+}
+
+ALWAYS_INLINE uptr SyncClock::capacity() const {
+  if (size_ == 0)
+    return 0;
+  uptr ratio = sizeof(ClockBlock::clock[0]) / sizeof(ClockBlock::table[0]);
+  // How many clock elements we can fit into the first level block.
+  // +1 for ref counter.
+  uptr top = ClockBlock::kClockCount - RoundUpTo(blocks_ + 1, ratio) / ratio;
+  return blocks_ * ClockBlock::kClockCount + top;
+}
+
+ALWAYS_INLINE u32 SyncClock::get_block(uptr bi) const {
+  DCHECK(size_);
+  DCHECK_LT(bi, blocks_);
+  return tab_->table[ClockBlock::kBlockIdx - bi];
+}
+
+ALWAYS_INLINE void SyncClock::append_block(u32 idx) {
+  uptr bi = blocks_++;
+  CHECK_EQ(get_block(bi), 0);
+  tab_->table[ClockBlock::kBlockIdx - bi] = idx;
+}
+
+// Used only by tests.
+u64 SyncClock::get(unsigned tid) const {
+  for (unsigned i = 0; i < kDirtyTids; i++) {
+    Dirty dirty = dirty_[i];
+    if (dirty.tid == tid)
+      return dirty.epoch;
+  }
+  return elem(tid).epoch;
+}
+
+// Used only by Iter test.
+u64 SyncClock::get_clean(unsigned tid) const {
+  return elem(tid).epoch;
+}
+
+void SyncClock::DebugDump(int(*printf)(const char *s, ...)) {
+  printf("clock=[");
+  for (uptr i = 0; i < size_; i++)
+    printf("%s%llu", i == 0 ? "" : ",", elem(i).epoch);
+  printf("] reused=[");
+  for (uptr i = 0; i < size_; i++)
+    printf("%s%llu", i == 0 ? "" : ",", elem(i).reused);
+  printf("] release_store_tid=%d/%d dirty_tids=%d[%llu]/%d[%llu]",
+      release_store_tid_, release_store_reused_,
+      dirty_[0].tid, dirty_[0].epoch,
+      dirty_[1].tid, dirty_[1].epoch);
+}
+
+void SyncClock::Iter::Next() {
+  // Finished with the current block, move on to the next one.
+  block_++;
+  if (block_ < parent_->blocks_) {
+    // Iterate over the next second level block.
+    u32 idx = parent_->get_block(block_);
+    ClockBlock *cb = ctx->clock_alloc.Map(idx);
+    pos_ = &cb->clock[0];
+    end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
+        ClockBlock::kClockCount);
+    return;
+  }
+  if (block_ == parent_->blocks_ &&
+      parent_->size_ > parent_->blocks_ * ClockBlock::kClockCount) {
+    // Iterate over elements in the first level block.
+    pos_ = &parent_->tab_->clock[0];
+    end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
+        ClockBlock::kClockCount);
+    return;
+  }
+  parent_ = nullptr;  // denotes end
+}
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_clock.h b/lib/tsan/tsan_clock.h
new file mode 100644
index 0000000000..736cdae06b
--- /dev/null
+++ b/lib/tsan/tsan_clock.h
@@ -0,0 +1,283 @@
+//===-- tsan_clock.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_CLOCK_H
+#define TSAN_CLOCK_H
+
+#include "tsan_defs.h"
+#include "tsan_dense_alloc.h"
+
+namespace __tsan {
+
+typedef DenseSlabAlloc<ClockBlock, 1<<16, 1<<10> ClockAlloc;
+typedef DenseSlabAllocCache ClockCache;
+
+// The clock that lives in sync variables (mutexes, atomics, etc).
+class SyncClock {
+ public:
+  SyncClock();
+  ~SyncClock();
+
+  uptr size() const;
+
+  // These are used only in tests.
+  u64 get(unsigned tid) const;
+  u64 get_clean(unsigned tid) const;
+
+  void Resize(ClockCache *c, uptr nclk);
+  void Reset(ClockCache *c);
+
+  void DebugDump(int(*printf)(const char *s, ...));
+
+  // Clock element iterator.
+  // Note: it iterates only over the table without regard to dirty entries.
+  class Iter {
+   public:
+    explicit Iter(SyncClock* parent);
+    Iter& operator++();
+    bool operator!=(const Iter& other);
+    ClockElem &operator*();
+
+   private:
+    SyncClock *parent_;
+    // [pos_, end_) is the current continuous range of clock elements.
+    ClockElem *pos_;
+    ClockElem *end_;
+    int block_;  // Current number of second level block.
+
+    NOINLINE void Next();
+  };
+
+  Iter begin();
+  Iter end();
+
+ private:
+  friend class ThreadClock;
+  friend class Iter;
+  static const uptr kDirtyTids = 2;
+
+  struct Dirty {
+    u64 epoch  : kClkBits;
+    u64 tid : 64 - kClkBits;  // kInvalidId if not active
+  };
+
+  unsigned release_store_tid_;
+  unsigned release_store_reused_;
+  Dirty dirty_[kDirtyTids];
+  // If size_ is 0, tab_ is nullptr.
+  // If size <= 64 (kClockCount), tab_ contains pointer to an array with
+  // 64 ClockElem's (ClockBlock::clock).
+  // Otherwise, tab_ points to an array with up to 127 u32 elements,
+  // each pointing to the second-level 512b block with 64 ClockElem's.
+  // Unused space in the first level ClockBlock is used to store additional
+  // clock elements.
+  // The last u32 element in the first level ClockBlock is always used as
+  // reference counter.
+  //
+  // See the following scheme for details.
+  // All memory blocks are 512 bytes (allocated from ClockAlloc).
+  // Clock (clk) elements are 64 bits.
+  // Idx and ref are 32 bits.
+  //
+  // tab_
+  //    |
+  //    \/
+  //    +----------------------------------------------------+
+  //    | clk128 | clk129 | ...unused... | idx1 | idx0 | ref |
+  //    +----------------------------------------------------+
+  //                                        |      |
+  //                                        |      \/
+  //                                        |      +----------------+
+  //                                        |      | clk0 ... clk63 |
+  //                                        |      +----------------+
+  //                                        \/
+  //                                        +------------------+
+  //                                        | clk64 ... clk127 |
+  //                                        +------------------+
+  //
+  // Note: dirty entries, if active, always override what's stored in the clock.
+  ClockBlock *tab_;
+  u32 tab_idx_;
+  u16 size_;
+  u16 blocks_;  // Number of second level blocks.
+
+  void Unshare(ClockCache *c);
+  bool IsShared() const;
+  bool Cachable() const;
+  void ResetImpl();
+  void FlushDirty();
+  uptr capacity() const;
+  u32 get_block(uptr bi) const;
+  void append_block(u32 idx);
+  ClockElem &elem(unsigned tid) const;
+};
+
+// The clock that lives in threads.
+class ThreadClock {
+ public:
+  typedef DenseSlabAllocCache Cache;
+
+  explicit ThreadClock(unsigned tid, unsigned reused = 0);
+
+  u64 get(unsigned tid) const;
+  void set(ClockCache *c, unsigned tid, u64 v);
+  void set(u64 v);
+  void tick();
+  uptr size() const;
+
+  void acquire(ClockCache *c, SyncClock *src);
+  void releaseStoreAcquire(ClockCache *c, SyncClock *src);
+  void release(ClockCache *c, SyncClock *dst);
+  void acq_rel(ClockCache *c, SyncClock *dst);
+  void ReleaseStore(ClockCache *c, SyncClock *dst);
+  void ResetCached(ClockCache *c);
+  void NoteGlobalAcquire(u64 v);
+
+  void DebugReset();
+  void DebugDump(int(*printf)(const char *s, ...));
+
+ private:
+  static const uptr kDirtyTids = SyncClock::kDirtyTids;
+  // Index of the thread associated with he clock ("current thread").
+  const unsigned tid_;
+  const unsigned reused_;  // tid_ reuse count.
+  // Current thread time when it acquired something from other threads.
+  u64 last_acquire_;
+
+  // Last time another thread has done a global acquire of this thread's clock.
+  // It helps to avoid problem described in:
+  // https://github.com/golang/go/issues/39186
+  // See test/tsan/java_finalizer2.cpp for a regression test.
+  // Note the failuire is _extremely_ hard to hit, so if you are trying
+  // to reproduce it, you may want to run something like:
+  // $ go get golang.org/x/tools/cmd/stress
+  // $ stress -p=64 ./a.out
+  //
+  // The crux of the problem is roughly as follows.
+  // A number of O(1) optimizations in the clocks algorithm assume proper
+  // transitive cumulative propagation of clock values. The AcquireGlobal
+  // operation may produce an inconsistent non-linearazable view of
+  // thread clocks. Namely, it may acquire a later value from a thread
+  // with a higher ID, but fail to acquire an earlier value from a thread
+  // with a lower ID. If a thread that executed AcquireGlobal then releases
+  // to a sync clock, it will spoil the sync clock with the inconsistent
+  // values. If another thread later releases to the sync clock, the optimized
+  // algorithm may break.
+  //
+  // The exact sequence of events that leads to the failure.
+  // - thread 1 executes AcquireGlobal
+  // - thread 1 acquires value 1 for thread 2
+  // - thread 2 increments clock to 2
+  // - thread 2 releases to sync object 1
+  // - thread 3 at time 1
+  // - thread 3 acquires from sync object 1
+  // - thread 3 increments clock to 2
+  // - thread 1 acquires value 2 for thread 3
+  // - thread 1 releases to sync object 2
+  // - sync object 2 clock has 1 for thread 2 and 2 for thread 3
+  // - thread 3 releases to sync object 2
+  // - thread 3 sees value 2 in the clock for itself
+  //   and decides that it has already released to the clock
+  //   and did not acquire anything from other threads after that
+  //   (the last_acquire_ check in release operation)
+  // - thread 3 does not update the value for thread 2 in the clock from 1 to 2
+  // - thread 4 acquires from sync object 2
+  // - thread 4 detects a false race with thread 2
+  //   as it should have been synchronized with thread 2 up to time 2,
+  //   but because of the broken clock it is now synchronized only up to time 1
+  //
+  // The global_acquire_ value helps to prevent this scenario.
+  // Namely, thread 3 will not trust any own clock values up to global_acquire_
+  // for the purposes of the last_acquire_ optimization.
+  atomic_uint64_t global_acquire_;
+
+  // Cached SyncClock (without dirty entries and release_store_tid_).
+  // We reuse it for subsequent store-release operations without intervening
+  // acquire operations. Since it is shared (and thus constant), clock value
+  // for the current thread is then stored in dirty entries in the SyncClock.
+  // We host a refernece to the table while it is cached here.
+  u32 cached_idx_;
+  u16 cached_size_;
+  u16 cached_blocks_;
+
+  // Number of active elements in the clk_ table (the rest is zeros).
+  uptr nclk_;
+  u64 clk_[kMaxTidInClock];  // Fixed size vector clock.
+
+  bool IsAlreadyAcquired(const SyncClock *src) const;
+  bool HasAcquiredAfterRelease(const SyncClock *dst) const;
+  void UpdateCurrentThread(ClockCache *c, SyncClock *dst) const;
+};
+
+ALWAYS_INLINE u64 ThreadClock::get(unsigned tid) const {
+  DCHECK_LT(tid, kMaxTidInClock);
+  return clk_[tid];
+}
+
+ALWAYS_INLINE void ThreadClock::set(u64 v) {
+  DCHECK_GE(v, clk_[tid_]);
+  clk_[tid_] = v;
+}
+
+ALWAYS_INLINE void ThreadClock::tick() {
+  clk_[tid_]++;
+}
+
+ALWAYS_INLINE uptr ThreadClock::size() const {
+  return nclk_;
+}
+
+ALWAYS_INLINE void ThreadClock::NoteGlobalAcquire(u64 v) {
+  // Here we rely on the fact that AcquireGlobal is protected by
+  // ThreadRegistryLock, thus only one thread at a time executes it
+  // and values passed to this function should not go backwards.
+  CHECK_LE(atomic_load_relaxed(&global_acquire_), v);
+  atomic_store_relaxed(&global_acquire_, v);
+}
+
+ALWAYS_INLINE SyncClock::Iter SyncClock::begin() {
+  return Iter(this);
+}
+
+ALWAYS_INLINE SyncClock::Iter SyncClock::end() {
+  return Iter(nullptr);
+}
+
+ALWAYS_INLINE uptr SyncClock::size() const {
+  return size_;
+}
+
+ALWAYS_INLINE SyncClock::Iter::Iter(SyncClock* parent)
+    : parent_(parent)
+    , pos_(nullptr)
+    , end_(nullptr)
+    , block_(-1) {
+  if (parent)
+    Next();
+}
+
+ALWAYS_INLINE SyncClock::Iter& SyncClock::Iter::operator++() {
+  pos_++;
+  if (UNLIKELY(pos_ >= end_))
+    Next();
+  return *this;
+}
+
+ALWAYS_INLINE bool SyncClock::Iter::operator!=(const SyncClock::Iter& other) {
+  return parent_ != other.parent_;
+}
+
+ALWAYS_INLINE ClockElem &SyncClock::Iter::operator*() {
+  return *pos_;
+}
+}  // namespace __tsan
+
+#endif  // TSAN_CLOCK_H
diff --git a/lib/tsan/tsan_debugging.cpp b/lib/tsan/tsan_debugging.cpp
new file mode 100644
index 0000000000..d3d6255090
--- /dev/null
+++ b/lib/tsan/tsan_debugging.cpp
@@ -0,0 +1,262 @@
+//===-- tsan_debugging.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// TSan debugging API implementation.
+//===----------------------------------------------------------------------===//
+#include "tsan_interface.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+using namespace __tsan;
+
+static const char *ReportTypeDescription(ReportType typ) {
+  switch (typ) {
+    case ReportTypeRace: return "data-race";
+    case ReportTypeVptrRace: return "data-race-vptr";
+    case ReportTypeUseAfterFree: return "heap-use-after-free";
+    case ReportTypeVptrUseAfterFree: return "heap-use-after-free-vptr";
+    case ReportTypeExternalRace: return "external-race";
+    case ReportTypeThreadLeak: return "thread-leak";
+    case ReportTypeMutexDestroyLocked: return "locked-mutex-destroy";
+    case ReportTypeMutexDoubleLock: return "mutex-double-lock";
+    case ReportTypeMutexInvalidAccess: return "mutex-invalid-access";
+    case ReportTypeMutexBadUnlock: return "mutex-bad-unlock";
+    case ReportTypeMutexBadReadLock: return "mutex-bad-read-lock";
+    case ReportTypeMutexBadReadUnlock: return "mutex-bad-read-unlock";
+    case ReportTypeSignalUnsafe: return "signal-unsafe-call";
+    case ReportTypeErrnoInSignal: return "errno-in-signal-handler";
+    case ReportTypeDeadlock: return "lock-order-inversion";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static const char *ReportLocationTypeDescription(ReportLocationType typ) {
+  switch (typ) {
+    case ReportLocationGlobal: return "global";
+    case ReportLocationHeap: return "heap";
+    case ReportLocationStack: return "stack";
+    case ReportLocationTLS: return "tls";
+    case ReportLocationFD: return "fd";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static void CopyTrace(SymbolizedStack *first_frame, void **trace,
+                      uptr trace_size) {
+  uptr i = 0;
+  for (SymbolizedStack *frame = first_frame; frame != nullptr;
+       frame = frame->next) {
+    trace[i++] = (void *)frame->info.address;
+    if (i >= trace_size) break;
+  }
+}
+
+// Meant to be called by the debugger.
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_report() {
+  return const_cast<ReportDesc*>(cur_thread()->current_report);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_data(void *report, const char **description, int *count,
+                           int *stack_count, int *mop_count, int *loc_count,
+                           int *mutex_count, int *thread_count,
+                           int *unique_tid_count, void **sleep_trace,
+                           uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  *description = ReportTypeDescription(rep->typ);
+  *count = rep->count;
+  *stack_count = rep->stacks.Size();
+  *mop_count = rep->mops.Size();
+  *loc_count = rep->locs.Size();
+  *mutex_count = rep->mutexes.Size();
+  *thread_count = rep->threads.Size();
+  *unique_tid_count = rep->unique_tids.Size();
+  if (rep->sleep) CopyTrace(rep->sleep->frames, sleep_trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_tag(void *report, uptr *tag) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  *tag = rep->tag;
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_stack(void *report, uptr idx, void **trace,
+                            uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->stacks.Size());
+  ReportStack *stack = rep->stacks[idx];
+  if (stack) CopyTrace(stack->frames, trace, trace_size);
+  return stack ? 1 : 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr,
+                          int *size, int *write, int *atomic, void **trace,
+                          uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->mops.Size());
+  ReportMop *mop = rep->mops[idx];
+  *tid = mop->tid;
+  *addr = (void *)mop->addr;
+  *size = mop->size;
+  *write = mop->write ? 1 : 0;
+  *atomic = mop->atomic ? 1 : 0;
+  if (mop->stack) CopyTrace(mop->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc(void *report, uptr idx, const char **type,
+                          void **addr, uptr *start, uptr *size, int *tid,
+                          int *fd, int *suppressable, void **trace,
+                          uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->locs.Size());
+  ReportLocation *loc = rep->locs[idx];
+  *type = ReportLocationTypeDescription(loc->type);
+  *addr = (void *)loc->global.start;
+  *start = loc->heap_chunk_start;
+  *size = loc->heap_chunk_size;
+  *tid = loc->tid;
+  *fd = loc->fd;
+  *suppressable = loc->suppressable;
+  if (loc->stack) CopyTrace(loc->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->locs.Size());
+  ReportLocation *loc = rep->locs[idx];
+  *object_type = GetObjectTypeFromTag(loc->external_tag);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
+                            int *destroyed, void **trace, uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->mutexes.Size());
+  ReportMutex *mutex = rep->mutexes[idx];
+  *mutex_id = mutex->id;
+  *addr = (void *)mutex->addr;
+  *destroyed = mutex->destroyed;
+  if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
+                             int *running, const char **name, int *parent_tid,
+                             void **trace, uptr trace_size) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->threads.Size());
+  ReportThread *thread = rep->threads[idx];
+  *tid = thread->id;
+  *os_id = thread->os_id;
+  *running = thread->running;
+  *name = thread->name;
+  *parent_tid = thread->parent_tid;
+  if (thread->stack) CopyTrace(thread->stack->frames, trace, trace_size);
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid) {
+  const ReportDesc *rep = (ReportDesc *)report;
+  CHECK_LT(idx, rep->unique_tids.Size());
+  *tid = rep->unique_tids[idx];
+  return 1;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address_ptr,
+                                  uptr *region_size_ptr) {
+  uptr region_address = 0;
+  uptr region_size = 0;
+  const char *region_kind = nullptr;
+  if (name && name_size > 0) name[0] = 0;
+
+  if (IsMetaMem(addr)) {
+    region_kind = "meta shadow";
+  } else if (IsShadowMem(addr)) {
+    region_kind = "shadow";
+  } else {
+    bool is_stack = false;
+    MBlock *b = 0;
+    Allocator *a = allocator();
+    if (a->PointerIsMine((void *)addr)) {
+      void *block_begin = a->GetBlockBegin((void *)addr);
+      if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+    }
+
+    if (b != 0) {
+      region_address = (uptr)allocator()->GetBlockBegin((void *)addr);
+      region_size = b->siz;
+      region_kind = "heap";
+    } else {
+      // TODO(kuba.brecka): We should not lock. This is supposed to be called
+      // from within the debugger when other threads are stopped.
+      ctx->thread_registry->Lock();
+      ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
+      ctx->thread_registry->Unlock();
+      if (tctx) {
+        region_kind = is_stack ? "stack" : "tls";
+      } else {
+        region_kind = "global";
+        DataInfo info;
+        if (Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) {
+          internal_strncpy(name, info.name, name_size);
+          region_address = info.start;
+          region_size = info.size;
+        }
+      }
+    }
+  }
+
+  CHECK(region_kind);
+  if (region_address_ptr) *region_address_ptr = region_address;
+  if (region_size_ptr) *region_size_ptr = region_size;
+  return region_kind;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id) {
+  MBlock *b = 0;
+  Allocator *a = allocator();
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b == 0) return 0;
+
+  *thread_id = b->tid;
+  // No locking.  This is supposed to be called from within the debugger when
+  // other threads are stopped.
+  ThreadContextBase *tctx = ctx->thread_registry->GetThreadLocked(b->tid);
+  *os_id = tctx->os_id;
+
+  StackTrace stack = StackDepotGet(b->stk);
+  size = Min(size, (uptr)stack.size);
+  for (uptr i = 0; i < size; i++) trace[i] = stack.trace[stack.size - i - 1];
+  return size;
+}
diff --git a/lib/tsan/tsan_defs.h b/lib/tsan/tsan_defs.h
new file mode 100644
index 0000000000..293d7deccc
--- /dev/null
+++ b/lib/tsan/tsan_defs.h
@@ -0,0 +1,195 @@
+//===-- tsan_defs.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_DEFS_H
+#define TSAN_DEFS_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "tsan_stat.h"
+#include "ubsan/ubsan_platform.h"
+
+// Setup defaults for compile definitions.
+#ifndef TSAN_NO_HISTORY
+# define TSAN_NO_HISTORY 0
+#endif
+
+#ifndef TSAN_COLLECT_STATS
+# define TSAN_COLLECT_STATS 0
+#endif
+
+#ifndef TSAN_CONTAINS_UBSAN
+# if CAN_SANITIZE_UB && !SANITIZER_GO
+#  define TSAN_CONTAINS_UBSAN 1
+# else
+#  define TSAN_CONTAINS_UBSAN 0
+# endif
+#endif
+
+namespace __tsan {
+
+const int kClkBits = 42;
+const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
+
+struct ClockElem {
+  u64 epoch  : kClkBits;
+  u64 reused : 64 - kClkBits;  // tid reuse count
+};
+
+struct ClockBlock {
+  static const uptr kSize = 512;
+  static const uptr kTableSize = kSize / sizeof(u32);
+  static const uptr kClockCount = kSize / sizeof(ClockElem);
+  static const uptr kRefIdx = kTableSize - 1;
+  static const uptr kBlockIdx = kTableSize - 2;
+
+  union {
+    u32       table[kTableSize];
+    ClockElem clock[kClockCount];
+  };
+
+  ClockBlock() {
+  }
+};
+
+const int kTidBits = 13;
+// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
+// occupied by reference counter, so total number of elements we can store
+// in SyncClock is kClockCount * (kTableSize - 1).
+const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
+#if !SANITIZER_GO
+const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
+#else
+const unsigned kMaxTidInClock = kMaxTid;  // Go does not track freed memory.
+#endif
+const uptr kShadowStackSize = 64 * 1024;
+
+// Count of shadow values in a shadow cell.
+const uptr kShadowCnt = 4;
+
+// That many user bytes are mapped onto a single shadow cell.
+const uptr kShadowCell = 8;
+
+// Size of a single shadow value (u64).
+const uptr kShadowSize = 8;
+
+// Shadow memory is kShadowMultiplier times larger than user memory.
+const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
+
+// That many user bytes are mapped onto a single meta shadow cell.
+// Must be less or equal to minimal memory allocator alignment.
+const uptr kMetaShadowCell = 8;
+
+// Size of a single meta shadow value (u32).
+const uptr kMetaShadowSize = 4;
+
+#if TSAN_NO_HISTORY
+const bool kCollectHistory = false;
+#else
+const bool kCollectHistory = true;
+#endif
+
+const u16 kInvalidTid = kMaxTid + 1;
+
+// The following "build consistency" machinery ensures that all source files
+// are built in the same configuration. Inconsistent builds lead to
+// hard to debug crashes.
+#if SANITIZER_DEBUG
+void build_consistency_debug();
+#else
+void build_consistency_release();
+#endif
+
+#if TSAN_COLLECT_STATS
+void build_consistency_stats();
+#else
+void build_consistency_nostats();
+#endif
+
+static inline void USED build_consistency() {
+#if SANITIZER_DEBUG
+  build_consistency_debug();
+#else
+  build_consistency_release();
+#endif
+#if TSAN_COLLECT_STATS
+  build_consistency_stats();
+#else
+  build_consistency_nostats();
+#endif
+}
+
+template<typename T>
+T min(T a, T b) {
+  return a < b ? a : b;
+}
+
+template<typename T>
+T max(T a, T b) {
+  return a > b ? a : b;
+}
+
+template<typename T>
+T RoundUp(T p, u64 align) {
+  DCHECK_EQ(align & (align - 1), 0);
+  return (T)(((u64)p + align - 1) & ~(align - 1));
+}
+
+template<typename T>
+T RoundDown(T p, u64 align) {
+  DCHECK_EQ(align & (align - 1), 0);
+  return (T)((u64)p & ~(align - 1));
+}
+
+// Zeroizes high part, returns 'bits' lsb bits.
+template<typename T>
+T GetLsb(T v, int bits) {
+  return (T)((u64)v & ((1ull << bits) - 1));
+}
+
+struct MD5Hash {
+  u64 hash[2];
+  bool operator==(const MD5Hash &other) const;
+};
+
+MD5Hash md5_hash(const void *data, uptr size);
+
+struct Processor;
+struct ThreadState;
+class ThreadContext;
+struct Context;
+struct ReportStack;
+class ReportDesc;
+class RegionAlloc;
+
+// Descriptor of user's memory block.
+struct MBlock {
+  u64  siz : 48;
+  u64  tag : 16;
+  u32  stk;
+  u16  tid;
+};
+
+COMPILER_CHECK(sizeof(MBlock) == 16);
+
+enum ExternalTag : uptr {
+  kExternalTagNone = 0,
+  kExternalTagSwiftModifyingAccess = 1,
+  kExternalTagFirstUserAvailable = 2,
+  kExternalTagMax = 1024,
+  // Don't set kExternalTagMax over 65,536, since MBlock only stores tags
+  // as 16-bit values, see tsan_defs.h.
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_DEFS_H
diff --git a/lib/tsan/tsan_dense_alloc.h b/lib/tsan/tsan_dense_alloc.h
new file mode 100644
index 0000000000..64fc50e95c
--- /dev/null
+++ b/lib/tsan/tsan_dense_alloc.h
@@ -0,0 +1,141 @@
+//===-- tsan_dense_alloc.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// A DenseSlabAlloc is a freelist-based allocator of fixed-size objects.
+// DenseSlabAllocCache is a thread-local cache for DenseSlabAlloc.
+// The only difference with traditional slab allocators is that DenseSlabAlloc
+// allocates/free indices of objects and provide a functionality to map
+// the index onto the real pointer. The index is u32, that is, 2 times smaller
+// than uptr (hense the Dense prefix).
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_DENSE_ALLOC_H
+#define TSAN_DENSE_ALLOC_H
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "tsan_defs.h"
+#include "tsan_mutex.h"
+
+namespace __tsan {
+
+class DenseSlabAllocCache {
+  static const uptr kSize = 128;
+  typedef u32 IndexT;
+  uptr pos;
+  IndexT cache[kSize];
+  template<typename T, uptr kL1Size, uptr kL2Size> friend class DenseSlabAlloc;
+};
+
+template<typename T, uptr kL1Size, uptr kL2Size>
+class DenseSlabAlloc {
+ public:
+  typedef DenseSlabAllocCache Cache;
+  typedef typename Cache::IndexT IndexT;
+
+  explicit DenseSlabAlloc(const char *name) {
+    // Check that kL1Size and kL2Size are sane.
+    CHECK_EQ(kL1Size & (kL1Size - 1), 0);
+    CHECK_EQ(kL2Size & (kL2Size - 1), 0);
+    CHECK_GE(1ull << (sizeof(IndexT) * 8), kL1Size * kL2Size);
+    // Check that it makes sense to use the dense alloc.
+    CHECK_GE(sizeof(T), sizeof(IndexT));
+    internal_memset(map_, 0, sizeof(map_));
+    freelist_ = 0;
+    fillpos_ = 0;
+    name_ = name;
+  }
+
+  ~DenseSlabAlloc() {
+    for (uptr i = 0; i < kL1Size; i++) {
+      if (map_[i] != 0)
+        UnmapOrDie(map_[i], kL2Size * sizeof(T));
+    }
+  }
+
+  IndexT Alloc(Cache *c) {
+    if (c->pos == 0)
+      Refill(c);
+    return c->cache[--c->pos];
+  }
+
+  void Free(Cache *c, IndexT idx) {
+    DCHECK_NE(idx, 0);
+    if (c->pos == Cache::kSize)
+      Drain(c);
+    c->cache[c->pos++] = idx;
+  }
+
+  T *Map(IndexT idx) {
+    DCHECK_NE(idx, 0);
+    DCHECK_LE(idx, kL1Size * kL2Size);
+    return &map_[idx / kL2Size][idx % kL2Size];
+  }
+
+  void FlushCache(Cache *c) {
+    SpinMutexLock lock(&mtx_);
+    while (c->pos) {
+      IndexT idx = c->cache[--c->pos];
+      *(IndexT*)Map(idx) = freelist_;
+      freelist_ = idx;
+    }
+  }
+
+  void InitCache(Cache *c) {
+    c->pos = 0;
+    internal_memset(c->cache, 0, sizeof(c->cache));
+  }
+
+ private:
+  T *map_[kL1Size];
+  SpinMutex mtx_;
+  IndexT freelist_;
+  uptr fillpos_;
+  const char *name_;
+
+  void Refill(Cache *c) {
+    SpinMutexLock lock(&mtx_);
+    if (freelist_ == 0) {
+      if (fillpos_ == kL1Size) {
+        Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+            name_, kL1Size, kL2Size);
+        Die();
+      }
+      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
+          name_, fillpos_, kL1Size, kL2Size);
+      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
+      // Reserve 0 as invalid index.
+      IndexT start = fillpos_ == 0 ? 1 : 0;
+      for (IndexT i = start; i < kL2Size; i++) {
+        new(batch + i) T;
+        *(IndexT*)(batch + i) = i + 1 + fillpos_ * kL2Size;
+      }
+      *(IndexT*)(batch + kL2Size - 1) = 0;
+      freelist_ = fillpos_ * kL2Size + start;
+      map_[fillpos_++] = batch;
+    }
+    for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {
+      IndexT idx = freelist_;
+      c->cache[c->pos++] = idx;
+      freelist_ = *(IndexT*)Map(idx);
+    }
+  }
+
+  void Drain(Cache *c) {
+    SpinMutexLock lock(&mtx_);
+    for (uptr i = 0; i < Cache::kSize / 2; i++) {
+      IndexT idx = c->cache[--c->pos];
+      *(IndexT*)Map(idx) = freelist_;
+      freelist_ = idx;
+    }
+  }
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_DENSE_ALLOC_H
diff --git a/lib/tsan/tsan_external.cpp b/lib/tsan/tsan_external.cpp
new file mode 100644
index 0000000000..0faa1ee93a
--- /dev/null
+++ b/lib/tsan/tsan_external.cpp
@@ -0,0 +1,124 @@
+//===-- tsan_external.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_rtl.h"
+#include "tsan_interceptors.h"
+
+namespace __tsan {
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+struct TagData {
+  const char *object_type;
+  const char *header;
+};
+
+static TagData registered_tags[kExternalTagMax] = {
+  {},
+  {"Swift variable", "Swift access race"},
+};
+static atomic_uint32_t used_tags{kExternalTagFirstUserAvailable};
+static TagData *GetTagData(uptr tag) {
+  // Invalid/corrupted tag?  Better return NULL and let the caller deal with it.
+  if (tag >= atomic_load(&used_tags, memory_order_relaxed)) return nullptr;
+  return &registered_tags[tag];
+}
+
+const char *GetObjectTypeFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->object_type : nullptr;
+}
+
+const char *GetReportHeaderFromTag(uptr tag) {
+  TagData *tag_data = GetTagData(tag);
+  return tag_data ? tag_data->header : nullptr;
+}
+
+void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
+  FuncEntry(thr, (uptr)&registered_tags[tag]);
+}
+
+uptr TagFromShadowStackFrame(uptr pc) {
+  uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
+  void *pc_ptr = (void *)pc;
+  if (pc_ptr < GetTagData(0) || pc_ptr > GetTagData(tag_count - 1))
+    return 0;
+  return (TagData *)pc_ptr - GetTagData(0);
+}
+
+#if !SANITIZER_GO
+
+typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
+void ExternalAccess(void *addr, void *caller_pc, void *tag, AccessFunc access) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  ThreadState *thr = cur_thread();
+  if (caller_pc) FuncEntry(thr, (uptr)caller_pc);
+  InsertShadowStackFrameForTag(thr, (uptr)tag);
+  bool in_ignored_lib;
+  if (!caller_pc || !libignore()->IsIgnored((uptr)caller_pc, &in_ignored_lib)) {
+    access(thr, CALLERPC, (uptr)addr, kSizeLog1);
+  }
+  FuncExit(thr);
+  if (caller_pc) FuncExit(thr);
+}
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type) {
+  uptr new_tag = atomic_fetch_add(&used_tags, 1, memory_order_relaxed);
+  CHECK_LT(new_tag, kExternalTagMax);
+  GetTagData(new_tag)->object_type = internal_strdup(object_type);
+  char header[127] = {0};
+  internal_snprintf(header, sizeof(header), "race on %s", object_type);
+  GetTagData(new_tag)->header = internal_strdup(header);
+  return (void *)new_tag;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header) {
+  CHECK_GE((uptr)tag, kExternalTagFirstUserAvailable);
+  CHECK_LT((uptr)tag, kExternalTagMax);
+  atomic_uintptr_t *header_ptr =
+      (atomic_uintptr_t *)&GetTagData((uptr)tag)->header;
+  header = internal_strdup(header);
+  char *old_header =
+      (char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
+  if (old_header) internal_free(old_header);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag) {
+  CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  Allocator *a = allocator();
+  MBlock *b = nullptr;
+  if (a->PointerIsMine((void *)addr)) {
+    void *block_begin = a->GetBlockBegin((void *)addr);
+    if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b) {
+    b->tag = (uptr)tag;
+  }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, caller_pc, tag, MemoryRead);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
+  ExternalAccess(addr, caller_pc, tag, MemoryWrite);
+}
+}  // extern "C"
+
+#endif  // !SANITIZER_GO
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_fd.cpp b/lib/tsan/tsan_fd.cpp
new file mode 100644
index 0000000000..50a6b56916
--- /dev/null
+++ b/lib/tsan/tsan_fd.cpp
@@ -0,0 +1,316 @@
+//===-- tsan_fd.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_fd.h"
+#include "tsan_rtl.h"
+#include <sanitizer_common/sanitizer_atomic.h>
+
+namespace __tsan {
+
+const int kTableSizeL1 = 1024;
+const int kTableSizeL2 = 1024;
+const int kTableSize = kTableSizeL1 * kTableSizeL2;
+
+struct FdSync {
+  atomic_uint64_t rc;
+};
+
+struct FdDesc {
+  FdSync *sync;
+  int creation_tid;
+  u32 creation_stack;
+};
+
+struct FdContext {
+  atomic_uintptr_t tab[kTableSizeL1];
+  // Addresses used for synchronization.
+  FdSync globsync;
+  FdSync filesync;
+  FdSync socksync;
+  u64 connectsync;
+};
+
+static FdContext fdctx;
+
+static bool bogusfd(int fd) {
+  // Apparently a bogus fd value.
+  return fd < 0 || fd >= kTableSize;
+}
+
+static FdSync *allocsync(ThreadState *thr, uptr pc) {
+  FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync),
+      kDefaultAlignment, false);
+  atomic_store(&s->rc, 1, memory_order_relaxed);
+  return s;
+}
+
+static FdSync *ref(FdSync *s) {
+  if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1)
+    atomic_fetch_add(&s->rc, 1, memory_order_relaxed);
+  return s;
+}
+
+static void unref(ThreadState *thr, uptr pc, FdSync *s) {
+  if (s && atomic_load(&s->rc, memory_order_relaxed) != (u64)-1) {
+    if (atomic_fetch_sub(&s->rc, 1, memory_order_acq_rel) == 1) {
+      CHECK_NE(s, &fdctx.globsync);
+      CHECK_NE(s, &fdctx.filesync);
+      CHECK_NE(s, &fdctx.socksync);
+      user_free(thr, pc, s, false);
+    }
+  }
+}
+
+static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) {
+  CHECK_GE(fd, 0);
+  CHECK_LT(fd, kTableSize);
+  atomic_uintptr_t *pl1 = &fdctx.tab[fd / kTableSizeL2];
+  uptr l1 = atomic_load(pl1, memory_order_consume);
+  if (l1 == 0) {
+    uptr size = kTableSizeL2 * sizeof(FdDesc);
+    // We need this to reside in user memory to properly catch races on it.
+    void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false);
+    internal_memset(p, 0, size);
+    MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size);
+    if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel))
+      l1 = (uptr)p;
+    else
+      user_free(thr, pc, p, false);
+  }
+  FdDesc *fds = reinterpret_cast<FdDesc *>(l1);
+  return &fds[fd % kTableSizeL2];
+}
+
+// pd must be already ref'ed.
+static void init(ThreadState *thr, uptr pc, int fd, FdSync *s,
+    bool write = true) {
+  FdDesc *d = fddesc(thr, pc, fd);
+  // As a matter of fact, we don't intercept all close calls.
+  // See e.g. libc __res_iclose().
+  if (d->sync) {
+    unref(thr, pc, d->sync);
+    d->sync = 0;
+  }
+  if (flags()->io_sync == 0) {
+    unref(thr, pc, s);
+  } else if (flags()->io_sync == 1) {
+    d->sync = s;
+  } else if (flags()->io_sync == 2) {
+    unref(thr, pc, s);
+    d->sync = &fdctx.globsync;
+  }
+  d->creation_tid = thr->tid;
+  d->creation_stack = CurrentStackId(thr, pc);
+  if (write) {
+    // To catch races between fd usage and open.
+    MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);
+  } else {
+    // See the dup-related comment in FdClose.
+    MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+  }
+}
+
+void FdInit() {
+  atomic_store(&fdctx.globsync.rc, (u64)-1, memory_order_relaxed);
+  atomic_store(&fdctx.filesync.rc, (u64)-1, memory_order_relaxed);
+  atomic_store(&fdctx.socksync.rc, (u64)-1, memory_order_relaxed);
+}
+
+void FdOnFork(ThreadState *thr, uptr pc) {
+  // On fork() we need to reset all fd's, because the child is going
+  // close all them, and that will cause races between previous read/write
+  // and the close.
+  for (int l1 = 0; l1 < kTableSizeL1; l1++) {
+    FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
+    if (tab == 0)
+      break;
+    for (int l2 = 0; l2 < kTableSizeL2; l2++) {
+      FdDesc *d = &tab[l2];
+      MemoryResetRange(thr, pc, (uptr)d, 8);
+    }
+  }
+}
+
+bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack) {
+  for (int l1 = 0; l1 < kTableSizeL1; l1++) {
+    FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
+    if (tab == 0)
+      break;
+    if (addr >= (uptr)tab && addr < (uptr)(tab + kTableSizeL2)) {
+      int l2 = (addr - (uptr)tab) / sizeof(FdDesc);
+      FdDesc *d = &tab[l2];
+      *fd = l1 * kTableSizeL1 + l2;
+      *tid = d->creation_tid;
+      *stack = d->creation_stack;
+      return true;
+    }
+  }
+  return false;
+}
+
+void FdAcquire(ThreadState *thr, uptr pc, int fd) {
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  FdSync *s = d->sync;
+  DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);
+  MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+  if (s)
+    Acquire(thr, pc, (uptr)s);
+}
+
+void FdRelease(ThreadState *thr, uptr pc, int fd) {
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  FdSync *s = d->sync;
+  DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);
+  MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+  if (s)
+    Release(thr, pc, (uptr)s);
+}
+
+void FdAccess(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdAccess(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+}
+
+void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
+  DPrintf("#%d: FdClose(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  FdDesc *d = fddesc(thr, pc, fd);
+  if (write) {
+    // To catch races between fd usage and close.
+    MemoryWrite(thr, pc, (uptr)d, kSizeLog8);
+  } else {
+    // This path is used only by dup2/dup3 calls.
+    // We do read instead of write because there is a number of legitimate
+    // cases where write would lead to false positives:
+    // 1. Some software dups a closed pipe in place of a socket before closing
+    //    the socket (to prevent races actually).
+    // 2. Some daemons dup /dev/null in place of stdin/stdout.
+    // On the other hand we have not seen cases when write here catches real
+    // bugs.
+    MemoryRead(thr, pc, (uptr)d, kSizeLog8);
+  }
+  // We need to clear it, because if we do not intercept any call out there
+  // that creates fd, we will hit false postives.
+  MemoryResetRange(thr, pc, (uptr)d, 8);
+  unref(thr, pc, d->sync);
+  d->sync = 0;
+  d->creation_tid = 0;
+  d->creation_stack = 0;
+}
+
+void FdFileCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdFileCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, &fdctx.filesync);
+}
+
+void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) {
+  DPrintf("#%d: FdDup(%d, %d)\n", thr->tid, oldfd, newfd);
+  if (bogusfd(oldfd) || bogusfd(newfd))
+    return;
+  // Ignore the case when user dups not yet connected socket.
+  FdDesc *od = fddesc(thr, pc, oldfd);
+  MemoryRead(thr, pc, (uptr)od, kSizeLog8);
+  FdClose(thr, pc, newfd, write);
+  init(thr, pc, newfd, ref(od->sync), write);
+}
+
+void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd) {
+  DPrintf("#%d: FdCreatePipe(%d, %d)\n", thr->tid, rfd, wfd);
+  FdSync *s = allocsync(thr, pc);
+  init(thr, pc, rfd, ref(s));
+  init(thr, pc, wfd, ref(s));
+  unref(thr, pc, s);
+}
+
+void FdEventCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdEventCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, allocsync(thr, pc));
+}
+
+void FdSignalCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSignalCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, 0);
+}
+
+void FdInotifyCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdInotifyCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, 0);
+}
+
+void FdPollCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdPollCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, allocsync(thr, pc));
+}
+
+void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  // It can be a UDP socket.
+  init(thr, pc, fd, &fdctx.socksync);
+}
+
+void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd) {
+  DPrintf("#%d: FdSocketAccept(%d, %d)\n", thr->tid, fd, newfd);
+  if (bogusfd(fd))
+    return;
+  // Synchronize connect->accept.
+  Acquire(thr, pc, (uptr)&fdctx.connectsync);
+  init(thr, pc, newfd, &fdctx.socksync);
+}
+
+void FdSocketConnecting(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketConnecting(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  // Synchronize connect->accept.
+  Release(thr, pc, (uptr)&fdctx.connectsync);
+}
+
+void FdSocketConnect(ThreadState *thr, uptr pc, int fd) {
+  DPrintf("#%d: FdSocketConnect(%d)\n", thr->tid, fd);
+  if (bogusfd(fd))
+    return;
+  init(thr, pc, fd, &fdctx.socksync);
+}
+
+uptr File2addr(const char *path) {
+  (void)path;
+  static u64 addr;
+  return (uptr)&addr;
+}
+
+uptr Dir2addr(const char *path) {
+  (void)path;
+  static u64 addr;
+  return (uptr)&addr;
+}
+
+}  //  namespace __tsan
diff --git a/lib/tsan/tsan_fd.h b/lib/tsan/tsan_fd.h
new file mode 100644
index 0000000000..ce4f2f73ba
--- /dev/null
+++ b/lib/tsan/tsan_fd.h
@@ -0,0 +1,64 @@
+//===-- tsan_fd.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// This file handles synchronization via IO.
+// People use IO for synchronization along the lines of:
+//
+// int X;
+// int client_socket;  // initialized elsewhere
+// int server_socket;  // initialized elsewhere
+//
+// Thread 1:
+// X = 42;
+// send(client_socket, ...);
+//
+// Thread 2:
+// if (recv(server_socket, ...) > 0)
+//   assert(X == 42);
+//
+// This file determines the scope of the file descriptor (pipe, socket,
+// all local files, etc) and executes acquire and release operations on
+// the scope as necessary.  Some scopes are very fine grained (e.g. pipe
+// operations synchronize only with operations on the same pipe), while
+// others are corse-grained (e.g. all operations on local files synchronize
+// with each other).
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_FD_H
+#define TSAN_FD_H
+
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+void FdInit();
+void FdAcquire(ThreadState *thr, uptr pc, int fd);
+void FdRelease(ThreadState *thr, uptr pc, int fd);
+void FdAccess(ThreadState *thr, uptr pc, int fd);
+void FdClose(ThreadState *thr, uptr pc, int fd, bool write = true);
+void FdFileCreate(ThreadState *thr, uptr pc, int fd);
+void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write);
+void FdPipeCreate(ThreadState *thr, uptr pc, int rfd, int wfd);
+void FdEventCreate(ThreadState *thr, uptr pc, int fd);
+void FdSignalCreate(ThreadState *thr, uptr pc, int fd);
+void FdInotifyCreate(ThreadState *thr, uptr pc, int fd);
+void FdPollCreate(ThreadState *thr, uptr pc, int fd);
+void FdSocketCreate(ThreadState *thr, uptr pc, int fd);
+void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd);
+void FdSocketConnecting(ThreadState *thr, uptr pc, int fd);
+void FdSocketConnect(ThreadState *thr, uptr pc, int fd);
+bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack);
+void FdOnFork(ThreadState *thr, uptr pc);
+
+uptr File2addr(const char *path);
+uptr Dir2addr(const char *path);
+
+}  // namespace __tsan
+
+#endif  // TSAN_INTERFACE_H
diff --git a/lib/tsan/tsan_flags.cpp b/lib/tsan/tsan_flags.cpp
new file mode 100644
index 0000000000..44bf325cd3
--- /dev/null
+++ b/lib/tsan/tsan_flags.cpp
@@ -0,0 +1,125 @@
+//===-- tsan_flags.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "tsan_flags.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "ubsan/ubsan_flags.h"
+
+namespace __tsan {
+
+// Can be overriden in frontend.
+#ifdef TSAN_EXTERNAL_HOOKS
+extern "C" const char* __tsan_default_options();
+#else
+SANITIZER_WEAK_DEFAULT_IMPL
+const char *__tsan_default_options() {
+  return "";
+}
+#endif
+
+void Flags::SetDefaults() {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+  // DDFlags
+  second_deadlock_stack = false;
+}
+
+void RegisterTsanFlags(FlagParser *parser, Flags *f) {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &f->Name);
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+  // DDFlags
+  RegisterFlag(parser, "second_deadlock_stack",
+      "Report where each mutex is locked in deadlock reports",
+      &f->second_deadlock_stack);
+}
+
+void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
+  SetCommonFlagsDefaults();
+  {
+    // Override some common flags defaults.
+    CommonFlags cf;
+    cf.CopyFrom(*common_flags());
+    cf.allow_addr2line = true;
+    if (SANITIZER_GO) {
+      // Does not work as expected for Go: runtime handles SIGABRT and crashes.
+      cf.abort_on_error = false;
+      // Go does not have mutexes.
+      cf.detect_deadlocks = false;
+    }
+    cf.print_suppressions = false;
+    cf.stack_trace_format = "    #%n %f %S %M";
+    cf.exitcode = 66;
+    cf.intercept_tls_get_addr = true;
+    OverrideCommonFlags(cf);
+  }
+
+  f->SetDefaults();
+
+  FlagParser parser;
+  RegisterTsanFlags(&parser, f);
+  RegisterCommonFlags(&parser);
+
+#if TSAN_CONTAINS_UBSAN
+  __ubsan::Flags *uf = __ubsan::flags();
+  uf->SetDefaults();
+
+  FlagParser ubsan_parser;
+  __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+  RegisterCommonFlags(&ubsan_parser);
+#endif
+
+  // Let a frontend override.
+  parser.ParseString(__tsan_default_options());
+#if TSAN_CONTAINS_UBSAN
+  const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+  ubsan_parser.ParseString(ubsan_default_options);
+#endif
+  // Override from command line.
+  parser.ParseString(env, env_option_name);
+#if TSAN_CONTAINS_UBSAN
+  ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
+#endif
+
+  // Sanity check.
+  if (!f->report_bugs) {
+    f->report_thread_leaks = false;
+    f->report_destroy_locked = false;
+    f->report_signal_unsafe = false;
+  }
+
+  InitializeCommonFlags();
+
+  if (Verbosity()) ReportUnrecognizedFlags();
+
+  if (common_flags()->help) parser.PrintFlagDescriptions();
+
+  if (f->history_size < 0 || f->history_size > 7) {
+    Printf("ThreadSanitizer: incorrect value for history_size"
+           " (must be [0..7])\n");
+    Die();
+  }
+
+  if (f->io_sync < 0 || f->io_sync > 2) {
+    Printf("ThreadSanitizer: incorrect value for io_sync"
+           " (must be [0..2])\n");
+    Die();
+  }
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_flags.h b/lib/tsan/tsan_flags.h
new file mode 100644
index 0000000000..da27d5b992
--- /dev/null
+++ b/lib/tsan/tsan_flags.h
@@ -0,0 +1,34 @@
+//===-- tsan_flags.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+// NOTE: This file may be included into user code.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_FLAGS_H
+#define TSAN_FLAGS_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+
+namespace __tsan {
+
+struct Flags : DDFlags {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+
+  void SetDefaults();
+  void ParseFromString(const char *str);
+};
+
+void InitializeFlags(Flags *flags, const char *env,
+                     const char *env_option_name = nullptr);
+}  // namespace __tsan
+
+#endif  // TSAN_FLAGS_H
diff --git a/lib/tsan/tsan_flags.inc b/lib/tsan/tsan_flags.inc
new file mode 100644
index 0000000000..2105c75448
--- /dev/null
+++ b/lib/tsan/tsan_flags.inc
@@ -0,0 +1,85 @@
+//===-- tsan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// TSan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_FLAG
+# error "Define TSAN_FLAG prior to including this file!"
+#endif
+
+// TSAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+TSAN_FLAG(bool, enable_annotations, true,
+          "Enable dynamic annotations, otherwise they are no-ops.")
+// Suppress a race report if we've already output another race report
+// with the same stack.
+TSAN_FLAG(bool, suppress_equal_stacks, true,
+          "Suppress a race report if we've already output another race report "
+          "with the same stack.")
+TSAN_FLAG(bool, suppress_equal_addresses, true,
+          "Suppress a race report if we've already output another race report "
+          "on the same address.")
+
+TSAN_FLAG(bool, report_bugs, true,
+          "Turns off bug reporting entirely (useful for benchmarking).")
+TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
+TSAN_FLAG(bool, report_destroy_locked, true,
+          "Report destruction of a locked mutex?")
+TSAN_FLAG(bool, report_mutex_bugs, true,
+          "Report incorrect usages of mutexes and mutex annotations?")
+TSAN_FLAG(bool, report_signal_unsafe, true,
+          "Report violations of async signal-safety "
+          "(e.g. malloc() call from a signal handler).")
+TSAN_FLAG(bool, report_atomic_races, true,
+          "Report races between atomic and plain memory accesses.")
+TSAN_FLAG(
+    bool, force_seq_cst_atomics, false,
+    "If set, all atomics are effectively sequentially consistent (seq_cst), "
+    "regardless of what user actually specified.")
+TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.")
+TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
+TSAN_FLAG(int, atexit_sleep_ms, 1000,
+          "Sleep in main thread before exiting for that many ms "
+          "(useful to catch \"at exit\" races).")
+TSAN_FLAG(const char *, profile_memory, "",
+          "If set, periodically write memory profile to that file.")
+TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.")
+TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.")
+TSAN_FLAG(
+    int, memory_limit_mb, 0,
+    "Resident memory limit in MB to aim at."
+    "If the process consumes more memory, then TSan will flush shadow memory.")
+TSAN_FLAG(bool, stop_on_start, false,
+          "Stops on start until __tsan_resume() is called (for debugging).")
+TSAN_FLAG(bool, running_on_valgrind, false,
+          "Controls whether RunningOnValgrind() returns true or false.")
+// There are a lot of goroutines in Go, so we use smaller history.
+TSAN_FLAG(
+    int, history_size, SANITIZER_GO ? 1 : 3,
+    "Per-thread history size, controls how many previous memory accesses "
+    "are remembered per thread.  Possible values are [0..7]. "
+    "history_size=0 amounts to 32K memory accesses.  Each next value doubles "
+    "the amount of memory accesses, up to history_size=7 that amounts to "
+    "4M memory accesses.  The default value is 2 (128K memory accesses).")
+TSAN_FLAG(int, io_sync, 1,
+          "Controls level of synchronization implied by IO operations. "
+          "0 - no synchronization "
+          "1 - reasonable level of synchronization (write->read)"
+          "2 - global synchronization of all IO operations.")
+TSAN_FLAG(bool, die_after_fork, true,
+          "Die after multi-threaded fork if the child creates new threads.")
+TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
+TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false,
+          "Ignore reads and writes from all interceptors.")
+TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
+          "Interceptors should only detect races when called from instrumented "
+          "modules.")
+TSAN_FLAG(bool, shared_ptr_interceptor, true,
+          "Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
diff --git a/lib/tsan/tsan_ignoreset.cpp b/lib/tsan/tsan_ignoreset.cpp
new file mode 100644
index 0000000000..f6e41f6686
--- /dev/null
+++ b/lib/tsan/tsan_ignoreset.cpp
@@ -0,0 +1,46 @@
+//===-- tsan_ignoreset.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_ignoreset.h"
+
+namespace __tsan {
+
+const uptr IgnoreSet::kMaxSize;
+
+IgnoreSet::IgnoreSet()
+    : size_() {
+}
+
+void IgnoreSet::Add(u32 stack_id) {
+  if (size_ == kMaxSize)
+    return;
+  for (uptr i = 0; i < size_; i++) {
+    if (stacks_[i] == stack_id)
+      return;
+  }
+  stacks_[size_++] = stack_id;
+}
+
+void IgnoreSet::Reset() {
+  size_ = 0;
+}
+
+uptr IgnoreSet::Size() const {
+  return size_;
+}
+
+u32 IgnoreSet::At(uptr i) const {
+  CHECK_LT(i, size_);
+  CHECK_LE(size_, kMaxSize);
+  return stacks_[i];
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_ignoreset.h b/lib/tsan/tsan_ignoreset.h
new file mode 100644
index 0000000000..3e318bd674
--- /dev/null
+++ b/lib/tsan/tsan_ignoreset.h
@@ -0,0 +1,37 @@
+//===-- tsan_ignoreset.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// IgnoreSet holds a set of stack traces where ignores were enabled.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_IGNORESET_H
+#define TSAN_IGNORESET_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class IgnoreSet {
+ public:
+  static const uptr kMaxSize = 16;
+
+  IgnoreSet();
+  void Add(u32 stack_id);
+  void Reset();
+  uptr Size() const;
+  u32 At(uptr i) const;
+
+ private:
+  uptr size_;
+  u32 stacks_[kMaxSize];
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_IGNORESET_H
diff --git a/lib/tsan/tsan_interceptors.h b/lib/tsan/tsan_interceptors.h
new file mode 100644
index 0000000000..88d1edd775
--- /dev/null
+++ b/lib/tsan/tsan_interceptors.h
@@ -0,0 +1,76 @@
+#ifndef TSAN_INTERCEPTORS_H
+#define TSAN_INTERCEPTORS_H
+
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+class ScopedInterceptor {
+ public:
+  ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
+  ~ScopedInterceptor();
+  void DisableIgnores();
+  void EnableIgnores();
+ private:
+  ThreadState *const thr_;
+  const uptr pc_;
+  bool in_ignored_lib_;
+  bool ignoring_;
+};
+
+LibIgnore *libignore();
+
+#if !SANITIZER_GO
+INLINE bool in_symbolizer() {
+  cur_thread_init();
+  return UNLIKELY(cur_thread()->in_symbolizer);
+}
+#endif
+
+}  // namespace __tsan
+
+#define SCOPED_INTERCEPTOR_RAW(func, ...) \
+    cur_thread_init(); \
+    ThreadState *thr = cur_thread(); \
+    const uptr caller_pc = GET_CALLER_PC(); \
+    ScopedInterceptor si(thr, #func, caller_pc); \
+    const uptr pc = StackTrace::GetCurrentPc(); \
+    (void)pc; \
+/**/
+
+#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
+    SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
+    if (REAL(func) == 0) { \
+      Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
+      Die(); \
+    }                                                    \
+    if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \
+      return REAL(func)(__VA_ARGS__); \
+/**/
+
+#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
+    si.DisableIgnores();
+
+#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \
+    si.EnableIgnores();
+
+#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
+
+#if SANITIZER_NETBSD
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func));
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func));
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \
+  TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
+  ALIAS(WRAPPER_NAME(pthread_##func2));
+#else
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)
+#endif
+
+#endif  // TSAN_INTERCEPTORS_H
diff --git a/lib/tsan/tsan_interceptors_posix.cpp b/lib/tsan/tsan_interceptors_posix.cpp
new file mode 100644
index 0000000000..9c3e0369bc
--- /dev/null
+++ b/lib/tsan/tsan_interceptors_posix.cpp
@@ -0,0 +1,2854 @@
+//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.inc
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_interface.h"
+#include "tsan_platform.h"
+#include "tsan_suppressions.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_fd.h"
+
+using namespace __tsan;
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+#define stdout __stdoutp
+#define stderr __stderrp
+#endif
+
+#if SANITIZER_NETBSD
+#define dirfd(dirp) (*(int *)(dirp))
+#define fileno_unlocked(fp)              \
+  (((__sanitizer_FILE *)fp)->_file == -1 \
+       ? -1                              \
+       : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file))
+
+#define stdout ((__sanitizer_FILE*)&__sF[1])
+#define stderr ((__sanitizer_FILE*)&__sF[2])
+
+#define nanosleep __nanosleep50
+#define vfork __vfork14
+#endif
+
+#if SANITIZER_ANDROID
+#define mallopt(a, b)
+#endif
+
+#ifdef __mips__
+const int kSigCount = 129;
+#else
+const int kSigCount = 65;
+#endif
+
+#ifdef __mips__
+struct ucontext_t {
+  u64 opaque[768 / sizeof(u64) + 1];
+};
+#else
+struct ucontext_t {
+  // The size is determined by looking at sizeof of real ucontext_t on linux.
+  u64 opaque[936 / sizeof(u64) + 1];
+};
+#endif
+
+#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1
+#define PTHREAD_ABI_BASE  "GLIBC_2.3.2"
+#elif defined(__aarch64__) || SANITIZER_PPC64V2
+#define PTHREAD_ABI_BASE  "GLIBC_2.17"
+#endif
+
+extern "C" int pthread_attr_init(void *attr);
+extern "C" int pthread_attr_destroy(void *attr);
+DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
+extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
+extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
+extern "C" int pthread_setspecific(unsigned key, const void *v);
+DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
+DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+extern "C" void *pthread_self();
+extern "C" void _exit(int status);
+#if !SANITIZER_NETBSD
+extern "C" int fileno_unlocked(void *stream);
+extern "C" int dirfd(void *dirp);
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD
+extern "C" int mallopt(int param, int value);
+#endif
+#if SANITIZER_NETBSD
+extern __sanitizer_FILE __sF[];
+#else
+extern __sanitizer_FILE *stdout, *stderr;
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+const int PTHREAD_MUTEX_RECURSIVE = 1;
+const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
+#else
+const int PTHREAD_MUTEX_RECURSIVE = 2;
+const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+const int EPOLL_CTL_ADD = 1;
+#endif
+const int SIGILL = 4;
+const int SIGTRAP = 5;
+const int SIGABRT = 6;
+const int SIGFPE = 8;
+const int SIGSEGV = 11;
+const int SIGPIPE = 13;
+const int SIGTERM = 15;
+#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+const int SIGBUS = 10;
+const int SIGSYS = 12;
+#else
+const int SIGBUS = 7;
+const int SIGSYS = 31;
+#endif
+void *const MAP_FAILED = (void*)-1;
+#if SANITIZER_NETBSD
+const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
+#elif !SANITIZER_MAC
+const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
+#endif
+const int MAP_FIXED = 0x10;
+typedef long long_t;
+
+// From /usr/include/unistd.h
+# define F_ULOCK 0      /* Unlock a previously locked region.  */
+# define F_LOCK  1      /* Lock a region for exclusive use.  */
+# define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
+# define F_TEST  3      /* Test a region for other processes locks.  */
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
+const int SA_SIGINFO = 0x40;
+const int SIG_SETMASK = 3;
+#elif defined(__mips__)
+const int SA_SIGINFO = 8;
+const int SIG_SETMASK = 3;
+#else
+const int SA_SIGINFO = 4;
+const int SIG_SETMASK = 2;
+#endif
+
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
+  (cur_thread_init(), !cur_thread()->is_inited)
+
+namespace __tsan {
+struct SignalDesc {
+  bool armed;
+  bool sigaction;
+  __sanitizer_siginfo siginfo;
+  ucontext_t ctx;
+};
+
+struct ThreadSignalContext {
+  int int_signal_send;
+  atomic_uintptr_t in_blocking_func;
+  atomic_uintptr_t have_pending_signals;
+  SignalDesc pending_signals[kSigCount];
+  // emptyset and oldset are too big for stack.
+  __sanitizer_sigset_t emptyset;
+  __sanitizer_sigset_t oldset;
+};
+
+// The sole reason tsan wraps atexit callbacks is to establish synchronization
+// between callback setup and callback execution.
+struct AtExitCtx {
+  void (*f)();
+  void *arg;
+};
+
+// InterceptorContext holds all global data required for interceptors.
+// It's explicitly constructed in InitializeInterceptors with placement new
+// and is never destroyed. This allows usage of members with non-trivial
+// constructors and destructors.
+struct InterceptorContext {
+  // The object is 64-byte aligned, because we want hot data to be located
+  // in a single cache line if possible (it's accessed in every interceptor).
+  ALIGNED(64) LibIgnore libignore;
+  __sanitizer_sigaction sigactions[kSigCount];
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+  unsigned finalize_key;
+#endif
+
+  BlockingMutex atexit_mu;
+  Vector<struct AtExitCtx *> AtExitStack;
+
+  InterceptorContext()
+      : libignore(LINKER_INITIALIZED), AtExitStack() {
+  }
+};
+
+static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
+InterceptorContext *interceptor_ctx() {
+  return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
+}
+
+LibIgnore *libignore() {
+  return &interceptor_ctx()->libignore;
+}
+
+void InitializeLibIgnore() {
+  const SuppressionContext &supp = *Suppressions();
+  const uptr n = supp.SuppressionCount();
+  for (uptr i = 0; i < n; i++) {
+    const Suppression *s = supp.SuppressionAt(i);
+    if (0 == internal_strcmp(s->type, kSuppressionLib))
+      libignore()->AddIgnoredLibrary(s->templ);
+  }
+  if (flags()->ignore_noninstrumented_modules)
+    libignore()->IgnoreNoninstrumentedModules(true);
+  libignore()->OnLibraryLoaded(0);
+}
+
+// The following two hooks can be used by for cooperative scheduling when
+// locking.
+#ifdef TSAN_EXTERNAL_HOOKS
+void OnPotentiallyBlockingRegionBegin();
+void OnPotentiallyBlockingRegionEnd();
+#else
+SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
+SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
+#endif
+
+}  // namespace __tsan
+
+static ThreadSignalContext *SigCtx(ThreadState *thr) {
+  ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx;
+  if (ctx == 0 && !thr->is_dead) {
+    ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext");
+    MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
+    thr->signal_ctx = ctx;
+  }
+  return ctx;
+}
+
+ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
+                                     uptr pc)
+    : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
+  Initialize(thr);
+  if (!thr_->is_inited) return;
+  if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
+  DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
+  ignoring_ =
+      !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses ||
+                                libignore()->IsIgnored(pc, &in_ignored_lib_));
+  EnableIgnores();
+}
+
+ScopedInterceptor::~ScopedInterceptor() {
+  if (!thr_->is_inited) return;
+  DisableIgnores();
+  if (!thr_->ignore_interceptors) {
+    ProcessPendingSignals(thr_);
+    FuncExit(thr_);
+    CheckNoLocks(thr_);
+  }
+}
+
+void ScopedInterceptor::EnableIgnores() {
+  if (ignoring_) {
+    ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
+    if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
+    if (in_ignored_lib_) {
+      DCHECK(!thr_->in_ignored_lib);
+      thr_->in_ignored_lib = true;
+    }
+  }
+}
+
+void ScopedInterceptor::DisableIgnores() {
+  if (ignoring_) {
+    ThreadIgnoreEnd(thr_, pc_);
+    if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--;
+    if (in_ignored_lib_) {
+      DCHECK(thr_->in_ignored_lib);
+      thr_->in_ignored_lib = false;
+    }
+  }
+}
+
+#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
+#if SANITIZER_FREEBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+#elif SANITIZER_NETBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
+         INTERCEPT_FUNCTION(__libc_##func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
+         INTERCEPT_FUNCTION(__libc_thr_##func)
+#else
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+#endif
+
+#define READ_STRING_OF_LEN(thr, pc, s, len, n)                 \
+  MemoryAccessRange((thr), (pc), (uptr)(s),                         \
+    common_flags()->strict_string_checks ? (len) + 1 : (n), false)
+
+#define READ_STRING(thr, pc, s, n)                             \
+    READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n))
+
+#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
+
+struct BlockingCall {
+  explicit BlockingCall(ThreadState *thr)
+      : thr(thr)
+      , ctx(SigCtx(thr)) {
+    for (;;) {
+      atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
+      if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0)
+        break;
+      atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+      ProcessPendingSignals(thr);
+    }
+    // When we are in a "blocking call", we process signals asynchronously
+    // (right when they arrive). In this context we do not expect to be
+    // executing any user/runtime code. The known interceptor sequence when
+    // this is not true is: pthread_join -> munmap(stack). It's fine
+    // to ignore munmap in this case -- we handle stack shadow separately.
+    thr->ignore_interceptors++;
+  }
+
+  ~BlockingCall() {
+    thr->ignore_interceptors--;
+    atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+  }
+
+  ThreadState *thr;
+  ThreadSignalContext *ctx;
+};
+
+TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
+  SCOPED_TSAN_INTERCEPTOR(sleep, sec);
+  unsigned res = BLOCK_REAL(sleep)(sec);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, usleep, long_t usec) {
+  SCOPED_TSAN_INTERCEPTOR(usleep, usec);
+  int res = BLOCK_REAL(usleep)(usec);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
+  SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
+  int res = BLOCK_REAL(nanosleep)(req, rem);
+  AfterSleep(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pause, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(pause, fake);
+  return BLOCK_REAL(pause)(fake);
+}
+
+static void at_exit_wrapper() {
+  AtExitCtx *ctx;
+  {
+    // Ensure thread-safety.
+    BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+
+    // Pop AtExitCtx from the top of the stack of callback functions
+    uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
+    ctx = interceptor_ctx()->AtExitStack[element];
+    interceptor_ctx()->AtExitStack.PopBack();
+  }
+
+  Acquire(cur_thread(), (uptr)0, (uptr)ctx);
+  ((void(*)())ctx->f)();
+  InternalFree(ctx);
+}
+
+static void cxa_at_exit_wrapper(void *arg) {
+  Acquire(cur_thread(), 0, (uptr)arg);
+  AtExitCtx *ctx = (AtExitCtx*)arg;
+  ((void(*)(void *arg))ctx->f)(ctx->arg);
+  InternalFree(ctx);
+}
+
+static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
+      void *arg, void *dso);
+
+#if !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
+  if (in_symbolizer())
+    return 0;
+  // We want to setup the atexit callback even if we are in ignored lib
+  // or after fork.
+  SCOPED_INTERCEPTOR_RAW(atexit, f);
+  return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0);
+}
+#endif
+
+TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
+  if (in_symbolizer())
+    return 0;
+  SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
+  return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
+}
+
+static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
+      void *arg, void *dso) {
+  AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
+  ctx->f = f;
+  ctx->arg = arg;
+  Release(thr, pc, (uptr)ctx);
+  // Memory allocation in __cxa_atexit will race with free during exit,
+  // because we do not see synchronization around atexit callback list.
+  ThreadIgnoreBegin(thr, pc);
+  int res;
+  if (!dso) {
+    // NetBSD does not preserve the 2nd argument if dso is equal to 0
+    // Store ctx in a local stack-like structure
+
+    // Ensure thread-safety.
+    BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+
+    res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0);
+    // Push AtExitCtx on the top of the stack of callback functions
+    if (!res) {
+      interceptor_ctx()->AtExitStack.PushBack(ctx);
+    }
+  } else {
+    res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);
+  }
+  ThreadIgnoreEnd(thr, pc);
+  return res;
+}
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+static void on_exit_wrapper(int status, void *arg) {
+  ThreadState *thr = cur_thread();
+  uptr pc = 0;
+  Acquire(thr, pc, (uptr)arg);
+  AtExitCtx *ctx = (AtExitCtx*)arg;
+  ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg);
+  InternalFree(ctx);
+}
+
+TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
+  if (in_symbolizer())
+    return 0;
+  SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
+  AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
+  ctx->f = (void(*)())f;
+  ctx->arg = arg;
+  Release(thr, pc, (uptr)ctx);
+  // Memory allocation in __cxa_atexit will race with free during exit,
+  // because we do not see synchronization around atexit callback list.
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(on_exit)(on_exit_wrapper, ctx);
+  ThreadIgnoreEnd(thr, pc);
+  return res;
+}
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT
+#endif
+
+// Cleanup old bufs.
+static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
+  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
+    JmpBuf *buf = &thr->jmp_bufs[i];
+    if (buf->sp <= sp) {
+      uptr sz = thr->jmp_bufs.Size();
+      internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf));
+      thr->jmp_bufs.PopBack();
+      i--;
+    }
+  }
+}
+
+static void SetJmp(ThreadState *thr, uptr sp) {
+  if (!thr->is_inited)  // called from libc guts during bootstrap
+    return;
+  // Cleanup old bufs.
+  JmpBufGarbageCollect(thr, sp);
+  // Remember the buf.
+  JmpBuf *buf = thr->jmp_bufs.PushBack();
+  buf->sp = sp;
+  buf->shadow_stack_pos = thr->shadow_stack_pos;
+  ThreadSignalContext *sctx = SigCtx(thr);
+  buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
+  buf->in_blocking_func = sctx ?
+      atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
+      false;
+  buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
+      memory_order_relaxed);
+}
+
+static void LongJmp(ThreadState *thr, uptr *env) {
+  uptr sp = ExtractLongJmpSp(env);
+  // Find the saved buf with matching sp.
+  for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
+    JmpBuf *buf = &thr->jmp_bufs[i];
+    if (buf->sp == sp) {
+      CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
+      // Unwind the stack.
+      while (thr->shadow_stack_pos > buf->shadow_stack_pos)
+        FuncExit(thr);
+      ThreadSignalContext *sctx = SigCtx(thr);
+      if (sctx) {
+        sctx->int_signal_send = buf->int_signal_send;
+        atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,
+            memory_order_relaxed);
+      }
+      atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
+          memory_order_relaxed);
+      JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
+      return;
+    }
+  }
+  Printf("ThreadSanitizer: can't find longjmp buf\n");
+  CHECK(0);
+}
+
+// FIXME: put everything below into a common extern "C" block?
+extern "C" void __tsan_setjmp(uptr sp) {
+  cur_thread_init();
+  SetJmp(cur_thread(), sp);
+}
+
+#if SANITIZER_MAC
+TSAN_INTERCEPTOR(int, setjmp, void *env);
+TSAN_INTERCEPTOR(int, _setjmp, void *env);
+TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
+#else  // SANITIZER_MAC
+
+#if SANITIZER_NETBSD
+#define setjmp_symname __setjmp14
+#define sigsetjmp_symname __sigsetjmp14
+#else
+#define setjmp_symname setjmp
+#define sigsetjmp_symname sigsetjmp
+#endif
+
+#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
+#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
+#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
+#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
+
+#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
+#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
+
+// Not called.  Merely to satisfy TSAN_INTERCEPT().
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int TSAN_INTERCEPTOR_SETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+// FIXME: any reason to have a separate declaration?
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int __interceptor__setjmp(void *env);
+extern "C" int __interceptor__setjmp(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
+  CHECK(0);
+  return 0;
+}
+
+#if !SANITIZER_NETBSD
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+int __interceptor___sigsetjmp(void *env);
+extern "C" int __interceptor___sigsetjmp(void *env) {
+  CHECK(0);
+  return 0;
+}
+#endif
+
+extern "C" int setjmp_symname(void *env);
+extern "C" int _setjmp(void *env);
+extern "C" int sigsetjmp_symname(void *env);
+#if !SANITIZER_NETBSD
+extern "C" int __sigsetjmp(void *env);
+#endif
+DEFINE_REAL(int, setjmp_symname, void *env)
+DEFINE_REAL(int, _setjmp, void *env)
+DEFINE_REAL(int, sigsetjmp_symname, void *env)
+#if !SANITIZER_NETBSD
+DEFINE_REAL(int, __sigsetjmp, void *env)
+#endif
+#endif  // SANITIZER_MAC
+
+#if SANITIZER_NETBSD
+#define longjmp_symname __longjmp14
+#define siglongjmp_symname __siglongjmp14
+#else
+#define longjmp_symname longjmp
+#define siglongjmp_symname siglongjmp
+#endif
+
+TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) {
+  // Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
+  // bad things will happen. We will jump over ScopedInterceptor dtor and can
+  // leave thr->in_ignored_lib set.
+  {
+    SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(longjmp_symname)(env, val);
+}
+
+TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) {
+  {
+    SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(siglongjmp_symname)(env, val);
+}
+
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
+  {
+    SCOPED_INTERCEPTOR_RAW(_longjmp, env, val);
+  }
+  LongJmp(cur_thread(), env);
+  REAL(_longjmp)(env, val);
+}
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(void*, malloc, uptr size) {
+  if (in_symbolizer())
+    return InternalAlloc(size);
+  void *p = 0;
+  {
+    SCOPED_INTERCEPTOR_RAW(malloc, size);
+    p = user_alloc(thr, pc, size);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
+  SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
+  return user_memalign(thr, pc, align, sz);
+}
+
+TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
+  if (in_symbolizer())
+    return InternalCalloc(size, n);
+  void *p = 0;
+  {
+    SCOPED_INTERCEPTOR_RAW(calloc, size, n);
+    p = user_calloc(thr, pc, size, n);
+  }
+  invoke_malloc_hook(p, n * size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
+  if (in_symbolizer())
+    return InternalRealloc(p, size);
+  if (p)
+    invoke_free_hook(p);
+  {
+    SCOPED_INTERCEPTOR_RAW(realloc, p, size);
+    p = user_realloc(thr, pc, p, size);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) {
+  if (in_symbolizer())
+    return InternalReallocArray(p, size, n);
+  if (p)
+    invoke_free_hook(p);
+  {
+    SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);
+    p = user_reallocarray(thr, pc, p, size, n);
+  }
+  invoke_malloc_hook(p, size);
+  return p;
+}
+
+TSAN_INTERCEPTOR(void, free, void *p) {
+  if (p == 0)
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free, p);
+  user_free(thr, pc, p);
+}
+
+TSAN_INTERCEPTOR(void, cfree, void *p) {
+  if (p == 0)
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(cfree, p);
+  user_free(thr, pc, p);
+}
+
+TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
+  SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
+  return user_alloc_usable_size(p);
+}
+#endif
+
+TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
+  SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);
+  uptr srclen = internal_strlen(src);
+  MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
+  MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
+  return REAL(strcpy)(dst, src);
+}
+
+TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
+  SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
+  uptr srclen = internal_strnlen(src, n);
+  MemoryAccessRange(thr, pc, (uptr)dst, n, true);
+  MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
+  return REAL(strncpy)(dst, src, n);
+}
+
+TSAN_INTERCEPTOR(char*, strdup, const char *str) {
+  SCOPED_TSAN_INTERCEPTOR(strdup, str);
+  // strdup will call malloc, so no instrumentation is required here.
+  return REAL(strdup)(str);
+}
+
+// Zero out addr if it points into shadow memory and was provided as a hint
+// only, i.e., MAP_FIXED is not set.
+static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
+  if (*addr) {
+    if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
+      if (flags & MAP_FIXED) {
+        errno = errno_EINVAL;
+        return false;
+      } else {
+        *addr = 0;
+      }
+    }
+  }
+  return true;
+}
+
+template <class Mmap>
+static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap,
+                              void *addr, SIZE_T sz, int prot, int flags,
+                              int fd, OFF64_T off) {
+  if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED;
+  void *res = real_mmap(addr, sz, prot, flags, fd, off);
+  if (res != MAP_FAILED) {
+    if (fd > 0) FdAccess(thr, pc, fd);
+    MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
+  SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
+  UnmapShadow(thr, (uptr)addr, sz);
+  int res = REAL(munmap)(addr, sz);
+  return res;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
+  SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
+  return user_memalign(thr, pc, align, sz);
+}
+#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
+#else
+#define TSAN_MAYBE_INTERCEPT_MEMALIGN
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
+  if (in_symbolizer())
+    return InternalAlloc(sz, nullptr, align);
+  SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
+  return user_aligned_alloc(thr, pc, align, sz);
+}
+
+TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
+  if (in_symbolizer())
+    return InternalAlloc(sz, nullptr, GetPageSizeCached());
+  SCOPED_INTERCEPTOR_RAW(valloc, sz);
+  return user_valloc(thr, pc, sz);
+}
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
+  if (in_symbolizer()) {
+    uptr PageSize = GetPageSizeCached();
+    sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+    return InternalAlloc(sz, nullptr, PageSize);
+  }
+  SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
+  return user_pvalloc(thr, pc, sz);
+}
+#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
+#else
+#define TSAN_MAYBE_INTERCEPT_PVALLOC
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
+  if (in_symbolizer()) {
+    void *p = InternalAlloc(sz, nullptr, align);
+    if (!p)
+      return errno_ENOMEM;
+    *memptr = p;
+    return 0;
+  }
+  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
+  return user_posix_memalign(thr, pc, memptr, align, sz);
+}
+#endif
+
+// __cxa_guard_acquire and friends need to be intercepted in a special way -
+// regular interceptors will break statically-linked libstdc++. Linux
+// interceptors are especially defined as weak functions (so that they don't
+// cause link errors when user defines them as well). So they silently
+// auto-disable themselves when such symbol is already present in the binary. If
+// we link libstdc++ statically, it will bring own __cxa_guard_acquire which
+// will silently replace our interceptor.  That's why on Linux we simply export
+// these interceptors with INTERFACE_ATTRIBUTE.
+// On OS X, we don't support statically linking, so we just use a regular
+// interceptor.
+#if SANITIZER_MAC
+#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
+#else
+#define STDCXX_INTERCEPTOR(rettype, name, ...) \
+  extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__)
+#endif
+
+// Used in thread-safe function static initialization.
+STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
+  OnPotentiallyBlockingRegionBegin();
+  auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd);
+  for (;;) {
+    u32 cmp = atomic_load(g, memory_order_acquire);
+    if (cmp == 0) {
+      if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
+        return 1;
+    } else if (cmp == 1) {
+      Acquire(thr, pc, (uptr)g);
+      return 0;
+    } else {
+      internal_sched_yield();
+    }
+  }
+}
+
+STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
+  Release(thr, pc, (uptr)g);
+  atomic_store(g, 1, memory_order_release);
+}
+
+STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
+  SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
+  atomic_store(g, 0, memory_order_relaxed);
+}
+
+namespace __tsan {
+void DestroyThreadState() {
+  ThreadState *thr = cur_thread();
+  Processor *proc = thr->proc();
+  ThreadFinish(thr);
+  ProcUnwire(proc, thr);
+  ProcDestroy(proc);
+  DTLS_Destroy();
+  cur_thread_finalize();
+}
+
+void PlatformCleanUpThreadState(ThreadState *thr) {
+  ThreadSignalContext *sctx = thr->signal_ctx;
+  if (sctx) {
+    thr->signal_ctx = 0;
+    UnmapOrDie(sctx, sizeof(*sctx));
+  }
+}
+}  // namespace __tsan
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+static void thread_finalize(void *v) {
+  uptr iter = (uptr)v;
+  if (iter > 1) {
+    if (pthread_setspecific(interceptor_ctx()->finalize_key,
+        (void*)(iter - 1))) {
+      Printf("ThreadSanitizer: failed to set thread key\n");
+      Die();
+    }
+    return;
+  }
+  DestroyThreadState();
+}
+#endif
+
+
+struct ThreadParam {
+  void* (*callback)(void *arg);
+  void *param;
+  atomic_uintptr_t tid;
+};
+
+extern "C" void *__tsan_thread_start_func(void *arg) {
+  ThreadParam *p = (ThreadParam*)arg;
+  void* (*callback)(void *arg) = p->callback;
+  void *param = p->param;
+  int tid = 0;
+  {
+    cur_thread_init();
+    ThreadState *thr = cur_thread();
+    // Thread-local state is not initialized yet.
+    ScopedIgnoreInterceptors ignore;
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+    ThreadIgnoreBegin(thr, 0);
+    if (pthread_setspecific(interceptor_ctx()->finalize_key,
+                            (void *)GetPthreadDestructorIterations())) {
+      Printf("ThreadSanitizer: failed to set thread key\n");
+      Die();
+    }
+    ThreadIgnoreEnd(thr, 0);
+#endif
+    while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
+      internal_sched_yield();
+    Processor *proc = ProcCreate();
+    ProcWire(proc, thr);
+    ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
+    atomic_store(&p->tid, 0, memory_order_release);
+  }
+  void *res = callback(param);
+  // Prevent the callback from being tail called,
+  // it mixes up stack traces.
+  volatile int foo = 42;
+  foo++;
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_create,
+    void *th, void *attr, void *(*callback)(void*), void * param) {
+  SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
+
+  MaybeSpawnBackgroundThread();
+
+  if (ctx->after_multithreaded_fork) {
+    if (flags()->die_after_fork) {
+      Report("ThreadSanitizer: starting new threads after multi-threaded "
+          "fork is not supported. Dying (set die_after_fork=0 to override)\n");
+      Die();
+    } else {
+      VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
+          "fork is not supported (pid %d). Continuing because of "
+          "die_after_fork=0, but you are on your own\n", internal_getpid());
+    }
+  }
+  __sanitizer_pthread_attr_t myattr;
+  if (attr == 0) {
+    pthread_attr_init(&myattr);
+    attr = &myattr;
+  }
+  int detached = 0;
+  REAL(pthread_attr_getdetachstate)(attr, &detached);
+  AdjustStackSize(attr);
+
+  ThreadParam p;
+  p.callback = callback;
+  p.param = param;
+  atomic_store(&p.tid, 0, memory_order_relaxed);
+  int res = -1;
+  {
+    // Otherwise we see false positives in pthread stack manipulation.
+    ScopedIgnoreInterceptors ignore;
+    ThreadIgnoreBegin(thr, pc);
+    res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
+    ThreadIgnoreEnd(thr, pc);
+  }
+  if (res == 0) {
+    int tid = ThreadCreate(thr, pc, *(uptr*)th, IsStateDetached(detached));
+    CHECK_NE(tid, 0);
+    // Synchronization on p.tid serves two purposes:
+    // 1. ThreadCreate must finish before the new thread starts.
+    //    Otherwise the new thread can call pthread_detach, but the pthread_t
+    //    identifier is not yet registered in ThreadRegistry by ThreadCreate.
+    // 2. ThreadStart must finish before this thread continues.
+    //    Otherwise, this thread can call pthread_detach and reset thr->sync
+    //    before the new thread got a chance to acquire from it in ThreadStart.
+    atomic_store(&p.tid, tid, memory_order_release);
+    while (atomic_load(&p.tid, memory_order_acquire) != 0)
+      internal_sched_yield();
+  }
+  if (attr == &myattr)
+    pthread_attr_destroy(&myattr);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
+  SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
+  int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = BLOCK_REAL(pthread_join)(th, ret);
+  ThreadIgnoreEnd(thr, pc);
+  if (res == 0) {
+    ThreadJoin(thr, pc, tid);
+  }
+  return res;
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
+TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
+  SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
+  int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  int res = REAL(pthread_detach)(th);
+  if (res == 0) {
+    ThreadDetach(thr, pc, tid);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
+  {
+    SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+    CHECK_EQ(thr, &cur_thread_placeholder);
+#endif
+  }
+  REAL(pthread_exit)(retval);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) {
+  SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
+  int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(pthread_tryjoin_np)(th, ret);
+  ThreadIgnoreEnd(thr, pc);
+  if (res == 0)
+    ThreadJoin(thr, pc, tid);
+  else
+    ThreadNotJoined(thr, pc, tid, (uptr)th);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret,
+                 const struct timespec *abstime) {
+  SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
+  int tid = ThreadConsumeTid(thr, pc, (uptr)th);
+  ThreadIgnoreBegin(thr, pc);
+  int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
+  ThreadIgnoreEnd(thr, pc);
+  if (res == 0)
+    ThreadJoin(thr, pc, tid);
+  else
+    ThreadNotJoined(thr, pc, tid, (uptr)th);
+  return res;
+}
+#endif
+
+// Problem:
+// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
+// pthread_cond_t has different size in the different versions.
+// If call new REAL functions for old pthread_cond_t, they will corrupt memory
+// after pthread_cond_t (old cond is smaller).
+// If we call old REAL functions for new pthread_cond_t, we will lose  some
+// functionality (e.g. old functions do not support waiting against
+// CLOCK_REALTIME).
+// Proper handling would require to have 2 versions of interceptors as well.
+// But this is messy, in particular requires linker scripts when sanitizer
+// runtime is linked into a shared library.
+// Instead we assume we don't have dynamic libraries built against old
+// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
+// that allows to work with old libraries (but this mode does not support
+// some features, e.g. pthread_condattr_getpshared).
+static void *init_cond(void *c, bool force = false) {
+  // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
+  // So we allocate additional memory on the side large enough to hold
+  // any pthread_cond_t object. Always call new REAL functions, but pass
+  // the aux object to them.
+  // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
+  // first word of pthread_cond_t to zero.
+  // It's all relevant only for linux.
+  if (!common_flags()->legacy_pthread_cond)
+    return c;
+  atomic_uintptr_t *p = (atomic_uintptr_t*)c;
+  uptr cond = atomic_load(p, memory_order_acquire);
+  if (!force && cond != 0)
+    return (void*)cond;
+  void *newcond = WRAP(malloc)(pthread_cond_t_sz);
+  internal_memset(newcond, 0, pthread_cond_t_sz);
+  if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
+      memory_order_acq_rel))
+    return newcond;
+  WRAP(free)(newcond);
+  return (void*)cond;
+}
+
+struct CondMutexUnlockCtx {
+  ScopedInterceptor *si;
+  ThreadState *thr;
+  uptr pc;
+  void *m;
+};
+
+static void cond_mutex_unlock(CondMutexUnlockCtx *arg) {
+  // pthread_cond_wait interceptor has enabled async signal delivery
+  // (see BlockingCall below). Disable async signals since we are running
+  // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
+  // since the thread is cancelled, so we have to manually execute them
+  // (the thread still can run some user code due to pthread_cleanup_push).
+  ThreadSignalContext *ctx = SigCtx(arg->thr);
+  CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
+  atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
+  MutexPostLock(arg->thr, arg->pc, (uptr)arg->m, MutexFlagDoPreLockOnPostLock);
+  // Undo BlockingCall ctor effects.
+  arg->thr->ignore_interceptors--;
+  arg->si->~ScopedInterceptor();
+}
+
+INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
+  void *cond = init_cond(c, true);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
+  return REAL(pthread_cond_init)(cond, a);
+}
+
+static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si,
+                     int (*fn)(void *c, void *m, void *abstime), void *c,
+                     void *m, void *t) {
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  MutexUnlock(thr, pc, (uptr)m);
+  CondMutexUnlockCtx arg = {si, thr, pc, m};
+  int res = 0;
+  // This ensures that we handle mutex lock even in case of pthread_cancel.
+  // See test/tsan/cond_cancel.cpp.
+  {
+    // Enable signal delivery while the thread is blocked.
+    BlockingCall bc(thr);
+    res = call_pthread_cancel_with_cleanup(
+        fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg);
+  }
+  if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m);
+  MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
+  return res;
+}
+
+INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
+  return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL(
+                                     pthread_cond_wait),
+                   cond, m, 0);
+}
+
+INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
+  return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m,
+                   abstime);
+}
+
+#if SANITIZER_MAC
+INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m,
+            void *reltime) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime);
+  return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond,
+                   m, reltime);
+}
+#endif
+
+INTERCEPTOR(int, pthread_cond_signal, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  return REAL(pthread_cond_signal)(cond);
+}
+
+INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
+  return REAL(pthread_cond_broadcast)(cond);
+}
+
+INTERCEPTOR(int, pthread_cond_destroy, void *c) {
+  void *cond = init_cond(c);
+  SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond);
+  MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
+  int res = REAL(pthread_cond_destroy)(cond);
+  if (common_flags()->legacy_pthread_cond) {
+    // Free our aux cond and zero the pointer to not leave dangling pointers.
+    WRAP(free)(cond);
+    atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
+  int res = REAL(pthread_mutex_init)(m, a);
+  if (res == 0) {
+    u32 flagz = 0;
+    if (a) {
+      int type = 0;
+      if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
+        if (type == PTHREAD_MUTEX_RECURSIVE ||
+            type == PTHREAD_MUTEX_RECURSIVE_NP)
+          flagz |= MutexFlagWriteReentrant;
+    }
+    MutexCreate(thr, pc, (uptr)m, flagz);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
+  int res = REAL(pthread_mutex_destroy)(m);
+  if (res == 0 || res == errno_EBUSY) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
+  int res = REAL(pthread_mutex_trylock)(m);
+  if (res == errno_EOWNERDEAD)
+    MutexRepair(thr, pc, (uptr)m);
+  if (res == 0 || res == errno_EOWNERDEAD)
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
+  int res = REAL(pthread_mutex_timedlock)(m, abstime);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+#endif
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
+  int res = REAL(pthread_spin_init)(m, pshared);
+  if (res == 0) {
+    MutexCreate(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
+  int res = REAL(pthread_spin_destroy)(m);
+  if (res == 0) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
+  MutexPreLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_spin_lock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
+  int res = REAL(pthread_spin_trylock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
+  MutexUnlock(thr, pc, (uptr)m);
+  int res = REAL(pthread_spin_unlock)(m);
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
+  int res = REAL(pthread_rwlock_init)(m, a);
+  if (res == 0) {
+    MutexCreate(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
+  int res = REAL(pthread_rwlock_destroy)(m);
+  if (res == 0) {
+    MutexDestroy(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
+  MutexPreReadLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_rdlock)(m);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
+  int res = REAL(pthread_rwlock_tryrdlock)(m);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
+  int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
+  if (res == 0) {
+    MutexPostReadLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
+  MutexPreLock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_wrlock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m);
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
+  int res = REAL(pthread_rwlock_trywrlock)(m);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
+  int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
+  if (res == 0) {
+    MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
+  MutexReadOrWriteUnlock(thr, pc, (uptr)m);
+  int res = REAL(pthread_rwlock_unlock)(m);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
+  MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
+  int res = REAL(pthread_barrier_init)(b, a, count);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
+  MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
+  int res = REAL(pthread_barrier_destroy)(b);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
+  Release(thr, pc, (uptr)b);
+  MemoryRead(thr, pc, (uptr)b, kSizeLog1);
+  int res = REAL(pthread_barrier_wait)(b);
+  MemoryRead(thr, pc, (uptr)b, kSizeLog1);
+  if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
+    Acquire(thr, pc, (uptr)b);
+  }
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
+  SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
+  if (o == 0 || f == 0)
+    return errno_EINVAL;
+  atomic_uint32_t *a;
+
+  if (SANITIZER_MAC)
+    a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t)));
+  else if (SANITIZER_NETBSD)
+    a = static_cast<atomic_uint32_t*>
+          ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz));
+  else
+    a = static_cast<atomic_uint32_t*>(o);
+
+  u32 v = atomic_load(a, memory_order_acquire);
+  if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
+                                               memory_order_relaxed)) {
+    (*f)();
+    if (!thr->in_ignored_lib)
+      Release(thr, pc, (uptr)o);
+    atomic_store(a, 2, memory_order_release);
+  } else {
+    while (v != 2) {
+      internal_sched_yield();
+      v = atomic_load(a, memory_order_acquire);
+    }
+    if (!thr->in_ignored_lib)
+      Acquire(thr, pc, (uptr)o);
+  }
+  return 0;
+}
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat)(version, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
+#else
+#define TSAN_MAYBE_INTERCEPT___FXSTAT
+#endif
+
+TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
+  SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(fstat)(fd, buf);
+#else
+  SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat)(0, fd, buf);
+#endif
+}
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat64)(version, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
+#else
+#define TSAN_MAYBE_INTERCEPT___FXSTAT64
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat64)(0, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
+#else
+#define TSAN_MAYBE_INTERCEPT_FSTAT64
+#endif
+
+TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(open)(name, flags, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(open64)(name, flags, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
+#else
+#define TSAN_MAYBE_INTERCEPT_OPEN64
+#endif
+
+TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(creat)(name, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
+  SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
+  READ_STRING(thr, pc, name, 0);
+  int fd = REAL(creat64)(name, mode);
+  if (fd >= 0)
+    FdFileCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
+#else
+#define TSAN_MAYBE_INTERCEPT_CREAT64
+#endif
+
+TSAN_INTERCEPTOR(int, dup, int oldfd) {
+  SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
+  int newfd = REAL(dup)(oldfd);
+  if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
+    FdDup(thr, pc, oldfd, newfd, true);
+  return newfd;
+}
+
+TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
+  SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
+  int newfd2 = REAL(dup2)(oldfd, newfd);
+  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
+    FdDup(thr, pc, oldfd, newfd2, false);
+  return newfd2;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
+  int newfd2 = REAL(dup3)(oldfd, newfd, flags);
+  if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
+    FdDup(thr, pc, oldfd, newfd2, false);
+  return newfd2;
+}
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
+  int fd = REAL(eventfd)(initval, flags);
+  if (fd >= 0)
+    FdEventCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
+#else
+#define TSAN_MAYBE_INTERCEPT_EVENTFD
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  fd = REAL(signalfd)(fd, mask, flags);
+  if (fd >= 0)
+    FdSignalCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
+#else
+#define TSAN_MAYBE_INTERCEPT_SIGNALFD
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, inotify_init, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
+  int fd = REAL(inotify_init)(fake);
+  if (fd >= 0)
+    FdInotifyCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
+#else
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
+#endif
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
+  int fd = REAL(inotify_init1)(flags);
+  if (fd >= 0)
+    FdInotifyCreate(thr, pc, fd);
+  return fd;
+}
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
+#else
+#define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
+#endif
+
+TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
+  SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
+  int fd = REAL(socket)(domain, type, protocol);
+  if (fd >= 0)
+    FdSocketCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
+  SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
+  int res = REAL(socketpair)(domain, type, protocol, fd);
+  if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
+    FdPipeCreate(thr, pc, fd[0], fd[1]);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
+  SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
+  FdSocketConnecting(thr, pc, fd);
+  int res = REAL(connect)(fd, addr, addrlen);
+  if (res == 0 && fd >= 0)
+    FdSocketConnect(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
+  SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
+  int res = REAL(bind)(fd, addr, addrlen);
+  if (fd > 0 && res == 0)
+    FdAccess(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
+  SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
+  int res = REAL(listen)(fd, backlog);
+  if (fd > 0 && res == 0)
+    FdAccess(thr, pc, fd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, close, int fd) {
+  SCOPED_TSAN_INTERCEPTOR(close, fd);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  return REAL(close)(fd);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, __close, int fd) {
+  SCOPED_TSAN_INTERCEPTOR(__close, fd);
+  if (fd >= 0)
+    FdClose(thr, pc, fd);
+  return REAL(__close)(fd);
+}
+#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
+#else
+#define TSAN_MAYBE_INTERCEPT___CLOSE
+#endif
+
+// glibc guts
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
+  SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
+  int fds[64];
+  int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
+  for (int i = 0; i < cnt; i++) {
+    if (fds[i] > 0)
+      FdClose(thr, pc, fds[i]);
+  }
+  REAL(__res_iclose)(state, free_addr);
+}
+#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
+#else
+#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
+#endif
+
+TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
+  SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
+  int res = REAL(pipe)(pipefd);
+  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
+    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
+  return res;
+}
+
+#if !SANITIZER_MAC
+TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
+  int res = REAL(pipe2)(pipefd, flags);
+  if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
+    FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
+  return res;
+}
+#endif
+
+TSAN_INTERCEPTOR(int, unlink, char *path) {
+  SCOPED_TSAN_INTERCEPTOR(unlink, path);
+  Release(thr, pc, File2addr(path));
+  int res = REAL(unlink)(path);
+  return res;
+}
+
+TSAN_INTERCEPTOR(void*, tmpfile, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(tmpfile, fake);
+  void *res = REAL(tmpfile)(fake);
+  if (res) {
+    int fd = fileno_unlocked(res);
+    if (fd >= 0)
+      FdFileCreate(thr, pc, fd);
+  }
+  return res;
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake);
+  void *res = REAL(tmpfile64)(fake);
+  if (res) {
+    int fd = fileno_unlocked(res);
+    if (fd >= 0)
+      FdFileCreate(thr, pc, fd);
+  }
+  return res;
+}
+#define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
+#else
+#define TSAN_MAYBE_INTERCEPT_TMPFILE64
+#endif
+
+static void FlushStreams() {
+  // Flushing all the streams here may freeze the process if a child thread is
+  // performing file stream operations at the same time.
+  REAL(fflush)(stdout);
+  REAL(fflush)(stderr);
+}
+
+TSAN_INTERCEPTOR(void, abort, int fake) {
+  SCOPED_TSAN_INTERCEPTOR(abort, fake);
+  FlushStreams();
+  REAL(abort)(fake);
+}
+
+TSAN_INTERCEPTOR(int, rmdir, char *path) {
+  SCOPED_TSAN_INTERCEPTOR(rmdir, path);
+  Release(thr, pc, Dir2addr(path));
+  int res = REAL(rmdir)(path);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, closedir, void *dirp) {
+  SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
+  if (dirp) {
+    int fd = dirfd(dirp);
+    FdClose(thr, pc, fd);
+  }
+  return REAL(closedir)(dirp);
+}
+
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, epoll_create, int size) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
+  int fd = REAL(epoll_create)(size);
+  if (fd >= 0)
+    FdPollCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
+  int fd = REAL(epoll_create1)(flags);
+  if (fd >= 0)
+    FdPollCreate(thr, pc, fd);
+  return fd;
+}
+
+TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  if (epfd >= 0 && fd >= 0)
+    FdAccess(thr, pc, fd);
+  if (op == EPOLL_CTL_ADD && epfd >= 0)
+    FdRelease(thr, pc, epfd);
+  int res = REAL(epoll_ctl)(epfd, op, fd, ev);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
+  if (res > 0 && epfd >= 0)
+    FdAcquire(thr, pc, epfd);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,
+                 void *sigmask) {
+  SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask);
+  if (epfd >= 0)
+    FdAccess(thr, pc, epfd);
+  int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask);
+  if (res > 0 && epfd >= 0)
+    FdAcquire(thr, pc, epfd);
+  return res;
+}
+
+#define TSAN_MAYBE_INTERCEPT_EPOLL \
+    TSAN_INTERCEPT(epoll_create); \
+    TSAN_INTERCEPT(epoll_create1); \
+    TSAN_INTERCEPT(epoll_ctl); \
+    TSAN_INTERCEPT(epoll_wait); \
+    TSAN_INTERCEPT(epoll_pwait)
+#else
+#define TSAN_MAYBE_INTERCEPT_EPOLL
+#endif
+
+// The following functions are intercepted merely to process pending signals.
+// If program blocks signal X, we must deliver the signal before the function
+// returns. Similarly, if program unblocks a signal (or returns from sigsuspend)
+// it's better to deliver the signal straight away.
+TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
+  return REAL(sigsuspend)(mask);
+}
+
+TSAN_INTERCEPTOR(int, sigblock, int mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigblock, mask);
+  return REAL(sigblock)(mask);
+}
+
+TSAN_INTERCEPTOR(int, sigsetmask, int mask) {
+  SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask);
+  return REAL(sigsetmask)(mask);
+}
+
+TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
+    __sanitizer_sigset_t *oldset) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset);
+  return REAL(pthread_sigmask)(how, set, oldset);
+}
+
+namespace __tsan {
+
+static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
+                                  bool sigact, int sig,
+                                  __sanitizer_siginfo *info, void *uctx) {
+  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
+  if (acquire)
+    Acquire(thr, 0, (uptr)&sigactions[sig]);
+  // Signals are generally asynchronous, so if we receive a signals when
+  // ignores are enabled we should disable ignores. This is critical for sync
+  // and interceptors, because otherwise we can miss syncronization and report
+  // false races.
+  int ignore_reads_and_writes = thr->ignore_reads_and_writes;
+  int ignore_interceptors = thr->ignore_interceptors;
+  int ignore_sync = thr->ignore_sync;
+  if (!ctx->after_multithreaded_fork) {
+    thr->ignore_reads_and_writes = 0;
+    thr->fast_state.ClearIgnoreBit();
+    thr->ignore_interceptors = 0;
+    thr->ignore_sync = 0;
+  }
+  // Ensure that the handler does not spoil errno.
+  const int saved_errno = errno;
+  errno = 99;
+  // This code races with sigaction. Be careful to not read sa_sigaction twice.
+  // Also need to remember pc for reporting before the call,
+  // because the handler can reset it.
+  volatile uptr pc =
+      sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler;
+  if (pc != sig_dfl && pc != sig_ign) {
+    if (sigact)
+      ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
+    else
+      ((__sanitizer_sighandler_ptr)pc)(sig);
+  }
+  if (!ctx->after_multithreaded_fork) {
+    thr->ignore_reads_and_writes = ignore_reads_and_writes;
+    if (ignore_reads_and_writes)
+      thr->fast_state.SetIgnoreBit();
+    thr->ignore_interceptors = ignore_interceptors;
+    thr->ignore_sync = ignore_sync;
+  }
+  // We do not detect errno spoiling for SIGTERM,
+  // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
+  // tsan reports false positive in such case.
+  // It's difficult to properly detect this situation (reraise),
+  // because in async signal processing case (when handler is called directly
+  // from rtl_generic_sighandler) we have not yet received the reraised
+  // signal; and it looks too fragile to intercept all ways to reraise a signal.
+  if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
+    VarSizeStackTrace stack;
+    // StackTrace::GetNestInstructionPc(pc) is used because return address is
+    // expected, OutputReport() will undo this.
+    ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
+    ThreadRegistryLock l(ctx->thread_registry);
+    ScopedReport rep(ReportTypeErrnoInSignal);
+    if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
+      rep.AddStack(stack, true);
+      OutputReport(thr, rep);
+    }
+  }
+  errno = saved_errno;
+}
+
+void ProcessPendingSignals(ThreadState *thr) {
+  ThreadSignalContext *sctx = SigCtx(thr);
+  if (sctx == 0 ||
+      atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0)
+    return;
+  atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed);
+  atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
+  internal_sigfillset(&sctx->emptyset);
+  int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
+  CHECK_EQ(res, 0);
+  for (int sig = 0; sig < kSigCount; sig++) {
+    SignalDesc *signal = &sctx->pending_signals[sig];
+    if (signal->armed) {
+      signal->armed = false;
+      CallUserSignalHandler(thr, false, true, signal->sigaction, sig,
+          &signal->siginfo, &signal->ctx);
+    }
+  }
+  res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
+  CHECK_EQ(res, 0);
+  atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
+}
+
+}  // namespace __tsan
+
+static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
+  return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP ||
+         sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
+         // If we are sending signal to ourselves, we must process it now.
+         (sctx && sig == sctx->int_signal_send);
+}
+
+void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
+                                          __sanitizer_siginfo *info,
+                                          void *ctx) {
+  cur_thread_init();
+  ThreadState *thr = cur_thread();
+  ThreadSignalContext *sctx = SigCtx(thr);
+  if (sig < 0 || sig >= kSigCount) {
+    VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
+    return;
+  }
+  // Don't mess with synchronous signals.
+  const bool sync = is_sync_signal(sctx, sig);
+  if (sync ||
+      // If we are in blocking function, we can safely process it now
+      // (but check if we are in a recursive interceptor,
+      // i.e. pthread_join()->munmap()).
+      (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {
+    atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
+    if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {
+      atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);
+      CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx);
+      atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);
+    } else {
+      // Be very conservative with when we do acquire in this case.
+      // It's unsafe to do acquire in async handlers, because ThreadState
+      // can be in inconsistent state.
+      // SIGSYS looks relatively safe -- it's synchronous and can actually
+      // need some global state.
+      bool acq = (sig == SIGSYS);
+      CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx);
+    }
+    atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
+    return;
+  }
+
+  if (sctx == 0)
+    return;
+  SignalDesc *signal = &sctx->pending_signals[sig];
+  if (signal->armed == false) {
+    signal->armed = true;
+    signal->sigaction = sigact;
+    if (info)
+      internal_memcpy(&signal->siginfo, info, sizeof(*info));
+    if (ctx)
+      internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
+    atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed);
+  }
+}
+
+static void rtl_sighandler(int sig) {
+  rtl_generic_sighandler(false, sig, 0, 0);
+}
+
+static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) {
+  rtl_generic_sighandler(true, sig, info, ctx);
+}
+
+TSAN_INTERCEPTOR(int, raise, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(raise, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  sctx->int_signal_send = sig;
+  int res = REAL(raise)(sig);
+  CHECK_EQ(sctx->int_signal_send, sig);
+  sctx->int_signal_send = prev;
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  if (pid == (int)internal_getpid()) {
+    sctx->int_signal_send = sig;
+  }
+  int res = REAL(kill)(pid, sig);
+  if (pid == (int)internal_getpid()) {
+    CHECK_EQ(sctx->int_signal_send, sig);
+    sctx->int_signal_send = prev;
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
+  SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
+  ThreadSignalContext *sctx = SigCtx(thr);
+  CHECK_NE(sctx, 0);
+  int prev = sctx->int_signal_send;
+  if (tid == pthread_self()) {
+    sctx->int_signal_send = sig;
+  }
+  int res = REAL(pthread_kill)(tid, sig);
+  if (tid == pthread_self()) {
+    CHECK_EQ(sctx->int_signal_send, sig);
+    sctx->int_signal_send = prev;
+  }
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
+  SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
+  // It's intercepted merely to process pending signals.
+  return REAL(gettimeofday)(tv, tz);
+}
+
+TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
+    void *hints, void *rv) {
+  SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
+  // We miss atomic synchronization in getaddrinfo,
+  // and can report false race between malloc and free
+  // inside of getaddrinfo. So ignore memory accesses.
+  ThreadIgnoreBegin(thr, pc);
+  int res = REAL(getaddrinfo)(node, service, hints, rv);
+  ThreadIgnoreEnd(thr, pc);
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, fork, int fake) {
+  if (in_symbolizer())
+    return REAL(fork)(fake);
+  SCOPED_INTERCEPTOR_RAW(fork, fake);
+  ForkBefore(thr, pc);
+  int pid;
+  {
+    // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
+    // we'll assert in CheckNoLocks() unless we ignore interceptors.
+    ScopedIgnoreInterceptors ignore;
+    pid = REAL(fork)(fake);
+  }
+  if (pid == 0) {
+    // child
+    ForkChildAfter(thr, pc);
+    FdOnFork(thr, pc);
+  } else if (pid > 0) {
+    // parent
+    ForkParentAfter(thr, pc);
+  } else {
+    // error
+    ForkParentAfter(thr, pc);
+  }
+  return pid;
+}
+
+TSAN_INTERCEPTOR(int, vfork, int fake) {
+  // Some programs (e.g. openjdk) call close for all file descriptors
+  // in the child process. Under tsan it leads to false positives, because
+  // address space is shared, so the parent process also thinks that
+  // the descriptors are closed (while they are actually not).
+  // This leads to false positives due to missed synchronization.
+  // Strictly saying this is undefined behavior, because vfork child is not
+  // allowed to call any functions other than exec/exit. But this is what
+  // openjdk does, so we want to handle it.
+  // We could disable interceptors in the child process. But it's not possible
+  // to simply intercept and wrap vfork, because vfork child is not allowed
+  // to return from the function that calls vfork, and that's exactly what
+  // we would do. So this would require some assembly trickery as well.
+  // Instead we simply turn vfork into fork.
+  return WRAP(fork)(fake);
+}
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size,
+                                    void *data);
+struct dl_iterate_phdr_data {
+  ThreadState *thr;
+  uptr pc;
+  dl_iterate_phdr_cb_t cb;
+  void *data;
+};
+
+static bool IsAppNotRodata(uptr addr) {
+  return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata;
+}
+
+static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
+                              void *data) {
+  dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
+  // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
+  // accessible in dl_iterate_phdr callback. But we don't see synchronization
+  // inside of dynamic linker, so we "unpoison" it here in order to not
+  // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough
+  // because some libc functions call __libc_dlopen.
+  if (info && IsAppNotRodata((uptr)info->dlpi_name))
+    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
+                     internal_strlen(info->dlpi_name));
+  int res = cbdata->cb(info, size, cbdata->data);
+  // Perform the check one more time in case info->dlpi_name was overwritten
+  // by user callback.
+  if (info && IsAppNotRodata((uptr)info->dlpi_name))
+    MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name,
+                     internal_strlen(info->dlpi_name));
+  return res;
+}
+
+TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) {
+  SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data);
+  dl_iterate_phdr_data cbdata;
+  cbdata.thr = thr;
+  cbdata.pc = pc;
+  cbdata.cb = cb;
+  cbdata.data = data;
+  int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata);
+  return res;
+}
+#endif
+
+static int OnExit(ThreadState *thr) {
+  int status = Finalize(thr);
+  FlushStreams();
+  return status;
+}
+
+struct TsanInterceptorContext {
+  ThreadState *thr;
+  const uptr caller_pc;
+  const uptr pc;
+};
+
+#if !SANITIZER_MAC
+static void HandleRecvmsg(ThreadState *thr, uptr pc,
+    __sanitizer_msghdr *msg) {
+  int fds[64];
+  int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
+  for (int i = 0; i < cnt; i++)
+    FdEventCreate(thr, pc, fds[i]);
+}
+#endif
+
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+// Causes interceptor recursion (getaddrinfo() and fopen())
+#undef SANITIZER_INTERCEPT_GETADDRINFO
+// We define our own.
+#if SANITIZER_INTERCEPT_TLS_GET_ADDR
+#define NEED_TLS_GET_ADDR
+#endif
+#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
+#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
+
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
+  INTERCEPT_FUNCTION_VER(name, ver)
+
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
+  MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
+                    ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
+                    true)
+
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
+  MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
+                    ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
+                    false)
+
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)      \
+  SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);         \
+  TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
+  ctx = (void *)&_ctx;                                \
+  (void) ctx;
+
+#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
+  SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
+  TsanInterceptorContext _ctx = {thr, caller_pc, pc};     \
+  ctx = (void *)&_ctx;                                    \
+  (void) ctx;
+
+#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
+  if (path)                                           \
+    Acquire(thr, pc, File2addr(path));                \
+  if (file) {                                         \
+    int fd = fileno_unlocked(file);                   \
+    if (fd >= 0) FdFileCreate(thr, pc, fd);           \
+  }
+
+#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
+  if (file) {                                    \
+    int fd = fileno_unlocked(file);              \
+    if (fd >= 0) FdClose(thr, pc, fd);           \
+  }
+
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
+  libignore()->OnLibraryLoaded(filename)
+
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
+  libignore()->OnLibraryUnloaded()
+
+#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
+  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
+
+#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \
+  Release(((TsanInterceptorContext *) ctx)->thr, pc, u)
+
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
+
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+  FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+  FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
+  FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
+
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+  FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
+
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
+  ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
+
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+  __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
+
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
+
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
+  OnExit(((TsanInterceptorContext *) ctx)->thr)
+
+#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
+  MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
+  MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
+  MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
+  MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
+            ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
+  MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
+                     ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
+
+#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd,  \
+                                     off)                                   \
+  do {                                                                      \
+    return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \
+                            off);                                           \
+  } while (false)
+
+#if !SANITIZER_MAC
+#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
+  HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
+      ((TsanInterceptorContext *)ctx)->pc, msg)
+#endif
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
+  if (TsanThread *t = GetCurrentThread()) {                                    \
+    *begin = t->tls_begin();                                                   \
+    *end = t->tls_end();                                                       \
+  } else {                                                                     \
+    *begin = *end = 0;                                                         \
+  }
+
+#define COMMON_INTERCEPTOR_USER_CALLBACK_START() \
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START()
+
+#define COMMON_INTERCEPTOR_USER_CALLBACK_END() \
+  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END()
+
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+
+static int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+                          __sanitizer_sigaction *old);
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+                                              __sanitizer_sighandler_ptr h);
+
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
+  { return sigaction_impl(signo, act, oldact); }
+
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
+  { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
+
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+                   __sanitizer_sigaction *old) {
+  // Note: if we call REAL(sigaction) directly for any reason without proxying
+  // the signal handler through rtl_sigaction, very bad things will happen.
+  // The handler will run synchronously and corrupt tsan per-thread state.
+  SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
+  __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
+  __sanitizer_sigaction old_stored;
+  if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored));
+  __sanitizer_sigaction newact;
+  if (act) {
+    // Copy act into sigactions[sig].
+    // Can't use struct copy, because compiler can emit call to memcpy.
+    // Can't use internal_memcpy, because it copies byte-by-byte,
+    // and signal handler reads the handler concurrently. It it can read
+    // some bytes from old value and some bytes from new value.
+    // Use volatile to prevent insertion of memcpy.
+    sigactions[sig].handler =
+        *(volatile __sanitizer_sighandler_ptr const *)&act->handler;
+    sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags;
+    internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
+                    sizeof(sigactions[sig].sa_mask));
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+    sigactions[sig].sa_restorer = act->sa_restorer;
+#endif
+    internal_memcpy(&newact, act, sizeof(newact));
+    internal_sigfillset(&newact.sa_mask);
+    if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) {
+      if (newact.sa_flags & SA_SIGINFO)
+        newact.sigaction = rtl_sigaction;
+      else
+        newact.handler = rtl_sighandler;
+    }
+    ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
+    act = &newact;
+  }
+  int res = REAL(sigaction)(sig, act, old);
+  if (res == 0 && old) {
+    uptr cb = (uptr)old->sigaction;
+    if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) {
+      internal_memcpy(old, &old_stored, sizeof(*old));
+    }
+  }
+  return res;
+}
+
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+                                              __sanitizer_sighandler_ptr h) {
+  __sanitizer_sigaction act;
+  act.handler = h;
+  internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask));
+  act.sa_flags = 0;
+  __sanitizer_sigaction old;
+  int res = sigaction_symname(sig, &act, &old);
+  if (res) return (__sanitizer_sighandler_ptr)sig_err;
+  return old.handler;
+}
+
+#define TSAN_SYSCALL() \
+  ThreadState *thr = cur_thread(); \
+  if (thr->ignore_interceptors) \
+    return; \
+  ScopedSyscall scoped_syscall(thr) \
+/**/
+
+struct ScopedSyscall {
+  ThreadState *thr;
+
+  explicit ScopedSyscall(ThreadState *thr)
+      : thr(thr) {
+    Initialize(thr);
+  }
+
+  ~ScopedSyscall() {
+    ProcessPendingSignals(thr);
+  }
+};
+
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
+  TSAN_SYSCALL();
+  MemoryAccessRange(thr, pc, p, s, write);
+}
+
+static void syscall_acquire(uptr pc, uptr addr) {
+  TSAN_SYSCALL();
+  Acquire(thr, pc, addr);
+  DPrintf("syscall_acquire(%p)\n", addr);
+}
+
+static void syscall_release(uptr pc, uptr addr) {
+  TSAN_SYSCALL();
+  DPrintf("syscall_release(%p)\n", addr);
+  Release(thr, pc, addr);
+}
+
+static void syscall_fd_close(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  FdClose(thr, pc, fd);
+}
+
+static USED void syscall_fd_acquire(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  FdAcquire(thr, pc, fd);
+  DPrintf("syscall_fd_acquire(%p)\n", fd);
+}
+
+static USED void syscall_fd_release(uptr pc, int fd) {
+  TSAN_SYSCALL();
+  DPrintf("syscall_fd_release(%p)\n", fd);
+  FdRelease(thr, pc, fd);
+}
+
+static void syscall_pre_fork(uptr pc) {
+  TSAN_SYSCALL();
+  ForkBefore(thr, pc);
+}
+
+static void syscall_post_fork(uptr pc, int pid) {
+  TSAN_SYSCALL();
+  if (pid == 0) {
+    // child
+    ForkChildAfter(thr, pc);
+    FdOnFork(thr, pc);
+  } else if (pid > 0) {
+    // parent
+    ForkParentAfter(thr, pc);
+  } else {
+    // error
+    ForkParentAfter(thr, pc);
+  }
+}
+#endif
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
+  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
+
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+  syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
+
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+  do {                                       \
+    (void)(p);                               \
+    (void)(s);                               \
+  } while (false)
+
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+  do {                                        \
+    (void)(p);                                \
+    (void)(s);                                \
+  } while (false)
+
+#define COMMON_SYSCALL_ACQUIRE(addr) \
+    syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
+
+#define COMMON_SYSCALL_RELEASE(addr) \
+    syscall_release(GET_CALLER_PC(), (uptr)(addr))
+
+#define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
+
+#define COMMON_SYSCALL_PRE_FORK() \
+  syscall_pre_fork(GET_CALLER_PC())
+
+#define COMMON_SYSCALL_POST_FORK(res) \
+  syscall_post_fork(GET_CALLER_PC(), res)
+
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
+
+#ifdef NEED_TLS_GET_ADDR
+// Define own interceptor instead of sanitizer_common's for three reasons:
+// 1. It must not process pending signals.
+//    Signal handlers may contain MOVDQA instruction (see below).
+// 2. It must be as simple as possible to not contain MOVDQA.
+// 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which
+//    is empty for tsan (meant only for msan).
+// Note: __tls_get_addr can be called with mis-aligned stack due to:
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
+// So the interceptor must work with mis-aligned stack, in particular, does not
+// execute MOVDQA with stack addresses.
+TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
+  void *res = REAL(__tls_get_addr)(arg);
+  ThreadState *thr = cur_thread();
+  if (!thr)
+    return res;
+  DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
+                                        thr->tls_addr + thr->tls_size);
+  if (!dtv)
+    return res;
+  // New DTLS block has been allocated.
+  MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+  return res;
+}
+#endif
+
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _lwp_exit) {
+  SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
+  DestroyThreadState();
+  REAL(_lwp_exit)();
+}
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT
+#endif
+
+#if SANITIZER_FREEBSD
+TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
+  SCOPED_TSAN_INTERCEPTOR(thr_exit, state);
+  DestroyThreadState();
+  REAL(thr_exit(state));
+}
+#define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT_THR_EXIT
+#endif
+
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m)
+TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)())
+TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b,
+  void *c)
+
+namespace __tsan {
+
+static void finalize(void *arg) {
+  ThreadState *thr = cur_thread();
+  int status = Finalize(thr);
+  // Make sure the output is not lost.
+  FlushStreams();
+  if (status)
+    Die();
+}
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+static void unreachable() {
+  Report("FATAL: ThreadSanitizer: unreachable called\n");
+  Die();
+}
+#endif
+
+// Define default implementation since interception of libdispatch  is optional.
+SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
+
+void InitializeInterceptors() {
+#if !SANITIZER_MAC
+  // We need to setup it early, because functions like dlsym() can call it.
+  REAL(memset) = internal_memset;
+  REAL(memcpy) = internal_memcpy;
+#endif
+
+  // Instruct libc malloc to consume less memory.
+#if SANITIZER_LINUX
+  mallopt(1, 0);  // M_MXFAST
+  mallopt(-3, 32*1024);  // M_MMAP_THRESHOLD
+#endif
+
+  new(interceptor_ctx()) InterceptorContext();
+
+  InitializeCommonInterceptors();
+  InitializeSignalInterceptors();
+  InitializeLibdispatchInterceptors();
+
+#if !SANITIZER_MAC
+  // We can not use TSAN_INTERCEPT to get setjmp addr,
+  // because it does &setjmp and setjmp is not present in some versions of libc.
+  using __interception::InterceptFunction;
+  InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
+  InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
+  InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
+                    0);
+#if !SANITIZER_NETBSD
+  InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
+#endif
+#endif
+
+  TSAN_INTERCEPT(longjmp_symname);
+  TSAN_INTERCEPT(siglongjmp_symname);
+#if SANITIZER_NETBSD
+  TSAN_INTERCEPT(_longjmp);
+#endif
+
+  TSAN_INTERCEPT(malloc);
+  TSAN_INTERCEPT(__libc_memalign);
+  TSAN_INTERCEPT(calloc);
+  TSAN_INTERCEPT(realloc);
+  TSAN_INTERCEPT(reallocarray);
+  TSAN_INTERCEPT(free);
+  TSAN_INTERCEPT(cfree);
+  TSAN_INTERCEPT(munmap);
+  TSAN_MAYBE_INTERCEPT_MEMALIGN;
+  TSAN_INTERCEPT(valloc);
+  TSAN_MAYBE_INTERCEPT_PVALLOC;
+  TSAN_INTERCEPT(posix_memalign);
+
+  TSAN_INTERCEPT(strcpy);
+  TSAN_INTERCEPT(strncpy);
+  TSAN_INTERCEPT(strdup);
+
+  TSAN_INTERCEPT(pthread_create);
+  TSAN_INTERCEPT(pthread_join);
+  TSAN_INTERCEPT(pthread_detach);
+  TSAN_INTERCEPT(pthread_exit);
+  #if SANITIZER_LINUX
+  TSAN_INTERCEPT(pthread_tryjoin_np);
+  TSAN_INTERCEPT(pthread_timedjoin_np);
+  #endif
+
+  TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
+  TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
+
+  TSAN_INTERCEPT(pthread_mutex_init);
+  TSAN_INTERCEPT(pthread_mutex_destroy);
+  TSAN_INTERCEPT(pthread_mutex_trylock);
+  TSAN_INTERCEPT(pthread_mutex_timedlock);
+
+  TSAN_INTERCEPT(pthread_spin_init);
+  TSAN_INTERCEPT(pthread_spin_destroy);
+  TSAN_INTERCEPT(pthread_spin_lock);
+  TSAN_INTERCEPT(pthread_spin_trylock);
+  TSAN_INTERCEPT(pthread_spin_unlock);
+
+  TSAN_INTERCEPT(pthread_rwlock_init);
+  TSAN_INTERCEPT(pthread_rwlock_destroy);
+  TSAN_INTERCEPT(pthread_rwlock_rdlock);
+  TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
+  TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
+  TSAN_INTERCEPT(pthread_rwlock_wrlock);
+  TSAN_INTERCEPT(pthread_rwlock_trywrlock);
+  TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
+  TSAN_INTERCEPT(pthread_rwlock_unlock);
+
+  TSAN_INTERCEPT(pthread_barrier_init);
+  TSAN_INTERCEPT(pthread_barrier_destroy);
+  TSAN_INTERCEPT(pthread_barrier_wait);
+
+  TSAN_INTERCEPT(pthread_once);
+
+  TSAN_INTERCEPT(fstat);
+  TSAN_MAYBE_INTERCEPT___FXSTAT;
+  TSAN_MAYBE_INTERCEPT_FSTAT64;
+  TSAN_MAYBE_INTERCEPT___FXSTAT64;
+  TSAN_INTERCEPT(open);
+  TSAN_MAYBE_INTERCEPT_OPEN64;
+  TSAN_INTERCEPT(creat);
+  TSAN_MAYBE_INTERCEPT_CREAT64;
+  TSAN_INTERCEPT(dup);
+  TSAN_INTERCEPT(dup2);
+  TSAN_INTERCEPT(dup3);
+  TSAN_MAYBE_INTERCEPT_EVENTFD;
+  TSAN_MAYBE_INTERCEPT_SIGNALFD;
+  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT;
+  TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1;
+  TSAN_INTERCEPT(socket);
+  TSAN_INTERCEPT(socketpair);
+  TSAN_INTERCEPT(connect);
+  TSAN_INTERCEPT(bind);
+  TSAN_INTERCEPT(listen);
+  TSAN_MAYBE_INTERCEPT_EPOLL;
+  TSAN_INTERCEPT(close);
+  TSAN_MAYBE_INTERCEPT___CLOSE;
+  TSAN_MAYBE_INTERCEPT___RES_ICLOSE;
+  TSAN_INTERCEPT(pipe);
+  TSAN_INTERCEPT(pipe2);
+
+  TSAN_INTERCEPT(unlink);
+  TSAN_INTERCEPT(tmpfile);
+  TSAN_MAYBE_INTERCEPT_TMPFILE64;
+  TSAN_INTERCEPT(abort);
+  TSAN_INTERCEPT(rmdir);
+  TSAN_INTERCEPT(closedir);
+
+  TSAN_INTERCEPT(sigsuspend);
+  TSAN_INTERCEPT(sigblock);
+  TSAN_INTERCEPT(sigsetmask);
+  TSAN_INTERCEPT(pthread_sigmask);
+  TSAN_INTERCEPT(raise);
+  TSAN_INTERCEPT(kill);
+  TSAN_INTERCEPT(pthread_kill);
+  TSAN_INTERCEPT(sleep);
+  TSAN_INTERCEPT(usleep);
+  TSAN_INTERCEPT(nanosleep);
+  TSAN_INTERCEPT(pause);
+  TSAN_INTERCEPT(gettimeofday);
+  TSAN_INTERCEPT(getaddrinfo);
+
+  TSAN_INTERCEPT(fork);
+  TSAN_INTERCEPT(vfork);
+#if !SANITIZER_ANDROID
+  TSAN_INTERCEPT(dl_iterate_phdr);
+#endif
+  TSAN_MAYBE_INTERCEPT_ON_EXIT;
+  TSAN_INTERCEPT(__cxa_atexit);
+  TSAN_INTERCEPT(_exit);
+
+#ifdef NEED_TLS_GET_ADDR
+  TSAN_INTERCEPT(__tls_get_addr);
+#endif
+
+  TSAN_MAYBE_INTERCEPT__LWP_EXIT;
+  TSAN_MAYBE_INTERCEPT_THR_EXIT;
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+  // Need to setup it, because interceptors check that the function is resolved.
+  // But atexit is emitted directly into the module, so can't be resolved.
+  REAL(atexit) = (int(*)(void(*)()))unreachable;
+#endif
+
+  if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
+    Printf("ThreadSanitizer: failed to setup atexit callback\n");
+    Die();
+  }
+
+#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
+  if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
+    Printf("ThreadSanitizer: failed to create thread key\n");
+    Die();
+  }
+#endif
+
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once);
+  TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask);
+
+  FdInit();
+}
+
+}  // namespace __tsan
+
+// Invisible barrier for tests.
+// There were several unsuccessful iterations for this functionality:
+// 1. Initially it was implemented in user code using
+//    REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on
+//    MacOS. Futexes are linux-specific for this matter.
+// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
+//    "as-if synchronized via sleep" messages in reports which failed some
+//    output tests.
+// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
+//    visible events, which lead to "failed to restore stack trace" failures.
+// Note that no_sanitize_thread attribute does not turn off atomic interception
+// so attaching it to the function defined in user code does not help.
+// That's why we now have what we have.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_testonly_barrier_init(u64 *barrier, u32 count) {
+  if (count >= (1 << 8)) {
+      Printf("barrier_init: count is too large (%d)\n", count);
+      Die();
+  }
+  // 8 lsb is thread count, the remaining are count of entered threads.
+  *barrier = count;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_testonly_barrier_wait(u64 *barrier) {
+  unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED);
+  unsigned old_epoch = (old >> 8) / (old & 0xff);
+  for (;;) {
+    unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED);
+    unsigned cur_epoch = (cur >> 8) / (cur & 0xff);
+    if (cur_epoch != old_epoch)
+      return;
+    internal_sched_yield();
+  }
+}
diff --git a/lib/tsan/tsan_interface.cpp b/lib/tsan/tsan_interface.cpp
new file mode 100644
index 0000000000..2b3a0889b7
--- /dev/null
+++ b/lib/tsan/tsan_interface.cpp
@@ -0,0 +1,160 @@
+//===-- tsan_interface.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interface.h"
+#include "tsan_interface_ann.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+typedef u16 uint16_t;
+typedef u32 uint32_t;
+typedef u64 uint64_t;
+
+void __tsan_init() {
+  cur_thread_init();
+  Initialize(cur_thread());
+}
+
+void __tsan_flush_memory() {
+  FlushShadowMemory();
+}
+
+void __tsan_read16(void *addr) {
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
+}
+
+void __tsan_write16(void *addr) {
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
+}
+
+void __tsan_read16_pc(void *addr, void *pc) {
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr + 8, kSizeLog8);
+}
+
+void __tsan_write16_pc(void *addr, void *pc) {
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr + 8, kSizeLog8);
+}
+
+// __tsan_unaligned_read/write calls are emitted by compiler.
+
+void __tsan_unaligned_read2(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false);
+}
+
+void __tsan_unaligned_read4(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false);
+}
+
+void __tsan_unaligned_read8(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false);
+}
+
+void __tsan_unaligned_read16(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false);
+}
+
+void __tsan_unaligned_write2(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false);
+}
+
+void __tsan_unaligned_write4(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false);
+}
+
+void __tsan_unaligned_write8(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false);
+}
+
+void __tsan_unaligned_write16(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false);
+}
+
+// __sanitizer_unaligned_load/store are for user instrumentation.
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *addr) {
+  __tsan_unaligned_read2(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *addr) {
+  __tsan_unaligned_read4(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *addr) {
+  __tsan_unaligned_read8(addr);
+  return *addr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
+  __tsan_unaligned_write2(addr);
+  *addr = v;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
+  __tsan_unaligned_write4(addr);
+  *addr = v;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
+  __tsan_unaligned_write8(addr);
+  *addr = v;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_fiber() {
+  return cur_thread();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_create_fiber(unsigned flags) {
+  return FiberCreate(cur_thread(), CALLERPC, flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_destroy_fiber(void *fiber) {
+  FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber));
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_switch_to_fiber(void *fiber, unsigned flags) {
+  FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_set_fiber_name(void *fiber, const char *name) {
+  ThreadSetName(static_cast<ThreadState *>(fiber), name);
+}
+}  // extern "C"
+
+void __tsan_acquire(void *addr) {
+  Acquire(cur_thread(), CALLERPC, (uptr)addr);
+}
+
+void __tsan_release(void *addr) {
+  Release(cur_thread(), CALLERPC, (uptr)addr);
+}
diff --git a/lib/tsan/tsan_interface.h b/lib/tsan/tsan_interface.h
new file mode 100644
index 0000000000..6d7286ca5b
--- /dev/null
+++ b/lib/tsan/tsan_interface.h
@@ -0,0 +1,422 @@
+//===-- tsan_interface.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// The functions declared in this header will be inserted by the instrumentation
+// module.
+// This header can be included by the instrumented program or by TSan tests.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_H
+#define TSAN_INTERFACE_H
+
+#include <sanitizer_common/sanitizer_internal_defs.h>
+using __sanitizer::uptr;
+using __sanitizer::tid_t;
+
+// This header should NOT include any other headers.
+// All functions in this header are extern "C" and start with __tsan_.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if !SANITIZER_GO
+
+// This function should be called at the very beginning of the process,
+// before any instrumented code is executed and before any call to malloc.
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read4_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read8_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read16_pc(void *addr, void *pc);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write1_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write2_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8_pc(void *addr, void *pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16_pc(void *addr, void *pc);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_vptr_update(void **vptr_p, void *new_val);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin();
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_external_register_tag(const char *object_type);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_register_header(void *tag, const char *header);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_assign_tag(void *addr, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_read(void *addr, void *caller_pc, void *tag);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_external_write(void *addr, void *caller_pc, void *tag);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_read_range(void *addr, unsigned long size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_write_range(void *addr, unsigned long size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+
+// User may provide function that would be called right when TSan detects
+// an error. The argument 'report' is an opaque pointer that can be used to
+// gather additional information using other TSan report API functions.
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_on_report(void *report);
+
+// If TSan is currently reporting a detected issue on the current thread,
+// returns an opaque pointer to the current report. Otherwise returns NULL.
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_report();
+
+// Returns a report's description (issue type), number of duplicate issues
+// found, counts of array data (stack traces, memory operations, locations,
+// mutexes, threads, unique thread IDs) and a stack trace of a sleep() call (if
+// one was involved in the issue).
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_data(void *report, const char **description, int *count,
+                           int *stack_count, int *mop_count, int *loc_count,
+                           int *mutex_count, int *thread_count,
+                           int *unique_tid_count, void **sleep_trace,
+                           uptr trace_size);
+
+/// Retrieves the "tag" from a report (for external-race report types). External
+/// races can be associated with a tag which give them more meaning. For example
+/// tag value '1' means "Swift access race". Tag value '0' indicated a plain
+/// external race.
+///
+/// \param report opaque pointer to the current report (obtained as argument in
+///               __tsan_on_report, or from __tsan_get_current_report)
+/// \param [out] tag points to storage that will be filled with the tag value
+///
+/// \returns non-zero value on success, zero on failure
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_tag(void *report, uptr *tag);
+
+// Returns information about stack traces included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_stack(void *report, uptr idx, void **trace,
+                            uptr trace_size);
+
+// Returns information about memory operations included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mop(void *report, uptr idx, int *tid, void **addr,
+                          int *size, int *write, int *atomic, void **trace,
+                          uptr trace_size);
+
+// Returns information about locations included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc(void *report, uptr idx, const char **type,
+                          void **addr, uptr *start, uptr *size, int *tid,
+                          int *fd, int *suppressable, void **trace,
+                          uptr trace_size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_loc_object_type(void *report, uptr idx,
+                                      const char **object_type);
+
+// Returns information about mutexes included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_mutex(void *report, uptr idx, uptr *mutex_id, void **addr,
+                            int *destroyed, void **trace, uptr trace_size);
+
+// Returns information about threads included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_thread(void *report, uptr idx, int *tid, tid_t *os_id,
+                             int *running, const char **name, int *parent_tid,
+                             void **trace, uptr trace_size);
+
+// Returns information about unique thread IDs included in the report.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_report_unique_tid(void *report, uptr idx, int *tid);
+
+// Returns the type of the pointer (heap, stack, global, ...) and if possible
+// also the starting address (e.g. of a heap allocation) and size.
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
+                                  uptr *region_address, uptr *region_size);
+
+// Returns the allocation stack for a heap pointer.
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
+                           tid_t *os_id);
+
+#endif  // SANITIZER_GO
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+namespace __tsan {
+
+// These should match declarations from public tsan_interface_atomic.h header.
+typedef unsigned char      a8;
+typedef unsigned short a16;
+typedef unsigned int       a32;
+typedef unsigned long long a64;
+#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
+    || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64)
+__extension__ typedef __int128 a128;
+# define __TSAN_HAS_INT128 1
+#else
+# define __TSAN_HAS_INT128 0
+#endif
+
+// Part of ABI, do not change.
+// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic
+typedef enum {
+  mo_relaxed,
+  mo_consume,
+  mo_acquire,
+  mo_release,
+  mo_acq_rel,
+  mo_seq_cst
+} morder;
+
+struct ThreadState;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_load(const volatile a8 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_load(const volatile a16 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_load(const volatile a32 *a, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_load(const volatile a64 *a, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_load(const volatile a128 *a, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
+                                           morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
+                                            morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
+                                            morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
+                                            morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
+                                             morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo,
+                                         morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
+                                          morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
+                                          morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
+                                          morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
+                                           morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo,
+                                       morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
+                                         morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
+                                         morder mo, morder fmo);
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
+                                         morder mo, morder fmo);
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
+                                           morder mo, morder fmo);
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_thread_fence(morder mo);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_signal_fence(morder mo);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
+                                         u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
+                                         u8 *a);
+}  // extern "C"
+
+}  // namespace __tsan
+
+#endif  // TSAN_INTERFACE_H
diff --git a/lib/tsan/tsan_interface_ann.cpp b/lib/tsan/tsan_interface_ann.cpp
new file mode 100644
index 0000000000..99516d94bb
--- /dev/null
+++ b/lib/tsan/tsan_interface_ann.cpp
@@ -0,0 +1,552 @@
+//===-- tsan_interface_ann.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_interface_ann.h"
+#include "tsan_mutex.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+#include "tsan_platform.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+namespace __tsan {
+
+class ScopedAnnotation {
+ public:
+  ScopedAnnotation(ThreadState *thr, const char *aname, uptr pc)
+      : thr_(thr) {
+    FuncEntry(thr_, pc);
+    DPrintf("#%d: annotation %s()\n", thr_->tid, aname);
+  }
+
+  ~ScopedAnnotation() {
+    FuncExit(thr_);
+    CheckNoLocks(thr_);
+  }
+ private:
+  ThreadState *const thr_;
+};
+
+#define SCOPED_ANNOTATION_RET(typ, ret) \
+    if (!flags()->enable_annotations) \
+      return ret; \
+    ThreadState *thr = cur_thread(); \
+    const uptr caller_pc = (uptr)__builtin_return_address(0); \
+    StatInc(thr, StatAnnotation); \
+    StatInc(thr, Stat##typ); \
+    ScopedAnnotation sa(thr, __func__, caller_pc); \
+    const uptr pc = StackTrace::GetCurrentPc(); \
+    (void)pc; \
+/**/
+
+#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
+
+static const int kMaxDescLen = 128;
+
+struct ExpectRace {
+  ExpectRace *next;
+  ExpectRace *prev;
+  atomic_uintptr_t hitcount;
+  atomic_uintptr_t addcount;
+  uptr addr;
+  uptr size;
+  char *file;
+  int line;
+  char desc[kMaxDescLen];
+};
+
+struct DynamicAnnContext {
+  Mutex mtx;
+  ExpectRace expect;
+  ExpectRace benign;
+
+  DynamicAnnContext()
+    : mtx(MutexTypeAnnotations, StatMtxAnnotations) {
+  }
+};
+
+static DynamicAnnContext *dyn_ann_ctx;
+static char dyn_ann_ctx_placeholder[sizeof(DynamicAnnContext)] ALIGNED(64);
+
+static void AddExpectRace(ExpectRace *list,
+    char *f, int l, uptr addr, uptr size, char *desc) {
+  ExpectRace *race = list->next;
+  for (; race != list; race = race->next) {
+    if (race->addr == addr && race->size == size) {
+      atomic_store_relaxed(&race->addcount,
+          atomic_load_relaxed(&race->addcount) + 1);
+      return;
+    }
+  }
+  race = (ExpectRace*)internal_alloc(MBlockExpectRace, sizeof(ExpectRace));
+  race->addr = addr;
+  race->size = size;
+  race->file = f;
+  race->line = l;
+  race->desc[0] = 0;
+  atomic_store_relaxed(&race->hitcount, 0);
+  atomic_store_relaxed(&race->addcount, 1);
+  if (desc) {
+    int i = 0;
+    for (; i < kMaxDescLen - 1 && desc[i]; i++)
+      race->desc[i] = desc[i];
+    race->desc[i] = 0;
+  }
+  race->prev = list;
+  race->next = list->next;
+  race->next->prev = race;
+  list->next = race;
+}
+
+static ExpectRace *FindRace(ExpectRace *list, uptr addr, uptr size) {
+  for (ExpectRace *race = list->next; race != list; race = race->next) {
+    uptr maxbegin = max(race->addr, addr);
+    uptr minend = min(race->addr + race->size, addr + size);
+    if (maxbegin < minend)
+      return race;
+  }
+  return 0;
+}
+
+static bool CheckContains(ExpectRace *list, uptr addr, uptr size) {
+  ExpectRace *race = FindRace(list, addr, size);
+  if (race == 0)
+    return false;
+  DPrintf("Hit expected/benign race: %s addr=%zx:%d %s:%d\n",
+      race->desc, race->addr, (int)race->size, race->file, race->line);
+  atomic_fetch_add(&race->hitcount, 1, memory_order_relaxed);
+  return true;
+}
+
+static void InitList(ExpectRace *list) {
+  list->next = list;
+  list->prev = list;
+}
+
+void InitializeDynamicAnnotations() {
+  dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext;
+  InitList(&dyn_ann_ctx->expect);
+  InitList(&dyn_ann_ctx->benign);
+}
+
+bool IsExpectedReport(uptr addr, uptr size) {
+  ReadLock lock(&dyn_ann_ctx->mtx);
+  if (CheckContains(&dyn_ann_ctx->expect, addr, size))
+    return true;
+  if (CheckContains(&dyn_ann_ctx->benign, addr, size))
+    return true;
+  return false;
+}
+
+static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched,
+    int *unique_count, int *hit_count, atomic_uintptr_t ExpectRace::*counter) {
+  ExpectRace *list = &dyn_ann_ctx->benign;
+  for (ExpectRace *race = list->next; race != list; race = race->next) {
+    (*unique_count)++;
+    const uptr cnt = atomic_load_relaxed(&(race->*counter));
+    if (cnt == 0)
+      continue;
+    *hit_count += cnt;
+    uptr i = 0;
+    for (; i < matched->Size(); i++) {
+      ExpectRace *race0 = &(*matched)[i];
+      if (race->line == race0->line
+          && internal_strcmp(race->file, race0->file) == 0
+          && internal_strcmp(race->desc, race0->desc) == 0) {
+        atomic_fetch_add(&(race0->*counter), cnt, memory_order_relaxed);
+        break;
+      }
+    }
+    if (i == matched->Size())
+      matched->PushBack(*race);
+  }
+}
+
+void PrintMatchedBenignRaces() {
+  Lock lock(&dyn_ann_ctx->mtx);
+  int unique_count = 0;
+  int hit_count = 0;
+  int add_count = 0;
+  Vector<ExpectRace> hit_matched;
+  CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count,
+      &ExpectRace::hitcount);
+  Vector<ExpectRace> add_matched;
+  CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count,
+      &ExpectRace::addcount);
+  if (hit_matched.Size()) {
+    Printf("ThreadSanitizer: Matched %d \"benign\" races (pid=%d):\n",
+        hit_count, (int)internal_getpid());
+    for (uptr i = 0; i < hit_matched.Size(); i++) {
+      Printf("%d %s:%d %s\n",
+          atomic_load_relaxed(&hit_matched[i].hitcount),
+          hit_matched[i].file, hit_matched[i].line, hit_matched[i].desc);
+    }
+  }
+  if (hit_matched.Size()) {
+    Printf("ThreadSanitizer: Annotated %d \"benign\" races, %d unique"
+           " (pid=%d):\n",
+        add_count, unique_count, (int)internal_getpid());
+    for (uptr i = 0; i < add_matched.Size(); i++) {
+      Printf("%d %s:%d %s\n",
+          atomic_load_relaxed(&add_matched[i].addcount),
+          add_matched[i].file, add_matched[i].line, add_matched[i].desc);
+    }
+  }
+}
+
+static void ReportMissedExpectedRace(ExpectRace *race) {
+  Printf("==================\n");
+  Printf("WARNING: ThreadSanitizer: missed expected data race\n");
+  Printf("  %s addr=%zx %s:%d\n",
+      race->desc, race->addr, race->file, race->line);
+  Printf("==================\n");
+}
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+void INTERFACE_ATTRIBUTE AnnotateHappensBefore(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensBefore);
+  Release(thr, pc, addr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensAfter);
+  Acquire(thr, pc, addr);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) {
+  SCOPED_ANNOTATION(AnnotateCondVarSignal);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) {
+  SCOPED_ANNOTATION(AnnotateCondVarSignalAll);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) {
+  SCOPED_ANNOTATION(AnnotateMutexIsNotPHB);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
+                                             uptr lock) {
+  SCOPED_ANNOTATION(AnnotateCondVarWait);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockCreate);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockCreateStatic(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockCreateStatic);
+  MutexCreate(thr, pc, m, MutexFlagWriteReentrant | MutexFlagLinkerInit);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockDestroy(char *f, int l, uptr m) {
+  SCOPED_ANNOTATION(AnnotateRWLockDestroy);
+  MutexDestroy(thr, pc, m);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockAcquired(char *f, int l, uptr m,
+                                                uptr is_w) {
+  SCOPED_ANNOTATION(AnnotateRWLockAcquired);
+  if (is_w)
+    MutexPostLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
+  else
+    MutexPostReadLock(thr, pc, m, MutexFlagDoPreLockOnPostLock);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m,
+                                                uptr is_w) {
+  SCOPED_ANNOTATION(AnnotateRWLockReleased);
+  if (is_w)
+    MutexUnlock(thr, pc, m);
+  else
+    MutexReadUnlock(thr, pc, m);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) {
+  SCOPED_ANNOTATION(AnnotateTraceMemory);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateFlushState);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem,
+                                           uptr size) {
+  SCOPED_ANNOTATION(AnnotateNewMemory);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) {
+  SCOPED_ANNOTATION(AnnotateNoOp);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateFlushExpectedRaces);
+  Lock lock(&dyn_ann_ctx->mtx);
+  while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) {
+    ExpectRace *race = dyn_ann_ctx->expect.next;
+    if (atomic_load_relaxed(&race->hitcount) == 0) {
+      ctx->nmissed_expected++;
+      ReportMissedExpectedRace(race);
+    }
+    race->prev->next = race->next;
+    race->next->prev = race->prev;
+    internal_free(race);
+  }
+}
+
+void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection(
+    char *f, int l, int enable) {
+  SCOPED_ANNOTATION(AnnotateEnableRaceDetection);
+  // FIXME: Reconsider this functionality later. It may be irrelevant.
+}
+
+void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar(
+    char *f, int l, uptr mu) {
+  SCOPED_ANNOTATION(AnnotateMutexIsUsedAsCondVar);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQGet(
+    char *f, int l, uptr pcq) {
+  SCOPED_ANNOTATION(AnnotatePCQGet);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQPut(
+    char *f, int l, uptr pcq) {
+  SCOPED_ANNOTATION(AnnotatePCQPut);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQDestroy(
+    char *f, int l, uptr pcq) {
+  SCOPED_ANNOTATION(AnnotatePCQDestroy);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePCQCreate(
+    char *f, int l, uptr pcq) {
+  SCOPED_ANNOTATION(AnnotatePCQCreate);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateExpectRace(
+    char *f, int l, uptr mem, char *desc) {
+  SCOPED_ANNOTATION(AnnotateExpectRace);
+  Lock lock(&dyn_ann_ctx->mtx);
+  AddExpectRace(&dyn_ann_ctx->expect,
+                f, l, mem, 1, desc);
+  DPrintf("Add expected race: %s addr=%zx %s:%d\n", desc, mem, f, l);
+}
+
+static void BenignRaceImpl(
+    char *f, int l, uptr mem, uptr size, char *desc) {
+  Lock lock(&dyn_ann_ctx->mtx);
+  AddExpectRace(&dyn_ann_ctx->benign,
+                f, l, mem, size, desc);
+  DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l);
+}
+
+// FIXME: Turn it off later. WTF is benign race?1?? Go talk to Hans Boehm.
+void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized(
+    char *f, int l, uptr mem, uptr size, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRaceSized);
+  BenignRaceImpl(f, l, mem, size, desc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateBenignRace(
+    char *f, int l, uptr mem, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRace);
+  BenignRaceImpl(f, l, mem, 1, desc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin);
+  ThreadIgnoreBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreWritesBegin);
+  ThreadIgnoreBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreSyncBegin);
+  ThreadIgnoreSyncBegin(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
+  SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
+  ThreadIgnoreSyncEnd(thr, pc);
+}
+
+void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(
+    char *f, int l, uptr addr, uptr size) {
+  SCOPED_ANNOTATION(AnnotatePublishMemoryRange);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange(
+    char *f, int l, uptr addr, uptr size) {
+  SCOPED_ANNOTATION(AnnotateUnpublishMemoryRange);
+}
+
+void INTERFACE_ATTRIBUTE AnnotateThreadName(
+    char *f, int l, char *name) {
+  SCOPED_ANNOTATION(AnnotateThreadName);
+  ThreadSetName(thr, name);
+}
+
+// We deliberately omit the implementation of WTFAnnotateHappensBefore() and
+// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate
+// atomic operations, which should be handled by ThreadSanitizer correctly.
+void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensBefore);
+}
+
+void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) {
+  SCOPED_ANNOTATION(AnnotateHappensAfter);
+}
+
+void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized(
+    char *f, int l, uptr mem, uptr sz, char *desc) {
+  SCOPED_ANNOTATION(AnnotateBenignRaceSized);
+  BenignRaceImpl(f, l, mem, sz, desc);
+}
+
+int INTERFACE_ATTRIBUTE RunningOnValgrind() {
+  return flags()->running_on_valgrind;
+}
+
+double __attribute__((weak)) INTERFACE_ATTRIBUTE ValgrindSlowdown(void) {
+  return 10.0;
+}
+
+const char INTERFACE_ATTRIBUTE* ThreadSanitizerQuery(const char *query) {
+  if (internal_strcmp(query, "pure_happens_before") == 0)
+    return "1";
+  else
+    return "0";
+}
+
+void INTERFACE_ATTRIBUTE
+AnnotateMemoryIsInitialized(char *f, int l, uptr mem, uptr sz) {}
+void INTERFACE_ATTRIBUTE
+AnnotateMemoryIsUninitialized(char *f, int l, uptr mem, uptr sz) {}
+
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_create(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_create);
+  MutexCreate(thr, pc, (uptr)m, flagz & MutexCreationFlagMask);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_destroy(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_destroy);
+  MutexDestroy(thr, pc, (uptr)m, flagz);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_lock);
+  if (!(flagz & MutexFlagTryLock)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPreReadLock(thr, pc, (uptr)m);
+    else
+      MutexPreLock(thr, pc, (uptr)m);
+  }
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_lock);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+  if (!(flagz & MutexFlagTryLockFailed)) {
+    if (flagz & MutexFlagReadLock)
+      MutexPostReadLock(thr, pc, (uptr)m, flagz);
+    else
+      MutexPostLock(thr, pc, (uptr)m, flagz, rec);
+  }
+}
+
+INTERFACE_ATTRIBUTE
+int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION_RET(__tsan_mutex_pre_unlock, 0);
+  int ret = 0;
+  if (flagz & MutexFlagReadLock) {
+    CHECK(!(flagz & MutexFlagRecursiveUnlock));
+    MutexReadUnlock(thr, pc, (uptr)m);
+  } else {
+    ret = MutexUnlock(thr, pc, (uptr)m, flagz);
+  }
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  return ret;
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_signal);
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
+  // Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
+  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreEnd(thr, pc);
+}
+
+INTERFACE_ATTRIBUTE
+void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
+  SCOPED_ANNOTATION(__tsan_mutex_post_divert);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+}
+}  // extern "C"
diff --git a/lib/tsan/tsan_interface_ann.h b/lib/tsan/tsan_interface_ann.h
new file mode 100644
index 0000000000..458d61f533
--- /dev/null
+++ b/lib/tsan/tsan_interface_ann.h
@@ -0,0 +1,32 @@
+//===-- tsan_interface_ann.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interface for dynamic annotations.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_ANN_H
+#define TSAN_INTERFACE_ANN_H
+
+#include <sanitizer_common/sanitizer_internal_defs.h>
+
+// This header should NOT include any other headers.
+// All functions in this header are extern "C" and start with __tsan_.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_acquire(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_release(void *addr);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TSAN_INTERFACE_ANN_H
diff --git a/lib/tsan/tsan_interface_atomic.cpp b/lib/tsan/tsan_interface_atomic.cpp
new file mode 100644
index 0000000000..3f459aff53
--- /dev/null
+++ b/lib/tsan/tsan_interface_atomic.cpp
@@ -0,0 +1,955 @@
+//===-- tsan_interface_atomic.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+// ThreadSanitizer atomic operations are based on C++11/C1x standards.
+// For background see C++11 standard.  A slightly older, publicly
+// available draft of the standard (not entirely up-to-date, but close enough
+// for casual browsing) is available here:
+// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
+// The following page contains more background information:
+// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "tsan_flags.h"
+#include "tsan_interface.h"
+#include "tsan_rtl.h"
+
+using namespace __tsan;
+
+#if !SANITIZER_GO && __TSAN_HAS_INT128
+// Protects emulation of 128-bit atomic operations.
+static StaticSpinMutex mutex128;
+#endif
+
+static bool IsLoadOrder(morder mo) {
+  return mo == mo_relaxed || mo == mo_consume
+      || mo == mo_acquire || mo == mo_seq_cst;
+}
+
+static bool IsStoreOrder(morder mo) {
+  return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
+}
+
+static bool IsReleaseOrder(morder mo) {
+  return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+static bool IsAcquireOrder(morder mo) {
+  return mo == mo_consume || mo == mo_acquire
+      || mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+static bool IsAcqRelOrder(morder mo) {
+  return mo == mo_acq_rel || mo == mo_seq_cst;
+}
+
+template<typename T> T func_xchg(volatile T *v, T op) {
+  T res = __sync_lock_test_and_set(v, op);
+  // __sync_lock_test_and_set does not contain full barrier.
+  __sync_synchronize();
+  return res;
+}
+
+template<typename T> T func_add(volatile T *v, T op) {
+  return __sync_fetch_and_add(v, op);
+}
+
+template<typename T> T func_sub(volatile T *v, T op) {
+  return __sync_fetch_and_sub(v, op);
+}
+
+template<typename T> T func_and(volatile T *v, T op) {
+  return __sync_fetch_and_and(v, op);
+}
+
+template<typename T> T func_or(volatile T *v, T op) {
+  return __sync_fetch_and_or(v, op);
+}
+
+template<typename T> T func_xor(volatile T *v, T op) {
+  return __sync_fetch_and_xor(v, op);
+}
+
+template<typename T> T func_nand(volatile T *v, T op) {
+  // clang does not support __sync_fetch_and_nand.
+  T cmp = *v;
+  for (;;) {
+    T newv = ~(cmp & op);
+    T cur = __sync_val_compare_and_swap(v, cmp, newv);
+    if (cmp == cur)
+      return cmp;
+    cmp = cur;
+  }
+}
+
+template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
+  return __sync_val_compare_and_swap(v, cmp, xch);
+}
+
+// clang does not support 128-bit atomic ops.
+// Atomic ops are executed under tsan internal mutex,
+// here we assume that the atomic variables are not accessed
+// from non-instrumented code.
+#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
+    && __TSAN_HAS_INT128
+a128 func_xchg(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = op;
+  return cmp;
+}
+
+a128 func_add(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp + op;
+  return cmp;
+}
+
+a128 func_sub(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp - op;
+  return cmp;
+}
+
+a128 func_and(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp & op;
+  return cmp;
+}
+
+a128 func_or(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp | op;
+  return cmp;
+}
+
+a128 func_xor(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = cmp ^ op;
+  return cmp;
+}
+
+a128 func_nand(volatile a128 *v, a128 op) {
+  SpinMutexLock lock(&mutex128);
+  a128 cmp = *v;
+  *v = ~(cmp & op);
+  return cmp;
+}
+
+a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
+  SpinMutexLock lock(&mutex128);
+  a128 cur = *v;
+  if (cur == cmp)
+    *v = xch;
+  return cur;
+}
+#endif
+
+template<typename T>
+static int SizeLog() {
+  if (sizeof(T) <= 1)
+    return kSizeLog1;
+  else if (sizeof(T) <= 2)
+    return kSizeLog2;
+  else if (sizeof(T) <= 4)
+    return kSizeLog4;
+  else
+    return kSizeLog8;
+  // For 16-byte atomics we also use 8-byte memory access,
+  // this leads to false negatives only in very obscure cases.
+}
+
+#if !SANITIZER_GO
+static atomic_uint8_t *to_atomic(const volatile a8 *a) {
+  return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
+}
+
+static atomic_uint16_t *to_atomic(const volatile a16 *a) {
+  return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
+}
+#endif
+
+static atomic_uint32_t *to_atomic(const volatile a32 *a) {
+  return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
+}
+
+static atomic_uint64_t *to_atomic(const volatile a64 *a) {
+  return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
+}
+
+static memory_order to_mo(morder mo) {
+  switch (mo) {
+  case mo_relaxed: return memory_order_relaxed;
+  case mo_consume: return memory_order_consume;
+  case mo_acquire: return memory_order_acquire;
+  case mo_release: return memory_order_release;
+  case mo_acq_rel: return memory_order_acq_rel;
+  case mo_seq_cst: return memory_order_seq_cst;
+  }
+  CHECK(0);
+  return memory_order_seq_cst;
+}
+
+template<typename T>
+static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
+  return atomic_load(to_atomic(a), to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128 && !SANITIZER_GO
+static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
+  SpinMutexLock lock(&mutex128);
+  return *a;
+}
+#endif
+
+template<typename T>
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
+  CHECK(IsLoadOrder(mo));
+  // This fast-path is critical for performance.
+  // Assume the access is atomic.
+  if (!IsAcquireOrder(mo)) {
+    MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
+    return NoTsanAtomicLoad(a, mo);
+  }
+  // Don't create sync object if it does not exist yet. For example, an atomic
+  // pointer is initialized to nullptr and then periodically acquire-loaded.
+  T v = NoTsanAtomicLoad(a, mo);
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+  if (s) {
+    AcquireImpl(thr, pc, &s->clock);
+    // Re-read under sync mutex because we need a consistent snapshot
+    // of the value and the clock we acquire.
+    v = NoTsanAtomicLoad(a, mo);
+    s->mtx.ReadUnlock();
+  }
+  MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
+  return v;
+}
+
+template<typename T>
+static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
+  atomic_store(to_atomic(a), v, to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128 && !SANITIZER_GO
+static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
+  SpinMutexLock lock(&mutex128);
+  *a = v;
+}
+#endif
+
+template<typename T>
+static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  CHECK(IsStoreOrder(mo));
+  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
+  // This fast-path is critical for performance.
+  // Assume the access is atomic.
+  // Strictly saying even relaxed store cuts off release sequence,
+  // so must reset the clock.
+  if (!IsReleaseOrder(mo)) {
+    NoTsanAtomicStore(a, v, mo);
+    return;
+  }
+  __sync_synchronize();
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreImpl(thr, pc, &s->clock);
+  NoTsanAtomicStore(a, v, mo);
+  s->mtx.Unlock();
+}
+
+template<typename T, T (*F)(volatile T *v, T op)>
+static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
+  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
+  SyncVar *s = 0;
+  if (mo != mo_relaxed) {
+    s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
+    thr->fast_state.IncrementEpoch();
+    // Can't increment epoch w/o writing to the trace as well.
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+    if (IsAcqRelOrder(mo))
+      AcquireReleaseImpl(thr, pc, &s->clock);
+    else if (IsReleaseOrder(mo))
+      ReleaseImpl(thr, pc, &s->clock);
+    else if (IsAcquireOrder(mo))
+      AcquireImpl(thr, pc, &s->clock);
+  }
+  v = F(a, v);
+  if (s)
+    s->mtx.Unlock();
+  return v;
+}
+
+template<typename T>
+static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
+  return func_xchg(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
+  return func_add(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
+  return func_sub(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
+  return func_and(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
+  return func_or(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
+  return func_xor(a, v);
+}
+
+template<typename T>
+static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
+  return func_nand(a, v);
+}
+
+template<typename T>
+static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
+    morder mo) {
+  return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
+}
+
+template<typename T>
+static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
+  return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
+}
+
+#if __TSAN_HAS_INT128
+static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  a128 old = *c;
+  a128 cur = func_cas(a, old, v);
+  if (cur == old)
+    return true;
+  *c = cur;
+  return false;
+}
+#endif
+
+template<typename T>
+static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
+  NoTsanAtomicCAS(a, &c, v, mo, fmo);
+  return c;
+}
+
+template<typename T>
+static bool AtomicCAS(ThreadState *thr, uptr pc,
+    volatile T *a, T *c, T v, morder mo, morder fmo) {
+  (void)fmo;  // Unused because llvm does not pass it yet.
+  MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
+  SyncVar *s = 0;
+  bool write_lock = mo != mo_acquire && mo != mo_consume;
+  if (mo != mo_relaxed) {
+    s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
+    thr->fast_state.IncrementEpoch();
+    // Can't increment epoch w/o writing to the trace as well.
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+    if (IsAcqRelOrder(mo))
+      AcquireReleaseImpl(thr, pc, &s->clock);
+    else if (IsReleaseOrder(mo))
+      ReleaseImpl(thr, pc, &s->clock);
+    else if (IsAcquireOrder(mo))
+      AcquireImpl(thr, pc, &s->clock);
+  }
+  T cc = *c;
+  T pr = func_cas(a, cc, v);
+  if (s) {
+    if (write_lock)
+      s->mtx.Unlock();
+    else
+      s->mtx.ReadUnlock();
+  }
+  if (pr == cc)
+    return true;
+  *c = pr;
+  return false;
+}
+
+template<typename T>
+static T AtomicCAS(ThreadState *thr, uptr pc,
+    volatile T *a, T c, T v, morder mo, morder fmo) {
+  AtomicCAS(thr, pc, a, &c, v, mo, fmo);
+  return c;
+}
+
+#if !SANITIZER_GO
+static void NoTsanAtomicFence(morder mo) {
+  __sync_synchronize();
+}
+
+static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
+  // FIXME(dvyukov): not implemented.
+  __sync_synchronize();
+}
+#endif
+
+// Interface functions follow.
+#if !SANITIZER_GO
+
+// C/C++
+
+static morder convert_morder(morder mo) {
+  if (flags()->force_seq_cst_atomics)
+    return (morder)mo_seq_cst;
+
+  // Filter out additional memory order flags:
+  // MEMMODEL_SYNC        = 1 << 15
+  // __ATOMIC_HLE_ACQUIRE = 1 << 16
+  // __ATOMIC_HLE_RELEASE = 1 << 17
+  //
+  // HLE is an optimization, and we pretend that elision always fails.
+  // MEMMODEL_SYNC is used when lowering __sync_ atomics,
+  // since we use __sync_ atomics for actual atomic operations,
+  // we can safely ignore it as well. It also subtly affects semantics,
+  // but we don't model the difference.
+  return (morder)(mo & 0x7fff);
+}
+
+#define SCOPED_ATOMIC(func, ...) \
+    ThreadState *const thr = cur_thread(); \
+    if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \
+      ProcessPendingSignals(thr); \
+      return NoTsanAtomic##func(__VA_ARGS__); \
+    } \
+    const uptr callpc = (uptr)__builtin_return_address(0); \
+    uptr pc = StackTrace::GetCurrentPc(); \
+    mo = convert_morder(mo); \
+    AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
+    ScopedAtomic sa(thr, callpc, a, mo, __func__); \
+    return Atomic##func(thr, pc, __VA_ARGS__); \
+/**/
+
+class ScopedAtomic {
+ public:
+  ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
+               morder mo, const char *func)
+      : thr_(thr) {
+    FuncEntry(thr_, pc);
+    DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
+  }
+  ~ScopedAtomic() {
+    ProcessPendingSignals(thr_);
+    FuncExit(thr_);
+  }
+ private:
+  ThreadState *thr_;
+};
+
+static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
+  StatInc(thr, StatAtomic);
+  StatInc(thr, t);
+  StatInc(thr, size == 1 ? StatAtomic1
+             : size == 2 ? StatAtomic2
+             : size == 4 ? StatAtomic4
+             : size == 8 ? StatAtomic8
+             :             StatAtomic16);
+  StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
+             : mo == mo_consume ? StatAtomicConsume
+             : mo == mo_acquire ? StatAtomicAcquire
+             : mo == mo_release ? StatAtomicRelease
+             : mo == mo_acq_rel ? StatAtomicAcq_Rel
+             :                    StatAtomicSeq_Cst);
+}
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
+  SCOPED_ATOMIC(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
+  SCOPED_ATOMIC(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
+  SCOPED_ATOMIC(Load, a, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
+  SCOPED_ATOMIC(Load, a, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
+  SCOPED_ATOMIC(Load, a, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(Store, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(Store, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(Store, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(Exchange, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(Exchange, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(Exchange, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchAdd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchAdd, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchAdd, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchSub, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchSub, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchSub, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchAnd, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchAnd, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchAnd, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchOr, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchOr, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchOr, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchXor, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchXor, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchXor, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
+  SCOPED_ATOMIC(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
+  SCOPED_ATOMIC(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
+  SCOPED_ATOMIC(FetchNand, a, v, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
+  SCOPED_ATOMIC(FetchNand, a, v, mo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
+  SCOPED_ATOMIC(FetchNand, a, v, mo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+
+#if __TSAN_HAS_INT128
+SANITIZER_INTERFACE_ATTRIBUTE
+a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
+    morder mo, morder fmo) {
+  SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
+}
+#endif
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_thread_fence(morder mo) {
+  char* a = 0;
+  SCOPED_ATOMIC(Fence, mo);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_atomic_signal_fence(morder mo) {
+}
+}  // extern "C"
+
+#else  // #if !SANITIZER_GO
+
+// Go
+
+#define ATOMIC(func, ...) \
+    if (thr->ignore_sync) { \
+      NoTsanAtomic##func(__VA_ARGS__); \
+    } else { \
+      FuncEntry(thr, cpc); \
+      Atomic##func(thr, pc, __VA_ARGS__); \
+      FuncExit(thr); \
+    } \
+/**/
+
+#define ATOMIC_RET(func, ret, ...) \
+    if (thr->ignore_sync) { \
+      (ret) = NoTsanAtomic##func(__VA_ARGS__); \
+    } else { \
+      FuncEntry(thr, cpc); \
+      (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
+      FuncExit(thr); \
+    } \
+/**/
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_compare_exchange(
+    ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  a32 cur = 0;
+  a32 cmp = *(a32*)(a+8);
+  ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
+  *(bool*)(a+16) = (cur == cmp);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_compare_exchange(
+    ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  a64 cur = 0;
+  a64 cmp = *(a64*)(a+8);
+  ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
+  *(bool*)(a+24) = (cur == cmp);
+}
+}  // extern "C"
+#endif  // #if !SANITIZER_GO
diff --git a/lib/tsan/tsan_interface_inl.h b/lib/tsan/tsan_interface_inl.h
new file mode 100644
index 0000000000..f955ddf992
--- /dev/null
+++ b/lib/tsan/tsan_interface_inl.h
@@ -0,0 +1,132 @@
+//===-- tsan_interface_inl.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interface.h"
+#include "tsan_rtl.h"
+
+#define CALLERPC ((uptr)__builtin_return_address(0))
+
+using namespace __tsan;
+
+void __tsan_read1(void *addr) {
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
+}
+
+void __tsan_read2(void *addr) {
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
+}
+
+void __tsan_read4(void *addr) {
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
+}
+
+void __tsan_read8(void *addr) {
+  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
+}
+
+void __tsan_write1(void *addr) {
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
+}
+
+void __tsan_write2(void *addr) {
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
+}
+
+void __tsan_write4(void *addr) {
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
+}
+
+void __tsan_write8(void *addr) {
+  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
+}
+
+void __tsan_read1_pc(void *addr, void *pc) {
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
+}
+
+void __tsan_read2_pc(void *addr, void *pc) {
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
+}
+
+void __tsan_read4_pc(void *addr, void *pc) {
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
+}
+
+void __tsan_read8_pc(void *addr, void *pc) {
+  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+}
+
+void __tsan_write1_pc(void *addr, void *pc) {
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
+}
+
+void __tsan_write2_pc(void *addr, void *pc) {
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
+}
+
+void __tsan_write4_pc(void *addr, void *pc) {
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
+}
+
+void __tsan_write8_pc(void *addr, void *pc) {
+  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+}
+
+void __tsan_vptr_update(void **vptr_p, void *new_val) {
+  CHECK_EQ(sizeof(vptr_p), 8);
+  if (*vptr_p != new_val) {
+    ThreadState *thr = cur_thread();
+    thr->is_vptr_access = true;
+    MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
+    thr->is_vptr_access = false;
+  }
+}
+
+void __tsan_vptr_read(void **vptr_p) {
+  CHECK_EQ(sizeof(vptr_p), 8);
+  ThreadState *thr = cur_thread();
+  thr->is_vptr_access = true;
+  MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
+  thr->is_vptr_access = false;
+}
+
+void __tsan_func_entry(void *pc) {
+  FuncEntry(cur_thread(), (uptr)pc);
+}
+
+void __tsan_func_exit() {
+  FuncExit(cur_thread());
+}
+
+void __tsan_ignore_thread_begin() {
+  ThreadIgnoreBegin(cur_thread(), CALLERPC);
+}
+
+void __tsan_ignore_thread_end() {
+  ThreadIgnoreEnd(cur_thread(), CALLERPC);
+}
+
+void __tsan_read_range(void *addr, uptr size) {
+  MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
+}
+
+void __tsan_write_range(void *addr, uptr size) {
+  MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
+}
+
+void __tsan_read_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, false);
+}
+
+void __tsan_write_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, true);
+}
diff --git a/lib/tsan/tsan_interface_java.cpp b/lib/tsan/tsan_interface_java.cpp
new file mode 100644
index 0000000000..081c6ff102
--- /dev/null
+++ b/lib/tsan/tsan_interface_java.cpp
@@ -0,0 +1,267 @@
+//===-- tsan_interface_java.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_interface_java.h"
+#include "tsan_rtl.h"
+#include "tsan_mutex.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+using namespace __tsan;
+
+const jptr kHeapAlignment = 8;
+
+namespace __tsan {
+
+struct JavaContext {
+  const uptr heap_begin;
+  const uptr heap_size;
+
+  JavaContext(jptr heap_begin, jptr heap_size)
+      : heap_begin(heap_begin)
+      , heap_size(heap_size) {
+  }
+};
+
+class ScopedJavaFunc {
+ public:
+  ScopedJavaFunc(ThreadState *thr, uptr pc)
+      : thr_(thr) {
+    Initialize(thr_);
+    FuncEntry(thr, pc);
+  }
+
+  ~ScopedJavaFunc() {
+    FuncExit(thr_);
+    // FIXME(dvyukov): process pending signals.
+  }
+
+ private:
+  ThreadState *thr_;
+};
+
+static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];
+static JavaContext *jctx;
+
+}  // namespace __tsan
+
+#define SCOPED_JAVA_FUNC(func) \
+  ThreadState *thr = cur_thread(); \
+  const uptr caller_pc = GET_CALLER_PC(); \
+  const uptr pc = StackTrace::GetCurrentPc(); \
+  (void)pc; \
+  ScopedJavaFunc scoped(thr, caller_pc); \
+/**/
+
+void __tsan_java_init(jptr heap_begin, jptr heap_size) {
+  SCOPED_JAVA_FUNC(__tsan_java_init);
+  DPrintf("#%d: java_init(%p, %p)\n", thr->tid, heap_begin, heap_size);
+  CHECK_EQ(jctx, 0);
+  CHECK_GT(heap_begin, 0);
+  CHECK_GT(heap_size, 0);
+  CHECK_EQ(heap_begin % kHeapAlignment, 0);
+  CHECK_EQ(heap_size % kHeapAlignment, 0);
+  CHECK_LT(heap_begin, heap_begin + heap_size);
+  jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);
+}
+
+int  __tsan_java_fini() {
+  SCOPED_JAVA_FUNC(__tsan_java_fini);
+  DPrintf("#%d: java_fini()\n", thr->tid);
+  CHECK_NE(jctx, 0);
+  // FIXME(dvyukov): this does not call atexit() callbacks.
+  int status = Finalize(thr);
+  DPrintf("#%d: java_fini() = %d\n", thr->tid, status);
+  return status;
+}
+
+void __tsan_java_alloc(jptr ptr, jptr size) {
+  SCOPED_JAVA_FUNC(__tsan_java_alloc);
+  DPrintf("#%d: java_alloc(%p, %p)\n", thr->tid, ptr, size);
+  CHECK_NE(jctx, 0);
+  CHECK_NE(size, 0);
+  CHECK_EQ(ptr % kHeapAlignment, 0);
+  CHECK_EQ(size % kHeapAlignment, 0);
+  CHECK_GE(ptr, jctx->heap_begin);
+  CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+  OnUserAlloc(thr, pc, ptr, size, false);
+}
+
+void __tsan_java_free(jptr ptr, jptr size) {
+  SCOPED_JAVA_FUNC(__tsan_java_free);
+  DPrintf("#%d: java_free(%p, %p)\n", thr->tid, ptr, size);
+  CHECK_NE(jctx, 0);
+  CHECK_NE(size, 0);
+  CHECK_EQ(ptr % kHeapAlignment, 0);
+  CHECK_EQ(size % kHeapAlignment, 0);
+  CHECK_GE(ptr, jctx->heap_begin);
+  CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
+
+  ctx->metamap.FreeRange(thr->proc(), ptr, size);
+}
+
+void __tsan_java_move(jptr src, jptr dst, jptr size) {
+  SCOPED_JAVA_FUNC(__tsan_java_move);
+  DPrintf("#%d: java_move(%p, %p, %p)\n", thr->tid, src, dst, size);
+  CHECK_NE(jctx, 0);
+  CHECK_NE(size, 0);
+  CHECK_EQ(src % kHeapAlignment, 0);
+  CHECK_EQ(dst % kHeapAlignment, 0);
+  CHECK_EQ(size % kHeapAlignment, 0);
+  CHECK_GE(src, jctx->heap_begin);
+  CHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
+  CHECK_GE(dst, jctx->heap_begin);
+  CHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
+  CHECK_NE(dst, src);
+  CHECK_NE(size, 0);
+
+  // Assuming it's not running concurrently with threads that do
+  // memory accesses and mutex operations (stop-the-world phase).
+  ctx->metamap.MoveMemory(src, dst, size);
+
+  // Move shadow.
+  u64 *s = (u64*)MemToShadow(src);
+  u64 *d = (u64*)MemToShadow(dst);
+  u64 *send = (u64*)MemToShadow(src + size);
+  uptr inc = 1;
+  if (dst > src) {
+    s = (u64*)MemToShadow(src + size) - 1;
+    d = (u64*)MemToShadow(dst + size) - 1;
+    send = (u64*)MemToShadow(src) - 1;
+    inc = -1;
+  }
+  for (; s != send; s += inc, d += inc) {
+    *d = *s;
+    *s = 0;
+  }
+}
+
+jptr __tsan_java_find(jptr *from_ptr, jptr to) {
+  SCOPED_JAVA_FUNC(__tsan_java_find);
+  DPrintf("#%d: java_find(&%p, %p)\n", *from_ptr, to);
+  CHECK_EQ((*from_ptr) % kHeapAlignment, 0);
+  CHECK_EQ(to % kHeapAlignment, 0);
+  CHECK_GE(*from_ptr, jctx->heap_begin);
+  CHECK_LE(to, jctx->heap_begin + jctx->heap_size);
+  for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
+    MBlock *b = ctx->metamap.GetBlock(from);
+    if (b) {
+      *from_ptr = from;
+      return b->siz;
+    }
+  }
+  return 0;
+}
+
+void __tsan_java_finalize() {
+  SCOPED_JAVA_FUNC(__tsan_java_finalize);
+  DPrintf("#%d: java_mutex_finalize()\n", thr->tid);
+  AcquireGlobal(thr, 0);
+}
+
+void __tsan_java_mutex_lock(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_lock);
+  DPrintf("#%d: java_mutex_lock(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
+      MutexFlagDoPreLockOnPostLock);
+}
+
+void __tsan_java_mutex_unlock(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock);
+  DPrintf("#%d: java_mutex_unlock(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexUnlock(thr, pc, addr);
+}
+
+void __tsan_java_mutex_read_lock(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_read_lock);
+  DPrintf("#%d: java_mutex_read_lock(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexPostReadLock(thr, pc, addr, MutexFlagLinkerInit |
+      MutexFlagWriteReentrant | MutexFlagDoPreLockOnPostLock);
+}
+
+void __tsan_java_mutex_read_unlock(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_read_unlock);
+  DPrintf("#%d: java_mutex_read_unlock(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  MutexReadUnlock(thr, pc, addr);
+}
+
+void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_lock_rec);
+  DPrintf("#%d: java_mutex_lock_rec(%p, %d)\n", thr->tid, addr, rec);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+  CHECK_GT(rec, 0);
+
+  MutexPostLock(thr, pc, addr, MutexFlagLinkerInit | MutexFlagWriteReentrant |
+      MutexFlagDoPreLockOnPostLock | MutexFlagRecursiveLock, rec);
+}
+
+int __tsan_java_mutex_unlock_rec(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock_rec);
+  DPrintf("#%d: java_mutex_unlock_rec(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  return MutexUnlock(thr, pc, addr, MutexFlagRecursiveUnlock);
+}
+
+void __tsan_java_acquire(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_acquire);
+  DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Acquire(thr, caller_pc, addr);
+}
+
+void __tsan_java_release(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_release);
+  DPrintf("#%d: java_release(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Release(thr, caller_pc, addr);
+}
+
+void __tsan_java_release_store(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_release);
+  DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  ReleaseStore(thr, caller_pc, addr);
+}
diff --git a/lib/tsan/tsan_interface_java.h b/lib/tsan/tsan_interface_java.h
new file mode 100644
index 0000000000..51b445251e
--- /dev/null
+++ b/lib/tsan/tsan_interface_java.h
@@ -0,0 +1,99 @@
+//===-- tsan_interface_java.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interface for verification of Java or mixed Java/C++ programs.
+// The interface is intended to be used from within a JVM and notify TSan
+// about such events like Java locks and GC memory compaction.
+//
+// For plain memory accesses and function entry/exit a JVM is intended to use
+// C++ interfaces: __tsan_readN/writeN and __tsan_func_enter/exit.
+//
+// For volatile memory accesses and atomic operations JVM is intended to use
+// standard atomics API: __tsan_atomicN_load/store/etc.
+//
+// For usage examples see lit_tests/java_*.cpp
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_INTERFACE_JAVA_H
+#define TSAN_INTERFACE_JAVA_H
+
+#ifndef INTERFACE_ATTRIBUTE
+# define INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef unsigned long jptr;
+
+// Must be called before any other callback from Java.
+void __tsan_java_init(jptr heap_begin, jptr heap_size) INTERFACE_ATTRIBUTE;
+// Must be called when the application exits.
+// Not necessary the last callback (concurrently running threads are OK).
+// Returns exit status or 0 if tsan does not want to override it.
+int  __tsan_java_fini() INTERFACE_ATTRIBUTE;
+
+// Callback for memory allocations.
+// May be omitted for allocations that are not subject to data races
+// nor contain synchronization objects (e.g. String).
+void __tsan_java_alloc(jptr ptr, jptr size) INTERFACE_ATTRIBUTE;
+// Callback for memory free.
+// Can be aggregated for several objects (preferably).
+void __tsan_java_free(jptr ptr, jptr size) INTERFACE_ATTRIBUTE;
+// Callback for memory move by GC.
+// Can be aggregated for several objects (preferably).
+// The ranges can overlap.
+void __tsan_java_move(jptr src, jptr dst, jptr size) INTERFACE_ATTRIBUTE;
+// This function must be called on the finalizer thread
+// before executing a batch of finalizers.
+// It ensures necessary synchronization between
+// java object creation and finalization.
+void __tsan_java_finalize() INTERFACE_ATTRIBUTE;
+// Finds the first allocated memory block in the [*from_ptr, to) range, saves
+// its address in *from_ptr and returns its size. Returns 0 if there are no
+// allocated memory blocks in the range.
+jptr __tsan_java_find(jptr *from_ptr, jptr to) INTERFACE_ATTRIBUTE;
+
+// Mutex lock.
+// Addr is any unique address associated with the mutex.
+// Can be called on recursive reentry.
+void __tsan_java_mutex_lock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex unlock.
+void __tsan_java_mutex_unlock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex read lock.
+void __tsan_java_mutex_read_lock(jptr addr) INTERFACE_ATTRIBUTE;
+// Mutex read unlock.
+void __tsan_java_mutex_read_unlock(jptr addr) INTERFACE_ATTRIBUTE;
+// Recursive mutex lock, intended for handling of Object.wait().
+// The 'rec' value must be obtained from the previous
+// __tsan_java_mutex_unlock_rec().
+void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE;
+// Recursive mutex unlock, intended for handling of Object.wait().
+// The return value says how many times this thread called lock()
+// w/o a pairing unlock() (i.e. how many recursive levels it unlocked).
+// It must be passed back to __tsan_java_mutex_lock_rec() to restore
+// the same recursion level.
+int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE;
+
+// Raw acquire/release primitives.
+// Can be used to establish happens-before edges on volatile/final fields,
+// in atomic operations, etc. release_store is the same as release, but it
+// breaks release sequence on addr (see C++ standard 1.10/7 for details).
+void __tsan_java_acquire(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release_store(jptr addr) INTERFACE_ATTRIBUTE;
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#undef INTERFACE_ATTRIBUTE
+
+#endif  // #ifndef TSAN_INTERFACE_JAVA_H
diff --git a/lib/tsan/tsan_malloc_mac.cpp b/lib/tsan/tsan_malloc_mac.cpp
new file mode 100644
index 0000000000..0e861bf1f9
--- /dev/null
+++ b/lib/tsan/tsan_malloc_mac.cpp
@@ -0,0 +1,71 @@
+//===-- tsan_malloc_mac.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific malloc interception.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_common/sanitizer_errno.h"
+#include "tsan_interceptors.h"
+#include "tsan_stack_trace.h"
+
+using namespace __tsan;
+#define COMMON_MALLOC_ZONE_NAME "tsan"
+#define COMMON_MALLOC_ENTER()
+#define COMMON_MALLOC_SANITIZER_INITIALIZED (cur_thread()->is_inited)
+#define COMMON_MALLOC_FORCE_LOCK()
+#define COMMON_MALLOC_FORCE_UNLOCK()
+#define COMMON_MALLOC_MEMALIGN(alignment, size) \
+  void *p =                                     \
+      user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
+#define COMMON_MALLOC_MALLOC(size)                             \
+  if (in_symbolizer()) return InternalAlloc(size);             \
+  SCOPED_INTERCEPTOR_RAW(malloc, size);                        \
+  void *p = user_alloc(thr, pc, size)
+#define COMMON_MALLOC_REALLOC(ptr, size)                              \
+  if (in_symbolizer()) return InternalRealloc(ptr, size);             \
+  SCOPED_INTERCEPTOR_RAW(realloc, ptr, size);                         \
+  void *p = user_realloc(thr, pc, ptr, size)
+#define COMMON_MALLOC_CALLOC(count, size)                              \
+  if (in_symbolizer()) return InternalCalloc(count, size);             \
+  SCOPED_INTERCEPTOR_RAW(calloc, size, count);                         \
+  void *p = user_calloc(thr, pc, size, count)
+#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size)      \
+  if (in_symbolizer()) {                                           \
+    void *p = InternalAlloc(size, nullptr, alignment);             \
+    if (!p) return errno_ENOMEM;                                   \
+    *memptr = p;                                                   \
+    return 0;                                                      \
+  }                                                                \
+  SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, alignment, size); \
+  int res = user_posix_memalign(thr, pc, memptr, alignment, size);
+#define COMMON_MALLOC_VALLOC(size)                            \
+  if (in_symbolizer())                                        \
+    return InternalAlloc(size, nullptr, GetPageSizeCached()); \
+  SCOPED_INTERCEPTOR_RAW(valloc, size);                       \
+  void *p = user_valloc(thr, pc, size)
+#define COMMON_MALLOC_FREE(ptr)                              \
+  if (in_symbolizer()) return InternalFree(ptr);             \
+  SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
+  user_free(thr, pc, ptr)
+#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
+#define COMMON_MALLOC_FILL_STATS(zone, stats)
+#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
+  (void)zone_name; \
+  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
+#define COMMON_MALLOC_NAMESPACE __tsan
+#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
+#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
+
+#include "sanitizer_common/sanitizer_malloc_mac.inc"
+
+#endif
diff --git a/lib/tsan/tsan_md5.cpp b/lib/tsan/tsan_md5.cpp
new file mode 100644
index 0000000000..72857b773f
--- /dev/null
+++ b/lib/tsan/tsan_md5.cpp
@@ -0,0 +1,250 @@
+//===-- tsan_md5.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+#define F(x, y, z)      ((z) ^ ((x) & ((y) ^ (z))))
+#define G(x, y, z)      ((y) ^ ((z) & ((x) ^ (y))))
+#define H(x, y, z)      ((x) ^ (y) ^ (z))
+#define I(x, y, z)      ((y) ^ ((x) | ~(z)))
+
+#define STEP(f, a, b, c, d, x, t, s) \
+  (a) += f((b), (c), (d)) + (x) + (t); \
+  (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
+  (a) += (b);
+
+#define SET(n) \
+  (*(const MD5_u32plus *)&ptr[(n) * 4])
+#define GET(n) \
+  SET(n)
+
+typedef unsigned int MD5_u32plus;
+typedef unsigned long ulong_t;
+
+typedef struct {
+  MD5_u32plus lo, hi;
+  MD5_u32plus a, b, c, d;
+  unsigned char buffer[64];
+  MD5_u32plus block[16];
+} MD5_CTX;
+
+static const void *body(MD5_CTX *ctx, const void *data, ulong_t size) {
+  const unsigned char *ptr = (const unsigned char *)data;
+  MD5_u32plus a, b, c, d;
+  MD5_u32plus saved_a, saved_b, saved_c, saved_d;
+
+  a = ctx->a;
+  b = ctx->b;
+  c = ctx->c;
+  d = ctx->d;
+
+  do {
+    saved_a = a;
+    saved_b = b;
+    saved_c = c;
+    saved_d = d;
+
+    STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
+    STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
+    STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
+    STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
+    STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
+    STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
+    STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
+    STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
+    STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
+    STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
+    STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
+    STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
+    STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
+    STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
+    STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
+    STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
+
+    STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
+    STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
+    STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
+    STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
+    STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
+    STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
+    STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
+    STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
+    STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
+    STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
+    STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
+    STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
+    STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
+    STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
+    STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
+    STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
+
+    STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
+    STEP(H, d, a, b, c, GET(8), 0x8771f681, 11)
+    STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
+    STEP(H, b, c, d, a, GET(14), 0xfde5380c, 23)
+    STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
+    STEP(H, d, a, b, c, GET(4), 0x4bdecfa9, 11)
+    STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
+    STEP(H, b, c, d, a, GET(10), 0xbebfbc70, 23)
+    STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
+    STEP(H, d, a, b, c, GET(0), 0xeaa127fa, 11)
+    STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
+    STEP(H, b, c, d, a, GET(6), 0x04881d05, 23)
+    STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
+    STEP(H, d, a, b, c, GET(12), 0xe6db99e5, 11)
+    STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
+    STEP(H, b, c, d, a, GET(2), 0xc4ac5665, 23)
+
+    STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
+    STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
+    STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
+    STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
+    STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
+    STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
+    STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
+    STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
+    STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
+    STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
+    STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
+    STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
+    STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
+    STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
+    STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
+    STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
+
+    a += saved_a;
+    b += saved_b;
+    c += saved_c;
+    d += saved_d;
+
+    ptr += 64;
+  } while (size -= 64);
+
+  ctx->a = a;
+  ctx->b = b;
+  ctx->c = c;
+  ctx->d = d;
+
+  return ptr;
+}
+
+#undef F
+#undef G
+#undef H
+#undef I
+#undef STEP
+#undef SET
+#undef GET
+
+void MD5_Init(MD5_CTX *ctx) {
+  ctx->a = 0x67452301;
+  ctx->b = 0xefcdab89;
+  ctx->c = 0x98badcfe;
+  ctx->d = 0x10325476;
+
+  ctx->lo = 0;
+  ctx->hi = 0;
+}
+
+void MD5_Update(MD5_CTX *ctx, const void *data, ulong_t size) {
+  MD5_u32plus saved_lo;
+  ulong_t used, free;
+
+  saved_lo = ctx->lo;
+  if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+    ctx->hi++;
+  ctx->hi += size >> 29;
+
+  used = saved_lo & 0x3f;
+
+  if (used) {
+    free = 64 - used;
+
+    if (size < free) {
+      internal_memcpy(&ctx->buffer[used], data, size);
+      return;
+    }
+
+    internal_memcpy(&ctx->buffer[used], data, free);
+    data = (const unsigned char *)data + free;
+    size -= free;
+    body(ctx, ctx->buffer, 64);
+  }
+
+  if (size >= 64) {
+    data = body(ctx, data, size & ~(ulong_t)0x3f);
+    size &= 0x3f;
+  }
+
+  internal_memcpy(ctx->buffer, data, size);
+}
+
+void MD5_Final(unsigned char *result, MD5_CTX *ctx) {
+  ulong_t used, free;
+
+  used = ctx->lo & 0x3f;
+
+  ctx->buffer[used++] = 0x80;
+
+  free = 64 - used;
+
+  if (free < 8) {
+    internal_memset(&ctx->buffer[used], 0, free);
+    body(ctx, ctx->buffer, 64);
+    used = 0;
+    free = 64;
+  }
+
+  internal_memset(&ctx->buffer[used], 0, free - 8);
+
+  ctx->lo <<= 3;
+  ctx->buffer[56] = ctx->lo;
+  ctx->buffer[57] = ctx->lo >> 8;
+  ctx->buffer[58] = ctx->lo >> 16;
+  ctx->buffer[59] = ctx->lo >> 24;
+  ctx->buffer[60] = ctx->hi;
+  ctx->buffer[61] = ctx->hi >> 8;
+  ctx->buffer[62] = ctx->hi >> 16;
+  ctx->buffer[63] = ctx->hi >> 24;
+
+  body(ctx, ctx->buffer, 64);
+
+  result[0] = ctx->a;
+  result[1] = ctx->a >> 8;
+  result[2] = ctx->a >> 16;
+  result[3] = ctx->a >> 24;
+  result[4] = ctx->b;
+  result[5] = ctx->b >> 8;
+  result[6] = ctx->b >> 16;
+  result[7] = ctx->b >> 24;
+  result[8] = ctx->c;
+  result[9] = ctx->c >> 8;
+  result[10] = ctx->c >> 16;
+  result[11] = ctx->c >> 24;
+  result[12] = ctx->d;
+  result[13] = ctx->d >> 8;
+  result[14] = ctx->d >> 16;
+  result[15] = ctx->d >> 24;
+
+  internal_memset(ctx, 0, sizeof(*ctx));
+}
+
+MD5Hash md5_hash(const void *data, uptr size) {
+  MD5Hash res;
+  MD5_CTX ctx;
+  MD5_Init(&ctx);
+  MD5_Update(&ctx, data, size);
+  MD5_Final((unsigned char*)&res.hash[0], &ctx);
+  return res;
+}
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_mman.cpp b/lib/tsan/tsan_mman.cpp
new file mode 100644
index 0000000000..743e67bf2f
--- /dev/null
+++ b/lib/tsan/tsan_mman.cpp
@@ -0,0 +1,405 @@
+//===-- tsan_mman.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_mman.h"
+#include "tsan_rtl.h"
+#include "tsan_report.h"
+#include "tsan_flags.h"
+
+// May be overriden by front-end.
+SANITIZER_WEAK_DEFAULT_IMPL
+void __sanitizer_malloc_hook(void *ptr, uptr size) {
+  (void)ptr;
+  (void)size;
+}
+
+SANITIZER_WEAK_DEFAULT_IMPL
+void __sanitizer_free_hook(void *ptr) {
+  (void)ptr;
+}
+
+namespace __tsan {
+
+struct MapUnmapCallback {
+  void OnMap(uptr p, uptr size) const { }
+  void OnUnmap(uptr p, uptr size) const {
+    // We are about to unmap a chunk of user memory.
+    // Mark the corresponding shadow memory as not needed.
+    DontNeedShadowFor(p, size);
+    // Mark the corresponding meta shadow memory as not needed.
+    // Note the block does not contain any meta info at this point
+    // (this happens after free).
+    const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
+    const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
+    // Block came from LargeMmapAllocator, so must be large.
+    // We rely on this in the calculations below.
+    CHECK_GE(size, 2 * kPageSize);
+    uptr diff = RoundUp(p, kPageSize) - p;
+    if (diff != 0) {
+      p += diff;
+      size -= diff;
+    }
+    diff = p + size - RoundDown(p + size, kPageSize);
+    if (diff != 0)
+      size -= diff;
+    uptr p_meta = (uptr)MemToMeta(p);
+    ReleaseMemoryPagesToOS(p_meta, p_meta + size / kMetaRatio);
+  }
+};
+
+static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
+Allocator *allocator() {
+  return reinterpret_cast<Allocator*>(&allocator_placeholder);
+}
+
+struct GlobalProc {
+  Mutex mtx;
+  Processor *proc;
+
+  GlobalProc()
+      : mtx(MutexTypeGlobalProc, StatMtxGlobalProc)
+      , proc(ProcCreate()) {
+  }
+};
+
+static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
+GlobalProc *global_proc() {
+  return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
+}
+
+ScopedGlobalProcessor::ScopedGlobalProcessor() {
+  GlobalProc *gp = global_proc();
+  ThreadState *thr = cur_thread();
+  if (thr->proc())
+    return;
+  // If we don't have a proc, use the global one.
+  // There are currently only two known case where this path is triggered:
+  //   __interceptor_free
+  //   __nptl_deallocate_tsd
+  //   start_thread
+  //   clone
+  // and:
+  //   ResetRange
+  //   __interceptor_munmap
+  //   __deallocate_stack
+  //   start_thread
+  //   clone
+  // Ideally, we destroy thread state (and unwire proc) when a thread actually
+  // exits (i.e. when we join/wait it). Then we would not need the global proc
+  gp->mtx.Lock();
+  ProcWire(gp->proc, thr);
+}
+
+ScopedGlobalProcessor::~ScopedGlobalProcessor() {
+  GlobalProc *gp = global_proc();
+  ThreadState *thr = cur_thread();
+  if (thr->proc() != gp->proc)
+    return;
+  ProcUnwire(gp->proc, thr);
+  gp->mtx.Unlock();
+}
+
+static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
+static uptr max_user_defined_malloc_size;
+
+void InitializeAllocator() {
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
+  max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
+                                     ? common_flags()->max_allocation_size_mb
+                                           << 20
+                                     : kMaxAllowedMallocSize;
+}
+
+void InitializeAllocatorLate() {
+  new(global_proc()) GlobalProc();
+}
+
+void AllocatorProcStart(Processor *proc) {
+  allocator()->InitCache(&proc->alloc_cache);
+  internal_allocator()->InitCache(&proc->internal_alloc_cache);
+}
+
+void AllocatorProcFinish(Processor *proc) {
+  allocator()->DestroyCache(&proc->alloc_cache);
+  internal_allocator()->DestroyCache(&proc->internal_alloc_cache);
+}
+
+void AllocatorPrintStats() {
+  allocator()->PrintStats();
+}
+
+static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
+  if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
+      !flags()->report_signal_unsafe)
+    return;
+  VarSizeStackTrace stack;
+  ObtainCurrentStack(thr, pc, &stack);
+  if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
+    return;
+  ThreadRegistryLock l(ctx->thread_registry);
+  ScopedReport rep(ReportTypeSignalUnsafe);
+  rep.AddStack(stack, true);
+  OutputReport(thr, rep);
+}
+
+
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
+                          bool signal) {
+  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
+      sz > max_user_defined_malloc_size) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    uptr malloc_limit =
+        Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
+  }
+  void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
+  if (UNLIKELY(!p)) {
+    SetAllocatorOutOfMemory();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportOutOfMemory(sz, &stack);
+  }
+  if (ctx && ctx->initialized)
+    OnUserAlloc(thr, pc, (uptr)p, sz, true);
+  if (signal)
+    SignalUnsafeCall(thr, pc);
+  return p;
+}
+
+void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
+  ScopedGlobalProcessor sgp;
+  if (ctx && ctx->initialized)
+    OnUserFree(thr, pc, (uptr)p, true);
+  allocator()->Deallocate(&thr->proc()->alloc_cache, p);
+  if (signal)
+    SignalUnsafeCall(thr, pc);
+}
+
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
+}
+
+void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
+  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
+    if (AllocatorMayReturnNull())
+      return SetErrnoOnNull(nullptr);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportCallocOverflow(n, size, &stack);
+  }
+  void *p = user_alloc_internal(thr, pc, n * size);
+  if (p)
+    internal_memset(p, 0, n * size);
+  return SetErrnoOnNull(p);
+}
+
+void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr size, uptr n) {
+  if (UNLIKELY(CheckForCallocOverflow(size, n))) {
+    if (AllocatorMayReturnNull())
+      return SetErrnoOnNull(nullptr);
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportReallocArrayOverflow(size, n, &stack);
+  }
+  return user_realloc(thr, pc, p, size * n);
+}
+
+void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
+  DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
+  ctx->metamap.AllocBlock(thr, pc, p, sz);
+  if (write && thr->ignore_reads_and_writes == 0)
+    MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
+  else
+    MemoryResetRange(thr, pc, (uptr)p, sz);
+}
+
+void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
+  CHECK_NE(p, (void*)0);
+  uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
+  DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);
+  if (write && thr->ignore_reads_and_writes == 0)
+    MemoryRangeFreed(thr, pc, (uptr)p, sz);
+}
+
+void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
+  // FIXME: Handle "shrinking" more efficiently,
+  // it seems that some software actually does this.
+  if (!p)
+    return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
+  if (!sz) {
+    user_free(thr, pc, p);
+    return nullptr;
+  }
+  void *new_p = user_alloc_internal(thr, pc, sz);
+  if (new_p) {
+    uptr old_sz = user_alloc_usable_size(p);
+    internal_memcpy(new_p, p, min(old_sz, sz));
+    user_free(thr, pc, p);
+  }
+  return SetErrnoOnNull(new_p);
+}
+
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!IsPowerOfTwo(align))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAllocationAlignment(align, &stack);
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
+    if (AllocatorMayReturnNull())
+      return errno_EINVAL;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidPosixMemalignAlignment(align, &stack);
+  }
+  void *ptr = user_alloc_internal(thr, pc, sz, align);
+  if (UNLIKELY(!ptr))
+    // OOM error is already taken care of by user_alloc_internal.
+    return errno_ENOMEM;
+  CHECK(IsAligned((uptr)ptr, align));
+  *memptr = ptr;
+  return 0;
+}
+
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportInvalidAlignedAllocAlignment(sz, align, &stack);
+  }
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
+}
+
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
+  uptr PageSize = GetPageSizeCached();
+  if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
+    errno = errno_ENOMEM;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportPvallocOverflow(sz, &stack);
+  }
+  // pvalloc(0) should allocate one page.
+  sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+  return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
+}
+
+uptr user_alloc_usable_size(const void *p) {
+  if (p == 0)
+    return 0;
+  MBlock *b = ctx->metamap.GetBlock((uptr)p);
+  if (!b)
+    return 0;  // Not a valid pointer.
+  if (b->siz == 0)
+    return 1;  // Zero-sized allocations are actually 1 byte.
+  return b->siz;
+}
+
+void invoke_malloc_hook(void *ptr, uptr size) {
+  ThreadState *thr = cur_thread();
+  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
+    return;
+  __sanitizer_malloc_hook(ptr, size);
+  RunMallocHooks(ptr, size);
+}
+
+void invoke_free_hook(void *ptr) {
+  ThreadState *thr = cur_thread();
+  if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors)
+    return;
+  __sanitizer_free_hook(ptr);
+  RunFreeHooks(ptr);
+}
+
+void *internal_alloc(MBlockType typ, uptr sz) {
+  ThreadState *thr = cur_thread();
+  if (thr->nomalloc) {
+    thr->nomalloc = 0;  // CHECK calls internal_malloc().
+    CHECK(0);
+  }
+  return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
+}
+
+void internal_free(void *p) {
+  ThreadState *thr = cur_thread();
+  if (thr->nomalloc) {
+    thr->nomalloc = 0;  // CHECK calls internal_malloc().
+    CHECK(0);
+  }
+  InternalFree(p, &thr->proc()->internal_alloc_cache);
+}
+
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+uptr __sanitizer_get_current_allocated_bytes() {
+  uptr stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+  uptr stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() {
+  return 1;
+}
+
+uptr __sanitizer_get_unmapped_bytes() {
+  return 1;
+}
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) {
+  return size;
+}
+
+int __sanitizer_get_ownership(const void *p) {
+  return allocator()->GetBlockBegin(p) != 0;
+}
+
+uptr __sanitizer_get_allocated_size(const void *p) {
+  return user_alloc_usable_size(p);
+}
+
+void __tsan_on_thread_idle() {
+  ThreadState *thr = cur_thread();
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+  allocator()->SwallowCache(&thr->proc()->alloc_cache);
+  internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
+  ctx->metamap.OnProcIdle(thr->proc());
+}
+}  // extern "C"
diff --git a/lib/tsan/tsan_mman.h b/lib/tsan/tsan_mman.h
new file mode 100644
index 0000000000..a5280d4472
--- /dev/null
+++ b/lib/tsan/tsan_mman.h
@@ -0,0 +1,89 @@
+//===-- tsan_mman.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_MMAN_H
+#define TSAN_MMAN_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+const uptr kDefaultAlignment = 16;
+
+void InitializeAllocator();
+void InitializeAllocatorLate();
+void ReplaceSystemMalloc();
+void AllocatorProcStart(Processor *proc);
+void AllocatorProcFinish(Processor *proc);
+void AllocatorPrintStats();
+
+// For user allocations.
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
+                          uptr align = kDefaultAlignment, bool signal = true);
+// Does not accept NULL.
+void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
+// Interceptor implementations.
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
+void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
+void *user_reallocarray(ThreadState *thr, uptr pc, void *p, uptr sz, uptr n);
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz);
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+                        uptr sz);
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz);
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz);
+uptr user_alloc_usable_size(const void *p);
+
+// Invoking malloc/free hooks that may be installed by the user.
+void invoke_malloc_hook(void *ptr, uptr size);
+void invoke_free_hook(void *ptr);
+
+enum MBlockType {
+  MBlockScopedBuf,
+  MBlockString,
+  MBlockStackTrace,
+  MBlockShadowStack,
+  MBlockSync,
+  MBlockClock,
+  MBlockThreadContex,
+  MBlockDeadInfo,
+  MBlockRacyStacks,
+  MBlockRacyAddresses,
+  MBlockAtExit,
+  MBlockFlag,
+  MBlockReport,
+  MBlockReportMop,
+  MBlockReportThread,
+  MBlockReportMutex,
+  MBlockReportLoc,
+  MBlockReportStack,
+  MBlockSuppression,
+  MBlockExpectRace,
+  MBlockSignal,
+  MBlockJmpBuf,
+
+  // This must be the last.
+  MBlockTypeCount
+};
+
+// For internal data structures.
+void *internal_alloc(MBlockType typ, uptr sz);
+void internal_free(void *p);
+
+template <typename T>
+void DestroyAndFree(T *p) {
+  p->~T();
+  internal_free(p);
+}
+
+}  // namespace __tsan
+#endif  // TSAN_MMAN_H
diff --git a/lib/tsan/tsan_mutex.cpp b/lib/tsan/tsan_mutex.cpp
new file mode 100644
index 0000000000..7a0918f2a2
--- /dev/null
+++ b/lib/tsan/tsan_mutex.cpp
@@ -0,0 +1,289 @@
+//===-- tsan_mutex.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_libc.h"
+#include "tsan_mutex.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+// Simple reader-writer spin-mutex. Optimized for not-so-contended case.
+// Readers have preference, can possibly starvate writers.
+
+// The table fixes what mutexes can be locked under what mutexes.
+// E.g. if the row for MutexTypeThreads contains MutexTypeReport,
+// then Report mutex can be locked while under Threads mutex.
+// The leaf mutexes can be locked under any other mutexes.
+// Recursive locking is not supported.
+#if SANITIZER_DEBUG && !SANITIZER_GO
+const MutexType MutexTypeLeaf = (MutexType)-1;
+static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = {
+  /*0  MutexTypeInvalid*/     {},
+  /*1  MutexTypeTrace*/       {MutexTypeLeaf},
+  /*2  MutexTypeThreads*/     {MutexTypeReport},
+  /*3  MutexTypeReport*/      {MutexTypeSyncVar,
+                               MutexTypeMBlock, MutexTypeJavaMBlock},
+  /*4  MutexTypeSyncVar*/     {MutexTypeDDetector},
+  /*5  MutexTypeSyncTab*/     {},  // unused
+  /*6  MutexTypeSlab*/        {MutexTypeLeaf},
+  /*7  MutexTypeAnnotations*/ {},
+  /*8  MutexTypeAtExit*/      {MutexTypeSyncVar},
+  /*9  MutexTypeMBlock*/      {MutexTypeSyncVar},
+  /*10 MutexTypeJavaMBlock*/  {MutexTypeSyncVar},
+  /*11 MutexTypeDDetector*/   {},
+  /*12 MutexTypeFired*/       {MutexTypeLeaf},
+  /*13 MutexTypeRacy*/        {MutexTypeLeaf},
+  /*14 MutexTypeGlobalProc*/  {},
+};
+
+static bool CanLockAdj[MutexTypeCount][MutexTypeCount];
+#endif
+
+void InitializeMutex() {
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  // Build the "can lock" adjacency matrix.
+  // If [i][j]==true, then one can lock mutex j while under mutex i.
+  const int N = MutexTypeCount;
+  int cnt[N] = {};
+  bool leaf[N] = {};
+  for (int i = 1; i < N; i++) {
+    for (int j = 0; j < N; j++) {
+      MutexType z = CanLockTab[i][j];
+      if (z == MutexTypeInvalid)
+        continue;
+      if (z == MutexTypeLeaf) {
+        CHECK(!leaf[i]);
+        leaf[i] = true;
+        continue;
+      }
+      CHECK(!CanLockAdj[i][(int)z]);
+      CanLockAdj[i][(int)z] = true;
+      cnt[i]++;
+    }
+  }
+  for (int i = 0; i < N; i++) {
+    CHECK(!leaf[i] || cnt[i] == 0);
+  }
+  // Add leaf mutexes.
+  for (int i = 0; i < N; i++) {
+    if (!leaf[i])
+      continue;
+    for (int j = 0; j < N; j++) {
+      if (i == j || leaf[j] || j == MutexTypeInvalid)
+        continue;
+      CHECK(!CanLockAdj[j][i]);
+      CanLockAdj[j][i] = true;
+    }
+  }
+  // Build the transitive closure.
+  bool CanLockAdj2[MutexTypeCount][MutexTypeCount];
+  for (int i = 0; i < N; i++) {
+    for (int j = 0; j < N; j++) {
+      CanLockAdj2[i][j] = CanLockAdj[i][j];
+    }
+  }
+  for (int k = 0; k < N; k++) {
+    for (int i = 0; i < N; i++) {
+      for (int j = 0; j < N; j++) {
+        if (CanLockAdj2[i][k] && CanLockAdj2[k][j]) {
+          CanLockAdj2[i][j] = true;
+        }
+      }
+    }
+  }
+#if 0
+  Printf("Can lock graph:\n");
+  for (int i = 0; i < N; i++) {
+    for (int j = 0; j < N; j++) {
+      Printf("%d ", CanLockAdj[i][j]);
+    }
+    Printf("\n");
+  }
+  Printf("Can lock graph closure:\n");
+  for (int i = 0; i < N; i++) {
+    for (int j = 0; j < N; j++) {
+      Printf("%d ", CanLockAdj2[i][j]);
+    }
+    Printf("\n");
+  }
+#endif
+  // Verify that the graph is acyclic.
+  for (int i = 0; i < N; i++) {
+    if (CanLockAdj2[i][i]) {
+      Printf("Mutex %d participates in a cycle\n", i);
+      Die();
+    }
+  }
+#endif
+}
+
+InternalDeadlockDetector::InternalDeadlockDetector() {
+  // Rely on zero initialization because some mutexes can be locked before ctor.
+}
+
+#if SANITIZER_DEBUG && !SANITIZER_GO
+void InternalDeadlockDetector::Lock(MutexType t) {
+  // Printf("LOCK %d @%zu\n", t, seq_ + 1);
+  CHECK_GT(t, MutexTypeInvalid);
+  CHECK_LT(t, MutexTypeCount);
+  u64 max_seq = 0;
+  u64 max_idx = MutexTypeInvalid;
+  for (int i = 0; i != MutexTypeCount; i++) {
+    if (locked_[i] == 0)
+      continue;
+    CHECK_NE(locked_[i], max_seq);
+    if (max_seq < locked_[i]) {
+      max_seq = locked_[i];
+      max_idx = i;
+    }
+  }
+  locked_[t] = ++seq_;
+  if (max_idx == MutexTypeInvalid)
+    return;
+  // Printf("  last %d @%zu\n", max_idx, max_seq);
+  if (!CanLockAdj[max_idx][t]) {
+    Printf("ThreadSanitizer: internal deadlock detected\n");
+    Printf("ThreadSanitizer: can't lock %d while under %zu\n",
+               t, (uptr)max_idx);
+    CHECK(0);
+  }
+}
+
+void InternalDeadlockDetector::Unlock(MutexType t) {
+  // Printf("UNLO %d @%zu #%zu\n", t, seq_, locked_[t]);
+  CHECK(locked_[t]);
+  locked_[t] = 0;
+}
+
+void InternalDeadlockDetector::CheckNoLocks() {
+  for (int i = 0; i != MutexTypeCount; i++) {
+    CHECK_EQ(locked_[i], 0);
+  }
+}
+#endif
+
+void CheckNoLocks(ThreadState *thr) {
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  thr->internal_deadlock_detector.CheckNoLocks();
+#endif
+}
+
+const uptr kUnlocked = 0;
+const uptr kWriteLock = 1;
+const uptr kReadLock = 2;
+
+class Backoff {
+ public:
+  Backoff()
+    : iter_() {
+  }
+
+  bool Do() {
+    if (iter_++ < kActiveSpinIters)
+      proc_yield(kActiveSpinCnt);
+    else
+      internal_sched_yield();
+    return true;
+  }
+
+  u64 Contention() const {
+    u64 active = iter_ % kActiveSpinIters;
+    u64 passive = iter_ - active;
+    return active + 10 * passive;
+  }
+
+ private:
+  int iter_;
+  static const int kActiveSpinIters = 10;
+  static const int kActiveSpinCnt = 20;
+};
+
+Mutex::Mutex(MutexType type, StatType stat_type) {
+  CHECK_GT(type, MutexTypeInvalid);
+  CHECK_LT(type, MutexTypeCount);
+#if SANITIZER_DEBUG
+  type_ = type;
+#endif
+#if TSAN_COLLECT_STATS
+  stat_type_ = stat_type;
+#endif
+  atomic_store(&state_, kUnlocked, memory_order_relaxed);
+}
+
+Mutex::~Mutex() {
+  CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
+}
+
+void Mutex::Lock() {
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  cur_thread()->internal_deadlock_detector.Lock(type_);
+#endif
+  uptr cmp = kUnlocked;
+  if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
+                                     memory_order_acquire))
+    return;
+  for (Backoff backoff; backoff.Do();) {
+    if (atomic_load(&state_, memory_order_relaxed) == kUnlocked) {
+      cmp = kUnlocked;
+      if (atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
+                                       memory_order_acquire)) {
+#if TSAN_COLLECT_STATS && !SANITIZER_GO
+        StatInc(cur_thread(), stat_type_, backoff.Contention());
+#endif
+        return;
+      }
+    }
+  }
+}
+
+void Mutex::Unlock() {
+  uptr prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
+  (void)prev;
+  DCHECK_NE(prev & kWriteLock, 0);
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  cur_thread()->internal_deadlock_detector.Unlock(type_);
+#endif
+}
+
+void Mutex::ReadLock() {
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  cur_thread()->internal_deadlock_detector.Lock(type_);
+#endif
+  uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
+  if ((prev & kWriteLock) == 0)
+    return;
+  for (Backoff backoff; backoff.Do();) {
+    prev = atomic_load(&state_, memory_order_acquire);
+    if ((prev & kWriteLock) == 0) {
+#if TSAN_COLLECT_STATS && !SANITIZER_GO
+      StatInc(cur_thread(), stat_type_, backoff.Contention());
+#endif
+      return;
+    }
+  }
+}
+
+void Mutex::ReadUnlock() {
+  uptr prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
+  (void)prev;
+  DCHECK_EQ(prev & kWriteLock, 0);
+  DCHECK_GT(prev & ~kWriteLock, 0);
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  cur_thread()->internal_deadlock_detector.Unlock(type_);
+#endif
+}
+
+void Mutex::CheckLocked() {
+  CHECK_NE(atomic_load(&state_, memory_order_relaxed), 0);
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_mutex.h b/lib/tsan/tsan_mutex.h
new file mode 100644
index 0000000000..80fdc6ed57
--- /dev/null
+++ b/lib/tsan/tsan_mutex.h
@@ -0,0 +1,90 @@
+//===-- tsan_mutex.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_MUTEX_H
+#define TSAN_MUTEX_H
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+enum MutexType {
+  MutexTypeInvalid,
+  MutexTypeTrace,
+  MutexTypeThreads,
+  MutexTypeReport,
+  MutexTypeSyncVar,
+  MutexTypeSyncTab,
+  MutexTypeSlab,
+  MutexTypeAnnotations,
+  MutexTypeAtExit,
+  MutexTypeMBlock,
+  MutexTypeJavaMBlock,
+  MutexTypeDDetector,
+  MutexTypeFired,
+  MutexTypeRacy,
+  MutexTypeGlobalProc,
+
+  // This must be the last.
+  MutexTypeCount
+};
+
+class Mutex {
+ public:
+  explicit Mutex(MutexType type, StatType stat_type);
+  ~Mutex();
+
+  void Lock();
+  void Unlock();
+
+  void ReadLock();
+  void ReadUnlock();
+
+  void CheckLocked();
+
+ private:
+  atomic_uintptr_t state_;
+#if SANITIZER_DEBUG
+  MutexType type_;
+#endif
+#if TSAN_COLLECT_STATS
+  StatType stat_type_;
+#endif
+
+  Mutex(const Mutex&);
+  void operator = (const Mutex&);
+};
+
+typedef GenericScopedLock<Mutex> Lock;
+typedef GenericScopedReadLock<Mutex> ReadLock;
+
+class InternalDeadlockDetector {
+ public:
+  InternalDeadlockDetector();
+  void Lock(MutexType t);
+  void Unlock(MutexType t);
+  void CheckNoLocks();
+ private:
+  u64 seq_;
+  u64 locked_[MutexTypeCount];
+};
+
+void InitializeMutex();
+
+// Checks that the current thread does not hold any runtime locks
+// (e.g. when returning from an interceptor).
+void CheckNoLocks(ThreadState *thr);
+
+}  // namespace __tsan
+
+#endif  // TSAN_MUTEX_H
diff --git a/lib/tsan/tsan_mutexset.cpp b/lib/tsan/tsan_mutexset.cpp
new file mode 100644
index 0000000000..813fa3bca9
--- /dev/null
+++ b/lib/tsan/tsan_mutexset.cpp
@@ -0,0 +1,88 @@
+//===-- tsan_mutexset.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_mutexset.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+const uptr MutexSet::kMaxSize;
+
+MutexSet::MutexSet() {
+  size_ = 0;
+  internal_memset(&descs_, 0, sizeof(descs_));
+}
+
+void MutexSet::Add(u64 id, bool write, u64 epoch) {
+  // Look up existing mutex with the same id.
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      descs_[i].count++;
+      descs_[i].epoch = epoch;
+      return;
+    }
+  }
+  // On overflow, find the oldest mutex and drop it.
+  if (size_ == kMaxSize) {
+    u64 minepoch = (u64)-1;
+    u64 mini = (u64)-1;
+    for (uptr i = 0; i < size_; i++) {
+      if (descs_[i].epoch < minepoch) {
+        minepoch = descs_[i].epoch;
+        mini = i;
+      }
+    }
+    RemovePos(mini);
+    CHECK_EQ(size_, kMaxSize - 1);
+  }
+  // Add new mutex descriptor.
+  descs_[size_].id = id;
+  descs_[size_].write = write;
+  descs_[size_].epoch = epoch;
+  descs_[size_].count = 1;
+  size_++;
+}
+
+void MutexSet::Del(u64 id, bool write) {
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      if (--descs_[i].count == 0)
+        RemovePos(i);
+      return;
+    }
+  }
+}
+
+void MutexSet::Remove(u64 id) {
+  for (uptr i = 0; i < size_; i++) {
+    if (descs_[i].id == id) {
+      RemovePos(i);
+      return;
+    }
+  }
+}
+
+void MutexSet::RemovePos(uptr i) {
+  CHECK_LT(i, size_);
+  descs_[i] = descs_[size_ - 1];
+  size_--;
+}
+
+uptr MutexSet::Size() const {
+  return size_;
+}
+
+MutexSet::Desc MutexSet::Get(uptr i) const {
+  CHECK_LT(i, size_);
+  return descs_[i];
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_mutexset.h b/lib/tsan/tsan_mutexset.h
new file mode 100644
index 0000000000..d63881f402
--- /dev/null
+++ b/lib/tsan/tsan_mutexset.h
@@ -0,0 +1,69 @@
+//===-- tsan_mutexset.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// MutexSet holds the set of mutexes currently held by a thread.
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_MUTEXSET_H
+#define TSAN_MUTEXSET_H
+
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+class MutexSet {
+ public:
+  // Holds limited number of mutexes.
+  // The oldest mutexes are discarded on overflow.
+  static const uptr kMaxSize = 16;
+  struct Desc {
+    u64 id;
+    u64 epoch;
+    int count;
+    bool write;
+  };
+
+  MutexSet();
+  // The 'id' is obtained from SyncVar::GetId().
+  void Add(u64 id, bool write, u64 epoch);
+  void Del(u64 id, bool write);
+  void Remove(u64 id);  // Removes the mutex completely (if it's destroyed).
+  uptr Size() const;
+  Desc Get(uptr i) const;
+
+  void operator=(const MutexSet &other) {
+    internal_memcpy(this, &other, sizeof(*this));
+  }
+
+ private:
+#if !SANITIZER_GO
+  uptr size_;
+  Desc descs_[kMaxSize];
+#endif
+
+  void RemovePos(uptr i);
+  MutexSet(const MutexSet&);
+};
+
+// Go does not have mutexes, so do not spend memory and time.
+// (Go sync.Mutex is actually a semaphore -- can be unlocked
+// in different goroutine).
+#if SANITIZER_GO
+MutexSet::MutexSet() {}
+void MutexSet::Add(u64 id, bool write, u64 epoch) {}
+void MutexSet::Del(u64 id, bool write) {}
+void MutexSet::Remove(u64 id) {}
+void MutexSet::RemovePos(uptr i) {}
+uptr MutexSet::Size() const { return 0; }
+MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
+#endif
+
+}  // namespace __tsan
+
+#endif  // TSAN_MUTEXSET_H
diff --git a/lib/tsan/tsan_platform.h b/lib/tsan/tsan_platform.h
new file mode 100644
index 0000000000..7256d64e50
--- /dev/null
+++ b/lib/tsan/tsan_platform.h
@@ -0,0 +1,1028 @@
+//===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Platform-specific code.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_PLATFORM_H
+#define TSAN_PLATFORM_H
+
+#if !defined(__LP64__) && !defined(_WIN64)
+# error "Only 64-bit is supported"
+#endif
+
+#include "tsan_defs.h"
+#include "tsan_trace.h"
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+
+#if defined(__x86_64__)
+/*
+C/C++ on linux/x86_64 and freebsd/x86_64
+0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
+0040 0000 0000 - 0100 0000 0000: -
+0100 0000 0000 - 2000 0000 0000: shadow
+2000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 5500 0000 0000: -
+5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
+5680 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 7d00 0000 0000: -
+7b00 0000 0000 - 7c00 0000 0000: heap
+7c00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+
+C/C++ on netbsd/amd64 can reuse the same mapping:
+ * The address space starts from 0x1000 (option with 0x0) and ends with
+   0x7f7ffffff000.
+ * LoAppMem-kHeapMemEnd can be reused as it is.
+ * No VDSO support.
+ * No MidAppMem region.
+ * No additional HeapMem region.
+ * HiAppMem contains the stack, loader, shared libraries and heap.
+ * Stack on NetBSD/amd64 has prereserved 128MB.
+ * Heap grows downwards (top-down).
+ * ASLR must be disabled per-process or globally.
+
+*/
+struct Mapping {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x340000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x200000000000ull;
+  static const uptr kHeapMemBeg    = 0x7b0000000000ull;
+  static const uptr kHeapMemEnd    = 0x7c0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x008000000000ull;
+  static const uptr kMidAppMemBeg  = 0x550000000000ull;
+  static const uptr kMidAppMemEnd  = 0x568000000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x800000000000ull;
+  static const uptr kAppMemMsk     = 0x780000000000ull;
+  static const uptr kAppMemXor     = 0x040000000000ull;
+  static const uptr kVdsoBeg       = 0xf000000000000000ull;
+};
+
+#define TSAN_MID_APP_RANGE 1
+#elif defined(__mips64)
+/*
+C/C++ on linux/mips64 (40-bit VMA)
+0000 0000 00 - 0100 0000 00: -                                           (4 GB)
+0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
+0200 0000 00 - 2000 0000 00: -                                         (120 GB)
+2000 0000 00 - 4000 0000 00: shadow                                    (128 GB)
+4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
+5000 0000 00 - aa00 0000 00: -                                         (360 GB)
+aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
+ab00 0000 00 - b000 0000 00: -                                          (20 GB)
+b000 0000 00 - b200 0000 00: traces                                      (8 GB)
+b200 0000 00 - fe00 0000 00: -                                         (304 GB)
+fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
+ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
+ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
+*/
+struct Mapping {
+  static const uptr kMetaShadowBeg = 0x4000000000ull;
+  static const uptr kMetaShadowEnd = 0x5000000000ull;
+  static const uptr kTraceMemBeg   = 0xb000000000ull;
+  static const uptr kTraceMemEnd   = 0xb200000000ull;
+  static const uptr kShadowBeg     = 0x2000000000ull;
+  static const uptr kShadowEnd     = 0x4000000000ull;
+  static const uptr kHeapMemBeg    = 0xfe00000000ull;
+  static const uptr kHeapMemEnd    = 0xff00000000ull;
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kMidAppMemBeg  = 0xaa00000000ull;
+  static const uptr kMidAppMemEnd  = 0xab00000000ull;
+  static const uptr kHiAppMemBeg   = 0xff80000000ull;
+  static const uptr kHiAppMemEnd   = 0xffffffffffull;
+  static const uptr kAppMemMsk     = 0xf800000000ull;
+  static const uptr kAppMemXor     = 0x0800000000ull;
+  static const uptr kVdsoBeg       = 0xfffff00000ull;
+};
+
+#define TSAN_MID_APP_RANGE 1
+#elif defined(__aarch64__) && defined(__APPLE__)
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: -                                    (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
+0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
+0300 0000 00 - 0400 0000 00: -                                    (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
+0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
+0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
+0f00 0000 00 - 0fc0 0000 00: traces                               (3 GB)
+0fc0 0000 00 - 1000 0000 00: -
+*/
+struct Mapping {
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kHeapMemBeg    = 0x0200000000ull;
+  static const uptr kHeapMemEnd    = 0x0300000000ull;
+  static const uptr kShadowBeg     = 0x0400000000ull;
+  static const uptr kShadowEnd     = 0x0c00000000ull;
+  static const uptr kMetaShadowBeg = 0x0d00000000ull;
+  static const uptr kMetaShadowEnd = 0x0e00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f00000000ull;
+  static const uptr kTraceMemEnd   = 0x0fc0000000ull;
+  static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
+  static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
+  static const uptr kAppMemMsk     =          0x0ull;
+  static const uptr kAppMemXor     =          0x0ull;
+  static const uptr kVdsoBeg       = 0x7000000000000000ull;
+};
+
+#elif defined(__aarch64__)
+// AArch64 supports multiple VMA which leads to multiple address transformation
+// functions.  To support these multiple VMAS transformations and mappings TSAN
+// runtime for AArch64 uses an external memory read (vmaSize) to select which
+// mapping to use.  Although slower, it make a same instrumented binary run on
+// multiple kernels.
+
+/*
+C/C++ on linux/aarch64 (39-bit VMA)
+0000 0010 00 - 0100 0000 00: main binary
+0100 0000 00 - 0800 0000 00: -
+0800 0000 00 - 2000 0000 00: shadow memory
+2000 0000 00 - 3100 0000 00: -
+3100 0000 00 - 3400 0000 00: metainfo
+3400 0000 00 - 5500 0000 00: -
+5500 0000 00 - 5600 0000 00: main binary (PIE)
+5600 0000 00 - 6000 0000 00: -
+6000 0000 00 - 6200 0000 00: traces
+6200 0000 00 - 7d00 0000 00: -
+7c00 0000 00 - 7d00 0000 00: heap
+7d00 0000 00 - 7fff ffff ff: modules and main thread stack
+*/
+struct Mapping39 {
+  static const uptr kLoAppMemBeg   = 0x0000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0100000000ull;
+  static const uptr kShadowBeg     = 0x0800000000ull;
+  static const uptr kShadowEnd     = 0x2000000000ull;
+  static const uptr kMetaShadowBeg = 0x3100000000ull;
+  static const uptr kMetaShadowEnd = 0x3400000000ull;
+  static const uptr kMidAppMemBeg  = 0x5500000000ull;
+  static const uptr kMidAppMemEnd  = 0x5600000000ull;
+  static const uptr kTraceMemBeg   = 0x6000000000ull;
+  static const uptr kTraceMemEnd   = 0x6200000000ull;
+  static const uptr kHeapMemBeg    = 0x7c00000000ull;
+  static const uptr kHeapMemEnd    = 0x7d00000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e00000000ull;
+  static const uptr kHiAppMemEnd   = 0x7fffffffffull;
+  static const uptr kAppMemMsk     = 0x7800000000ull;
+  static const uptr kAppMemXor     = 0x0200000000ull;
+  static const uptr kVdsoBeg       = 0x7f00000000ull;
+};
+
+/*
+C/C++ on linux/aarch64 (42-bit VMA)
+00000 0010 00 - 01000 0000 00: main binary
+01000 0000 00 - 10000 0000 00: -
+10000 0000 00 - 20000 0000 00: shadow memory
+20000 0000 00 - 26000 0000 00: -
+26000 0000 00 - 28000 0000 00: metainfo
+28000 0000 00 - 2aa00 0000 00: -
+2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
+2ab00 0000 00 - 36200 0000 00: -
+36200 0000 00 - 36240 0000 00: traces
+36240 0000 00 - 3e000 0000 00: -
+3e000 0000 00 - 3f000 0000 00: heap
+3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
+*/
+struct Mapping42 {
+  static const uptr kLoAppMemBeg   = 0x00000001000ull;
+  static const uptr kLoAppMemEnd   = 0x01000000000ull;
+  static const uptr kShadowBeg     = 0x10000000000ull;
+  static const uptr kShadowEnd     = 0x20000000000ull;
+  static const uptr kMetaShadowBeg = 0x26000000000ull;
+  static const uptr kMetaShadowEnd = 0x28000000000ull;
+  static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
+  static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
+  static const uptr kTraceMemBeg   = 0x36200000000ull;
+  static const uptr kTraceMemEnd   = 0x36400000000ull;
+  static const uptr kHeapMemBeg    = 0x3e000000000ull;
+  static const uptr kHeapMemEnd    = 0x3f000000000ull;
+  static const uptr kHiAppMemBeg   = 0x3f000000000ull;
+  static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
+  static const uptr kAppMemMsk     = 0x3c000000000ull;
+  static const uptr kAppMemXor     = 0x04000000000ull;
+  static const uptr kVdsoBeg       = 0x37f00000000ull;
+};
+
+struct Mapping48 {
+  static const uptr kLoAppMemBeg   = 0x0000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0000200000000ull;
+  static const uptr kShadowBeg     = 0x0002000000000ull;
+  static const uptr kShadowEnd     = 0x0004000000000ull;
+  static const uptr kMetaShadowBeg = 0x0005000000000ull;
+  static const uptr kMetaShadowEnd = 0x0006000000000ull;
+  static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
+  static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f06000000000ull;
+  static const uptr kTraceMemEnd   = 0x0f06200000000ull;
+  static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
+  static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
+  static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
+  static const uptr kHiAppMemEnd   = 0x1000000000000ull;
+  static const uptr kAppMemMsk     = 0x0fff800000000ull;
+  static const uptr kAppMemXor     = 0x0000800000000ull;
+  static const uptr kVdsoBeg       = 0xffff000000000ull;
+};
+
+// Indicates the runtime will define the memory regions at runtime.
+#define TSAN_RUNTIME_VMA 1
+// Indicates that mapping defines a mid range memory segment.
+#define TSAN_MID_APP_RANGE 1
+#elif defined(__powerpc64__)
+// PPC64 supports multiple VMA which leads to multiple address transformation
+// functions.  To support these multiple VMAS transformations and mappings TSAN
+// runtime for PPC64 uses an external memory read (vmaSize) to select which
+// mapping to use.  Although slower, it make a same instrumented binary run on
+// multiple kernels.
+
+/*
+C/C++ on linux/powerpc64 (44-bit VMA)
+0000 0000 0100 - 0001 0000 0000: main binary
+0001 0000 0000 - 0001 0000 0000: -
+0001 0000 0000 - 0b00 0000 0000: shadow
+0b00 0000 0000 - 0b00 0000 0000: -
+0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
+0d00 0000 0000 - 0d00 0000 0000: -
+0d00 0000 0000 - 0f00 0000 0000: traces
+0f00 0000 0000 - 0f00 0000 0000: -
+0f00 0000 0000 - 0f50 0000 0000: heap
+0f50 0000 0000 - 0f60 0000 0000: -
+0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
+*/
+struct Mapping44 {
+  static const uptr kMetaShadowBeg = 0x0b0000000000ull;
+  static const uptr kMetaShadowEnd = 0x0d0000000000ull;
+  static const uptr kTraceMemBeg   = 0x0d0000000000ull;
+  static const uptr kTraceMemEnd   = 0x0f0000000000ull;
+  static const uptr kShadowBeg     = 0x000100000000ull;
+  static const uptr kShadowEnd     = 0x0b0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000000100ull;
+  static const uptr kLoAppMemEnd   = 0x000100000000ull;
+  static const uptr kHeapMemBeg    = 0x0f0000000000ull;
+  static const uptr kHeapMemEnd    = 0x0f5000000000ull;
+  static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
+  static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
+  static const uptr kAppMemMsk     = 0x0f0000000000ull;
+  static const uptr kAppMemXor     = 0x002100000000ull;
+  static const uptr kVdsoBeg       = 0x3c0000000000000ull;
+};
+
+/*
+C/C++ on linux/powerpc64 (46-bit VMA)
+0000 0000 1000 - 0100 0000 0000: main binary
+0100 0000 0000 - 0200 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
+2000 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2200 0000 0000: traces
+2200 0000 0000 - 3d00 0000 0000: -
+3d00 0000 0000 - 3e00 0000 0000: heap
+3e00 0000 0000 - 3e80 0000 0000: -
+3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
+*/
+struct Mapping46 {
+  static const uptr kMetaShadowBeg = 0x100000000000ull;
+  static const uptr kMetaShadowEnd = 0x200000000000ull;
+  static const uptr kTraceMemBeg   = 0x200000000000ull;
+  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kHeapMemBeg    = 0x3d0000000000ull;
+  static const uptr kHeapMemEnd    = 0x3e0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x010000000000ull;
+  static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
+  static const uptr kAppMemMsk     = 0x3c0000000000ull;
+  static const uptr kAppMemXor     = 0x020000000000ull;
+  static const uptr kVdsoBeg       = 0x7800000000000000ull;
+};
+
+/*
+C/C++ on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0100 0000 0000: main binary
+0100 0000 0000 - 0200 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
+2000 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2200 0000 0000: traces
+2200 0000 0000 - 7d00 0000 0000: -
+7d00 0000 0000 - 7e00 0000 0000: heap
+7e00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+*/
+struct Mapping47 {
+  static const uptr kMetaShadowBeg = 0x100000000000ull;
+  static const uptr kMetaShadowEnd = 0x200000000000ull;
+  static const uptr kTraceMemBeg   = 0x200000000000ull;
+  static const uptr kTraceMemEnd   = 0x220000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x100000000000ull;
+  static const uptr kHeapMemBeg    = 0x7d0000000000ull;
+  static const uptr kHeapMemEnd    = 0x7e0000000000ull;
+  static const uptr kLoAppMemBeg   = 0x000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x010000000000ull;
+  static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
+  static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
+  static const uptr kAppMemMsk     = 0x7c0000000000ull;
+  static const uptr kAppMemXor     = 0x020000000000ull;
+  static const uptr kVdsoBeg       = 0x7800000000000000ull;
+};
+
+// Indicates the runtime will define the memory regions at runtime.
+#define TSAN_RUNTIME_VMA 1
+#endif
+
+#elif SANITIZER_GO && !SANITIZER_WINDOWS && defined(__x86_64__)
+
+/* Go on linux, darwin and freebsd on x86_64
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2380 0000 0000: shadow
+2380 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+struct Mapping {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+#elif SANITIZER_GO && SANITIZER_WINDOWS
+
+/* Go on windows
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00f8 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 0100 0000 0000: -
+0100 0000 0000 - 0500 0000 0000: shadow
+0500 0000 0000 - 0560 0000 0000: -
+0560 0000 0000 - 0760 0000 0000: traces
+0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
+07d0 0000 0000 - 8000 0000 0000: -
+*/
+
+struct Mapping {
+  static const uptr kMetaShadowBeg = 0x076000000000ull;
+  static const uptr kMetaShadowEnd = 0x07d000000000ull;
+  static const uptr kTraceMemBeg   = 0x056000000000ull;
+  static const uptr kTraceMemEnd   = 0x076000000000ull;
+  static const uptr kShadowBeg     = 0x010000000000ull;
+  static const uptr kShadowEnd     = 0x050000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+#elif SANITIZER_GO && defined(__powerpc64__)
+
+/* Only Mapping46 and Mapping47 are currently supported for powercp64 on Go. */
+
+/* Go on linux/powerpc64 (46-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2380 0000 0000: shadow
+2380 0000 0000 - 2400 0000 0000: -
+2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
+3400 0000 0000 - 3600 0000 0000: -
+3600 0000 0000 - 3800 0000 0000: traces
+3800 0000 0000 - 4000 0000 0000: -
+*/
+
+struct Mapping46 {
+  static const uptr kMetaShadowBeg = 0x240000000000ull;
+  static const uptr kMetaShadowEnd = 0x340000000000ull;
+  static const uptr kTraceMemBeg   = 0x360000000000ull;
+  static const uptr kTraceMemEnd   = 0x380000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x238000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+/* Go on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+struct Mapping47 {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+#define TSAN_RUNTIME_VMA 1
+
+#elif SANITIZER_GO && defined(__aarch64__)
+
+/* Go on linux/aarch64 (48-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+
+struct Mapping {
+  static const uptr kMetaShadowBeg = 0x300000000000ull;
+  static const uptr kMetaShadowEnd = 0x400000000000ull;
+  static const uptr kTraceMemBeg   = 0x600000000000ull;
+  static const uptr kTraceMemEnd   = 0x620000000000ull;
+  static const uptr kShadowBeg     = 0x200000000000ull;
+  static const uptr kShadowEnd     = 0x300000000000ull;
+  static const uptr kAppMemBeg     = 0x000000001000ull;
+  static const uptr kAppMemEnd     = 0x00e000000000ull;
+};
+
+// Indicates the runtime will define the memory regions at runtime.
+#define TSAN_RUNTIME_VMA 1
+
+#else
+# error "Unknown platform"
+#endif
+
+
+#ifdef TSAN_RUNTIME_VMA
+extern uptr vmaSize;
+#endif
+
+
+enum MappingType {
+  MAPPING_LO_APP_BEG,
+  MAPPING_LO_APP_END,
+  MAPPING_HI_APP_BEG,
+  MAPPING_HI_APP_END,
+#ifdef TSAN_MID_APP_RANGE
+  MAPPING_MID_APP_BEG,
+  MAPPING_MID_APP_END,
+#endif
+  MAPPING_HEAP_BEG,
+  MAPPING_HEAP_END,
+  MAPPING_APP_BEG,
+  MAPPING_APP_END,
+  MAPPING_SHADOW_BEG,
+  MAPPING_SHADOW_END,
+  MAPPING_META_SHADOW_BEG,
+  MAPPING_META_SHADOW_END,
+  MAPPING_TRACE_BEG,
+  MAPPING_TRACE_END,
+  MAPPING_VDSO_BEG,
+};
+
+template<typename Mapping, int Type>
+uptr MappingImpl(void) {
+  switch (Type) {
+#if !SANITIZER_GO
+    case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
+    case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
+# ifdef TSAN_MID_APP_RANGE
+    case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
+    case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
+# endif
+    case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
+    case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
+    case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
+    case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
+    case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
+#else
+    case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
+    case MAPPING_APP_END: return Mapping::kAppMemEnd;
+#endif
+    case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
+    case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
+    case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
+    case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
+    case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
+    case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
+  }
+}
+
+template<int Type>
+uptr MappingArchImpl(void) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return MappingImpl<Mapping39, Type>();
+    case 42: return MappingImpl<Mapping42, Type>();
+    case 48: return MappingImpl<Mapping48, Type>();
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return MappingImpl<Mapping44, Type>();
+#endif
+    case 46: return MappingImpl<Mapping46, Type>();
+    case 47: return MappingImpl<Mapping47, Type>();
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return MappingImpl<Mapping, Type>();
+#endif
+}
+
+#if !SANITIZER_GO
+ALWAYS_INLINE
+uptr LoAppMemBeg(void) {
+  return MappingArchImpl<MAPPING_LO_APP_BEG>();
+}
+ALWAYS_INLINE
+uptr LoAppMemEnd(void) {
+  return MappingArchImpl<MAPPING_LO_APP_END>();
+}
+
+#ifdef TSAN_MID_APP_RANGE
+ALWAYS_INLINE
+uptr MidAppMemBeg(void) {
+  return MappingArchImpl<MAPPING_MID_APP_BEG>();
+}
+ALWAYS_INLINE
+uptr MidAppMemEnd(void) {
+  return MappingArchImpl<MAPPING_MID_APP_END>();
+}
+#endif
+
+ALWAYS_INLINE
+uptr HeapMemBeg(void) {
+  return MappingArchImpl<MAPPING_HEAP_BEG>();
+}
+ALWAYS_INLINE
+uptr HeapMemEnd(void) {
+  return MappingArchImpl<MAPPING_HEAP_END>();
+}
+
+ALWAYS_INLINE
+uptr HiAppMemBeg(void) {
+  return MappingArchImpl<MAPPING_HI_APP_BEG>();
+}
+ALWAYS_INLINE
+uptr HiAppMemEnd(void) {
+  return MappingArchImpl<MAPPING_HI_APP_END>();
+}
+
+ALWAYS_INLINE
+uptr VdsoBeg(void) {
+  return MappingArchImpl<MAPPING_VDSO_BEG>();
+}
+
+#else
+
+ALWAYS_INLINE
+uptr AppMemBeg(void) {
+  return MappingArchImpl<MAPPING_APP_BEG>();
+}
+ALWAYS_INLINE
+uptr AppMemEnd(void) {
+  return MappingArchImpl<MAPPING_APP_END>();
+}
+
+#endif
+
+static inline
+bool GetUserRegion(int i, uptr *start, uptr *end) {
+  switch (i) {
+  default:
+    return false;
+#if !SANITIZER_GO
+  case 0:
+    *start = LoAppMemBeg();
+    *end = LoAppMemEnd();
+    return true;
+  case 1:
+    *start = HiAppMemBeg();
+    *end = HiAppMemEnd();
+    return true;
+  case 2:
+    *start = HeapMemBeg();
+    *end = HeapMemEnd();
+    return true;
+# ifdef TSAN_MID_APP_RANGE
+  case 3:
+    *start = MidAppMemBeg();
+    *end = MidAppMemEnd();
+    return true;
+# endif
+#else
+  case 0:
+    *start = AppMemBeg();
+    *end = AppMemEnd();
+    return true;
+#endif
+  }
+}
+
+ALWAYS_INLINE
+uptr ShadowBeg(void) {
+  return MappingArchImpl<MAPPING_SHADOW_BEG>();
+}
+ALWAYS_INLINE
+uptr ShadowEnd(void) {
+  return MappingArchImpl<MAPPING_SHADOW_END>();
+}
+
+ALWAYS_INLINE
+uptr MetaShadowBeg(void) {
+  return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
+}
+ALWAYS_INLINE
+uptr MetaShadowEnd(void) {
+  return MappingArchImpl<MAPPING_META_SHADOW_END>();
+}
+
+ALWAYS_INLINE
+uptr TraceMemBeg(void) {
+  return MappingArchImpl<MAPPING_TRACE_BEG>();
+}
+ALWAYS_INLINE
+uptr TraceMemEnd(void) {
+  return MappingArchImpl<MAPPING_TRACE_END>();
+}
+
+
+template<typename Mapping>
+bool IsAppMemImpl(uptr mem) {
+#if !SANITIZER_GO
+  return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
+# ifdef TSAN_MID_APP_RANGE
+         (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
+# endif
+         (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
+         (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
+#else
+  return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
+#endif
+}
+
+ALWAYS_INLINE
+bool IsAppMem(uptr mem) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return IsAppMemImpl<Mapping39>(mem);
+    case 42: return IsAppMemImpl<Mapping42>(mem);
+    case 48: return IsAppMemImpl<Mapping48>(mem);
+  }
+  DCHECK(0);
+  return false;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return IsAppMemImpl<Mapping44>(mem);
+#endif
+    case 46: return IsAppMemImpl<Mapping46>(mem);
+    case 47: return IsAppMemImpl<Mapping47>(mem);
+  }
+  DCHECK(0);
+  return false;
+#else
+  return IsAppMemImpl<Mapping>(mem);
+#endif
+}
+
+
+template<typename Mapping>
+bool IsShadowMemImpl(uptr mem) {
+  return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
+}
+
+ALWAYS_INLINE
+bool IsShadowMem(uptr mem) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return IsShadowMemImpl<Mapping39>(mem);
+    case 42: return IsShadowMemImpl<Mapping42>(mem);
+    case 48: return IsShadowMemImpl<Mapping48>(mem);
+  }
+  DCHECK(0);
+  return false;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return IsShadowMemImpl<Mapping44>(mem);
+#endif
+    case 46: return IsShadowMemImpl<Mapping46>(mem);
+    case 47: return IsShadowMemImpl<Mapping47>(mem);
+  }
+  DCHECK(0);
+  return false;
+#else
+  return IsShadowMemImpl<Mapping>(mem);
+#endif
+}
+
+
+template<typename Mapping>
+bool IsMetaMemImpl(uptr mem) {
+  return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
+}
+
+ALWAYS_INLINE
+bool IsMetaMem(uptr mem) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return IsMetaMemImpl<Mapping39>(mem);
+    case 42: return IsMetaMemImpl<Mapping42>(mem);
+    case 48: return IsMetaMemImpl<Mapping48>(mem);
+  }
+  DCHECK(0);
+  return false;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return IsMetaMemImpl<Mapping44>(mem);
+#endif
+    case 46: return IsMetaMemImpl<Mapping46>(mem);
+    case 47: return IsMetaMemImpl<Mapping47>(mem);
+  }
+  DCHECK(0);
+  return false;
+#else
+  return IsMetaMemImpl<Mapping>(mem);
+#endif
+}
+
+
+template<typename Mapping>
+uptr MemToShadowImpl(uptr x) {
+  DCHECK(IsAppMem(x));
+#if !SANITIZER_GO
+  return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1)))
+      ^ Mapping::kAppMemXor) * kShadowCnt;
+#else
+# ifndef SANITIZER_WINDOWS
+  return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg;
+# else
+  return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
+# endif
+#endif
+}
+
+ALWAYS_INLINE
+uptr MemToShadow(uptr x) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return MemToShadowImpl<Mapping39>(x);
+    case 42: return MemToShadowImpl<Mapping42>(x);
+    case 48: return MemToShadowImpl<Mapping48>(x);
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return MemToShadowImpl<Mapping44>(x);
+#endif
+    case 46: return MemToShadowImpl<Mapping46>(x);
+    case 47: return MemToShadowImpl<Mapping47>(x);
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return MemToShadowImpl<Mapping>(x);
+#endif
+}
+
+
+template<typename Mapping>
+u32 *MemToMetaImpl(uptr x) {
+  DCHECK(IsAppMem(x));
+#if !SANITIZER_GO
+  return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
+      kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
+#else
+# ifndef SANITIZER_WINDOWS
+  return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
+      kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
+# else
+  return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
+      kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
+# endif
+#endif
+}
+
+ALWAYS_INLINE
+u32 *MemToMeta(uptr x) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return MemToMetaImpl<Mapping39>(x);
+    case 42: return MemToMetaImpl<Mapping42>(x);
+    case 48: return MemToMetaImpl<Mapping48>(x);
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return MemToMetaImpl<Mapping44>(x);
+#endif
+    case 46: return MemToMetaImpl<Mapping46>(x);
+    case 47: return MemToMetaImpl<Mapping47>(x);
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return MemToMetaImpl<Mapping>(x);
+#endif
+}
+
+
+template<typename Mapping>
+uptr ShadowToMemImpl(uptr s) {
+  DCHECK(IsShadowMem(s));
+#if !SANITIZER_GO
+  // The shadow mapping is non-linear and we've lost some bits, so we don't have
+  // an easy way to restore the original app address. But the mapping is a
+  // bijection, so we try to restore the address as belonging to low/mid/high
+  // range consecutively and see if shadow->app->shadow mapping gives us the
+  // same address.
+  uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
+  if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
+      MemToShadow(p) == s)
+    return p;
+# ifdef TSAN_MID_APP_RANGE
+  p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
+      (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
+  if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
+      MemToShadow(p) == s)
+    return p;
+# endif
+  return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
+#else  // #if !SANITIZER_GO
+# ifndef SANITIZER_WINDOWS
+  return (s & ~Mapping::kShadowBeg) / kShadowCnt;
+# else
+  return (s - Mapping::kShadowBeg) / kShadowCnt;
+# endif // SANITIZER_WINDOWS
+#endif
+}
+
+ALWAYS_INLINE
+uptr ShadowToMem(uptr s) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return ShadowToMemImpl<Mapping39>(s);
+    case 42: return ShadowToMemImpl<Mapping42>(s);
+    case 48: return ShadowToMemImpl<Mapping48>(s);
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return ShadowToMemImpl<Mapping44>(s);
+#endif
+    case 46: return ShadowToMemImpl<Mapping46>(s);
+    case 47: return ShadowToMemImpl<Mapping47>(s);
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return ShadowToMemImpl<Mapping>(s);
+#endif
+}
+
+
+
+// The additional page is to catch shadow stack overflow as paging fault.
+// Windows wants 64K alignment for mmaps.
+const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
+    + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
+
+template<typename Mapping>
+uptr GetThreadTraceImpl(int tid) {
+  uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
+  DCHECK_LT(p, Mapping::kTraceMemEnd);
+  return p;
+}
+
+ALWAYS_INLINE
+uptr GetThreadTrace(int tid) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return GetThreadTraceImpl<Mapping39>(tid);
+    case 42: return GetThreadTraceImpl<Mapping42>(tid);
+    case 48: return GetThreadTraceImpl<Mapping48>(tid);
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return GetThreadTraceImpl<Mapping44>(tid);
+#endif
+    case 46: return GetThreadTraceImpl<Mapping46>(tid);
+    case 47: return GetThreadTraceImpl<Mapping47>(tid);
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return GetThreadTraceImpl<Mapping>(tid);
+#endif
+}
+
+
+template<typename Mapping>
+uptr GetThreadTraceHeaderImpl(int tid) {
+  uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
+      + kTraceSize * sizeof(Event);
+  DCHECK_LT(p, Mapping::kTraceMemEnd);
+  return p;
+}
+
+ALWAYS_INLINE
+uptr GetThreadTraceHeader(int tid) {
+#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
+  switch (vmaSize) {
+    case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
+    case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
+    case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
+  }
+  DCHECK(0);
+  return 0;
+#elif defined(__powerpc64__)
+  switch (vmaSize) {
+#if !SANITIZER_GO
+    case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
+#endif
+    case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
+    case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
+  }
+  DCHECK(0);
+  return 0;
+#else
+  return GetThreadTraceHeaderImpl<Mapping>(tid);
+#endif
+}
+
+void InitializePlatform();
+void InitializePlatformEarly();
+void CheckAndProtect();
+void InitializeShadowMemoryPlatform();
+void FlushShadowMemory();
+void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
+int ExtractResolvFDs(void *state, int *fds, int nfd);
+int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
+uptr ExtractLongJmpSp(uptr *env);
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
+
+int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
+    void *abstime), void *c, void *m, void *abstime,
+    void(*cleanup)(void *arg), void *arg);
+
+void DestroyThreadState();
+void PlatformCleanUpThreadState(ThreadState *thr);
+
+}  // namespace __tsan
+
+#endif  // TSAN_PLATFORM_H
diff --git a/lib/tsan/tsan_ppc_regs.h b/lib/tsan/tsan_ppc_regs.h
new file mode 100644
index 0000000000..5b43f3ddad
--- /dev/null
+++ b/lib/tsan/tsan_ppc_regs.h
@@ -0,0 +1,96 @@
+#define r0 0
+#define r1 1
+#define r2 2
+#define r3 3
+#define r4 4
+#define r5 5
+#define r6 6
+#define r7 7
+#define r8 8
+#define r9 9
+#define r10 10
+#define r11 11
+#define r12 12
+#define r13 13
+#define r14 14
+#define r15 15
+#define r16 16
+#define r17 17
+#define r18 18
+#define r19 19
+#define r20 20
+#define r21 21
+#define r22 22
+#define r23 23
+#define r24 24
+#define r25 25
+#define r26 26
+#define r27 27
+#define r28 28
+#define r29 29
+#define r30 30
+#define r31 31
+#define f0 0
+#define f1 1
+#define f2 2
+#define f3 3
+#define f4 4
+#define f5 5
+#define f6 6
+#define f7 7
+#define f8 8
+#define f9 9
+#define f10 10
+#define f11 11
+#define f12 12
+#define f13 13
+#define f14 14
+#define f15 15
+#define f16 16
+#define f17 17
+#define f18 18
+#define f19 19
+#define f20 20
+#define f21 21
+#define f22 22
+#define f23 23
+#define f24 24
+#define f25 25
+#define f26 26
+#define f27 27
+#define f28 28
+#define f29 29
+#define f30 30
+#define f31 31
+#define v0 0
+#define v1 1
+#define v2 2
+#define v3 3
+#define v4 4
+#define v5 5
+#define v6 6
+#define v7 7
+#define v8 8
+#define v9 9
+#define v10 10
+#define v11 11
+#define v12 12
+#define v13 13
+#define v14 14
+#define v15 15
+#define v16 16
+#define v17 17
+#define v18 18
+#define v19 19
+#define v20 20
+#define v21 21
+#define v22 22
+#define v23 23
+#define v24 24
+#define v25 25
+#define v26 26
+#define v27 27
+#define v28 28
+#define v29 29
+#define v30 30
+#define v31 31
diff --git a/lib/tsan/tsan_preinit.cpp b/lib/tsan/tsan_preinit.cpp
new file mode 100644
index 0000000000..205bdbf93b
--- /dev/null
+++ b/lib/tsan/tsan_preinit.cpp
@@ -0,0 +1,26 @@
+//===-- tsan_preinit.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer.
+//
+// Call __tsan_init at the very early stage of process startup.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "tsan_interface.h"
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+
+// The symbol is called __local_tsan_preinit, because it's not intended to be
+// exported.
+// This code linked into the main executable when -fsanitize=thread is in
+// the link flags. It can only use exported interface functions.
+__attribute__((section(".preinit_array"), used))
+void (*__local_tsan_preinit)(void) = __tsan_init;
+
+#endif
diff --git a/lib/tsan/tsan_report.cpp b/lib/tsan/tsan_report.cpp
new file mode 100644
index 0000000000..368f1ca8ad
--- /dev/null
+++ b/lib/tsan/tsan_report.cpp
@@ -0,0 +1,486 @@
+//===-- tsan_report.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_report.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stacktrace_printer.h"
+
+namespace __tsan {
+
+ReportStack::ReportStack() : frames(nullptr), suppressable(false) {}
+
+ReportStack *ReportStack::New() {
+  void *mem = internal_alloc(MBlockReportStack, sizeof(ReportStack));
+  return new(mem) ReportStack();
+}
+
+ReportLocation::ReportLocation(ReportLocationType type)
+    : type(type), global(), heap_chunk_start(0), heap_chunk_size(0), tid(0),
+      fd(0), suppressable(false), stack(nullptr) {}
+
+ReportLocation *ReportLocation::New(ReportLocationType type) {
+  void *mem = internal_alloc(MBlockReportStack, sizeof(ReportLocation));
+  return new(mem) ReportLocation(type);
+}
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+  Decorator() : SanitizerCommonDecorator() { }
+  const char *Access()     { return Blue(); }
+  const char *ThreadDescription()    { return Cyan(); }
+  const char *Location()   { return Green(); }
+  const char *Sleep()   { return Yellow(); }
+  const char *Mutex()   { return Magenta(); }
+};
+
+ReportDesc::ReportDesc()
+    : tag(kExternalTagNone)
+    , stacks()
+    , mops()
+    , locs()
+    , mutexes()
+    , threads()
+    , unique_tids()
+    , sleep()
+    , count() {
+}
+
+ReportMop::ReportMop()
+    : mset() {
+}
+
+ReportDesc::~ReportDesc() {
+  // FIXME(dvyukov): it must be leaking a lot of memory.
+}
+
+#if !SANITIZER_GO
+
+const int kThreadBufSize = 32;
+const char *thread_name(char *buf, int tid) {
+  if (tid == 0)
+    return "main thread";
+  internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
+  return buf;
+}
+
+static const char *ReportTypeString(ReportType typ, uptr tag) {
+  switch (typ) {
+    case ReportTypeRace:
+      return "data race";
+    case ReportTypeVptrRace:
+      return "data race on vptr (ctor/dtor vs virtual call)";
+    case ReportTypeUseAfterFree:
+      return "heap-use-after-free";
+    case ReportTypeVptrUseAfterFree:
+      return "heap-use-after-free (virtual call vs free)";
+    case ReportTypeExternalRace: {
+      const char *str = GetReportHeaderFromTag(tag);
+      return str ? str : "race on external object";
+    }
+    case ReportTypeThreadLeak:
+      return "thread leak";
+    case ReportTypeMutexDestroyLocked:
+      return "destroy of a locked mutex";
+    case ReportTypeMutexDoubleLock:
+      return "double lock of a mutex";
+    case ReportTypeMutexInvalidAccess:
+      return "use of an invalid mutex (e.g. uninitialized or destroyed)";
+    case ReportTypeMutexBadUnlock:
+      return "unlock of an unlocked mutex (or by a wrong thread)";
+    case ReportTypeMutexBadReadLock:
+      return "read lock of a write locked mutex";
+    case ReportTypeMutexBadReadUnlock:
+      return "read unlock of a write locked mutex";
+    case ReportTypeSignalUnsafe:
+      return "signal-unsafe call inside of a signal";
+    case ReportTypeErrnoInSignal:
+      return "signal handler spoils errno";
+    case ReportTypeDeadlock:
+      return "lock-order-inversion (potential deadlock)";
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+#if SANITIZER_MAC
+static const char *const kInterposedFunctionPrefix = "wrap_";
+#else
+static const char *const kInterposedFunctionPrefix = "__interceptor_";
+#endif
+
+void PrintStack(const ReportStack *ent) {
+  if (ent == 0 || ent->frames == 0) {
+    Printf("    [failed to restore the stack]\n\n");
+    return;
+  }
+  SymbolizedStack *frame = ent->frames;
+  for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
+    InternalScopedString res(2 * GetPageSizeCached());
+    RenderFrame(&res, common_flags()->stack_trace_format, i, frame->info,
+                common_flags()->symbolize_vs_style,
+                common_flags()->strip_path_prefix, kInterposedFunctionPrefix);
+    Printf("%s\n", res.data());
+  }
+  Printf("\n");
+}
+
+static void PrintMutexSet(Vector<ReportMopMutex> const& mset) {
+  for (uptr i = 0; i < mset.Size(); i++) {
+    if (i == 0)
+      Printf(" (mutexes:");
+    const ReportMopMutex m = mset[i];
+    Printf(" %s M%llu", m.write ? "write" : "read", m.id);
+    Printf(i == mset.Size() - 1 ? ")" : ",");
+  }
+}
+
+static const char *MopDesc(bool first, bool write, bool atomic) {
+  return atomic ? (first ? (write ? "Atomic write" : "Atomic read")
+                : (write ? "Previous atomic write" : "Previous atomic read"))
+                : (first ? (write ? "Write" : "Read")
+                : (write ? "Previous write" : "Previous read"));
+}
+
+static const char *ExternalMopDesc(bool first, bool write) {
+  return first ? (write ? "Modifying" : "Read-only")
+               : (write ? "Previous modifying" : "Previous read-only");
+}
+
+static void PrintMop(const ReportMop *mop, bool first) {
+  Decorator d;
+  char thrbuf[kThreadBufSize];
+  Printf("%s", d.Access());
+  if (mop->external_tag == kExternalTagNone) {
+    Printf("  %s of size %d at %p by %s",
+           MopDesc(first, mop->write, mop->atomic), mop->size,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  } else {
+    const char *object_type = GetObjectTypeFromTag(mop->external_tag);
+    if (object_type == nullptr)
+        object_type = "external object";
+    Printf("  %s access of %s at %p by %s",
+           ExternalMopDesc(first, mop->write), object_type,
+           (void *)mop->addr, thread_name(thrbuf, mop->tid));
+  }
+  PrintMutexSet(mop->mset);
+  Printf(":\n");
+  Printf("%s", d.Default());
+  PrintStack(mop->stack);
+}
+
+static void PrintLocation(const ReportLocation *loc) {
+  Decorator d;
+  char thrbuf[kThreadBufSize];
+  bool print_stack = false;
+  Printf("%s", d.Location());
+  if (loc->type == ReportLocationGlobal) {
+    const DataInfo &global = loc->global;
+    if (global.size != 0)
+      Printf("  Location is global '%s' of size %zu at %p (%s+%p)\n\n",
+             global.name, global.size, global.start,
+             StripModuleName(global.module), global.module_offset);
+    else
+      Printf("  Location is global '%s' at %p (%s+%p)\n\n", global.name,
+             global.start, StripModuleName(global.module),
+             global.module_offset);
+  } else if (loc->type == ReportLocationHeap) {
+    char thrbuf[kThreadBufSize];
+    const char *object_type = GetObjectTypeFromTag(loc->external_tag);
+    if (!object_type) {
+      Printf("  Location is heap block of size %zu at %p allocated by %s:\n",
+             loc->heap_chunk_size, loc->heap_chunk_start,
+             thread_name(thrbuf, loc->tid));
+    } else {
+      Printf("  Location is %s of size %zu at %p allocated by %s:\n",
+             object_type, loc->heap_chunk_size, loc->heap_chunk_start,
+             thread_name(thrbuf, loc->tid));
+    }
+    print_stack = true;
+  } else if (loc->type == ReportLocationStack) {
+    Printf("  Location is stack of %s.\n\n", thread_name(thrbuf, loc->tid));
+  } else if (loc->type == ReportLocationTLS) {
+    Printf("  Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid));
+  } else if (loc->type == ReportLocationFD) {
+    Printf("  Location is file descriptor %d created by %s at:\n",
+        loc->fd, thread_name(thrbuf, loc->tid));
+    print_stack = true;
+  }
+  Printf("%s", d.Default());
+  if (print_stack)
+    PrintStack(loc->stack);
+}
+
+static void PrintMutexShort(const ReportMutex *rm, const char *after) {
+  Decorator d;
+  Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
+}
+
+static void PrintMutexShortWithAddress(const ReportMutex *rm,
+                                       const char *after) {
+  Decorator d;
+  Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
+}
+
+static void PrintMutex(const ReportMutex *rm) {
+  Decorator d;
+  if (rm->destroyed) {
+    Printf("%s", d.Mutex());
+    Printf("  Mutex M%llu is already destroyed.\n\n", rm->id);
+    Printf("%s", d.Default());
+  } else {
+    Printf("%s", d.Mutex());
+    Printf("  Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
+    Printf("%s", d.Default());
+    PrintStack(rm->stack);
+  }
+}
+
+static void PrintThread(const ReportThread *rt) {
+  Decorator d;
+  if (rt->id == 0)  // Little sense in describing the main thread.
+    return;
+  Printf("%s", d.ThreadDescription());
+  Printf("  Thread T%d", rt->id);
+  if (rt->name && rt->name[0] != '\0')
+    Printf(" '%s'", rt->name);
+  char thrbuf[kThreadBufSize];
+  const char *thread_status = rt->running ? "running" : "finished";
+  if (rt->thread_type == ThreadType::Worker) {
+    Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
+    Printf("\n");
+    Printf("%s", d.Default());
+    return;
+  }
+  Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
+         thread_name(thrbuf, rt->parent_tid));
+  if (rt->stack)
+    Printf(" at:");
+  Printf("\n");
+  Printf("%s", d.Default());
+  PrintStack(rt->stack);
+}
+
+static void PrintSleep(const ReportStack *s) {
+  Decorator d;
+  Printf("%s", d.Sleep());
+  Printf("  As if synchronized via sleep:\n");
+  Printf("%s", d.Default());
+  PrintStack(s);
+}
+
+static ReportStack *ChooseSummaryStack(const ReportDesc *rep) {
+  if (rep->mops.Size())
+    return rep->mops[0]->stack;
+  if (rep->stacks.Size())
+    return rep->stacks[0];
+  if (rep->mutexes.Size())
+    return rep->mutexes[0]->stack;
+  if (rep->threads.Size())
+    return rep->threads[0]->stack;
+  return 0;
+}
+
+static bool FrameIsInternal(const SymbolizedStack *frame) {
+  if (frame == 0)
+    return false;
+  const char *file = frame->info.file;
+  const char *module = frame->info.module;
+  if (file != 0 &&
+      (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
+       internal_strstr(file, "sanitizer_common_interceptors.inc") ||
+       internal_strstr(file, "tsan_interface_")))
+    return true;
+  if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_")))
+    return true;
+  return false;
+}
+
+static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) {
+  while (FrameIsInternal(frames) && frames->next)
+    frames = frames->next;
+  return frames;
+}
+
+void PrintReport(const ReportDesc *rep) {
+  Decorator d;
+  Printf("==================\n");
+  const char *rep_typ_str = ReportTypeString(rep->typ, rep->tag);
+  Printf("%s", d.Warning());
+  Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str,
+         (int)internal_getpid());
+  Printf("%s", d.Default());
+
+  if (rep->typ == ReportTypeDeadlock) {
+    char thrbuf[kThreadBufSize];
+    Printf("  Cycle in lock order graph: ");
+    for (uptr i = 0; i < rep->mutexes.Size(); i++)
+      PrintMutexShortWithAddress(rep->mutexes[i], " => ");
+    PrintMutexShort(rep->mutexes[0], "\n\n");
+    CHECK_GT(rep->mutexes.Size(), 0U);
+    CHECK_EQ(rep->mutexes.Size() * (flags()->second_deadlock_stack ? 2 : 1),
+             rep->stacks.Size());
+    for (uptr i = 0; i < rep->mutexes.Size(); i++) {
+      Printf("  Mutex ");
+      PrintMutexShort(rep->mutexes[(i + 1) % rep->mutexes.Size()],
+                      " acquired here while holding mutex ");
+      PrintMutexShort(rep->mutexes[i], " in ");
+      Printf("%s", d.ThreadDescription());
+      Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i]));
+      Printf("%s", d.Default());
+      if (flags()->second_deadlock_stack) {
+        PrintStack(rep->stacks[2*i]);
+        Printf("  Mutex ");
+        PrintMutexShort(rep->mutexes[i],
+                        " previously acquired by the same thread here:\n");
+        PrintStack(rep->stacks[2*i+1]);
+      } else {
+        PrintStack(rep->stacks[i]);
+        if (i == 0)
+          Printf("    Hint: use TSAN_OPTIONS=second_deadlock_stack=1 "
+                 "to get more informative warning message\n\n");
+      }
+    }
+  } else {
+    for (uptr i = 0; i < rep->stacks.Size(); i++) {
+      if (i)
+        Printf("  and:\n");
+      PrintStack(rep->stacks[i]);
+    }
+  }
+
+  for (uptr i = 0; i < rep->mops.Size(); i++)
+    PrintMop(rep->mops[i], i == 0);
+
+  if (rep->sleep)
+    PrintSleep(rep->sleep);
+
+  for (uptr i = 0; i < rep->locs.Size(); i++)
+    PrintLocation(rep->locs[i]);
+
+  if (rep->typ != ReportTypeDeadlock) {
+    for (uptr i = 0; i < rep->mutexes.Size(); i++)
+      PrintMutex(rep->mutexes[i]);
+  }
+
+  for (uptr i = 0; i < rep->threads.Size(); i++)
+    PrintThread(rep->threads[i]);
+
+  if (rep->typ == ReportTypeThreadLeak && rep->count > 1)
+    Printf("  And %d more similar thread leaks.\n\n", rep->count - 1);
+
+  if (ReportStack *stack = ChooseSummaryStack(rep)) {
+    if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames))
+      ReportErrorSummary(rep_typ_str, frame->info);
+  }
+
+  if (common_flags()->print_module_map == 2) PrintModuleMap();
+
+  Printf("==================\n");
+}
+
+#else  // #if !SANITIZER_GO
+
+const int kMainThreadId = 1;
+
+void PrintStack(const ReportStack *ent) {
+  if (ent == 0 || ent->frames == 0) {
+    Printf("  [failed to restore the stack]\n");
+    return;
+  }
+  SymbolizedStack *frame = ent->frames;
+  for (int i = 0; frame; frame = frame->next, i++) {
+    const AddressInfo &info = frame->info;
+    Printf("  %s()\n      %s:%d +0x%zx\n", info.function,
+        StripPathPrefix(info.file, common_flags()->strip_path_prefix),
+        info.line, (void *)info.module_offset);
+  }
+}
+
+static void PrintMop(const ReportMop *mop, bool first) {
+  Printf("\n");
+  Printf("%s at %p by ",
+      (first ? (mop->write ? "Write" : "Read")
+             : (mop->write ? "Previous write" : "Previous read")), mop->addr);
+  if (mop->tid == kMainThreadId)
+    Printf("main goroutine:\n");
+  else
+    Printf("goroutine %d:\n", mop->tid);
+  PrintStack(mop->stack);
+}
+
+static void PrintLocation(const ReportLocation *loc) {
+  switch (loc->type) {
+  case ReportLocationHeap: {
+    Printf("\n");
+    Printf("Heap block of size %zu at %p allocated by ",
+        loc->heap_chunk_size, loc->heap_chunk_start);
+    if (loc->tid == kMainThreadId)
+      Printf("main goroutine:\n");
+    else
+      Printf("goroutine %d:\n", loc->tid);
+    PrintStack(loc->stack);
+    break;
+  }
+  case ReportLocationGlobal: {
+    Printf("\n");
+    Printf("Global var %s of size %zu at %p declared at %s:%zu\n",
+        loc->global.name, loc->global.size, loc->global.start,
+        loc->global.file, loc->global.line);
+    break;
+  }
+  default:
+    break;
+  }
+}
+
+static void PrintThread(const ReportThread *rt) {
+  if (rt->id == kMainThreadId)
+    return;
+  Printf("\n");
+  Printf("Goroutine %d (%s) created at:\n",
+    rt->id, rt->running ? "running" : "finished");
+  PrintStack(rt->stack);
+}
+
+void PrintReport(const ReportDesc *rep) {
+  Printf("==================\n");
+  if (rep->typ == ReportTypeRace) {
+    Printf("WARNING: DATA RACE");
+    for (uptr i = 0; i < rep->mops.Size(); i++)
+      PrintMop(rep->mops[i], i == 0);
+    for (uptr i = 0; i < rep->locs.Size(); i++)
+      PrintLocation(rep->locs[i]);
+    for (uptr i = 0; i < rep->threads.Size(); i++)
+      PrintThread(rep->threads[i]);
+  } else if (rep->typ == ReportTypeDeadlock) {
+    Printf("WARNING: DEADLOCK\n");
+    for (uptr i = 0; i < rep->mutexes.Size(); i++) {
+      Printf("Goroutine %d lock mutex %d while holding mutex %d:\n",
+          999, rep->mutexes[i]->id,
+          rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+      PrintStack(rep->stacks[2*i]);
+      Printf("\n");
+      Printf("Mutex %d was previously locked here:\n",
+          rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+      PrintStack(rep->stacks[2*i + 1]);
+      Printf("\n");
+    }
+  }
+  Printf("==================\n");
+}
+
+#endif
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_report.h b/lib/tsan/tsan_report.h
new file mode 100644
index 0000000000..b4e4d89893
--- /dev/null
+++ b/lib/tsan/tsan_report.h
@@ -0,0 +1,135 @@
+//===-- tsan_report.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_REPORT_H
+#define TSAN_REPORT_H
+
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+enum ReportType {
+  ReportTypeRace,
+  ReportTypeVptrRace,
+  ReportTypeUseAfterFree,
+  ReportTypeVptrUseAfterFree,
+  ReportTypeExternalRace,
+  ReportTypeThreadLeak,
+  ReportTypeMutexDestroyLocked,
+  ReportTypeMutexDoubleLock,
+  ReportTypeMutexInvalidAccess,
+  ReportTypeMutexBadUnlock,
+  ReportTypeMutexBadReadLock,
+  ReportTypeMutexBadReadUnlock,
+  ReportTypeSignalUnsafe,
+  ReportTypeErrnoInSignal,
+  ReportTypeDeadlock
+};
+
+struct ReportStack {
+  SymbolizedStack *frames;
+  bool suppressable;
+  static ReportStack *New();
+
+ private:
+  ReportStack();
+};
+
+struct ReportMopMutex {
+  u64 id;
+  bool write;
+};
+
+struct ReportMop {
+  int tid;
+  uptr addr;
+  int size;
+  bool write;
+  bool atomic;
+  uptr external_tag;
+  Vector<ReportMopMutex> mset;
+  ReportStack *stack;
+
+  ReportMop();
+};
+
+enum ReportLocationType {
+  ReportLocationGlobal,
+  ReportLocationHeap,
+  ReportLocationStack,
+  ReportLocationTLS,
+  ReportLocationFD
+};
+
+struct ReportLocation {
+  ReportLocationType type;
+  DataInfo global;
+  uptr heap_chunk_start;
+  uptr heap_chunk_size;
+  uptr external_tag;
+  int tid;
+  int fd;
+  bool suppressable;
+  ReportStack *stack;
+
+  static ReportLocation *New(ReportLocationType type);
+ private:
+  explicit ReportLocation(ReportLocationType type);
+};
+
+struct ReportThread {
+  int id;
+  tid_t os_id;
+  bool running;
+  ThreadType thread_type;
+  char *name;
+  u32 parent_tid;
+  ReportStack *stack;
+};
+
+struct ReportMutex {
+  u64 id;
+  uptr addr;
+  bool destroyed;
+  ReportStack *stack;
+};
+
+class ReportDesc {
+ public:
+  ReportType typ;
+  uptr tag;
+  Vector<ReportStack*> stacks;
+  Vector<ReportMop*> mops;
+  Vector<ReportLocation*> locs;
+  Vector<ReportMutex*> mutexes;
+  Vector<ReportThread*> threads;
+  Vector<int> unique_tids;
+  ReportStack *sleep;
+  int count;
+
+  ReportDesc();
+  ~ReportDesc();
+
+ private:
+  ReportDesc(const ReportDesc&);
+  void operator = (const ReportDesc&);
+};
+
+// Format and output the report to the console/log. No additional logic.
+void PrintReport(const ReportDesc *rep);
+void PrintStack(const ReportStack *stack);
+
+}  // namespace __tsan
+
+#endif  // TSAN_REPORT_H
diff --git a/lib/tsan/tsan_rtl.cpp b/lib/tsan/tsan_rtl.cpp
new file mode 100644
index 0000000000..13c9b770f5
--- /dev/null
+++ b/lib/tsan/tsan_rtl.cpp
@@ -0,0 +1,1127 @@
+//===-- tsan_rtl.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Main file (entry points) for the TSan run-time.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "tsan_defs.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_suppressions.h"
+#include "tsan_symbolize.h"
+#include "ubsan/ubsan_init.h"
+
+#ifdef __SSE3__
+// <emmintrin.h> transitively includes <stdlib.h>,
+// and it's prohibited to include std headers into tsan runtime.
+// So we do this dirty trick.
+#define _MM_MALLOC_H_INCLUDED
+#define __MM_MALLOC_H
+#include <emmintrin.h>
+typedef __m128i m128;
+#endif
+
+volatile int __tsan_resumed = 0;
+
+extern "C" void __tsan_resume() {
+  __tsan_resumed = 1;
+}
+
+namespace __tsan {
+
+#if !SANITIZER_GO && !SANITIZER_MAC
+__attribute__((tls_model("initial-exec")))
+THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
+#endif
+static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
+Context *ctx;
+
+// Can be overriden by a front-end.
+#ifdef TSAN_EXTERNAL_HOOKS
+bool OnFinalize(bool failed);
+void OnInitialize();
+#else
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+bool OnFinalize(bool failed) {
+  return failed;
+}
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+void OnInitialize() {}
+#endif
+
+static char thread_registry_placeholder[sizeof(ThreadRegistry)];
+
+static ThreadContextBase *CreateThreadContext(u32 tid) {
+  // Map thread trace when context is created.
+  char name[50];
+  internal_snprintf(name, sizeof(name), "trace %u", tid);
+  MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
+  const uptr hdr = GetThreadTraceHeader(tid);
+  internal_snprintf(name, sizeof(name), "trace header %u", tid);
+  MapThreadTrace(hdr, sizeof(Trace), name);
+  new((void*)hdr) Trace();
+  // We are going to use only a small part of the trace with the default
+  // value of history_size. However, the constructor writes to the whole trace.
+  // Unmap the unused part.
+  uptr hdr_end = hdr + sizeof(Trace);
+  hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
+  hdr_end = RoundUp(hdr_end, GetPageSizeCached());
+  if (hdr_end < hdr + sizeof(Trace))
+    UnmapOrDie((void*)hdr_end, hdr + sizeof(Trace) - hdr_end);
+  void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
+  return new(mem) ThreadContext(tid);
+}
+
+#if !SANITIZER_GO
+static const u32 kThreadQuarantineSize = 16;
+#else
+static const u32 kThreadQuarantineSize = 64;
+#endif
+
+Context::Context()
+  : initialized()
+  , report_mtx(MutexTypeReport, StatMtxReport)
+  , nreported()
+  , nmissed_expected()
+  , thread_registry(new(thread_registry_placeholder) ThreadRegistry(
+      CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse))
+  , racy_mtx(MutexTypeRacy, StatMtxRacy)
+  , racy_stacks()
+  , racy_addresses()
+  , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
+  , clock_alloc("clock allocator") {
+  fired_suppressions.reserve(8);
+}
+
+// The objects are allocated in TLS, so one may rely on zero-initialization.
+ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
+                         unsigned reuse_count,
+                         uptr stk_addr, uptr stk_size,
+                         uptr tls_addr, uptr tls_size)
+  : fast_state(tid, epoch)
+  // Do not touch these, rely on zero initialization,
+  // they may be accessed before the ctor.
+  // , ignore_reads_and_writes()
+  // , ignore_interceptors()
+  , clock(tid, reuse_count)
+#if !SANITIZER_GO
+  , jmp_bufs()
+#endif
+  , tid(tid)
+  , unique_id(unique_id)
+  , stk_addr(stk_addr)
+  , stk_size(stk_size)
+  , tls_addr(tls_addr)
+  , tls_size(tls_size)
+#if !SANITIZER_GO
+  , last_sleep_clock(tid)
+#endif
+{
+}
+
+#if !SANITIZER_GO
+static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
+  uptr n_threads;
+  uptr n_running_threads;
+  ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
+  InternalMmapVector<char> buf(4096);
+  WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads);
+  WriteToFile(fd, buf.data(), internal_strlen(buf.data()));
+}
+
+static void *BackgroundThread(void *arg) {
+  // This is a non-initialized non-user thread, nothing to see here.
+  // We don't use ScopedIgnoreInterceptors, because we want ignores to be
+  // enabled even when the thread function exits (e.g. during pthread thread
+  // shutdown code).
+  cur_thread_init();
+  cur_thread()->ignore_interceptors++;
+  const u64 kMs2Ns = 1000 * 1000;
+
+  fd_t mprof_fd = kInvalidFd;
+  if (flags()->profile_memory && flags()->profile_memory[0]) {
+    if (internal_strcmp(flags()->profile_memory, "stdout") == 0) {
+      mprof_fd = 1;
+    } else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
+      mprof_fd = 2;
+    } else {
+      InternalScopedString filename(kMaxPathLength);
+      filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
+      fd_t fd = OpenFile(filename.data(), WrOnly);
+      if (fd == kInvalidFd) {
+        Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
+            &filename[0]);
+      } else {
+        mprof_fd = fd;
+      }
+    }
+  }
+
+  u64 last_flush = NanoTime();
+  uptr last_rss = 0;
+  for (int i = 0;
+      atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
+      i++) {
+    SleepForMillis(100);
+    u64 now = NanoTime();
+
+    // Flush memory if requested.
+    if (flags()->flush_memory_ms > 0) {
+      if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
+        VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
+        FlushShadowMemory();
+        last_flush = NanoTime();
+      }
+    }
+    // GetRSS can be expensive on huge programs, so don't do it every 100ms.
+    if (flags()->memory_limit_mb > 0) {
+      uptr rss = GetRSS();
+      uptr limit = uptr(flags()->memory_limit_mb) << 20;
+      VPrintf(1, "ThreadSanitizer: memory flush check"
+                 " RSS=%llu LAST=%llu LIMIT=%llu\n",
+              (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
+      if (2 * rss > limit + last_rss) {
+        VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
+        FlushShadowMemory();
+        rss = GetRSS();
+        VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
+      }
+      last_rss = rss;
+    }
+
+    // Write memory profile if requested.
+    if (mprof_fd != kInvalidFd)
+      MemoryProfiler(ctx, mprof_fd, i);
+
+    // Flush symbolizer cache if requested.
+    if (flags()->flush_symbolizer_ms > 0) {
+      u64 last = atomic_load(&ctx->last_symbolize_time_ns,
+                             memory_order_relaxed);
+      if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
+        Lock l(&ctx->report_mtx);
+        ScopedErrorReportLock l2;
+        SymbolizeFlush();
+        atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
+      }
+    }
+  }
+  return nullptr;
+}
+
+static void StartBackgroundThread() {
+  ctx->background_thread = internal_start_thread(&BackgroundThread, 0);
+}
+
+#ifndef __mips__
+static void StopBackgroundThread() {
+  atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed);
+  internal_join_thread(ctx->background_thread);
+  ctx->background_thread = 0;
+}
+#endif
+#endif
+
+void DontNeedShadowFor(uptr addr, uptr size) {
+  ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
+}
+
+#if !SANITIZER_GO
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
+  if (size == 0) return;
+  DontNeedShadowFor(addr, size);
+  ScopedGlobalProcessor sgp;
+  ctx->metamap.ResetRange(thr->proc(), addr, size);
+}
+#endif
+
+void MapShadow(uptr addr, uptr size) {
+  // Global data is not 64K aligned, but there are no adjacent mappings,
+  // so we can get away with unaligned mapping.
+  // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
+  const uptr kPageSize = GetPageSizeCached();
+  uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
+  uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
+  if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
+    Die();
+
+  // Meta shadow is 2:1, so tread carefully.
+  static bool data_mapped = false;
+  static uptr mapped_meta_end = 0;
+  uptr meta_begin = (uptr)MemToMeta(addr);
+  uptr meta_end = (uptr)MemToMeta(addr + size);
+  meta_begin = RoundDownTo(meta_begin, 64 << 10);
+  meta_end = RoundUpTo(meta_end, 64 << 10);
+  if (!data_mapped) {
+    // First call maps data+bss.
+    data_mapped = true;
+    if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+      Die();
+  } else {
+    // Mapping continous heap.
+    // Windows wants 64K alignment.
+    meta_begin = RoundDownTo(meta_begin, 64 << 10);
+    meta_end = RoundUpTo(meta_end, 64 << 10);
+    if (meta_end <= mapped_meta_end)
+      return;
+    if (meta_begin < mapped_meta_end)
+      meta_begin = mapped_meta_end;
+    if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+      Die();
+    mapped_meta_end = meta_end;
+  }
+  VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
+      addr, addr+size, meta_begin, meta_end);
+}
+
+void MapThreadTrace(uptr addr, uptr size, const char *name) {
+  DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
+  CHECK_GE(addr, TraceMemBeg());
+  CHECK_LE(addr + size, TraceMemEnd());
+  CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
+  if (!MmapFixedNoReserve(addr, size, name)) {
+    Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
+        addr, size);
+    Die();
+  }
+}
+
+static void CheckShadowMapping() {
+  uptr beg, end;
+  for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
+    // Skip cases for empty regions (heap definition for architectures that
+    // do not use 64-bit allocator).
+    if (beg == end)
+      continue;
+    VPrintf(3, "checking shadow region %p-%p\n", beg, end);
+    uptr prev = 0;
+    for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
+      for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
+        const uptr p = RoundDown(p0 + x, kShadowCell);
+        if (p < beg || p >= end)
+          continue;
+        const uptr s = MemToShadow(p);
+        const uptr m = (uptr)MemToMeta(p);
+        VPrintf(3, "  checking pointer %p: shadow=%p meta=%p\n", p, s, m);
+        CHECK(IsAppMem(p));
+        CHECK(IsShadowMem(s));
+        CHECK_EQ(p, ShadowToMem(s));
+        CHECK(IsMetaMem(m));
+        if (prev) {
+          // Ensure that shadow and meta mappings are linear within a single
+          // user range. Lots of code that processes memory ranges assumes it.
+          const uptr prev_s = MemToShadow(prev);
+          const uptr prev_m = (uptr)MemToMeta(prev);
+          CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
+          CHECK_EQ((m - prev_m) / kMetaShadowSize,
+                   (p - prev) / kMetaShadowCell);
+        }
+        prev = p;
+      }
+    }
+  }
+}
+
+#if !SANITIZER_GO
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+                common_flags()->fast_unwind_on_fatal);
+}
+
+static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+#endif
+
+void Initialize(ThreadState *thr) {
+  // Thread safe because done before all threads exist.
+  static bool is_initialized = false;
+  if (is_initialized)
+    return;
+  is_initialized = true;
+  // We are not ready to handle interceptors yet.
+  ScopedIgnoreInterceptors ignore;
+  SanitizerToolName = "ThreadSanitizer";
+  // Install tool-specific callbacks in sanitizer_common.
+  SetCheckFailedCallback(TsanCheckFailed);
+
+  ctx = new(ctx_placeholder) Context;
+  const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS";
+  const char *options = GetEnv(env_name);
+  CacheBinaryName();
+  CheckASLR();
+  InitializeFlags(&ctx->flags, options, env_name);
+  AvoidCVE_2016_2143();
+  __sanitizer::InitializePlatformEarly();
+  __tsan::InitializePlatformEarly();
+
+#if !SANITIZER_GO
+  // Re-exec ourselves if we need to set additional env or command line args.
+  MaybeReexec();
+
+  InitializeAllocator();
+  ReplaceSystemMalloc();
+#endif
+  if (common_flags()->detect_deadlocks)
+    ctx->dd = DDetector::Create(flags());
+  Processor *proc = ProcCreate();
+  ProcWire(proc, thr);
+  InitializeInterceptors();
+  CheckShadowMapping();
+  InitializePlatform();
+  InitializeMutex();
+  InitializeDynamicAnnotations();
+#if !SANITIZER_GO
+  InitializeShadowMemory();
+  InitializeAllocatorLate();
+  InstallDeadlySignalHandlers(TsanOnDeadlySignal);
+#endif
+  // Setup correct file descriptor for error reports.
+  __sanitizer_set_report_path(common_flags()->log_path);
+  InitializeSuppressions();
+#if !SANITIZER_GO
+  InitializeLibIgnore();
+  Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+#endif
+
+  VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
+          (int)internal_getpid());
+
+  // Initialize thread 0.
+  int tid = ThreadCreate(thr, 0, 0, true);
+  CHECK_EQ(tid, 0);
+  ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
+#if TSAN_CONTAINS_UBSAN
+  __ubsan::InitAsPlugin();
+#endif
+  ctx->initialized = true;
+
+#if !SANITIZER_GO
+  Symbolizer::LateInitialize();
+#endif
+
+  if (flags()->stop_on_start) {
+    Printf("ThreadSanitizer is suspended at startup (pid %d)."
+           " Call __tsan_resume().\n",
+           (int)internal_getpid());
+    while (__tsan_resumed == 0) {}
+  }
+
+  OnInitialize();
+}
+
+void MaybeSpawnBackgroundThread() {
+  // On MIPS, TSan initialization is run before
+  // __pthread_initialize_minimal_internal() is finished, so we can not spawn
+  // new threads.
+#if !SANITIZER_GO && !defined(__mips__)
+  static atomic_uint32_t bg_thread = {};
+  if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
+      atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
+    StartBackgroundThread();
+    SetSandboxingCallback(StopBackgroundThread);
+  }
+#endif
+}
+
+
+int Finalize(ThreadState *thr) {
+  bool failed = false;
+
+  if (common_flags()->print_module_map == 1) PrintModuleMap();
+
+  if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
+    SleepForMillis(flags()->atexit_sleep_ms);
+
+  // Wait for pending reports.
+  ctx->report_mtx.Lock();
+  { ScopedErrorReportLock l; }
+  ctx->report_mtx.Unlock();
+
+#if !SANITIZER_GO
+  if (Verbosity()) AllocatorPrintStats();
+#endif
+
+  ThreadFinalize(thr);
+
+  if (ctx->nreported) {
+    failed = true;
+#if !SANITIZER_GO
+    Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
+#else
+    Printf("Found %d data race(s)\n", ctx->nreported);
+#endif
+  }
+
+  if (ctx->nmissed_expected) {
+    failed = true;
+    Printf("ThreadSanitizer: missed %d expected races\n",
+        ctx->nmissed_expected);
+  }
+
+  if (common_flags()->print_suppressions)
+    PrintMatchedSuppressions();
+#if !SANITIZER_GO
+  if (flags()->print_benign)
+    PrintMatchedBenignRaces();
+#endif
+
+  failed = OnFinalize(failed);
+
+#if TSAN_COLLECT_STATS
+  StatAggregate(ctx->stat, thr->stat);
+  StatOutput(ctx->stat);
+#endif
+
+  return failed ? common_flags()->exitcode : 0;
+}
+
+#if !SANITIZER_GO
+void ForkBefore(ThreadState *thr, uptr pc) {
+  ctx->thread_registry->Lock();
+  ctx->report_mtx.Lock();
+  // Ignore memory accesses in the pthread_atfork callbacks.
+  // If any of them triggers a data race we will deadlock
+  // on the report_mtx.
+  // We could ignore interceptors and sync operations as well,
+  // but so far it's unclear if it will do more good or harm.
+  // Unnecessarily ignoring things can lead to false positives later.
+  ThreadIgnoreBegin(thr, pc);
+}
+
+void ForkParentAfter(ThreadState *thr, uptr pc) {
+  ThreadIgnoreEnd(thr, pc);  // Begin is in ForkBefore.
+  ctx->report_mtx.Unlock();
+  ctx->thread_registry->Unlock();
+}
+
+void ForkChildAfter(ThreadState *thr, uptr pc) {
+  ThreadIgnoreEnd(thr, pc);  // Begin is in ForkBefore.
+  ctx->report_mtx.Unlock();
+  ctx->thread_registry->Unlock();
+
+  uptr nthread = 0;
+  ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */);
+  VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
+      " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
+  if (nthread == 1) {
+    StartBackgroundThread();
+  } else {
+    // We've just forked a multi-threaded process. We cannot reasonably function
+    // after that (some mutexes may be locked before fork). So just enable
+    // ignores for everything in the hope that we will exec soon.
+    ctx->after_multithreaded_fork = true;
+    thr->ignore_interceptors++;
+    ThreadIgnoreBegin(thr, pc);
+    ThreadIgnoreSyncBegin(thr, pc);
+  }
+}
+#endif
+
+#if SANITIZER_GO
+NOINLINE
+void GrowShadowStack(ThreadState *thr) {
+  const int sz = thr->shadow_stack_end - thr->shadow_stack;
+  const int newsz = 2 * sz;
+  uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack,
+      newsz * sizeof(uptr));
+  internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
+  internal_free(thr->shadow_stack);
+  thr->shadow_stack = newstack;
+  thr->shadow_stack_pos = newstack + sz;
+  thr->shadow_stack_end = newstack + newsz;
+}
+#endif
+
+u32 CurrentStackId(ThreadState *thr, uptr pc) {
+  if (!thr->is_inited)  // May happen during bootstrap.
+    return 0;
+  if (pc != 0) {
+#if !SANITIZER_GO
+    DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+    if (thr->shadow_stack_pos == thr->shadow_stack_end)
+      GrowShadowStack(thr);
+#endif
+    thr->shadow_stack_pos[0] = pc;
+    thr->shadow_stack_pos++;
+  }
+  u32 id = StackDepotPut(
+      StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
+  if (pc != 0)
+    thr->shadow_stack_pos--;
+  return id;
+}
+
+void TraceSwitch(ThreadState *thr) {
+#if !SANITIZER_GO
+  if (ctx->after_multithreaded_fork)
+    return;
+#endif
+  thr->nomalloc++;
+  Trace *thr_trace = ThreadTrace(thr->tid);
+  Lock l(&thr_trace->mtx);
+  unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
+  TraceHeader *hdr = &thr_trace->headers[trace];
+  hdr->epoch0 = thr->fast_state.epoch();
+  ObtainCurrentStack(thr, 0, &hdr->stack0);
+  hdr->mset0 = thr->mset;
+  thr->nomalloc--;
+}
+
+Trace *ThreadTrace(int tid) {
+  return (Trace*)GetThreadTraceHeader(tid);
+}
+
+uptr TraceTopPC(ThreadState *thr) {
+  Event *events = (Event*)GetThreadTrace(thr->tid);
+  uptr pc = events[thr->fast_state.GetTracePos()];
+  return pc;
+}
+
+uptr TraceSize() {
+  return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
+}
+
+uptr TraceParts() {
+  return TraceSize() / kTracePartSize;
+}
+
+#if !SANITIZER_GO
+extern "C" void __tsan_trace_switch() {
+  TraceSwitch(cur_thread());
+}
+
+extern "C" void __tsan_report_race() {
+  ReportRace(cur_thread());
+}
+#endif
+
+ALWAYS_INLINE
+Shadow LoadShadow(u64 *p) {
+  u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
+  return Shadow(raw);
+}
+
+ALWAYS_INLINE
+void StoreShadow(u64 *sp, u64 s) {
+  atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
+}
+
+ALWAYS_INLINE
+void StoreIfNotYetStored(u64 *sp, u64 *s) {
+  StoreShadow(sp, *s);
+  *s = 0;
+}
+
+ALWAYS_INLINE
+void HandleRace(ThreadState *thr, u64 *shadow_mem,
+                              Shadow cur, Shadow old) {
+  thr->racy_state[0] = cur.raw();
+  thr->racy_state[1] = old.raw();
+  thr->racy_shadow_addr = shadow_mem;
+#if !SANITIZER_GO
+  HACKY_CALL(__tsan_report_race);
+#else
+  ReportRace(thr);
+#endif
+}
+
+static inline bool HappensBefore(Shadow old, ThreadState *thr) {
+  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
+}
+
+ALWAYS_INLINE
+void MemoryAccessImpl1(ThreadState *thr, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
+    u64 *shadow_mem, Shadow cur) {
+  StatInc(thr, StatMop);
+  StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
+  StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
+
+  // This potentially can live in an MMX/SSE scratch register.
+  // The required intrinsics are:
+  // __m128i _mm_move_epi64(__m128i*);
+  // _mm_storel_epi64(u64*, __m128i);
+  u64 store_word = cur.raw();
+  bool stored = false;
+
+  // scan all the shadow values and dispatch to 4 categories:
+  // same, replace, candidate and race (see comments below).
+  // we consider only 3 cases regarding access sizes:
+  // equal, intersect and not intersect. initially I considered
+  // larger and smaller as well, it allowed to replace some
+  // 'candidates' with 'same' or 'replace', but I think
+  // it's just not worth it (performance- and complexity-wise).
+
+  Shadow old(0);
+
+  // It release mode we manually unroll the loop,
+  // because empirically gcc generates better code this way.
+  // However, we can't afford unrolling in debug mode, because the function
+  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
+  // threads, which is not enough for the unrolled loop.
+#if SANITIZER_DEBUG
+  for (int idx = 0; idx < 4; idx++) {
+#include "tsan_update_shadow_word_inl.h"
+  }
+#else
+  int idx = 0;
+#include "tsan_update_shadow_word_inl.h"
+  idx = 1;
+  if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+  } else {
+#include "tsan_update_shadow_word_inl.h"
+  }
+  idx = 2;
+  if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+  } else {
+#include "tsan_update_shadow_word_inl.h"
+  }
+  idx = 3;
+  if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+  } else {
+#include "tsan_update_shadow_word_inl.h"
+  }
+#endif
+
+  // we did not find any races and had already stored
+  // the current access info, so we are done
+  if (LIKELY(stored))
+    return;
+  // choose a random candidate slot and replace it
+  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
+  StatInc(thr, StatShadowReplace);
+  return;
+ RACE:
+  HandleRace(thr, shadow_mem, cur, old);
+  return;
+}
+
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+    int size, bool kAccessIsWrite, bool kIsAtomic) {
+  while (size) {
+    int size1 = 1;
+    int kAccessSizeLog = kSizeLog1;
+    if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
+      size1 = 8;
+      kAccessSizeLog = kSizeLog8;
+    } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
+      size1 = 4;
+      kAccessSizeLog = kSizeLog4;
+    } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
+      size1 = 2;
+      kAccessSizeLog = kSizeLog2;
+    }
+    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
+    addr += size1;
+    size -= size1;
+  }
+}
+
+ALWAYS_INLINE
+bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  Shadow cur(a);
+  for (uptr i = 0; i < kShadowCnt; i++) {
+    Shadow old(LoadShadow(&s[i]));
+    if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
+        old.TidWithIgnore() == cur.TidWithIgnore() &&
+        old.epoch() > sync_epoch &&
+        old.IsAtomic() == cur.IsAtomic() &&
+        old.IsRead() <= cur.IsRead())
+      return true;
+  }
+  return false;
+}
+
+#if defined(__SSE3__)
+#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
+    _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
+    (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
+ALWAYS_INLINE
+bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  // This is an optimized version of ContainsSameAccessSlow.
+  // load current access into access[0:63]
+  const m128 access     = _mm_cvtsi64_si128(a);
+  // duplicate high part of access in addr0:
+  // addr0[0:31]        = access[32:63]
+  // addr0[32:63]       = access[32:63]
+  // addr0[64:95]       = access[32:63]
+  // addr0[96:127]      = access[32:63]
+  const m128 addr0      = SHUF(access, access, 1, 1, 1, 1);
+  // load 4 shadow slots
+  const m128 shadow0    = _mm_load_si128((__m128i*)s);
+  const m128 shadow1    = _mm_load_si128((__m128i*)s + 1);
+  // load high parts of 4 shadow slots into addr_vect:
+  // addr_vect[0:31]    = shadow0[32:63]
+  // addr_vect[32:63]   = shadow0[96:127]
+  // addr_vect[64:95]   = shadow1[32:63]
+  // addr_vect[96:127]  = shadow1[96:127]
+  m128 addr_vect        = SHUF(shadow0, shadow1, 1, 3, 1, 3);
+  if (!is_write) {
+    // set IsRead bit in addr_vect
+    const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
+    const m128 rw_mask  = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
+    addr_vect           = _mm_or_si128(addr_vect, rw_mask);
+  }
+  // addr0 == addr_vect?
+  const m128 addr_res   = _mm_cmpeq_epi32(addr0, addr_vect);
+  // epoch1[0:63]       = sync_epoch
+  const m128 epoch1     = _mm_cvtsi64_si128(sync_epoch);
+  // epoch[0:31]        = sync_epoch[0:31]
+  // epoch[32:63]       = sync_epoch[0:31]
+  // epoch[64:95]       = sync_epoch[0:31]
+  // epoch[96:127]      = sync_epoch[0:31]
+  const m128 epoch      = SHUF(epoch1, epoch1, 0, 0, 0, 0);
+  // load low parts of shadow cell epochs into epoch_vect:
+  // epoch_vect[0:31]   = shadow0[0:31]
+  // epoch_vect[32:63]  = shadow0[64:95]
+  // epoch_vect[64:95]  = shadow1[0:31]
+  // epoch_vect[96:127] = shadow1[64:95]
+  const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
+  // epoch_vect >= sync_epoch?
+  const m128 epoch_res  = _mm_cmpgt_epi32(epoch_vect, epoch);
+  // addr_res & epoch_res
+  const m128 res        = _mm_and_si128(addr_res, epoch_res);
+  // mask[0] = res[7]
+  // mask[1] = res[15]
+  // ...
+  // mask[15] = res[127]
+  const int mask        = _mm_movemask_epi8(res);
+  return mask != 0;
+}
+#endif
+
+ALWAYS_INLINE
+bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+#if defined(__SSE3__)
+  bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
+  // NOTE: this check can fail if the shadow is concurrently mutated
+  // by other threads. But it still can be useful if you modify
+  // ContainsSameAccessFast and want to ensure that it's not completely broken.
+  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
+  return res;
+#else
+  return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
+#endif
+}
+
+ALWAYS_INLINE USED
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
+  u64 *shadow_mem = (u64*)MemToShadow(addr);
+  DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
+      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
+      (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
+      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
+      (uptr)shadow_mem[0], (uptr)shadow_mem[1],
+      (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsShadowMem((uptr)shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem((uptr)shadow_mem));
+  }
+#endif
+
+  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    StatInc(thr, StatMop);
+    StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
+    StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
+    StatInc(thr, StatMopRodata);
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit())) {
+    StatInc(thr, StatMop);
+    StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
+    StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
+    StatInc(thr, StatMopIgnored);
+    return;
+  }
+
+  Shadow cur(fast_state);
+  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
+  cur.SetWrite(kAccessIsWrite);
+  cur.SetAtomic(kIsAtomic);
+
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
+      thr->fast_synch_epoch, kAccessIsWrite))) {
+    StatInc(thr, StatMop);
+    StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
+    StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
+    StatInc(thr, StatMopSame);
+    return;
+  }
+
+  if (kCollectHistory) {
+    fast_state.IncrementEpoch();
+    thr->fast_state = fast_state;
+    TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+    cur.IncrementEpoch();
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+      shadow_mem, cur);
+}
+
+// Called by MemoryAccessRange in tsan_rtl_thread.cpp
+ALWAYS_INLINE USED
+void MemoryAccessImpl(ThreadState *thr, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
+    u64 *shadow_mem, Shadow cur) {
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
+      thr->fast_synch_epoch, kAccessIsWrite))) {
+    StatInc(thr, StatMop);
+    StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
+    StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
+    StatInc(thr, StatMopSame);
+    return;
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+      shadow_mem, cur);
+}
+
+static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           u64 val) {
+  (void)thr;
+  (void)pc;
+  if (size == 0)
+    return;
+  // FIXME: fix me.
+  uptr offset = addr % kShadowCell;
+  if (offset) {
+    offset = kShadowCell - offset;
+    if (size <= offset)
+      return;
+    addr += offset;
+    size -= offset;
+  }
+  DCHECK_EQ(addr % 8, 0);
+  // If a user passes some insane arguments (memset(0)),
+  // let it just crash as usual.
+  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+    return;
+  // Don't want to touch lots of shadow memory.
+  // If a program maps 10MB stack, there is no need reset the whole range.
+  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
+  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
+    u64 *p = (u64*)MemToShadow(addr);
+    CHECK(IsShadowMem((uptr)p));
+    CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
+    // FIXME: may overwrite a part outside the region
+    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
+      p[i++] = val;
+      for (uptr j = 1; j < kShadowCnt; j++)
+        p[i++] = 0;
+    }
+  } else {
+    // The region is big, reset only beginning and end.
+    const uptr kPageSize = GetPageSizeCached();
+    u64 *begin = (u64*)MemToShadow(addr);
+    u64 *end = begin + size / kShadowCell * kShadowCnt;
+    u64 *p = begin;
+    // Set at least first kPageSize/2 to page boundary.
+    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++)
+        *p++ = 0;
+    }
+    // Reset middle part.
+    u64 *p1 = p;
+    p = RoundDown(end, kPageSize);
+    UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
+    if (!MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+      Die();
+    // Set the ending.
+    while (p < end) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++)
+        *p++ = 0;
+    }
+  }
+}
+
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  MemoryRangeSet(thr, pc, addr, size, 0);
+}
+
+void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  // Processing more than 1k (4k of shadow) is expensive,
+  // can cause excessive memory consumption (user does not necessary touch
+  // the whole range) and most likely unnecessary.
+  if (size > 1024)
+    size = 1024;
+  CHECK_EQ(thr->is_freeing, false);
+  thr->is_freeing = true;
+  MemoryAccessRange(thr, pc, addr, size, true);
+  thr->is_freeing = false;
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.MarkAsFreed();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size) {
+  if (thr->ignore_reads_and_writes == 0)
+    MemoryRangeImitateWrite(thr, pc, addr, size);
+  else
+    MemoryResetRange(thr, pc, addr, size);
+}
+
+ALWAYS_INLINE USED
+void FuncEntry(ThreadState *thr, uptr pc) {
+  StatInc(thr, StatFuncEnter);
+  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
+  }
+
+  // Shadow stack maintenance can be replaced with
+  // stack unwinding during trace switch (which presumably must be faster).
+  DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+  if (thr->shadow_stack_pos == thr->shadow_stack_end)
+    GrowShadowStack(thr);
+#endif
+  thr->shadow_stack_pos[0] = pc;
+  thr->shadow_stack_pos++;
+}
+
+ALWAYS_INLINE USED
+void FuncExit(ThreadState *thr) {
+  StatInc(thr, StatFuncExit);
+  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
+  }
+
+  DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#endif
+  thr->shadow_stack_pos--;
+}
+
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {
+  DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
+  thr->ignore_reads_and_writes++;
+  CHECK_GT(thr->ignore_reads_and_writes, 0);
+  thr->fast_state.SetIgnoreBit();
+#if !SANITIZER_GO
+  if (save_stack && !ctx->after_multithreaded_fork)
+    thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
+}
+
+void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_reads_and_writes, 0);
+  thr->ignore_reads_and_writes--;
+  if (thr->ignore_reads_and_writes == 0) {
+    thr->fast_state.ClearIgnoreBit();
+#if !SANITIZER_GO
+    thr->mop_ignore_set.Reset();
+#endif
+  }
+}
+
+#if !SANITIZER_GO
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+uptr __tsan_testonly_shadow_stack_current_size() {
+  ThreadState *thr = cur_thread();
+  return thr->shadow_stack_pos - thr->shadow_stack;
+}
+#endif
+
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {
+  DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
+  thr->ignore_sync++;
+  CHECK_GT(thr->ignore_sync, 0);
+#if !SANITIZER_GO
+  if (save_stack && !ctx->after_multithreaded_fork)
+    thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
+#endif
+}
+
+void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
+  CHECK_GT(thr->ignore_sync, 0);
+  thr->ignore_sync--;
+#if !SANITIZER_GO
+  if (thr->ignore_sync == 0)
+    thr->sync_ignore_set.Reset();
+#endif
+}
+
+bool MD5Hash::operator==(const MD5Hash &other) const {
+  return hash[0] == other.hash[0] && hash[1] == other.hash[1];
+}
+
+#if SANITIZER_DEBUG
+void build_consistency_debug() {}
+#else
+void build_consistency_release() {}
+#endif
+
+#if TSAN_COLLECT_STATS
+void build_consistency_stats() {}
+#else
+void build_consistency_nostats() {}
+#endif
+
+}  // namespace __tsan
+
+#if !SANITIZER_GO
+// Must be included in this file to make sure everything is inlined.
+#include "tsan_interface_inl.h"
+#endif
diff --git a/lib/tsan/tsan_rtl.h b/lib/tsan/tsan_rtl.h
new file mode 100644
index 0000000000..d3bb61ff87
--- /dev/null
+++ b/lib/tsan/tsan_rtl.h
@@ -0,0 +1,893 @@
+//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Main internal TSan header file.
+//
+// Ground rules:
+//   - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
+//     function-scope locals)
+//   - All functions/classes/etc reside in namespace __tsan, except for those
+//     declared in tsan_interface.h.
+//   - Platform-specific files should be used instead of ifdefs (*).
+//   - No system headers included in header files (*).
+//   - Platform specific headres included only into platform-specific files (*).
+//
+//  (*) Except when inlining is critical for performance.
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_RTL_H
+#define TSAN_RTL_H
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_asm.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+#include "sanitizer_common/sanitizer_libignore.h"
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+#include "sanitizer_common/sanitizer_vector.h"
+#include "tsan_clock.h"
+#include "tsan_defs.h"
+#include "tsan_flags.h"
+#include "tsan_mman.h"
+#include "tsan_sync.h"
+#include "tsan_trace.h"
+#include "tsan_report.h"
+#include "tsan_platform.h"
+#include "tsan_mutexset.h"
+#include "tsan_ignoreset.h"
+#include "tsan_stack_trace.h"
+
+#if SANITIZER_WORDSIZE != 64
+# error "ThreadSanitizer is supported only on 64-bit platforms"
+#endif
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+struct MapUnmapCallback;
+#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
+
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = 0;
+  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = 20;
+  using AddressSpaceView = LocalAddressSpaceView;
+  typedef __tsan::MapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+#else
+struct AP64 {  // Allocator64 parameters. Deliberately using a short name.
+  static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
+  static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
+  static const uptr kMetadataSize = 0;
+  typedef DefaultSizeClassMap SizeClassMap;
+  typedef __tsan::MapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+  using AddressSpaceView = LocalAddressSpaceView;
+};
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
+#endif
+typedef CombinedAllocator<PrimaryAllocator> Allocator;
+typedef Allocator::AllocatorCache AllocatorCache;
+Allocator *allocator();
+#endif
+
+void TsanCheckFailed(const char *file, int line, const char *cond,
+                     u64 v1, u64 v2);
+
+const u64 kShadowRodata = (u64)-1;  // .rodata shadow marker
+
+// FastState (from most significant bit):
+//   ignore          : 1
+//   tid             : kTidBits
+//   unused          : -
+//   history_size    : 3
+//   epoch           : kClkBits
+class FastState {
+ public:
+  FastState(u64 tid, u64 epoch) {
+    x_ = tid << kTidShift;
+    x_ |= epoch;
+    DCHECK_EQ(tid, this->tid());
+    DCHECK_EQ(epoch, this->epoch());
+    DCHECK_EQ(GetIgnoreBit(), false);
+  }
+
+  explicit FastState(u64 x)
+      : x_(x) {
+  }
+
+  u64 raw() const {
+    return x_;
+  }
+
+  u64 tid() const {
+    u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
+    return res;
+  }
+
+  u64 TidWithIgnore() const {
+    u64 res = x_ >> kTidShift;
+    return res;
+  }
+
+  u64 epoch() const {
+    u64 res = x_ & ((1ull << kClkBits) - 1);
+    return res;
+  }
+
+  void IncrementEpoch() {
+    u64 old_epoch = epoch();
+    x_ += 1;
+    DCHECK_EQ(old_epoch + 1, epoch());
+    (void)old_epoch;
+  }
+
+  void SetIgnoreBit() { x_ |= kIgnoreBit; }
+  void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
+  bool GetIgnoreBit() const { return (s64)x_ < 0; }
+
+  void SetHistorySize(int hs) {
+    CHECK_GE(hs, 0);
+    CHECK_LE(hs, 7);
+    x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift);
+  }
+
+  ALWAYS_INLINE
+  int GetHistorySize() const {
+    return (int)((x_ >> kHistoryShift) & kHistoryMask);
+  }
+
+  void ClearHistorySize() {
+    SetHistorySize(0);
+  }
+
+  ALWAYS_INLINE
+  u64 GetTracePos() const {
+    const int hs = GetHistorySize();
+    // When hs == 0, the trace consists of 2 parts.
+    const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
+    return epoch() & mask;
+  }
+
+ private:
+  friend class Shadow;
+  static const int kTidShift = 64 - kTidBits - 1;
+  static const u64 kIgnoreBit = 1ull << 63;
+  static const u64 kFreedBit = 1ull << 63;
+  static const u64 kHistoryShift = kClkBits;
+  static const u64 kHistoryMask = 7;
+  u64 x_;
+};
+
+// Shadow (from most significant bit):
+//   freed           : 1
+//   tid             : kTidBits
+//   is_atomic       : 1
+//   is_read         : 1
+//   size_log        : 2
+//   addr0           : 3
+//   epoch           : kClkBits
+class Shadow : public FastState {
+ public:
+  explicit Shadow(u64 x)
+      : FastState(x) {
+  }
+
+  explicit Shadow(const FastState &s)
+      : FastState(s.x_) {
+    ClearHistorySize();
+  }
+
+  void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
+    DCHECK_EQ((x_ >> kClkBits) & 31, 0);
+    DCHECK_LE(addr0, 7);
+    DCHECK_LE(kAccessSizeLog, 3);
+    x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits;
+    DCHECK_EQ(kAccessSizeLog, size_log());
+    DCHECK_EQ(addr0, this->addr0());
+  }
+
+  void SetWrite(unsigned kAccessIsWrite) {
+    DCHECK_EQ(x_ & kReadBit, 0);
+    if (!kAccessIsWrite)
+      x_ |= kReadBit;
+    DCHECK_EQ(kAccessIsWrite, IsWrite());
+  }
+
+  void SetAtomic(bool kIsAtomic) {
+    DCHECK(!IsAtomic());
+    if (kIsAtomic)
+      x_ |= kAtomicBit;
+    DCHECK_EQ(IsAtomic(), kIsAtomic);
+  }
+
+  bool IsAtomic() const {
+    return x_ & kAtomicBit;
+  }
+
+  bool IsZero() const {
+    return x_ == 0;
+  }
+
+  static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
+    u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
+    DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
+    return shifted_xor == 0;
+  }
+
+  static ALWAYS_INLINE
+  bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
+    u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
+    return masked_xor == 0;
+  }
+
+  static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
+      unsigned kS2AccessSize) {
+    bool res = false;
+    u64 diff = s1.addr0() - s2.addr0();
+    if ((s64)diff < 0) {  // s1.addr0 < s2.addr0
+      // if (s1.addr0() + size1) > s2.addr0()) return true;
+      if (s1.size() > -diff)
+        res = true;
+    } else {
+      // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
+      if (kS2AccessSize > diff)
+        res = true;
+    }
+    DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
+    DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
+    return res;
+  }
+
+  u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
+  u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
+  bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
+  bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
+
+  // The idea behind the freed bit is as follows.
+  // When the memory is freed (or otherwise unaccessible) we write to the shadow
+  // values with tid/epoch related to the free and the freed bit set.
+  // During memory accesses processing the freed bit is considered
+  // as msb of tid. So any access races with shadow with freed bit set
+  // (it is as if write from a thread with which we never synchronized before).
+  // This allows us to detect accesses to freed memory w/o additional
+  // overheads in memory access processing and at the same time restore
+  // tid/epoch of free.
+  void MarkAsFreed() {
+     x_ |= kFreedBit;
+  }
+
+  bool IsFreed() const {
+    return x_ & kFreedBit;
+  }
+
+  bool GetFreedAndReset() {
+    bool res = x_ & kFreedBit;
+    x_ &= ~kFreedBit;
+    return res;
+  }
+
+  bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
+    bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
+        | (u64(kIsAtomic) << kAtomicShift));
+    DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic));
+    return v;
+  }
+
+  bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
+    bool v = ((x_ >> kReadShift) & 3)
+        <= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
+    DCHECK_EQ(v, (IsAtomic() < kIsAtomic) ||
+        (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
+    return v;
+  }
+
+  bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
+    bool v = ((x_ >> kReadShift) & 3)
+        >= u64((kIsWrite ^ 1) | (kIsAtomic << 1));
+    DCHECK_EQ(v, (IsAtomic() > kIsAtomic) ||
+        (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
+    return v;
+  }
+
+ private:
+  static const u64 kReadShift   = 5 + kClkBits;
+  static const u64 kReadBit     = 1ull << kReadShift;
+  static const u64 kAtomicShift = 6 + kClkBits;
+  static const u64 kAtomicBit   = 1ull << kAtomicShift;
+
+  u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
+
+  static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
+    if (s1.addr0() == s2.addr0()) return true;
+    if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
+      return true;
+    if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
+      return true;
+    return false;
+  }
+};
+
+struct ThreadSignalContext;
+
+struct JmpBuf {
+  uptr sp;
+  int int_signal_send;
+  bool in_blocking_func;
+  uptr in_signal_handler;
+  uptr *shadow_stack_pos;
+};
+
+// A Processor represents a physical thread, or a P for Go.
+// It is used to store internal resources like allocate cache, and does not
+// participate in race-detection logic (invisible to end user).
+// In C++ it is tied to an OS thread just like ThreadState, however ideally
+// it should be tied to a CPU (this way we will have fewer allocator caches).
+// In Go it is tied to a P, so there are significantly fewer Processor's than
+// ThreadState's (which are tied to Gs).
+// A ThreadState must be wired with a Processor to handle events.
+struct Processor {
+  ThreadState *thr; // currently wired thread, or nullptr
+#if !SANITIZER_GO
+  AllocatorCache alloc_cache;
+  InternalAllocatorCache internal_alloc_cache;
+#endif
+  DenseSlabAllocCache block_cache;
+  DenseSlabAllocCache sync_cache;
+  DenseSlabAllocCache clock_cache;
+  DDPhysicalThread *dd_pt;
+};
+
+#if !SANITIZER_GO
+// ScopedGlobalProcessor temporary setups a global processor for the current
+// thread, if it does not have one. Intended for interceptors that can run
+// at the very thread end, when we already destroyed the thread processor.
+struct ScopedGlobalProcessor {
+  ScopedGlobalProcessor();
+  ~ScopedGlobalProcessor();
+};
+#endif
+
+// This struct is stored in TLS.
+struct ThreadState {
+  FastState fast_state;
+  // Synch epoch represents the threads's epoch before the last synchronization
+  // action. It allows to reduce number of shadow state updates.
+  // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
+  // if we are processing write to X from the same thread at epoch=200,
+  // we do nothing, because both writes happen in the same 'synch epoch'.
+  // That is, if another memory access does not race with the former write,
+  // it does not race with the latter as well.
+  // QUESTION: can we can squeeze this into ThreadState::Fast?
+  // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
+  // taken by epoch between synchs.
+  // This way we can save one load from tls.
+  u64 fast_synch_epoch;
+  // Technically `current` should be a separate THREADLOCAL variable;
+  // but it is placed here in order to share cache line with previous fields.
+  ThreadState* current;
+  // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
+  // We do not distinguish beteween ignoring reads and writes
+  // for better performance.
+  int ignore_reads_and_writes;
+  int ignore_sync;
+  int suppress_reports;
+  // Go does not support ignores.
+#if !SANITIZER_GO
+  IgnoreSet mop_ignore_set;
+  IgnoreSet sync_ignore_set;
+#endif
+  // C/C++ uses fixed size shadow stack embed into Trace.
+  // Go uses malloc-allocated shadow stack with dynamic size.
+  uptr *shadow_stack;
+  uptr *shadow_stack_end;
+  uptr *shadow_stack_pos;
+  u64 *racy_shadow_addr;
+  u64 racy_state[2];
+  MutexSet mset;
+  ThreadClock clock;
+#if !SANITIZER_GO
+  Vector<JmpBuf> jmp_bufs;
+  int ignore_interceptors;
+#endif
+#if TSAN_COLLECT_STATS
+  u64 stat[StatCnt];
+#endif
+  const int tid;
+  const int unique_id;
+  bool in_symbolizer;
+  bool in_ignored_lib;
+  bool is_inited;
+  bool is_dead;
+  bool is_freeing;
+  bool is_vptr_access;
+  const uptr stk_addr;
+  const uptr stk_size;
+  const uptr tls_addr;
+  const uptr tls_size;
+  ThreadContext *tctx;
+
+#if SANITIZER_DEBUG && !SANITIZER_GO
+  InternalDeadlockDetector internal_deadlock_detector;
+#endif
+  DDLogicalThread *dd_lt;
+
+  // Current wired Processor, or nullptr. Required to handle any events.
+  Processor *proc1;
+#if !SANITIZER_GO
+  Processor *proc() { return proc1; }
+#else
+  Processor *proc();
+#endif
+
+  atomic_uintptr_t in_signal_handler;
+  ThreadSignalContext *signal_ctx;
+
+#if !SANITIZER_GO
+  u32 last_sleep_stack_id;
+  ThreadClock last_sleep_clock;
+#endif
+
+  // Set in regions of runtime that must be signal-safe and fork-safe.
+  // If set, malloc must not be called.
+  int nomalloc;
+
+  const ReportDesc *current_report;
+
+  explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
+                       unsigned reuse_count,
+                       uptr stk_addr, uptr stk_size,
+                       uptr tls_addr, uptr tls_size);
+};
+
+#if !SANITIZER_GO
+#if SANITIZER_MAC || SANITIZER_ANDROID
+ThreadState *cur_thread();
+void set_cur_thread(ThreadState *thr);
+void cur_thread_finalize();
+INLINE void cur_thread_init() { }
+#else
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL char cur_thread_placeholder[];
+INLINE ThreadState *cur_thread() {
+  return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
+}
+INLINE void cur_thread_init() {
+  ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
+  if (UNLIKELY(!thr->current))
+    thr->current = thr;
+}
+INLINE void set_cur_thread(ThreadState *thr) {
+  reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
+}
+INLINE void cur_thread_finalize() { }
+#endif  // SANITIZER_MAC || SANITIZER_ANDROID
+#endif  // SANITIZER_GO
+
+class ThreadContext : public ThreadContextBase {
+ public:
+  explicit ThreadContext(int tid);
+  ~ThreadContext();
+  ThreadState *thr;
+  u32 creation_stack_id;
+  SyncClock sync;
+  // Epoch at which the thread had started.
+  // If we see an event from the thread stamped by an older epoch,
+  // the event is from a dead thread that shared tid with this thread.
+  u64 epoch0;
+  u64 epoch1;
+
+  // Override superclass callbacks.
+  void OnDead() override;
+  void OnJoined(void *arg) override;
+  void OnFinished() override;
+  void OnStarted(void *arg) override;
+  void OnCreated(void *arg) override;
+  void OnReset() override;
+  void OnDetached(void *arg) override;
+};
+
+struct RacyStacks {
+  MD5Hash hash[2];
+  bool operator==(const RacyStacks &other) const {
+    if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
+      return true;
+    if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
+      return true;
+    return false;
+  }
+};
+
+struct RacyAddress {
+  uptr addr_min;
+  uptr addr_max;
+};
+
+struct FiredSuppression {
+  ReportType type;
+  uptr pc_or_addr;
+  Suppression *supp;
+};
+
+struct Context {
+  Context();
+
+  bool initialized;
+#if !SANITIZER_GO
+  bool after_multithreaded_fork;
+#endif
+
+  MetaMap metamap;
+
+  Mutex report_mtx;
+  int nreported;
+  int nmissed_expected;
+  atomic_uint64_t last_symbolize_time_ns;
+
+  void *background_thread;
+  atomic_uint32_t stop_background_thread;
+
+  ThreadRegistry *thread_registry;
+
+  Mutex racy_mtx;
+  Vector<RacyStacks> racy_stacks;
+  Vector<RacyAddress> racy_addresses;
+  // Number of fired suppressions may be large enough.
+  Mutex fired_suppressions_mtx;
+  InternalMmapVector<FiredSuppression> fired_suppressions;
+  DDetector *dd;
+
+  ClockAlloc clock_alloc;
+
+  Flags flags;
+
+  u64 stat[StatCnt];
+  u64 int_alloc_cnt[MBlockTypeCount];
+  u64 int_alloc_siz[MBlockTypeCount];
+};
+
+extern Context *ctx;  // The one and the only global runtime context.
+
+ALWAYS_INLINE Flags *flags() {
+  return &ctx->flags;
+}
+
+struct ScopedIgnoreInterceptors {
+  ScopedIgnoreInterceptors() {
+#if !SANITIZER_GO
+    cur_thread()->ignore_interceptors++;
+#endif
+  }
+
+  ~ScopedIgnoreInterceptors() {
+#if !SANITIZER_GO
+    cur_thread()->ignore_interceptors--;
+#endif
+  }
+};
+
+const char *GetObjectTypeFromTag(uptr tag);
+const char *GetReportHeaderFromTag(uptr tag);
+uptr TagFromShadowStackFrame(uptr pc);
+
+class ScopedReportBase {
+ public:
+  void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
+                       const MutexSet *mset);
+  void AddStack(StackTrace stack, bool suppressable = false);
+  void AddThread(const ThreadContext *tctx, bool suppressable = false);
+  void AddThread(int unique_tid, bool suppressable = false);
+  void AddUniqueTid(int unique_tid);
+  void AddMutex(const SyncVar *s);
+  u64 AddMutex(u64 id);
+  void AddLocation(uptr addr, uptr size);
+  void AddSleep(u32 stack_id);
+  void SetCount(int count);
+
+  const ReportDesc *GetReport() const;
+
+ protected:
+  ScopedReportBase(ReportType typ, uptr tag);
+  ~ScopedReportBase();
+
+ private:
+  ReportDesc *rep_;
+  // Symbolizer makes lots of intercepted calls. If we try to process them,
+  // at best it will cause deadlocks on internal mutexes.
+  ScopedIgnoreInterceptors ignore_interceptors_;
+
+  void AddDeadMutex(u64 id);
+
+  ScopedReportBase(const ScopedReportBase &) = delete;
+  void operator=(const ScopedReportBase &) = delete;
+};
+
+class ScopedReport : public ScopedReportBase {
+ public:
+  explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
+  ~ScopedReport();
+
+ private:
+  ScopedErrorReportLock lock_;
+};
+
+ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
+void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
+                  MutexSet *mset, uptr *tag = nullptr);
+
+// The stack could look like:
+//   <start> | <main> | <foo> | tag | <bar>
+// This will extract the tag and keep:
+//   <start> | <main> | <foo> | <bar>
+template<typename StackTraceTy>
+void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
+  if (stack->size < 2) return;
+  uptr possible_tag_pc = stack->trace[stack->size - 2];
+  uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc);
+  if (possible_tag == kExternalTagNone) return;
+  stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
+  stack->size -= 1;
+  if (tag) *tag = possible_tag;
+}
+
+template<typename StackTraceTy>
+void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
+                        uptr *tag = nullptr) {
+  uptr size = thr->shadow_stack_pos - thr->shadow_stack;
+  uptr start = 0;
+  if (size + !!toppc > kStackTraceMax) {
+    start = size + !!toppc - kStackTraceMax;
+    size = kStackTraceMax - !!toppc;
+  }
+  stack->Init(&thr->shadow_stack[start], size, toppc);
+  ExtractTagFromStack(stack, tag);
+}
+
+#define GET_STACK_TRACE_FATAL(thr, pc) \
+  VarSizeStackTrace stack; \
+  ObtainCurrentStack(thr, pc, &stack); \
+  stack.ReverseOrder();
+
+#if TSAN_COLLECT_STATS
+void StatAggregate(u64 *dst, u64 *src);
+void StatOutput(u64 *stat);
+#endif
+
+void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) {
+#if TSAN_COLLECT_STATS
+  thr->stat[typ] += n;
+#endif
+}
+void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) {
+#if TSAN_COLLECT_STATS
+  thr->stat[typ] = n;
+#endif
+}
+
+void MapShadow(uptr addr, uptr size);
+void MapThreadTrace(uptr addr, uptr size, const char *name);
+void DontNeedShadowFor(uptr addr, uptr size);
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
+void InitializeShadowMemory();
+void InitializeInterceptors();
+void InitializeLibIgnore();
+void InitializeDynamicAnnotations();
+
+void ForkBefore(ThreadState *thr, uptr pc);
+void ForkParentAfter(ThreadState *thr, uptr pc);
+void ForkChildAfter(ThreadState *thr, uptr pc);
+
+void ReportRace(ThreadState *thr);
+bool OutputReport(ThreadState *thr, const ScopedReport &srep);
+bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
+bool IsExpectedReport(uptr addr, uptr size);
+void PrintMatchedBenignRaces();
+
+#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
+# define DPrintf Printf
+#else
+# define DPrintf(...)
+#endif
+
+#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
+# define DPrintf2 Printf
+#else
+# define DPrintf2(...)
+#endif
+
+u32 CurrentStackId(ThreadState *thr, uptr pc);
+ReportStack *SymbolizeStackId(u32 stack_id);
+void PrintCurrentStack(ThreadState *thr, uptr pc);
+void PrintCurrentStackSlow(uptr pc);  // uses libunwind
+
+void Initialize(ThreadState *thr);
+void MaybeSpawnBackgroundThread();
+int Finalize(ThreadState *thr);
+
+void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
+void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
+
+void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
+void MemoryAccessImpl(ThreadState *thr, uptr addr,
+    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
+    u64 *shadow_mem, Shadow cur);
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
+    uptr size, bool is_write);
+void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,
+    uptr size, uptr step, bool is_write);
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+    int size, bool kAccessIsWrite, bool kIsAtomic);
+
+const int kSizeLog1 = 0;
+const int kSizeLog2 = 1;
+const int kSizeLog4 = 2;
+const int kSizeLog8 = 3;
+
+void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
+                                     uptr addr, int kAccessSizeLog) {
+  MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
+}
+
+void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
+                                      uptr addr, int kAccessSizeLog) {
+  MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
+}
+
+void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
+                                           uptr addr, int kAccessSizeLog) {
+  MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
+}
+
+void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
+                                            uptr addr, int kAccessSizeLog) {
+  MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
+}
+
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size);
+
+void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
+void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
+void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
+void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
+
+void FuncEntry(ThreadState *thr, uptr pc);
+void FuncExit(ThreadState *thr);
+
+int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
+void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
+                 ThreadType thread_type);
+void ThreadFinish(ThreadState *thr);
+int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
+void ThreadJoin(ThreadState *thr, uptr pc, int tid);
+void ThreadDetach(ThreadState *thr, uptr pc, int tid);
+void ThreadFinalize(ThreadState *thr);
+void ThreadSetName(ThreadState *thr, const char *name);
+int ThreadCount(ThreadState *thr);
+void ProcessPendingSignals(ThreadState *thr);
+void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid);
+
+Processor *ProcCreate();
+void ProcDestroy(Processor *proc);
+void ProcWire(Processor *proc, ThreadState *thr);
+void ProcUnwire(Processor *proc, ThreadState *thr);
+
+// Note: the parameter is called flagz, because flags is already taken
+// by the global function that returns flags.
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
+    int rec = 1);
+int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr);  // call on EOWNERDEAD
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
+
+void Acquire(ThreadState *thr, uptr pc, uptr addr);
+// AcquireGlobal synchronizes the current thread with all other threads.
+// In terms of happens-before relation, it draws a HB edge from all threads
+// (where they happen to execute right now) to the current thread. We use it to
+// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
+// right before executing finalizers. This provides a coarse, but simple
+// approximation of the actual required synchronization.
+void AcquireGlobal(ThreadState *thr, uptr pc);
+void Release(ThreadState *thr, uptr pc, uptr addr);
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
+void AfterSleep(ThreadState *thr, uptr pc);
+void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c);
+void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
+
+// The hacky call uses custom calling convention and an assembly thunk.
+// It is considerably faster that a normal call for the caller
+// if it is not executed (it is intended for slow paths from hot functions).
+// The trick is that the call preserves all registers and the compiler
+// does not treat it as a call.
+// If it does not work for you, use normal call.
+#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
+// The caller may not create the stack frame for itself at all,
+// so we create a reserve stack frame for it (1024b must be enough).
+#define HACKY_CALL(f) \
+  __asm__ __volatile__("sub $1024, %%rsp;" \
+                       CFI_INL_ADJUST_CFA_OFFSET(1024) \
+                       ".hidden " #f "_thunk;" \
+                       "call " #f "_thunk;" \
+                       "add $1024, %%rsp;" \
+                       CFI_INL_ADJUST_CFA_OFFSET(-1024) \
+                       ::: "memory", "cc");
+#else
+#define HACKY_CALL(f) f()
+#endif
+
+void TraceSwitch(ThreadState *thr);
+uptr TraceTopPC(ThreadState *thr);
+uptr TraceSize();
+uptr TraceParts();
+Trace *ThreadTrace(int tid);
+
+extern "C" void __tsan_trace_switch();
+void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
+                                        EventType typ, u64 addr) {
+  if (!kCollectHistory)
+    return;
+  DCHECK_GE((int)typ, 0);
+  DCHECK_LE((int)typ, 7);
+  DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
+  StatInc(thr, StatEvents);
+  u64 pos = fs.GetTracePos();
+  if (UNLIKELY((pos % kTracePartSize) == 0)) {
+#if !SANITIZER_GO
+    HACKY_CALL(__tsan_trace_switch);
+#else
+    TraceSwitch(thr);
+#endif
+  }
+  Event *trace = (Event*)GetThreadTrace(fs.tid());
+  Event *evp = &trace[pos];
+  Event ev = (u64)addr | ((u64)typ << kEventPCBits);
+  *evp = ev;
+}
+
+#if !SANITIZER_GO
+uptr ALWAYS_INLINE HeapEnd() {
+  return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
+}
+#endif
+
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
+void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
+
+// These need to match __tsan_switch_to_fiber_* flags defined in
+// tsan_interface.h. See documentation there as well.
+enum FiberSwitchFlags {
+  FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_RTL_H
diff --git a/lib/tsan/tsan_rtl_mutex.cpp b/lib/tsan/tsan_rtl_mutex.cpp
new file mode 100644
index 0000000000..ebd0d72218
--- /dev/null
+++ b/lib/tsan/tsan_rtl_mutex.cpp
@@ -0,0 +1,562 @@
+//===-- tsan_rtl_mutex.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include <sanitizer_common/sanitizer_deadlock_detector_interface.h>
+#include <sanitizer_common/sanitizer_stackdepot.h>
+
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+#include "tsan_sync.h"
+#include "tsan_report.h"
+#include "tsan_symbolize.h"
+#include "tsan_platform.h"
+
+namespace __tsan {
+
+void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
+
+struct Callback : DDCallback {
+  ThreadState *thr;
+  uptr pc;
+
+  Callback(ThreadState *thr, uptr pc)
+      : thr(thr)
+      , pc(pc) {
+    DDCallback::pt = thr->proc()->dd_pt;
+    DDCallback::lt = thr->dd_lt;
+  }
+
+  u32 Unwind() override { return CurrentStackId(thr, pc); }
+  int UniqueTid() override { return thr->unique_id; }
+};
+
+void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
+  Callback cb(thr, pc);
+  ctx->dd->MutexInit(&cb, &s->dd);
+  s->dd.ctx = s->GetId();
+}
+
+static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
+    uptr addr, u64 mid) {
+  // In Go, these misuses are either impossible, or detected by std lib,
+  // or false positives (e.g. unlock in a different thread).
+  if (SANITIZER_GO)
+    return;
+  ThreadRegistryLock l(ctx->thread_registry);
+  ScopedReport rep(typ);
+  rep.AddMutex(mid);
+  VarSizeStackTrace trace;
+  ObtainCurrentStack(thr, pc, &trace);
+  rep.AddStack(trace, true);
+  rep.AddLocation(addr, 1);
+  OutputReport(thr, rep);
+}
+
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  StatInc(thr, StatMutexCreate);
+  if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
+    CHECK(!thr->is_freeing);
+    thr->is_freeing = true;
+    MemoryWrite(thr, pc, addr, kSizeLog1);
+    thr->is_freeing = false;
+  }
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  s->SetFlags(flagz & MutexCreationFlagMask);
+  if (!SANITIZER_GO && s->creation_stack_id == 0)
+    s->creation_stack_id = CurrentStackId(thr, pc);
+  s->mtx.Unlock();
+}
+
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
+  StatInc(thr, StatMutexDestroy);
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
+  if (s == 0)
+    return;
+  if ((flagz & MutexFlagLinkerInit)
+      || s->IsFlagSet(MutexFlagLinkerInit)
+      || ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
+    // Destroy is no-op for linker-initialized mutexes.
+    s->mtx.Unlock();
+    return;
+  }
+  if (common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ctx->dd->MutexDestroy(&cb, &s->dd);
+    ctx->dd->MutexInit(&cb, &s->dd);
+  }
+  bool unlock_locked = false;
+  if (flags()->report_destroy_locked
+      && s->owner_tid != SyncVar::kInvalidTid
+      && !s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
+    unlock_locked = true;
+  }
+  u64 mid = s->GetId();
+  u64 last_lock = s->last_lock;
+  if (!unlock_locked)
+    s->Reset(thr->proc());  // must not reset it before the report is printed
+  s->mtx.Unlock();
+  if (unlock_locked) {
+    ThreadRegistryLock l(ctx->thread_registry);
+    ScopedReport rep(ReportTypeMutexDestroyLocked);
+    rep.AddMutex(mid);
+    VarSizeStackTrace trace;
+    ObtainCurrentStack(thr, pc, &trace);
+    rep.AddStack(trace, true);
+    FastState last(last_lock);
+    RestoreStack(last.tid(), last.epoch(), &trace, 0);
+    rep.AddStack(trace, true);
+    rep.AddLocation(addr, 1);
+    OutputReport(thr, rep);
+
+    SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
+    if (s != 0) {
+      s->Reset(thr->proc());
+      s->mtx.Unlock();
+    }
+  }
+  thr->mset.Remove(mid);
+  // Imitate a memory write to catch unlock-destroy races.
+  // Do this outside of sync mutex, because it can report a race which locks
+  // sync mutexes.
+  if (IsAppMem(addr)) {
+    CHECK(!thr->is_freeing);
+    thr->is_freeing = true;
+    MemoryWrite(thr, pc, addr, kSizeLog1);
+    thr->is_freeing = false;
+  }
+  // s will be destroyed and freed in MetaMap::FreeBlock.
+}
+
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+    s->UpdateFlags(flagz);
+    if (s->owner_tid != thr->tid) {
+      Callback cb(thr, pc);
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+      s->mtx.ReadUnlock();
+      ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+    } else {
+      s->mtx.ReadUnlock();
+    }
+  }
+}
+
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
+  DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
+      thr->tid, addr, flagz, rec);
+  if (flagz & MutexFlagRecursiveLock)
+    CHECK_GT(rec, 0);
+  else
+    rec = 1;
+  if (IsAppMem(addr))
+    MemoryReadAtomic(thr, pc, addr, kSizeLog1);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  s->UpdateFlags(flagz);
+  thr->fast_state.IncrementEpoch();
+  TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
+  bool report_double_lock = false;
+  if (s->owner_tid == SyncVar::kInvalidTid) {
+    CHECK_EQ(s->recursion, 0);
+    s->owner_tid = thr->tid;
+    s->last_lock = thr->fast_state.raw();
+  } else if (s->owner_tid == thr->tid) {
+    CHECK_GT(s->recursion, 0);
+  } else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
+    report_double_lock = true;
+  }
+  const bool first = s->recursion == 0;
+  s->recursion += rec;
+  if (first) {
+    StatInc(thr, StatMutexLock);
+    AcquireImpl(thr, pc, &s->clock);
+    AcquireImpl(thr, pc, &s->read_clock);
+  } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
+    StatInc(thr, StatMutexRecLock);
+  }
+  thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
+  bool pre_lock = false;
+  if (first && common_flags()->detect_deadlocks) {
+    pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+        !(flagz & MutexFlagTryLock);
+    Callback cb(thr, pc);
+    if (pre_lock)
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
+    ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
+  }
+  u64 mid = s->GetId();
+  s->mtx.Unlock();
+  // Can't touch s after this point.
+  s = 0;
+  if (report_double_lock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
+  if (first && pre_lock && common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (IsAppMem(addr))
+    MemoryReadAtomic(thr, pc, addr, kSizeLog1);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  thr->fast_state.IncrementEpoch();
+  TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
+  int rec = 0;
+  bool report_bad_unlock = false;
+  if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      report_bad_unlock = true;
+    }
+  } else {
+    rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
+    s->recursion -= rec;
+    if (s->recursion == 0) {
+      StatInc(thr, StatMutexUnlock);
+      s->owner_tid = SyncVar::kInvalidTid;
+      ReleaseStoreImpl(thr, pc, &s->clock);
+    } else {
+      StatInc(thr, StatMutexRecUnlock);
+    }
+  }
+  thr->mset.Del(s->GetId(), true);
+  if (common_flags()->detect_deadlocks && s->recursion == 0 &&
+      !report_bad_unlock) {
+    Callback cb(thr, pc);
+    ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
+  }
+  u64 mid = s->GetId();
+  s->mtx.Unlock();
+  // Can't touch s after this point.
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks && !report_bad_unlock) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+  return rec;
+}
+
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
+    SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+    s->UpdateFlags(flagz);
+    Callback cb(thr, pc);
+    ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+    s->mtx.ReadUnlock();
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+  DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
+  StatInc(thr, StatMutexReadLock);
+  if (IsAppMem(addr))
+    MemoryReadAtomic(thr, pc, addr, kSizeLog1);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
+  s->UpdateFlags(flagz);
+  thr->fast_state.IncrementEpoch();
+  TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
+  bool report_bad_lock = false;
+  if (s->owner_tid != SyncVar::kInvalidTid) {
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      report_bad_lock = true;
+    }
+  }
+  AcquireImpl(thr, pc, &s->clock);
+  s->last_lock = thr->fast_state.raw();
+  thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
+  bool pre_lock = false;
+  if (common_flags()->detect_deadlocks) {
+    pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
+        !(flagz & MutexFlagTryLock);
+    Callback cb(thr, pc);
+    if (pre_lock)
+      ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
+    ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
+  }
+  u64 mid = s->GetId();
+  s->mtx.ReadUnlock();
+  // Can't touch s after this point.
+  s = 0;
+  if (report_bad_lock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
+  if (pre_lock  && common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
+  StatInc(thr, StatMutexReadUnlock);
+  if (IsAppMem(addr))
+    MemoryReadAtomic(thr, pc, addr, kSizeLog1);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  thr->fast_state.IncrementEpoch();
+  TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
+  bool report_bad_unlock = false;
+  if (s->owner_tid != SyncVar::kInvalidTid) {
+    if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
+      s->SetFlags(MutexFlagBroken);
+      report_bad_unlock = true;
+    }
+  }
+  ReleaseImpl(thr, pc, &s->read_clock);
+  if (common_flags()->detect_deadlocks && s->recursion == 0) {
+    Callback cb(thr, pc);
+    ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
+  }
+  u64 mid = s->GetId();
+  s->mtx.Unlock();
+  // Can't touch s after this point.
+  thr->mset.Del(mid, false);
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
+  if (IsAppMem(addr))
+    MemoryReadAtomic(thr, pc, addr, kSizeLog1);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  bool write = true;
+  bool report_bad_unlock = false;
+  if (s->owner_tid == SyncVar::kInvalidTid) {
+    // Seems to be read unlock.
+    write = false;
+    StatInc(thr, StatMutexReadUnlock);
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
+    ReleaseImpl(thr, pc, &s->read_clock);
+  } else if (s->owner_tid == thr->tid) {
+    // Seems to be write unlock.
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
+    CHECK_GT(s->recursion, 0);
+    s->recursion--;
+    if (s->recursion == 0) {
+      StatInc(thr, StatMutexUnlock);
+      s->owner_tid = SyncVar::kInvalidTid;
+      ReleaseStoreImpl(thr, pc, &s->clock);
+    } else {
+      StatInc(thr, StatMutexRecUnlock);
+    }
+  } else if (!s->IsFlagSet(MutexFlagBroken)) {
+    s->SetFlags(MutexFlagBroken);
+    report_bad_unlock = true;
+  }
+  thr->mset.Del(s->GetId(), write);
+  if (common_flags()->detect_deadlocks && s->recursion == 0) {
+    Callback cb(thr, pc);
+    ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
+  }
+  u64 mid = s->GetId();
+  s->mtx.Unlock();
+  // Can't touch s after this point.
+  if (report_bad_unlock)
+    ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
+  if (common_flags()->detect_deadlocks) {
+    Callback cb(thr, pc);
+    ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
+  }
+}
+
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  s->owner_tid = SyncVar::kInvalidTid;
+  s->recursion = 0;
+  s->mtx.Unlock();
+}
+
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  u64 mid = s->GetId();
+  s->mtx.Unlock();
+  ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid);
+}
+
+void Acquire(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, false);
+  if (!s)
+    return;
+  AcquireImpl(thr, pc, &s->clock);
+  s->mtx.ReadUnlock();
+}
+
+static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
+  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
+  if (tctx->status == ThreadStatusRunning) {
+    epoch = tctx->thr->fast_state.epoch();
+    tctx->thr->clock.NoteGlobalAcquire(epoch);
+  }
+  thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+}
+
+void AcquireGlobal(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: AcquireGlobal\n", thr->tid);
+  if (thr->ignore_sync)
+    return;
+  ThreadRegistryLock l(ctx->thread_registry);
+  ctx->thread_registry->RunCallbackForEachThreadLocked(
+      UpdateClockCallback, thr);
+}
+
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreAcquireImpl(thr, pc, &s->clock);
+  s->mtx.Unlock();
+}
+
+void Release(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: Release %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseImpl(thr, pc, &s->clock);
+  s->mtx.Unlock();
+}
+
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
+  DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
+  if (thr->ignore_sync)
+    return;
+  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
+  thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+  ReleaseStoreImpl(thr, pc, &s->clock);
+  s->mtx.Unlock();
+}
+
+#if !SANITIZER_GO
+static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
+  ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
+  if (tctx->status == ThreadStatusRunning)
+    epoch = tctx->thr->fast_state.epoch();
+  thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
+}
+
+void AfterSleep(ThreadState *thr, uptr pc) {
+  DPrintf("#%d: AfterSleep %zx\n", thr->tid);
+  if (thr->ignore_sync)
+    return;
+  thr->last_sleep_stack_id = CurrentStackId(thr, pc);
+  ThreadRegistryLock l(ctx->thread_registry);
+  ctx->thread_registry->RunCallbackForEachThreadLocked(
+      UpdateSleepClockCallback, thr);
+}
+#endif
+
+void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->clock.acquire(&thr->proc()->clock_cache, c);
+  StatInc(thr, StatSyncAcquire);
+}
+
+void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
+  StatInc(thr, StatSyncReleaseStoreAcquire);
+}
+
+void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.release(&thr->proc()->clock_cache, c);
+  StatInc(thr, StatSyncRelease);
+}
+
+void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
+  StatInc(thr, StatSyncRelease);
+}
+
+void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
+  if (thr->ignore_sync)
+    return;
+  thr->clock.set(thr->fast_state.epoch());
+  thr->fast_synch_epoch = thr->fast_state.epoch();
+  thr->clock.acq_rel(&thr->proc()->clock_cache, c);
+  StatInc(thr, StatSyncAcquire);
+  StatInc(thr, StatSyncRelease);
+}
+
+void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
+  if (r == 0)
+    return;
+  ThreadRegistryLock l(ctx->thread_registry);
+  ScopedReport rep(ReportTypeDeadlock);
+  for (int i = 0; i < r->n; i++) {
+    rep.AddMutex(r->loop[i].mtx_ctx0);
+    rep.AddUniqueTid((int)r->loop[i].thr_ctx);
+    rep.AddThread((int)r->loop[i].thr_ctx);
+  }
+  uptr dummy_pc = 0x42;
+  for (int i = 0; i < r->n; i++) {
+    for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
+      u32 stk = r->loop[i].stk[j];
+      if (stk && stk != 0xffffffff) {
+        rep.AddStack(StackDepotGet(stk), true);
+      } else {
+        // Sometimes we fail to extract the stack trace (FIXME: investigate),
+        // but we should still produce some stack trace in the report.
+        rep.AddStack(StackTrace(&dummy_pc, 1), true);
+      }
+    }
+  }
+  OutputReport(thr, rep);
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_rtl_proc.cpp b/lib/tsan/tsan_rtl_proc.cpp
new file mode 100644
index 0000000000..def61cca14
--- /dev/null
+++ b/lib/tsan/tsan_rtl_proc.cpp
@@ -0,0 +1,60 @@
+//===-- tsan_rtl_proc.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+
+namespace __tsan {
+
+Processor *ProcCreate() {
+  void *mem = InternalAlloc(sizeof(Processor));
+  internal_memset(mem, 0, sizeof(Processor));
+  Processor *proc = new(mem) Processor;
+  proc->thr = nullptr;
+#if !SANITIZER_GO
+  AllocatorProcStart(proc);
+#endif
+  if (common_flags()->detect_deadlocks)
+    proc->dd_pt = ctx->dd->CreatePhysicalThread();
+  return proc;
+}
+
+void ProcDestroy(Processor *proc) {
+  CHECK_EQ(proc->thr, nullptr);
+#if !SANITIZER_GO
+  AllocatorProcFinish(proc);
+#endif
+  ctx->clock_alloc.FlushCache(&proc->clock_cache);
+  ctx->metamap.OnProcIdle(proc);
+  if (common_flags()->detect_deadlocks)
+     ctx->dd->DestroyPhysicalThread(proc->dd_pt);
+  proc->~Processor();
+  InternalFree(proc);
+}
+
+void ProcWire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc1, nullptr);
+  CHECK_EQ(proc->thr, nullptr);
+  thr->proc1 = proc;
+  proc->thr = thr;
+}
+
+void ProcUnwire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc1, proc);
+  CHECK_EQ(proc->thr, thr);
+  thr->proc1 = nullptr;
+  proc->thr = nullptr;
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_rtl_report.cpp b/lib/tsan/tsan_rtl_report.cpp
new file mode 100644
index 0000000000..3354546c2a
--- /dev/null
+++ b/lib/tsan/tsan_rtl_report.cpp
@@ -0,0 +1,754 @@
+//===-- tsan_rtl_report.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "tsan_suppressions.h"
+#include "tsan_symbolize.h"
+#include "tsan_report.h"
+#include "tsan_sync.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+#include "tsan_fd.h"
+
+namespace __tsan {
+
+using namespace __sanitizer;
+
+static ReportStack *SymbolizeStack(StackTrace trace);
+
+void TsanCheckFailed(const char *file, int line, const char *cond,
+                     u64 v1, u64 v2) {
+  // There is high probability that interceptors will check-fail as well,
+  // on the other hand there is no sense in processing interceptors
+  // since we are going to die soon.
+  ScopedIgnoreInterceptors ignore;
+#if !SANITIZER_GO
+  cur_thread()->ignore_sync++;
+  cur_thread()->ignore_reads_and_writes++;
+#endif
+  Printf("FATAL: ThreadSanitizer CHECK failed: "
+         "%s:%d \"%s\" (0x%zx, 0x%zx)\n",
+         file, line, cond, (uptr)v1, (uptr)v2);
+  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
+  Die();
+}
+
+// Can be overriden by an application/test to intercept reports.
+#ifdef TSAN_EXTERNAL_HOOKS
+bool OnReport(const ReportDesc *rep, bool suppressed);
+#else
+SANITIZER_WEAK_CXX_DEFAULT_IMPL
+bool OnReport(const ReportDesc *rep, bool suppressed) {
+  (void)rep;
+  return suppressed;
+}
+#endif
+
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_on_report(const ReportDesc *rep) {
+  (void)rep;
+}
+
+static void StackStripMain(SymbolizedStack *frames) {
+  SymbolizedStack *last_frame = nullptr;
+  SymbolizedStack *last_frame2 = nullptr;
+  for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+    last_frame2 = last_frame;
+    last_frame = cur;
+  }
+
+  if (last_frame2 == 0)
+    return;
+#if !SANITIZER_GO
+  const char *last = last_frame->info.function;
+  const char *last2 = last_frame2->info.function;
+  // Strip frame above 'main'
+  if (last2 && 0 == internal_strcmp(last2, "main")) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+  // Strip our internal thread start routine.
+  } else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+  // Strip global ctors init.
+  } else if (last && 0 == internal_strcmp(last, "__do_global_ctors_aux")) {
+    last_frame->ClearAll();
+    last_frame2->next = nullptr;
+  // If both are 0, then we probably just failed to symbolize.
+  } else if (last || last2) {
+    // Ensure that we recovered stack completely. Trimmed stack
+    // can actually happen if we do not instrument some code,
+    // so it's only a debug print. However we must try hard to not miss it
+    // due to our fault.
+    DPrintf("Bottom stack frame is missed\n");
+  }
+#else
+  // The last frame always point into runtime (gosched0, goexit0, runtime.main).
+  last_frame->ClearAll();
+  last_frame2->next = nullptr;
+#endif
+}
+
+ReportStack *SymbolizeStackId(u32 stack_id) {
+  if (stack_id == 0)
+    return 0;
+  StackTrace stack = StackDepotGet(stack_id);
+  if (stack.trace == nullptr)
+    return nullptr;
+  return SymbolizeStack(stack);
+}
+
+static ReportStack *SymbolizeStack(StackTrace trace) {
+  if (trace.size == 0)
+    return 0;
+  SymbolizedStack *top = nullptr;
+  for (uptr si = 0; si < trace.size; si++) {
+    const uptr pc = trace.trace[si];
+    uptr pc1 = pc;
+    // We obtain the return address, but we're interested in the previous
+    // instruction.
+    if ((pc & kExternalPCBit) == 0)
+      pc1 = StackTrace::GetPreviousInstructionPc(pc);
+    SymbolizedStack *ent = SymbolizeCode(pc1);
+    CHECK_NE(ent, 0);
+    SymbolizedStack *last = ent;
+    while (last->next) {
+      last->info.address = pc;  // restore original pc for report
+      last = last->next;
+    }
+    last->info.address = pc;  // restore original pc for report
+    last->next = top;
+    top = ent;
+  }
+  StackStripMain(top);
+
+  ReportStack *stack = ReportStack::New();
+  stack->frames = top;
+  return stack;
+}
+
+ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
+  ctx->thread_registry->CheckLocked();
+  void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
+  rep_ = new(mem) ReportDesc;
+  rep_->typ = typ;
+  rep_->tag = tag;
+  ctx->report_mtx.Lock();
+}
+
+ScopedReportBase::~ScopedReportBase() {
+  ctx->report_mtx.Unlock();
+  DestroyAndFree(rep_);
+  rep_ = nullptr;
+}
+
+void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
+  ReportStack **rs = rep_->stacks.PushBack();
+  *rs = SymbolizeStack(stack);
+  (*rs)->suppressable = suppressable;
+}
+
+void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
+                                       StackTrace stack, const MutexSet *mset) {
+  void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
+  ReportMop *mop = new(mem) ReportMop;
+  rep_->mops.PushBack(mop);
+  mop->tid = s.tid();
+  mop->addr = addr + s.addr0();
+  mop->size = s.size();
+  mop->write = s.IsWrite();
+  mop->atomic = s.IsAtomic();
+  mop->stack = SymbolizeStack(stack);
+  mop->external_tag = external_tag;
+  if (mop->stack)
+    mop->stack->suppressable = true;
+  for (uptr i = 0; i < mset->Size(); i++) {
+    MutexSet::Desc d = mset->Get(i);
+    u64 mid = this->AddMutex(d.id);
+    ReportMopMutex mtx = {mid, d.write};
+    mop->mset.PushBack(mtx);
+  }
+}
+
+void ScopedReportBase::AddUniqueTid(int unique_tid) {
+  rep_->unique_tids.PushBack(unique_tid);
+}
+
+void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
+  for (uptr i = 0; i < rep_->threads.Size(); i++) {
+    if ((u32)rep_->threads[i]->id == tctx->tid)
+      return;
+  }
+  void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread));
+  ReportThread *rt = new(mem) ReportThread;
+  rep_->threads.PushBack(rt);
+  rt->id = tctx->tid;
+  rt->os_id = tctx->os_id;
+  rt->running = (tctx->status == ThreadStatusRunning);
+  rt->name = internal_strdup(tctx->name);
+  rt->parent_tid = tctx->parent_tid;
+  rt->thread_type = tctx->thread_type;
+  rt->stack = 0;
+  rt->stack = SymbolizeStackId(tctx->creation_stack_id);
+  if (rt->stack)
+    rt->stack->suppressable = suppressable;
+}
+
+#if !SANITIZER_GO
+static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) {
+  int unique_id = *(int *)arg;
+  return tctx->unique_id == (u32)unique_id;
+}
+
+static ThreadContext *FindThreadByUidLocked(int unique_id) {
+  ctx->thread_registry->CheckLocked();
+  return static_cast<ThreadContext *>(
+      ctx->thread_registry->FindThreadContextLocked(
+          FindThreadByUidLockedCallback, &unique_id));
+}
+
+static ThreadContext *FindThreadByTidLocked(int tid) {
+  ctx->thread_registry->CheckLocked();
+  return static_cast<ThreadContext*>(
+      ctx->thread_registry->GetThreadLocked(tid));
+}
+
+static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) {
+  uptr addr = (uptr)arg;
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  if (tctx->status != ThreadStatusRunning)
+    return false;
+  ThreadState *thr = tctx->thr;
+  CHECK(thr);
+  return ((addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) ||
+          (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size));
+}
+
+ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
+  ctx->thread_registry->CheckLocked();
+  ThreadContext *tctx = static_cast<ThreadContext*>(
+      ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls,
+                                                    (void*)addr));
+  if (!tctx)
+    return 0;
+  ThreadState *thr = tctx->thr;
+  CHECK(thr);
+  *is_stack = (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size);
+  return tctx;
+}
+#endif
+
+void ScopedReportBase::AddThread(int unique_tid, bool suppressable) {
+#if !SANITIZER_GO
+  if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
+    AddThread(tctx, suppressable);
+#endif
+}
+
+void ScopedReportBase::AddMutex(const SyncVar *s) {
+  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
+    if (rep_->mutexes[i]->id == s->uid)
+      return;
+  }
+  void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
+  ReportMutex *rm = new(mem) ReportMutex;
+  rep_->mutexes.PushBack(rm);
+  rm->id = s->uid;
+  rm->addr = s->addr;
+  rm->destroyed = false;
+  rm->stack = SymbolizeStackId(s->creation_stack_id);
+}
+
+u64 ScopedReportBase::AddMutex(u64 id) {
+  u64 uid = 0;
+  u64 mid = id;
+  uptr addr = SyncVar::SplitId(id, &uid);
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
+  // Check that the mutex is still alive.
+  // Another mutex can be created at the same address,
+  // so check uid as well.
+  if (s && s->CheckId(uid)) {
+    mid = s->uid;
+    AddMutex(s);
+  } else {
+    AddDeadMutex(id);
+  }
+  if (s)
+    s->mtx.Unlock();
+  return mid;
+}
+
+void ScopedReportBase::AddDeadMutex(u64 id) {
+  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
+    if (rep_->mutexes[i]->id == id)
+      return;
+  }
+  void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
+  ReportMutex *rm = new(mem) ReportMutex;
+  rep_->mutexes.PushBack(rm);
+  rm->id = id;
+  rm->addr = 0;
+  rm->destroyed = true;
+  rm->stack = 0;
+}
+
+void ScopedReportBase::AddLocation(uptr addr, uptr size) {
+  if (addr == 0)
+    return;
+#if !SANITIZER_GO
+  int fd = -1;
+  int creat_tid = kInvalidTid;
+  u32 creat_stack = 0;
+  if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
+    ReportLocation *loc = ReportLocation::New(ReportLocationFD);
+    loc->fd = fd;
+    loc->tid = creat_tid;
+    loc->stack = SymbolizeStackId(creat_stack);
+    rep_->locs.PushBack(loc);
+    ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
+    if (tctx)
+      AddThread(tctx);
+    return;
+  }
+  MBlock *b = 0;
+  Allocator *a = allocator();
+  if (a->PointerIsMine((void*)addr)) {
+    void *block_begin = a->GetBlockBegin((void*)addr);
+    if (block_begin)
+      b = ctx->metamap.GetBlock((uptr)block_begin);
+  }
+  if (b != 0) {
+    ThreadContext *tctx = FindThreadByTidLocked(b->tid);
+    ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
+    loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);
+    loc->heap_chunk_size = b->siz;
+    loc->external_tag = b->tag;
+    loc->tid = tctx ? tctx->tid : b->tid;
+    loc->stack = SymbolizeStackId(b->stk);
+    rep_->locs.PushBack(loc);
+    if (tctx)
+      AddThread(tctx);
+    return;
+  }
+  bool is_stack = false;
+  if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
+    ReportLocation *loc =
+        ReportLocation::New(is_stack ? ReportLocationStack : ReportLocationTLS);
+    loc->tid = tctx->tid;
+    rep_->locs.PushBack(loc);
+    AddThread(tctx);
+  }
+#endif
+  if (ReportLocation *loc = SymbolizeData(addr)) {
+    loc->suppressable = true;
+    rep_->locs.PushBack(loc);
+    return;
+  }
+}
+
+#if !SANITIZER_GO
+void ScopedReportBase::AddSleep(u32 stack_id) {
+  rep_->sleep = SymbolizeStackId(stack_id);
+}
+#endif
+
+void ScopedReportBase::SetCount(int count) { rep_->count = count; }
+
+const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
+
+ScopedReport::ScopedReport(ReportType typ, uptr tag)
+    : ScopedReportBase(typ, tag) {}
+
+ScopedReport::~ScopedReport() {}
+
+void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
+                  MutexSet *mset, uptr *tag) {
+  // This function restores stack trace and mutex set for the thread/epoch.
+  // It does so by getting stack trace and mutex set at the beginning of
+  // trace part, and then replaying the trace till the given epoch.
+  Trace* trace = ThreadTrace(tid);
+  ReadLock l(&trace->mtx);
+  const int partidx = (epoch / kTracePartSize) % TraceParts();
+  TraceHeader* hdr = &trace->headers[partidx];
+  if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize)
+    return;
+  CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
+  const u64 epoch0 = RoundDown(epoch, TraceSize());
+  const u64 eend = epoch % TraceSize();
+  const u64 ebegin = RoundDown(eend, kTracePartSize);
+  DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
+          tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
+  Vector<uptr> stack;
+  stack.Resize(hdr->stack0.size + 64);
+  for (uptr i = 0; i < hdr->stack0.size; i++) {
+    stack[i] = hdr->stack0.trace[i];
+    DPrintf2("  #%02zu: pc=%zx\n", i, stack[i]);
+  }
+  if (mset)
+    *mset = hdr->mset0;
+  uptr pos = hdr->stack0.size;
+  Event *events = (Event*)GetThreadTrace(tid);
+  for (uptr i = ebegin; i <= eend; i++) {
+    Event ev = events[i];
+    EventType typ = (EventType)(ev >> kEventPCBits);
+    uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
+    DPrintf2("  %zu typ=%d pc=%zx\n", i, typ, pc);
+    if (typ == EventTypeMop) {
+      stack[pos] = pc;
+    } else if (typ == EventTypeFuncEnter) {
+      if (stack.Size() < pos + 2)
+        stack.Resize(pos + 2);
+      stack[pos++] = pc;
+    } else if (typ == EventTypeFuncExit) {
+      if (pos > 0)
+        pos--;
+    }
+    if (mset) {
+      if (typ == EventTypeLock) {
+        mset->Add(pc, true, epoch0 + i);
+      } else if (typ == EventTypeUnlock) {
+        mset->Del(pc, true);
+      } else if (typ == EventTypeRLock) {
+        mset->Add(pc, false, epoch0 + i);
+      } else if (typ == EventTypeRUnlock) {
+        mset->Del(pc, false);
+      }
+    }
+    for (uptr j = 0; j <= pos; j++)
+      DPrintf2("      #%zu: %zx\n", j, stack[j]);
+  }
+  if (pos == 0 && stack[0] == 0)
+    return;
+  pos++;
+  stk->Init(&stack[0], pos);
+  ExtractTagFromStack(stk, tag);
+}
+
+static bool FindRacyStacks(const RacyStacks &hash) {
+  for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) {
+    if (hash == ctx->racy_stacks[i]) {
+      VPrintf(2, "ThreadSanitizer: suppressing report as doubled (stack)\n");
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {
+  if (!flags()->suppress_equal_stacks)
+    return false;
+  RacyStacks hash;
+  hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr));
+  hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr));
+  {
+    ReadLock lock(&ctx->racy_mtx);
+    if (FindRacyStacks(hash))
+      return true;
+  }
+  Lock lock(&ctx->racy_mtx);
+  if (FindRacyStacks(hash))
+    return true;
+  ctx->racy_stacks.PushBack(hash);
+  return false;
+}
+
+static bool FindRacyAddress(const RacyAddress &ra0) {
+  for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
+    RacyAddress ra2 = ctx->racy_addresses[i];
+    uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
+    uptr minend = min(ra0.addr_max, ra2.addr_max);
+    if (maxbeg < minend) {
+      VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) {
+  if (!flags()->suppress_equal_addresses)
+    return false;
+  RacyAddress ra0 = {addr_min, addr_max};
+  {
+    ReadLock lock(&ctx->racy_mtx);
+    if (FindRacyAddress(ra0))
+      return true;
+  }
+  Lock lock(&ctx->racy_mtx);
+  if (FindRacyAddress(ra0))
+    return true;
+  ctx->racy_addresses.PushBack(ra0);
+  return false;
+}
+
+bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
+  if (!flags()->report_bugs || thr->suppress_reports)
+    return false;
+  atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
+  const ReportDesc *rep = srep.GetReport();
+  CHECK_EQ(thr->current_report, nullptr);
+  thr->current_report = rep;
+  Suppression *supp = 0;
+  uptr pc_or_addr = 0;
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->stacks.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->stacks[i], &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->threads.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp);
+  for (uptr i = 0; pc_or_addr == 0 && i < rep->locs.Size(); i++)
+    pc_or_addr = IsSuppressed(rep->typ, rep->locs[i], &supp);
+  if (pc_or_addr != 0) {
+    Lock lock(&ctx->fired_suppressions_mtx);
+    FiredSuppression s = {srep.GetReport()->typ, pc_or_addr, supp};
+    ctx->fired_suppressions.push_back(s);
+  }
+  {
+    bool old_is_freeing = thr->is_freeing;
+    thr->is_freeing = false;
+    bool suppressed = OnReport(rep, pc_or_addr != 0);
+    thr->is_freeing = old_is_freeing;
+    if (suppressed) {
+      thr->current_report = nullptr;
+      return false;
+    }
+  }
+  PrintReport(rep);
+  __tsan_on_report(rep);
+  ctx->nreported++;
+  if (flags()->halt_on_error)
+    Die();
+  thr->current_report = nullptr;
+  return true;
+}
+
+bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace) {
+  ReadLock lock(&ctx->fired_suppressions_mtx);
+  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
+    if (ctx->fired_suppressions[k].type != type)
+      continue;
+    for (uptr j = 0; j < trace.size; j++) {
+      FiredSuppression *s = &ctx->fired_suppressions[k];
+      if (trace.trace[j] == s->pc_or_addr) {
+        if (s->supp)
+          atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
+  ReadLock lock(&ctx->fired_suppressions_mtx);
+  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
+    if (ctx->fired_suppressions[k].type != type)
+      continue;
+    FiredSuppression *s = &ctx->fired_suppressions[k];
+    if (addr == s->pc_or_addr) {
+      if (s->supp)
+        atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool RaceBetweenAtomicAndFree(ThreadState *thr) {
+  Shadow s0(thr->racy_state[0]);
+  Shadow s1(thr->racy_state[1]);
+  CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
+  if (!s0.IsAtomic() && !s1.IsAtomic())
+    return true;
+  if (s0.IsAtomic() && s1.IsFreed())
+    return true;
+  if (s1.IsAtomic() && thr->is_freeing)
+    return true;
+  return false;
+}
+
+void ReportRace(ThreadState *thr) {
+  CheckNoLocks(thr);
+
+  // Symbolizer makes lots of intercepted calls. If we try to process them,
+  // at best it will cause deadlocks on internal mutexes.
+  ScopedIgnoreInterceptors ignore;
+
+  if (!flags()->report_bugs)
+    return;
+  if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
+    return;
+
+  bool freed = false;
+  {
+    Shadow s(thr->racy_state[1]);
+    freed = s.GetFreedAndReset();
+    thr->racy_state[1] = s.raw();
+  }
+
+  uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);
+  uptr addr_min = 0;
+  uptr addr_max = 0;
+  {
+    uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
+    uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
+    uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
+    uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
+    addr_min = min(a0, a1);
+    addr_max = max(e0, e1);
+    if (IsExpectedReport(addr_min, addr_max - addr_min))
+      return;
+  }
+  if (HandleRacyAddress(thr, addr_min, addr_max))
+    return;
+
+  ReportType typ = ReportTypeRace;
+  if (thr->is_vptr_access && freed)
+    typ = ReportTypeVptrUseAfterFree;
+  else if (thr->is_vptr_access)
+    typ = ReportTypeVptrRace;
+  else if (freed)
+    typ = ReportTypeUseAfterFree;
+
+  if (IsFiredSuppression(ctx, typ, addr))
+    return;
+
+  const uptr kMop = 2;
+  VarSizeStackTrace traces[kMop];
+  uptr tags[kMop] = {kExternalTagNone};
+  uptr toppc = TraceTopPC(thr);
+  if (toppc >> kEventPCBits) {
+    // This is a work-around for a known issue.
+    // The scenario where this happens is rather elaborate and requires
+    // an instrumented __sanitizer_report_error_summary callback and
+    // a __tsan_symbolize_external callback and a race during a range memory
+    // access larger than 8 bytes. MemoryAccessRange adds the current PC to
+    // the trace and starts processing memory accesses. A first memory access
+    // triggers a race, we report it and call the instrumented
+    // __sanitizer_report_error_summary, which adds more stuff to the trace
+    // since it is intrumented. Then a second memory access in MemoryAccessRange
+    // also triggers a race and we get here and call TraceTopPC to get the
+    // current PC, however now it contains some unrelated events from the
+    // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
+    // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
+    // and the resulting PC has kExternalPCBit set, so we pass it to
+    // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
+    // rights to crash since the PC is completely bogus.
+    // test/tsan/double_race.cpp contains a test case for this.
+    toppc = 0;
+  }
+  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
+  if (IsFiredSuppression(ctx, typ, traces[0]))
+    return;
+
+  // MutexSet is too large to live on stack.
+  Vector<u64> mset_buffer;
+  mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
+  MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
+
+  Shadow s2(thr->racy_state[1]);
+  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
+  if (IsFiredSuppression(ctx, typ, traces[1]))
+    return;
+
+  if (HandleRacyStacks(thr, traces))
+    return;
+
+  // If any of the accesses has a tag, treat this as an "external" race.
+  uptr tag = kExternalTagNone;
+  for (uptr i = 0; i < kMop; i++) {
+    if (tags[i] != kExternalTagNone) {
+      typ = ReportTypeExternalRace;
+      tag = tags[i];
+      break;
+    }
+  }
+
+  ThreadRegistryLock l0(ctx->thread_registry);
+  ScopedReport rep(typ, tag);
+  for (uptr i = 0; i < kMop; i++) {
+    Shadow s(thr->racy_state[i]);
+    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
+                        i == 0 ? &thr->mset : mset2);
+  }
+
+  for (uptr i = 0; i < kMop; i++) {
+    FastState s(thr->racy_state[i]);
+    ThreadContext *tctx = static_cast<ThreadContext*>(
+        ctx->thread_registry->GetThreadLocked(s.tid()));
+    if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1)
+      continue;
+    rep.AddThread(tctx);
+  }
+
+  rep.AddLocation(addr_min, addr_max - addr_min);
+
+#if !SANITIZER_GO
+  {
+    Shadow s(thr->racy_state[1]);
+    if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
+      rep.AddSleep(thr->last_sleep_stack_id);
+  }
+#endif
+
+  if (!OutputReport(thr, rep))
+    return;
+
+}
+
+void PrintCurrentStack(ThreadState *thr, uptr pc) {
+  VarSizeStackTrace trace;
+  ObtainCurrentStack(thr, pc, &trace);
+  PrintStack(SymbolizeStack(trace));
+}
+
+// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes
+// __sanitizer_print_stack_trace exists in the actual unwinded stack, but
+// tail-call to PrintCurrentStackSlow breaks this assumption because
+// __sanitizer_print_stack_trace disappears after tail-call.
+// However, this solution is not reliable enough, please see dvyukov's comment
+// http://reviews.llvm.org/D19148#406208
+// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
+ALWAYS_INLINE
+void PrintCurrentStackSlow(uptr pc) {
+#if !SANITIZER_GO
+  uptr bp = GET_CURRENT_FRAME();
+  BufferedStackTrace *ptrace =
+      new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace)))
+          BufferedStackTrace();
+  ptrace->Unwind(pc, bp, nullptr, false);
+
+  for (uptr i = 0; i < ptrace->size / 2; i++) {
+    uptr tmp = ptrace->trace_buffer[i];
+    ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1];
+    ptrace->trace_buffer[ptrace->size - i - 1] = tmp;
+  }
+  PrintStack(SymbolizeStack(*ptrace));
+#endif
+}
+
+}  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
+}
+}  // extern "C"
diff --git a/lib/tsan/tsan_rtl_thread.cpp b/lib/tsan/tsan_rtl_thread.cpp
new file mode 100644
index 0000000000..d80146735e
--- /dev/null
+++ b/lib/tsan/tsan_rtl_thread.cpp
@@ -0,0 +1,462 @@
+//===-- tsan_rtl_thread.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_platform.h"
+#include "tsan_report.h"
+#include "tsan_sync.h"
+
+namespace __tsan {
+
+// ThreadContext implementation.
+
+ThreadContext::ThreadContext(int tid)
+  : ThreadContextBase(tid)
+  , thr()
+  , sync()
+  , epoch0()
+  , epoch1() {
+}
+
+#if !SANITIZER_GO
+ThreadContext::~ThreadContext() {
+}
+#endif
+
+void ThreadContext::OnDead() {
+  CHECK_EQ(sync.size(), 0);
+}
+
+void ThreadContext::OnJoined(void *arg) {
+  ThreadState *caller_thr = static_cast<ThreadState *>(arg);
+  AcquireImpl(caller_thr, 0, &sync);
+  sync.Reset(&caller_thr->proc()->clock_cache);
+}
+
+struct OnCreatedArgs {
+  ThreadState *thr;
+  uptr pc;
+};
+
+void ThreadContext::OnCreated(void *arg) {
+  thr = 0;
+  if (tid == 0)
+    return;
+  OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
+  if (!args->thr)  // GCD workers don't have a parent thread.
+    return;
+  args->thr->fast_state.IncrementEpoch();
+  // Can't increment epoch w/o writing to the trace as well.
+  TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
+  ReleaseImpl(args->thr, 0, &sync);
+  creation_stack_id = CurrentStackId(args->thr, args->pc);
+  if (reuse_count == 0)
+    StatInc(args->thr, StatThreadMaxTid);
+}
+
+void ThreadContext::OnReset() {
+  CHECK_EQ(sync.size(), 0);
+  uptr trace_p = GetThreadTrace(tid);
+  ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
+  //!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
+}
+
+void ThreadContext::OnDetached(void *arg) {
+  ThreadState *thr1 = static_cast<ThreadState*>(arg);
+  sync.Reset(&thr1->proc()->clock_cache);
+}
+
+struct OnStartedArgs {
+  ThreadState *thr;
+  uptr stk_addr;
+  uptr stk_size;
+  uptr tls_addr;
+  uptr tls_size;
+};
+
+void ThreadContext::OnStarted(void *arg) {
+  OnStartedArgs *args = static_cast<OnStartedArgs*>(arg);
+  thr = args->thr;
+  // RoundUp so that one trace part does not contain events
+  // from different threads.
+  epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
+  epoch1 = (u64)-1;
+  new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count,
+      args->stk_addr, args->stk_size, args->tls_addr, args->tls_size);
+#if !SANITIZER_GO
+  thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0];
+  thr->shadow_stack_pos = thr->shadow_stack;
+  thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize;
+#else
+  // Setup dynamic shadow stack.
+  const int kInitStackSize = 8;
+  thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
+      kInitStackSize * sizeof(uptr));
+  thr->shadow_stack_pos = thr->shadow_stack;
+  thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
+#endif
+  if (common_flags()->detect_deadlocks)
+    thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
+  thr->fast_state.SetHistorySize(flags()->history_size);
+  // Commit switch to the new part of the trace.
+  // TraceAddEvent will reset stack0/mset0 in the new part for us.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+
+  thr->fast_synch_epoch = epoch0;
+  AcquireImpl(thr, 0, &sync);
+  StatInc(thr, StatSyncAcquire);
+  sync.Reset(&thr->proc()->clock_cache);
+  thr->is_inited = true;
+  DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
+          "tls_addr=%zx tls_size=%zx\n",
+          tid, (uptr)epoch0, args->stk_addr, args->stk_size,
+          args->tls_addr, args->tls_size);
+}
+
+void ThreadContext::OnFinished() {
+#if SANITIZER_GO
+  internal_free(thr->shadow_stack);
+  thr->shadow_stack = nullptr;
+  thr->shadow_stack_pos = nullptr;
+  thr->shadow_stack_end = nullptr;
+#endif
+  if (!detached) {
+    thr->fast_state.IncrementEpoch();
+    // Can't increment epoch w/o writing to the trace as well.
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+    ReleaseImpl(thr, 0, &sync);
+  }
+  epoch1 = thr->fast_state.epoch();
+
+  if (common_flags()->detect_deadlocks)
+    ctx->dd->DestroyLogicalThread(thr->dd_lt);
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
+#if !SANITIZER_GO
+  PlatformCleanUpThreadState(thr);
+#endif
+  thr->~ThreadState();
+#if TSAN_COLLECT_STATS
+  StatAggregate(ctx->stat, thr->stat);
+#endif
+  thr = 0;
+}
+
+#if !SANITIZER_GO
+struct ThreadLeak {
+  ThreadContext *tctx;
+  int count;
+};
+
+static void MaybeReportThreadLeak(ThreadContextBase *tctx_base, void *arg) {
+  Vector<ThreadLeak> &leaks = *(Vector<ThreadLeak>*)arg;
+  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  if (tctx->detached || tctx->status != ThreadStatusFinished)
+    return;
+  for (uptr i = 0; i < leaks.Size(); i++) {
+    if (leaks[i].tctx->creation_stack_id == tctx->creation_stack_id) {
+      leaks[i].count++;
+      return;
+    }
+  }
+  ThreadLeak leak = {tctx, 1};
+  leaks.PushBack(leak);
+}
+#endif
+
+#if !SANITIZER_GO
+static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
+  if (tctx->tid == 0) {
+    Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
+  } else {
+    Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled,"
+      " created at:\n", tctx->tid, tctx->name);
+    PrintStack(SymbolizeStackId(tctx->creation_stack_id));
+  }
+  Printf("  One of the following ignores was not ended"
+      " (in order of probability)\n");
+  for (uptr i = 0; i < set->Size(); i++) {
+    Printf("  Ignore was enabled at:\n");
+    PrintStack(SymbolizeStackId(set->At(i)));
+  }
+  Die();
+}
+
+static void ThreadCheckIgnore(ThreadState *thr) {
+  if (ctx->after_multithreaded_fork)
+    return;
+  if (thr->ignore_reads_and_writes)
+    ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set);
+  if (thr->ignore_sync)
+    ReportIgnoresEnabled(thr->tctx, &thr->sync_ignore_set);
+}
+#else
+static void ThreadCheckIgnore(ThreadState *thr) {}
+#endif
+
+void ThreadFinalize(ThreadState *thr) {
+  ThreadCheckIgnore(thr);
+#if !SANITIZER_GO
+  if (!flags()->report_thread_leaks)
+    return;
+  ThreadRegistryLock l(ctx->thread_registry);
+  Vector<ThreadLeak> leaks;
+  ctx->thread_registry->RunCallbackForEachThreadLocked(
+      MaybeReportThreadLeak, &leaks);
+  for (uptr i = 0; i < leaks.Size(); i++) {
+    ScopedReport rep(ReportTypeThreadLeak);
+    rep.AddThread(leaks[i].tctx, true);
+    rep.SetCount(leaks[i].count);
+    OutputReport(thr, rep);
+  }
+#endif
+}
+
+int ThreadCount(ThreadState *thr) {
+  uptr result;
+  ctx->thread_registry->GetNumberOfThreads(0, 0, &result);
+  return (int)result;
+}
+
+int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
+  StatInc(thr, StatThreadCreate);
+  OnCreatedArgs args = { thr, pc };
+  u32 parent_tid = thr ? thr->tid : kInvalidTid;  // No parent for GCD workers.
+  int tid =
+      ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);
+  DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
+  StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads());
+  return tid;
+}
+
+void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
+                 ThreadType thread_type) {
+  uptr stk_addr = 0;
+  uptr stk_size = 0;
+  uptr tls_addr = 0;
+  uptr tls_size = 0;
+#if !SANITIZER_GO
+  if (thread_type != ThreadType::Fiber)
+    GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size);
+
+  if (tid) {
+    if (stk_addr && stk_size)
+      MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size);
+
+    if (tls_addr && tls_size) ImitateTlsWrite(thr, tls_addr, tls_size);
+  }
+#endif
+
+  ThreadRegistry *tr = ctx->thread_registry;
+  OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
+  tr->StartThread(tid, os_id, thread_type, &args);
+
+  tr->Lock();
+  thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
+  tr->Unlock();
+
+#if !SANITIZER_GO
+  if (ctx->after_multithreaded_fork) {
+    thr->ignore_interceptors++;
+    ThreadIgnoreBegin(thr, 0);
+    ThreadIgnoreSyncBegin(thr, 0);
+  }
+#endif
+}
+
+void ThreadFinish(ThreadState *thr) {
+  ThreadCheckIgnore(thr);
+  StatInc(thr, StatThreadFinish);
+  if (thr->stk_addr && thr->stk_size)
+    DontNeedShadowFor(thr->stk_addr, thr->stk_size);
+  if (thr->tls_addr && thr->tls_size)
+    DontNeedShadowFor(thr->tls_addr, thr->tls_size);
+  thr->is_dead = true;
+  ctx->thread_registry->FinishThread(thr->tid);
+}
+
+struct ConsumeThreadContext {
+  uptr uid;
+  ThreadContextBase *tctx;
+};
+
+static bool ConsumeThreadByUid(ThreadContextBase *tctx, void *arg) {
+  ConsumeThreadContext *findCtx = (ConsumeThreadContext *)arg;
+  if (tctx->user_id == findCtx->uid && tctx->status != ThreadStatusInvalid) {
+    if (findCtx->tctx) {
+      // Ensure that user_id is unique. If it's not the case we are screwed.
+      // Something went wrong before, but now there is no way to recover.
+      // Returning a wrong thread is not an option, it may lead to very hard
+      // to debug false positives (e.g. if we join a wrong thread).
+      Report("ThreadSanitizer: dup thread with used id 0x%zx\n", findCtx->uid);
+      Die();
+    }
+    findCtx->tctx = tctx;
+    tctx->user_id = 0;
+  }
+  return false;
+}
+
+int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
+  ConsumeThreadContext findCtx = {uid, nullptr};
+  ctx->thread_registry->FindThread(ConsumeThreadByUid, &findCtx);
+  int tid = findCtx.tctx ? findCtx.tctx->tid : ThreadRegistry::kUnknownTid;
+  DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, tid);
+  return tid;
+}
+
+void ThreadJoin(ThreadState *thr, uptr pc, int tid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
+  ctx->thread_registry->JoinThread(tid, thr);
+}
+
+void ThreadDetach(ThreadState *thr, uptr pc, int tid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  ctx->thread_registry->DetachThread(tid, thr);
+}
+
+void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) {
+  CHECK_GT(tid, 0);
+  CHECK_LT(tid, kMaxTid);
+  ctx->thread_registry->SetThreadUserId(tid, uid);
+}
+
+void ThreadSetName(ThreadState *thr, const char *name) {
+  ctx->thread_registry->SetThreadName(thr->tid, name);
+}
+
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
+                       uptr size, bool is_write) {
+  if (size == 0)
+    return;
+
+  u64 *shadow_mem = (u64*)MemToShadow(addr);
+  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
+      thr->tid, (void*)pc, (void*)addr,
+      (int)size, is_write);
+
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsAppMem(addr + size - 1)) {
+    Printf("Access to non app mem %zx\n", addr + size - 1);
+    DCHECK(IsAppMem(addr + size - 1));
+  }
+  if (!IsShadowMem((uptr)shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem((uptr)shadow_mem));
+  }
+  if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) {
+    Printf("Bad shadow addr %p (%zx)\n",
+               shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
+    DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1)));
+  }
+#endif
+
+  StatInc(thr, StatMopRange);
+
+  if (*shadow_mem == kShadowRodata) {
+    DCHECK(!is_write);
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    StatInc(thr, StatMopRangeRodata);
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (fast_state.GetIgnoreBit())
+    return;
+
+  fast_state.IncrementEpoch();
+  thr->fast_state = fast_state;
+  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+
+  bool unaligned = (addr % kShadowCell) != 0;
+
+  // Handle unaligned beginning, if any.
+  for (; addr % kShadowCell && size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
+        shadow_mem, cur);
+  }
+  if (unaligned)
+    shadow_mem += kShadowCnt;
+  // Handle middle part, if any.
+  for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
+    int const kAccessSizeLog = 3;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
+        shadow_mem, cur);
+    shadow_mem += kShadowCnt;
+  }
+  // Handle ending, if any.
+  for (; size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
+        shadow_mem, cur);
+  }
+}
+
+#if !SANITIZER_GO
+void FiberSwitchImpl(ThreadState *from, ThreadState *to) {
+  Processor *proc = from->proc();
+  ProcUnwire(proc, from);
+  ProcWire(proc, to);
+  set_cur_thread(to);
+}
+
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) {
+  void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState));
+  ThreadState *fiber = static_cast<ThreadState *>(mem);
+  internal_memset(fiber, 0, sizeof(*fiber));
+  int tid = ThreadCreate(thr, pc, 0, true);
+  FiberSwitchImpl(thr, fiber);
+  ThreadStart(fiber, tid, 0, ThreadType::Fiber);
+  FiberSwitchImpl(fiber, thr);
+  return fiber;
+}
+
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) {
+  FiberSwitchImpl(thr, fiber);
+  ThreadFinish(fiber);
+  FiberSwitchImpl(fiber, thr);
+  internal_free(fiber);
+}
+
+void FiberSwitch(ThreadState *thr, uptr pc,
+                 ThreadState *fiber, unsigned flags) {
+  if (!(flags & FiberSwitchFlagNoSync))
+    Release(thr, pc, (uptr)fiber);
+  FiberSwitchImpl(thr, fiber);
+  if (!(flags & FiberSwitchFlagNoSync))
+    Acquire(fiber, pc, (uptr)fiber);
+}
+#endif
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_stack_trace.cpp b/lib/tsan/tsan_stack_trace.cpp
new file mode 100644
index 0000000000..403a21ae4a
--- /dev/null
+++ b/lib/tsan/tsan_stack_trace.cpp
@@ -0,0 +1,63 @@
+//===-- tsan_stack_trace.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_stack_trace.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+VarSizeStackTrace::VarSizeStackTrace()
+    : StackTrace(nullptr, 0), trace_buffer(nullptr) {}
+
+VarSizeStackTrace::~VarSizeStackTrace() {
+  ResizeBuffer(0);
+}
+
+void VarSizeStackTrace::ResizeBuffer(uptr new_size) {
+  if (trace_buffer) {
+    internal_free(trace_buffer);
+  }
+  trace_buffer =
+      (new_size > 0)
+          ? (uptr *)internal_alloc(MBlockStackTrace,
+                                   new_size * sizeof(trace_buffer[0]))
+          : nullptr;
+  trace = trace_buffer;
+  size = new_size;
+}
+
+void VarSizeStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
+  ResizeBuffer(cnt + !!extra_top_pc);
+  internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0]));
+  if (extra_top_pc)
+    trace_buffer[cnt] = extra_top_pc;
+}
+
+void VarSizeStackTrace::ReverseOrder() {
+  for (u32 i = 0; i < (size >> 1); i++)
+    Swap(trace_buffer[i], trace_buffer[size - 1 - i]);
+}
+
+}  // namespace __tsan
+
+#if !SANITIZER_GO
+void __sanitizer::BufferedStackTrace::UnwindImpl(
+    uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
+  uptr top = 0;
+  uptr bottom = 0;
+  if (StackTrace::WillUseFastUnwind(request_fast)) {
+    GetThreadStackTopAndBottom(false, &top, &bottom);
+    Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
+  } else
+    Unwind(max_depth, pc, 0, context, 0, 0, false);
+}
+#endif  // SANITIZER_GO
diff --git a/lib/tsan/tsan_stack_trace.h b/lib/tsan/tsan_stack_trace.h
new file mode 100644
index 0000000000..3eb8ce156e
--- /dev/null
+++ b/lib/tsan/tsan_stack_trace.h
@@ -0,0 +1,42 @@
+//===-- tsan_stack_trace.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_STACK_TRACE_H
+#define TSAN_STACK_TRACE_H
+
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_defs.h"
+
+namespace __tsan {
+
+// StackTrace which calls malloc/free to allocate the buffer for
+// addresses in stack traces.
+struct VarSizeStackTrace : public StackTrace {
+  uptr *trace_buffer;  // Owned.
+
+  VarSizeStackTrace();
+  ~VarSizeStackTrace();
+  void Init(const uptr *pcs, uptr cnt, uptr extra_top_pc = 0);
+
+  // Reverses the current stack trace order, the top frame goes to the bottom,
+  // the last frame goes to the top.
+  void ReverseOrder();
+
+ private:
+  void ResizeBuffer(uptr new_size);
+
+  VarSizeStackTrace(const VarSizeStackTrace &);
+  void operator=(const VarSizeStackTrace &);
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_STACK_TRACE_H
diff --git a/lib/tsan/tsan_stat.cpp b/lib/tsan/tsan_stat.cpp
new file mode 100644
index 0000000000..78f3cce913
--- /dev/null
+++ b/lib/tsan/tsan_stat.cpp
@@ -0,0 +1,186 @@
+//===-- tsan_stat.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "tsan_stat.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+#if TSAN_COLLECT_STATS
+
+void StatAggregate(u64 *dst, u64 *src) {
+  for (int i = 0; i < StatCnt; i++)
+    dst[i] += src[i];
+}
+
+void StatOutput(u64 *stat) {
+  stat[StatShadowNonZero] = stat[StatShadowProcessed] - stat[StatShadowZero];
+
+  static const char *name[StatCnt] = {};
+  name[StatMop]                          = "Memory accesses                   ";
+  name[StatMopRead]                      = "  Including reads                 ";
+  name[StatMopWrite]                     = "            writes                ";
+  name[StatMop1]                         = "  Including size 1                ";
+  name[StatMop2]                         = "            size 2                ";
+  name[StatMop4]                         = "            size 4                ";
+  name[StatMop8]                         = "            size 8                ";
+  name[StatMopSame]                      = "  Including same                  ";
+  name[StatMopIgnored]                   = "  Including ignored               ";
+  name[StatMopRange]                     = "  Including range                 ";
+  name[StatMopRodata]                    = "  Including .rodata               ";
+  name[StatMopRangeRodata]               = "  Including .rodata range         ";
+  name[StatShadowProcessed]              = "Shadow processed                  ";
+  name[StatShadowZero]                   = "  Including empty                 ";
+  name[StatShadowNonZero]                = "  Including non empty             ";
+  name[StatShadowSameSize]               = "  Including same size             ";
+  name[StatShadowIntersect]              = "            intersect             ";
+  name[StatShadowNotIntersect]           = "            not intersect         ";
+  name[StatShadowSameThread]             = "  Including same thread           ";
+  name[StatShadowAnotherThread]          = "            another thread        ";
+  name[StatShadowReplace]                = "  Including evicted               ";
+
+  name[StatFuncEnter]                    = "Function entries                  ";
+  name[StatFuncExit]                     = "Function exits                    ";
+  name[StatEvents]                       = "Events collected                  ";
+
+  name[StatThreadCreate]                 = "Total threads created             ";
+  name[StatThreadFinish]                 = "  threads finished                ";
+  name[StatThreadReuse]                  = "  threads reused                  ";
+  name[StatThreadMaxTid]                 = "  max tid                         ";
+  name[StatThreadMaxAlive]               = "  max alive threads               ";
+
+  name[StatMutexCreate]                  = "Mutexes created                   ";
+  name[StatMutexDestroy]                 = "  destroyed                       ";
+  name[StatMutexLock]                    = "  lock                            ";
+  name[StatMutexUnlock]                  = "  unlock                          ";
+  name[StatMutexRecLock]                 = "  recursive lock                  ";
+  name[StatMutexRecUnlock]               = "  recursive unlock                ";
+  name[StatMutexReadLock]                = "  read lock                       ";
+  name[StatMutexReadUnlock]              = "  read unlock                     ";
+
+  name[StatSyncCreated]                  = "Sync objects created              ";
+  name[StatSyncDestroyed]                = "             destroyed            ";
+  name[StatSyncAcquire]                  = "             acquired             ";
+  name[StatSyncRelease]                  = "             released             ";
+
+  name[StatClockAcquire]                 = "Clock acquire                     ";
+  name[StatClockAcquireEmpty]            = "  empty clock                     ";
+  name[StatClockAcquireFastRelease]      = "  fast from release-store         ";
+  name[StatClockAcquireFull]             = "  full (slow)                     ";
+  name[StatClockAcquiredSomething]       = "  acquired something              ";
+  name[StatClockRelease]                 = "Clock release                     ";
+  name[StatClockReleaseResize]           = "  resize                          ";
+  name[StatClockReleaseFast]             = "  fast                            ";
+  name[StatClockReleaseSlow]             = "  dirty overflow (slow)           ";
+  name[StatClockReleaseFull]             = "  full (slow)                     ";
+  name[StatClockReleaseAcquired]         = "  was acquired                    ";
+  name[StatClockReleaseClearTail]        = "  clear tail                      ";
+  name[StatClockStore]                   = "Clock release store               ";
+  name[StatClockStoreResize]             = "  resize                          ";
+  name[StatClockStoreFast]               = "  fast                            ";
+  name[StatClockStoreFull]               = "  slow                            ";
+  name[StatClockStoreTail]               = "  clear tail                      ";
+  name[StatClockAcquireRelease]          = "Clock acquire-release             ";
+
+  name[StatAtomic]                       = "Atomic operations                 ";
+  name[StatAtomicLoad]                   = "  Including load                  ";
+  name[StatAtomicStore]                  = "            store                 ";
+  name[StatAtomicExchange]               = "            exchange              ";
+  name[StatAtomicFetchAdd]               = "            fetch_add             ";
+  name[StatAtomicFetchSub]               = "            fetch_sub             ";
+  name[StatAtomicFetchAnd]               = "            fetch_and             ";
+  name[StatAtomicFetchOr]                = "            fetch_or              ";
+  name[StatAtomicFetchXor]               = "            fetch_xor             ";
+  name[StatAtomicFetchNand]              = "            fetch_nand            ";
+  name[StatAtomicCAS]                    = "            compare_exchange      ";
+  name[StatAtomicFence]                  = "            fence                 ";
+  name[StatAtomicRelaxed]                = "  Including relaxed               ";
+  name[StatAtomicConsume]                = "            consume               ";
+  name[StatAtomicAcquire]                = "            acquire               ";
+  name[StatAtomicRelease]                = "            release               ";
+  name[StatAtomicAcq_Rel]                = "            acq_rel               ";
+  name[StatAtomicSeq_Cst]                = "            seq_cst               ";
+  name[StatAtomic1]                      = "  Including size 1                ";
+  name[StatAtomic2]                      = "            size 2                ";
+  name[StatAtomic4]                      = "            size 4                ";
+  name[StatAtomic8]                      = "            size 8                ";
+  name[StatAtomic16]                     = "            size 16               ";
+
+  name[StatAnnotation]                   = "Dynamic annotations               ";
+  name[StatAnnotateHappensBefore]        = "  HappensBefore                   ";
+  name[StatAnnotateHappensAfter]         = "  HappensAfter                    ";
+  name[StatAnnotateCondVarSignal]        = "  CondVarSignal                   ";
+  name[StatAnnotateCondVarSignalAll]     = "  CondVarSignalAll                ";
+  name[StatAnnotateMutexIsNotPHB]        = "  MutexIsNotPHB                   ";
+  name[StatAnnotateCondVarWait]          = "  CondVarWait                     ";
+  name[StatAnnotateRWLockCreate]         = "  RWLockCreate                    ";
+  name[StatAnnotateRWLockCreateStatic]   = "  StatAnnotateRWLockCreateStatic  ";
+  name[StatAnnotateRWLockDestroy]        = "  RWLockDestroy                   ";
+  name[StatAnnotateRWLockAcquired]       = "  RWLockAcquired                  ";
+  name[StatAnnotateRWLockReleased]       = "  RWLockReleased                  ";
+  name[StatAnnotateTraceMemory]          = "  TraceMemory                     ";
+  name[StatAnnotateFlushState]           = "  FlushState                      ";
+  name[StatAnnotateNewMemory]            = "  NewMemory                       ";
+  name[StatAnnotateNoOp]                 = "  NoOp                            ";
+  name[StatAnnotateFlushExpectedRaces]   = "  FlushExpectedRaces              ";
+  name[StatAnnotateEnableRaceDetection]  = "  EnableRaceDetection             ";
+  name[StatAnnotateMutexIsUsedAsCondVar] = "  MutexIsUsedAsCondVar            ";
+  name[StatAnnotatePCQGet]               = "  PCQGet                          ";
+  name[StatAnnotatePCQPut]               = "  PCQPut                          ";
+  name[StatAnnotatePCQDestroy]           = "  PCQDestroy                      ";
+  name[StatAnnotatePCQCreate]            = "  PCQCreate                       ";
+  name[StatAnnotateExpectRace]           = "  ExpectRace                      ";
+  name[StatAnnotateBenignRaceSized]      = "  BenignRaceSized                 ";
+  name[StatAnnotateBenignRace]           = "  BenignRace                      ";
+  name[StatAnnotateIgnoreReadsBegin]     = "  IgnoreReadsBegin                ";
+  name[StatAnnotateIgnoreReadsEnd]       = "  IgnoreReadsEnd                  ";
+  name[StatAnnotateIgnoreWritesBegin]    = "  IgnoreWritesBegin               ";
+  name[StatAnnotateIgnoreWritesEnd]      = "  IgnoreWritesEnd                 ";
+  name[StatAnnotateIgnoreSyncBegin]      = "  IgnoreSyncBegin                 ";
+  name[StatAnnotateIgnoreSyncEnd]        = "  IgnoreSyncEnd                   ";
+  name[StatAnnotatePublishMemoryRange]   = "  PublishMemoryRange              ";
+  name[StatAnnotateUnpublishMemoryRange] = "  UnpublishMemoryRange            ";
+  name[StatAnnotateThreadName]           = "  ThreadName                      ";
+  name[Stat__tsan_mutex_create]          = "  __tsan_mutex_create             ";
+  name[Stat__tsan_mutex_destroy]         = "  __tsan_mutex_destroy            ";
+  name[Stat__tsan_mutex_pre_lock]        = "  __tsan_mutex_pre_lock           ";
+  name[Stat__tsan_mutex_post_lock]       = "  __tsan_mutex_post_lock          ";
+  name[Stat__tsan_mutex_pre_unlock]      = "  __tsan_mutex_pre_unlock         ";
+  name[Stat__tsan_mutex_post_unlock]     = "  __tsan_mutex_post_unlock        ";
+  name[Stat__tsan_mutex_pre_signal]      = "  __tsan_mutex_pre_signal         ";
+  name[Stat__tsan_mutex_post_signal]     = "  __tsan_mutex_post_signal        ";
+  name[Stat__tsan_mutex_pre_divert]      = "  __tsan_mutex_pre_divert         ";
+  name[Stat__tsan_mutex_post_divert]     = "  __tsan_mutex_post_divert        ";
+
+  name[StatMtxTotal]                     = "Contentionz                       ";
+  name[StatMtxTrace]                     = "  Trace                           ";
+  name[StatMtxThreads]                   = "  Threads                         ";
+  name[StatMtxReport]                    = "  Report                          ";
+  name[StatMtxSyncVar]                   = "  SyncVar                         ";
+  name[StatMtxSyncTab]                   = "  SyncTab                         ";
+  name[StatMtxSlab]                      = "  Slab                            ";
+  name[StatMtxAtExit]                    = "  Atexit                          ";
+  name[StatMtxAnnotations]               = "  Annotations                     ";
+  name[StatMtxMBlock]                    = "  MBlock                          ";
+  name[StatMtxDeadlockDetector]          = "  DeadlockDetector                ";
+  name[StatMtxFired]                     = "  FiredSuppressions               ";
+  name[StatMtxRacy]                      = "  RacyStacks                      ";
+  name[StatMtxFD]                        = "  FD                              ";
+  name[StatMtxGlobalProc]                = "  GlobalProc                      ";
+
+  Printf("Statistics:\n");
+  for (int i = 0; i < StatCnt; i++)
+    Printf("%s: %16zu\n", name[i], (uptr)stat[i]);
+}
+
+#endif
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_stat.h b/lib/tsan/tsan_stat.h
new file mode 100644
index 0000000000..8b26a59bb2
--- /dev/null
+++ b/lib/tsan/tsan_stat.h
@@ -0,0 +1,191 @@
+//===-- tsan_stat.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_STAT_H
+#define TSAN_STAT_H
+
+namespace __tsan {
+
+enum StatType {
+  // Memory access processing related stuff.
+  StatMop,
+  StatMopRead,
+  StatMopWrite,
+  StatMop1,  // These must be consequtive.
+  StatMop2,
+  StatMop4,
+  StatMop8,
+  StatMopSame,
+  StatMopIgnored,
+  StatMopRange,
+  StatMopRodata,
+  StatMopRangeRodata,
+  StatShadowProcessed,
+  StatShadowZero,
+  StatShadowNonZero,  // Derived.
+  StatShadowSameSize,
+  StatShadowIntersect,
+  StatShadowNotIntersect,
+  StatShadowSameThread,
+  StatShadowAnotherThread,
+  StatShadowReplace,
+
+  // Func processing.
+  StatFuncEnter,
+  StatFuncExit,
+
+  // Trace processing.
+  StatEvents,
+
+  // Threads.
+  StatThreadCreate,
+  StatThreadFinish,
+  StatThreadReuse,
+  StatThreadMaxTid,
+  StatThreadMaxAlive,
+
+  // Mutexes.
+  StatMutexCreate,
+  StatMutexDestroy,
+  StatMutexLock,
+  StatMutexUnlock,
+  StatMutexRecLock,
+  StatMutexRecUnlock,
+  StatMutexReadLock,
+  StatMutexReadUnlock,
+
+  // Synchronization.
+  StatSyncCreated,
+  StatSyncDestroyed,
+  StatSyncAcquire,
+  StatSyncRelease,
+  StatSyncReleaseStoreAcquire,
+
+  // Clocks - acquire.
+  StatClockAcquire,
+  StatClockAcquireEmpty,
+  StatClockAcquireFastRelease,
+  StatClockAcquireFull,
+  StatClockAcquiredSomething,
+  // Clocks - release.
+  StatClockRelease,
+  StatClockReleaseResize,
+  StatClockReleaseFast,
+  StatClockReleaseSlow,
+  StatClockReleaseFull,
+  StatClockReleaseAcquired,
+  StatClockReleaseClearTail,
+  // Clocks - release store.
+  StatClockStore,
+  StatClockStoreResize,
+  StatClockStoreFast,
+  StatClockStoreFull,
+  StatClockStoreTail,
+  // Clocks - acquire-release.
+  StatClockAcquireRelease,
+
+  // Atomics.
+  StatAtomic,
+  StatAtomicLoad,
+  StatAtomicStore,
+  StatAtomicExchange,
+  StatAtomicFetchAdd,
+  StatAtomicFetchSub,
+  StatAtomicFetchAnd,
+  StatAtomicFetchOr,
+  StatAtomicFetchXor,
+  StatAtomicFetchNand,
+  StatAtomicCAS,
+  StatAtomicFence,
+  StatAtomicRelaxed,
+  StatAtomicConsume,
+  StatAtomicAcquire,
+  StatAtomicRelease,
+  StatAtomicAcq_Rel,
+  StatAtomicSeq_Cst,
+  StatAtomic1,
+  StatAtomic2,
+  StatAtomic4,
+  StatAtomic8,
+  StatAtomic16,
+
+  // Dynamic annotations.
+  StatAnnotation,
+  StatAnnotateHappensBefore,
+  StatAnnotateHappensAfter,
+  StatAnnotateCondVarSignal,
+  StatAnnotateCondVarSignalAll,
+  StatAnnotateMutexIsNotPHB,
+  StatAnnotateCondVarWait,
+  StatAnnotateRWLockCreate,
+  StatAnnotateRWLockCreateStatic,
+  StatAnnotateRWLockDestroy,
+  StatAnnotateRWLockAcquired,
+  StatAnnotateRWLockReleased,
+  StatAnnotateTraceMemory,
+  StatAnnotateFlushState,
+  StatAnnotateNewMemory,
+  StatAnnotateNoOp,
+  StatAnnotateFlushExpectedRaces,
+  StatAnnotateEnableRaceDetection,
+  StatAnnotateMutexIsUsedAsCondVar,
+  StatAnnotatePCQGet,
+  StatAnnotatePCQPut,
+  StatAnnotatePCQDestroy,
+  StatAnnotatePCQCreate,
+  StatAnnotateExpectRace,
+  StatAnnotateBenignRaceSized,
+  StatAnnotateBenignRace,
+  StatAnnotateIgnoreReadsBegin,
+  StatAnnotateIgnoreReadsEnd,
+  StatAnnotateIgnoreWritesBegin,
+  StatAnnotateIgnoreWritesEnd,
+  StatAnnotateIgnoreSyncBegin,
+  StatAnnotateIgnoreSyncEnd,
+  StatAnnotatePublishMemoryRange,
+  StatAnnotateUnpublishMemoryRange,
+  StatAnnotateThreadName,
+  Stat__tsan_mutex_create,
+  Stat__tsan_mutex_destroy,
+  Stat__tsan_mutex_pre_lock,
+  Stat__tsan_mutex_post_lock,
+  Stat__tsan_mutex_pre_unlock,
+  Stat__tsan_mutex_post_unlock,
+  Stat__tsan_mutex_pre_signal,
+  Stat__tsan_mutex_post_signal,
+  Stat__tsan_mutex_pre_divert,
+  Stat__tsan_mutex_post_divert,
+
+  // Internal mutex contentionz.
+  StatMtxTotal,
+  StatMtxTrace,
+  StatMtxThreads,
+  StatMtxReport,
+  StatMtxSyncVar,
+  StatMtxSyncTab,
+  StatMtxSlab,
+  StatMtxAnnotations,
+  StatMtxAtExit,
+  StatMtxMBlock,
+  StatMtxDeadlockDetector,
+  StatMtxFired,
+  StatMtxRacy,
+  StatMtxFD,
+  StatMtxGlobalProc,
+
+  // This must be the last.
+  StatCnt
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_STAT_H
diff --git a/lib/tsan/tsan_suppressions.cpp b/lib/tsan/tsan_suppressions.cpp
new file mode 100644
index 0000000000..a1c1bf81bf
--- /dev/null
+++ b/lib/tsan/tsan_suppressions.cpp
@@ -0,0 +1,161 @@
+//===-- tsan_suppressions.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "tsan_suppressions.h"
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+#include "tsan_mman.h"
+#include "tsan_platform.h"
+
+#if !SANITIZER_GO
+// Suppressions for true/false positives in standard libraries.
+static const char *const std_suppressions =
+// Libstdc++ 4.4 has data races in std::string.
+// See http://crbug.com/181502 for an example.
+"race:^_M_rep$\n"
+"race:^_M_is_leaked$\n"
+// False positive when using std <thread>.
+// Happens because we miss atomic synchronization in libstdc++.
+// See http://llvm.org/bugs/show_bug.cgi?id=17066 for details.
+"race:std::_Sp_counted_ptr_inplace<std::thread::_Impl\n";
+
+// Can be overriden in frontend.
+SANITIZER_WEAK_DEFAULT_IMPL
+const char *__tsan_default_suppressions() {
+  return 0;
+}
+#endif
+
+namespace __tsan {
+
+ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
+static SuppressionContext *suppression_ctx = nullptr;
+static const char *kSuppressionTypes[] = {
+    kSuppressionRace,   kSuppressionRaceTop, kSuppressionMutex,
+    kSuppressionThread, kSuppressionSignal, kSuppressionLib,
+    kSuppressionDeadlock};
+
+void InitializeSuppressions() {
+  CHECK_EQ(nullptr, suppression_ctx);
+  suppression_ctx = new (suppression_placeholder)
+      SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
+  suppression_ctx->ParseFromFile(flags()->suppressions);
+#if !SANITIZER_GO
+  suppression_ctx->Parse(__tsan_default_suppressions());
+  suppression_ctx->Parse(std_suppressions);
+#endif
+}
+
+SuppressionContext *Suppressions() {
+  CHECK(suppression_ctx);
+  return suppression_ctx;
+}
+
+static const char *conv(ReportType typ) {
+  switch (typ) {
+    case ReportTypeRace:
+    case ReportTypeVptrRace:
+    case ReportTypeUseAfterFree:
+    case ReportTypeVptrUseAfterFree:
+    case ReportTypeExternalRace:
+      return kSuppressionRace;
+    case ReportTypeThreadLeak:
+      return kSuppressionThread;
+    case ReportTypeMutexDestroyLocked:
+    case ReportTypeMutexDoubleLock:
+    case ReportTypeMutexInvalidAccess:
+    case ReportTypeMutexBadUnlock:
+    case ReportTypeMutexBadReadLock:
+    case ReportTypeMutexBadReadUnlock:
+      return kSuppressionMutex;
+    case ReportTypeSignalUnsafe:
+    case ReportTypeErrnoInSignal:
+      return kSuppressionSignal;
+    case ReportTypeDeadlock:
+      return kSuppressionDeadlock;
+    // No default case so compiler warns us if we miss one
+  }
+  UNREACHABLE("missing case");
+}
+
+static uptr IsSuppressed(const char *stype, const AddressInfo &info,
+    Suppression **sp) {
+  if (suppression_ctx->Match(info.function, stype, sp) ||
+      suppression_ctx->Match(info.file, stype, sp) ||
+      suppression_ctx->Match(info.module, stype, sp)) {
+    VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", (*sp)->templ);
+    atomic_fetch_add(&(*sp)->hit_count, 1, memory_order_relaxed);
+    return info.address;
+  }
+  return 0;
+}
+
+uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp) {
+  CHECK(suppression_ctx);
+  if (!suppression_ctx->SuppressionCount() || stack == 0 ||
+      !stack->suppressable)
+    return 0;
+  const char *stype = conv(typ);
+  if (0 == internal_strcmp(stype, kSuppressionNone))
+    return 0;
+  for (const SymbolizedStack *frame = stack->frames; frame;
+      frame = frame->next) {
+    uptr pc = IsSuppressed(stype, frame->info, sp);
+    if (pc != 0)
+      return pc;
+  }
+  if (0 == internal_strcmp(stype, kSuppressionRace) && stack->frames != nullptr)
+    return IsSuppressed(kSuppressionRaceTop, stack->frames->info, sp);
+  return 0;
+}
+
+uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) {
+  CHECK(suppression_ctx);
+  if (!suppression_ctx->SuppressionCount() || loc == 0 ||
+      loc->type != ReportLocationGlobal || !loc->suppressable)
+    return 0;
+  const char *stype = conv(typ);
+  if (0 == internal_strcmp(stype, kSuppressionNone))
+    return 0;
+  Suppression *s;
+  const DataInfo &global = loc->global;
+  if (suppression_ctx->Match(global.name, stype, &s) ||
+      suppression_ctx->Match(global.module, stype, &s)) {
+      VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", s->templ);
+      atomic_fetch_add(&s->hit_count, 1, memory_order_relaxed);
+      *sp = s;
+      return global.start;
+  }
+  return 0;
+}
+
+void PrintMatchedSuppressions() {
+  InternalMmapVector<Suppression *> matched;
+  CHECK(suppression_ctx);
+  suppression_ctx->GetMatched(&matched);
+  if (!matched.size())
+    return;
+  int hit_count = 0;
+  for (uptr i = 0; i < matched.size(); i++)
+    hit_count += atomic_load_relaxed(&matched[i]->hit_count);
+  Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count,
+         (int)internal_getpid());
+  for (uptr i = 0; i < matched.size(); i++) {
+    Printf("%d %s:%s\n", atomic_load_relaxed(&matched[i]->hit_count),
+           matched[i]->type, matched[i]->templ);
+  }
+}
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_suppressions.h b/lib/tsan/tsan_suppressions.h
new file mode 100644
index 0000000000..f430aeb6c4
--- /dev/null
+++ b/lib/tsan/tsan_suppressions.h
@@ -0,0 +1,37 @@
+//===-- tsan_suppressions.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SUPPRESSIONS_H
+#define TSAN_SUPPRESSIONS_H
+
+#include "sanitizer_common/sanitizer_suppressions.h"
+#include "tsan_report.h"
+
+namespace __tsan {
+
+const char kSuppressionNone[] = "none";
+const char kSuppressionRace[] = "race";
+const char kSuppressionRaceTop[] = "race_top";
+const char kSuppressionMutex[] = "mutex";
+const char kSuppressionThread[] = "thread";
+const char kSuppressionSignal[] = "signal";
+const char kSuppressionLib[] = "called_from_lib";
+const char kSuppressionDeadlock[] = "deadlock";
+
+void InitializeSuppressions();
+SuppressionContext *Suppressions();
+void PrintMatchedSuppressions();
+uptr IsSuppressed(ReportType typ, const ReportStack *stack, Suppression **sp);
+uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp);
+
+}  // namespace __tsan
+
+#endif  // TSAN_SUPPRESSIONS_H
diff --git a/lib/tsan/tsan_symbolize.cpp b/lib/tsan/tsan_symbolize.cpp
new file mode 100644
index 0000000000..6478f3a754
--- /dev/null
+++ b/lib/tsan/tsan_symbolize.cpp
@@ -0,0 +1,122 @@
+//===-- tsan_symbolize.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_symbolize.h"
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "tsan_flags.h"
+#include "tsan_report.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+void EnterSymbolizer() {
+  ThreadState *thr = cur_thread();
+  CHECK(!thr->in_symbolizer);
+  thr->in_symbolizer = true;
+  thr->ignore_interceptors++;
+}
+
+void ExitSymbolizer() {
+  ThreadState *thr = cur_thread();
+  CHECK(thr->in_symbolizer);
+  thr->in_symbolizer = false;
+  thr->ignore_interceptors--;
+}
+
+// Legacy API.
+// May be overriden by JIT/JAVA/etc,
+// whatever produces PCs marked with kExternalPCBit.
+SANITIZER_WEAK_DEFAULT_IMPL
+bool __tsan_symbolize_external(uptr pc, char *func_buf, uptr func_siz,
+                               char *file_buf, uptr file_siz, int *line,
+                               int *col) {
+  return false;
+}
+
+// New API: call __tsan_symbolize_external_ex only when it exists.
+// Once old clients are gone, provide dummy implementation.
+SANITIZER_WEAK_DEFAULT_IMPL
+void __tsan_symbolize_external_ex(uptr pc,
+                                  void (*add_frame)(void *, const char *,
+                                                    const char *, int, int),
+                                  void *ctx) {}
+
+struct SymbolizedStackBuilder {
+  SymbolizedStack *head;
+  SymbolizedStack *tail;
+  uptr addr;
+};
+
+static void AddFrame(void *ctx, const char *function_name, const char *file,
+                     int line, int column) {
+  SymbolizedStackBuilder *ssb = (struct SymbolizedStackBuilder *)ctx;
+  if (ssb->tail) {
+    ssb->tail->next = SymbolizedStack::New(ssb->addr);
+    ssb->tail = ssb->tail->next;
+  } else {
+    ssb->head = ssb->tail = SymbolizedStack::New(ssb->addr);
+  }
+  AddressInfo *info = &ssb->tail->info;
+  if (function_name) {
+    info->function = internal_strdup(function_name);
+  }
+  if (file) {
+    info->file = internal_strdup(file);
+  }
+  info->line = line;
+  info->column = column;
+}
+
+SymbolizedStack *SymbolizeCode(uptr addr) {
+  // Check if PC comes from non-native land.
+  if (addr & kExternalPCBit) {
+    SymbolizedStackBuilder ssb = {nullptr, nullptr, addr};
+    __tsan_symbolize_external_ex(addr, AddFrame, &ssb);
+    if (ssb.head)
+      return ssb.head;
+    // Legacy code: remove along with the declaration above
+    // once all clients using this API are gone.
+    // Declare static to not consume too much stack space.
+    // We symbolize reports in a single thread, so this is fine.
+    static char func_buf[1024];
+    static char file_buf[1024];
+    int line, col;
+    SymbolizedStack *frame = SymbolizedStack::New(addr);
+    if (__tsan_symbolize_external(addr, func_buf, sizeof(func_buf), file_buf,
+                                  sizeof(file_buf), &line, &col)) {
+      frame->info.function = internal_strdup(func_buf);
+      frame->info.file = internal_strdup(file_buf);
+      frame->info.line = line;
+      frame->info.column = col;
+    }
+    return frame;
+  }
+  return Symbolizer::GetOrInit()->SymbolizePC(addr);
+}
+
+ReportLocation *SymbolizeData(uptr addr) {
+  DataInfo info;
+  if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))
+    return 0;
+  ReportLocation *ent = ReportLocation::New(ReportLocationGlobal);
+  internal_memcpy(&ent->global, &info, sizeof(info));
+  return ent;
+}
+
+void SymbolizeFlush() {
+  Symbolizer::GetOrInit()->Flush();
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_symbolize.h b/lib/tsan/tsan_symbolize.h
new file mode 100644
index 0000000000..7adaa04dc2
--- /dev/null
+++ b/lib/tsan/tsan_symbolize.h
@@ -0,0 +1,30 @@
+//===-- tsan_symbolize.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SYMBOLIZE_H
+#define TSAN_SYMBOLIZE_H
+
+#include "tsan_defs.h"
+#include "tsan_report.h"
+
+namespace __tsan {
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+SymbolizedStack *SymbolizeCode(uptr addr);
+ReportLocation *SymbolizeData(uptr addr);
+void SymbolizeFlush();
+
+ReportStack *NewReportStackEntry(uptr addr);
+
+}  // namespace __tsan
+
+#endif  // TSAN_SYMBOLIZE_H
diff --git a/lib/tsan/tsan_sync.cpp b/lib/tsan/tsan_sync.cpp
new file mode 100644
index 0000000000..7f686dc5fc
--- /dev/null
+++ b/lib/tsan/tsan_sync.cpp
@@ -0,0 +1,296 @@
+//===-- tsan_sync.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_sync.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+
+namespace __tsan {
+
+void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
+
+SyncVar::SyncVar()
+    : mtx(MutexTypeSyncVar, StatMtxSyncVar) {
+  Reset(0);
+}
+
+void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
+  this->addr = addr;
+  this->uid = uid;
+  this->next = 0;
+
+  creation_stack_id = 0;
+  if (!SANITIZER_GO)  // Go does not use them
+    creation_stack_id = CurrentStackId(thr, pc);
+  if (common_flags()->detect_deadlocks)
+    DDMutexInit(thr, pc, this);
+}
+
+void SyncVar::Reset(Processor *proc) {
+  uid = 0;
+  creation_stack_id = 0;
+  owner_tid = kInvalidTid;
+  last_lock = 0;
+  recursion = 0;
+  atomic_store_relaxed(&flags, 0);
+
+  if (proc == 0) {
+    CHECK_EQ(clock.size(), 0);
+    CHECK_EQ(read_clock.size(), 0);
+  } else {
+    clock.Reset(&proc->clock_cache);
+    read_clock.Reset(&proc->clock_cache);
+  }
+}
+
+MetaMap::MetaMap()
+    : block_alloc_("heap block allocator")
+    , sync_alloc_("sync allocator") {
+  atomic_store(&uid_gen_, 0, memory_order_relaxed);
+}
+
+void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
+  u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
+  MBlock *b = block_alloc_.Map(idx);
+  b->siz = sz;
+  b->tag = 0;
+  b->tid = thr->tid;
+  b->stk = CurrentStackId(thr, pc);
+  u32 *meta = MemToMeta(p);
+  DCHECK_EQ(*meta, 0);
+  *meta = idx | kFlagBlock;
+}
+
+uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
+  MBlock* b = GetBlock(p);
+  if (b == 0)
+    return 0;
+  uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
+  FreeRange(proc, p, sz);
+  return sz;
+}
+
+bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
+  bool has_something = false;
+  u32 *meta = MemToMeta(p);
+  u32 *end = MemToMeta(p + sz);
+  if (end == meta)
+    end++;
+  for (; meta < end; meta++) {
+    u32 idx = *meta;
+    if (idx == 0) {
+      // Note: don't write to meta in this case -- the block can be huge.
+      continue;
+    }
+    *meta = 0;
+    has_something = true;
+    while (idx != 0) {
+      if (idx & kFlagBlock) {
+        block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask);
+        break;
+      } else if (idx & kFlagSync) {
+        DCHECK(idx & kFlagSync);
+        SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
+        u32 next = s->next;
+        s->Reset(proc);
+        sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
+        idx = next;
+      } else {
+        CHECK(0);
+      }
+    }
+  }
+  return has_something;
+}
+
+// ResetRange removes all meta objects from the range.
+// It is called for large mmap-ed regions. The function is best-effort wrt
+// freeing of meta objects, because we don't want to page in the whole range
+// which can be huge. The function probes pages one-by-one until it finds a page
+// without meta objects, at this point it stops freeing meta objects. Because
+// thread stacks grow top-down, we do the same starting from end as well.
+void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
+  if (SANITIZER_GO) {
+    // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
+    // so we do the optimization only for C/C++.
+    FreeRange(proc, p, sz);
+    return;
+  }
+  const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
+  const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
+  if (sz <= 4 * kPageSize) {
+    // If the range is small, just do the normal free procedure.
+    FreeRange(proc, p, sz);
+    return;
+  }
+  // First, round both ends of the range to page size.
+  uptr diff = RoundUp(p, kPageSize) - p;
+  if (diff != 0) {
+    FreeRange(proc, p, diff);
+    p += diff;
+    sz -= diff;
+  }
+  diff = p + sz - RoundDown(p + sz, kPageSize);
+  if (diff != 0) {
+    FreeRange(proc, p + sz - diff, diff);
+    sz -= diff;
+  }
+  // Now we must have a non-empty page-aligned range.
+  CHECK_GT(sz, 0);
+  CHECK_EQ(p, RoundUp(p, kPageSize));
+  CHECK_EQ(sz, RoundUp(sz, kPageSize));
+  const uptr p0 = p;
+  const uptr sz0 = sz;
+  // Probe start of the range.
+  for (uptr checked = 0; sz > 0; checked += kPageSize) {
+    bool has_something = FreeRange(proc, p, kPageSize);
+    p += kPageSize;
+    sz -= kPageSize;
+    if (!has_something && checked > (128 << 10))
+      break;
+  }
+  // Probe end of the range.
+  for (uptr checked = 0; sz > 0; checked += kPageSize) {
+    bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);
+    sz -= kPageSize;
+    // Stacks grow down, so sync object are most likely at the end of the region
+    // (if it is a stack). The very end of the stack is TLS and tsan increases
+    // TLS by at least 256K, so check at least 512K.
+    if (!has_something && checked > (512 << 10))
+      break;
+  }
+  // Finally, page out the whole range (including the parts that we've just
+  // freed). Note: we can't simply madvise, because we need to leave a zeroed
+  // range (otherwise __tsan_java_move can crash if it encounters a left-over
+  // meta objects in java heap).
+  uptr metap = (uptr)MemToMeta(p0);
+  uptr metasz = sz0 / kMetaRatio;
+  UnmapOrDie((void*)metap, metasz);
+  if (!MmapFixedNoReserve(metap, metasz))
+    Die();
+}
+
+MBlock* MetaMap::GetBlock(uptr p) {
+  u32 *meta = MemToMeta(p);
+  u32 idx = *meta;
+  for (;;) {
+    if (idx == 0)
+      return 0;
+    if (idx & kFlagBlock)
+      return block_alloc_.Map(idx & ~kFlagMask);
+    DCHECK(idx & kFlagSync);
+    SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
+    idx = s->next;
+  }
+}
+
+SyncVar* MetaMap::GetOrCreateAndLock(ThreadState *thr, uptr pc,
+                              uptr addr, bool write_lock) {
+  return GetAndLock(thr, pc, addr, write_lock, true);
+}
+
+SyncVar* MetaMap::GetIfExistsAndLock(uptr addr, bool write_lock) {
+  return GetAndLock(0, 0, addr, write_lock, false);
+}
+
+SyncVar* MetaMap::GetAndLock(ThreadState *thr, uptr pc,
+                             uptr addr, bool write_lock, bool create) {
+  u32 *meta = MemToMeta(addr);
+  u32 idx0 = *meta;
+  u32 myidx = 0;
+  SyncVar *mys = 0;
+  for (;;) {
+    u32 idx = idx0;
+    for (;;) {
+      if (idx == 0)
+        break;
+      if (idx & kFlagBlock)
+        break;
+      DCHECK(idx & kFlagSync);
+      SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
+      if (s->addr == addr) {
+        if (myidx != 0) {
+          mys->Reset(thr->proc());
+          sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
+        }
+        if (write_lock)
+          s->mtx.Lock();
+        else
+          s->mtx.ReadLock();
+        return s;
+      }
+      idx = s->next;
+    }
+    if (!create)
+      return 0;
+    if (*meta != idx0) {
+      idx0 = *meta;
+      continue;
+    }
+
+    if (myidx == 0) {
+      const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
+      myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
+      mys = sync_alloc_.Map(myidx);
+      mys->Init(thr, pc, addr, uid);
+    }
+    mys->next = idx0;
+    if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
+        myidx | kFlagSync, memory_order_release)) {
+      if (write_lock)
+        mys->mtx.Lock();
+      else
+        mys->mtx.ReadLock();
+      return mys;
+    }
+  }
+}
+
+void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
+  // src and dst can overlap,
+  // there are no concurrent accesses to the regions (e.g. stop-the-world).
+  CHECK_NE(src, dst);
+  CHECK_NE(sz, 0);
+  uptr diff = dst - src;
+  u32 *src_meta = MemToMeta(src);
+  u32 *dst_meta = MemToMeta(dst);
+  u32 *src_meta_end = MemToMeta(src + sz);
+  uptr inc = 1;
+  if (dst > src) {
+    src_meta = MemToMeta(src + sz) - 1;
+    dst_meta = MemToMeta(dst + sz) - 1;
+    src_meta_end = MemToMeta(src) - 1;
+    inc = -1;
+  }
+  for (; src_meta != src_meta_end; src_meta += inc, dst_meta += inc) {
+    CHECK_EQ(*dst_meta, 0);
+    u32 idx = *src_meta;
+    *src_meta = 0;
+    *dst_meta = idx;
+    // Patch the addresses in sync objects.
+    while (idx != 0) {
+      if (idx & kFlagBlock)
+        break;
+      CHECK(idx & kFlagSync);
+      SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
+      s->addr += diff;
+      idx = s->next;
+    }
+  }
+}
+
+void MetaMap::OnProcIdle(Processor *proc) {
+  block_alloc_.FlushCache(&proc->block_cache);
+  sync_alloc_.FlushCache(&proc->sync_cache);
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_sync.h b/lib/tsan/tsan_sync.h
new file mode 100644
index 0000000000..47f2739d8d
--- /dev/null
+++ b/lib/tsan/tsan_sync.h
@@ -0,0 +1,145 @@
+//===-- tsan_sync.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_SYNC_H
+#define TSAN_SYNC_H
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
+#include "tsan_defs.h"
+#include "tsan_clock.h"
+#include "tsan_mutex.h"
+#include "tsan_dense_alloc.h"
+
+namespace __tsan {
+
+// These need to match __tsan_mutex_* flags defined in tsan_interface.h.
+// See documentation there as well.
+enum MutexFlags {
+  MutexFlagLinkerInit          = 1 << 0, // __tsan_mutex_linker_init
+  MutexFlagWriteReentrant      = 1 << 1, // __tsan_mutex_write_reentrant
+  MutexFlagReadReentrant       = 1 << 2, // __tsan_mutex_read_reentrant
+  MutexFlagReadLock            = 1 << 3, // __tsan_mutex_read_lock
+  MutexFlagTryLock             = 1 << 4, // __tsan_mutex_try_lock
+  MutexFlagTryLockFailed       = 1 << 5, // __tsan_mutex_try_lock_failed
+  MutexFlagRecursiveLock       = 1 << 6, // __tsan_mutex_recursive_lock
+  MutexFlagRecursiveUnlock     = 1 << 7, // __tsan_mutex_recursive_unlock
+  MutexFlagNotStatic           = 1 << 8, // __tsan_mutex_not_static
+
+  // The following flags are runtime private.
+  // Mutex API misuse was detected, so don't report any more.
+  MutexFlagBroken              = 1 << 30,
+  // We did not intercept pre lock event, so handle it on post lock.
+  MutexFlagDoPreLockOnPostLock = 1 << 29,
+  // Must list all mutex creation flags.
+  MutexCreationFlagMask        = MutexFlagLinkerInit |
+                                 MutexFlagWriteReentrant |
+                                 MutexFlagReadReentrant |
+                                 MutexFlagNotStatic,
+};
+
+struct SyncVar {
+  SyncVar();
+
+  static const int kInvalidTid = -1;
+
+  uptr addr;  // overwritten by DenseSlabAlloc freelist
+  Mutex mtx;
+  u64 uid;  // Globally unique id.
+  u32 creation_stack_id;
+  int owner_tid;  // Set only by exclusive owners.
+  u64 last_lock;
+  int recursion;
+  atomic_uint32_t flags;
+  u32 next;  // in MetaMap
+  DDMutex dd;
+  SyncClock read_clock;  // Used for rw mutexes only.
+  // The clock is placed last, so that it is situated on a different cache line
+  // with the mtx. This reduces contention for hot sync objects.
+  SyncClock clock;
+
+  void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
+  void Reset(Processor *proc);
+
+  u64 GetId() const {
+    // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
+    return GetLsb((u64)addr | (uid << 48), 60);
+  }
+  bool CheckId(u64 uid) const {
+    CHECK_EQ(uid, GetLsb(uid, 14));
+    return GetLsb(this->uid, 14) == uid;
+  }
+  static uptr SplitId(u64 id, u64 *uid) {
+    *uid = id >> 48;
+    return (uptr)GetLsb(id, 48);
+  }
+
+  bool IsFlagSet(u32 f) const {
+    return atomic_load_relaxed(&flags) & f;
+  }
+
+  void SetFlags(u32 f) {
+    atomic_store_relaxed(&flags, atomic_load_relaxed(&flags) | f);
+  }
+
+  void UpdateFlags(u32 flagz) {
+    // Filter out operation flags.
+    if (!(flagz & MutexCreationFlagMask))
+      return;
+    u32 current = atomic_load_relaxed(&flags);
+    if (current & MutexCreationFlagMask)
+      return;
+    // Note: this can be called from MutexPostReadLock which holds only read
+    // lock on the SyncVar.
+    atomic_store_relaxed(&flags, current | (flagz & MutexCreationFlagMask));
+  }
+};
+
+/* MetaMap allows to map arbitrary user pointers onto various descriptors.
+   Currently it maps pointers to heap block descriptors and sync var descs.
+   It uses 1/2 direct shadow, see tsan_platform.h.
+*/
+class MetaMap {
+ public:
+  MetaMap();
+
+  void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
+  uptr FreeBlock(Processor *proc, uptr p);
+  bool FreeRange(Processor *proc, uptr p, uptr sz);
+  void ResetRange(Processor *proc, uptr p, uptr sz);
+  MBlock* GetBlock(uptr p);
+
+  SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc,
+                              uptr addr, bool write_lock);
+  SyncVar* GetIfExistsAndLock(uptr addr, bool write_lock);
+
+  void MoveMemory(uptr src, uptr dst, uptr sz);
+
+  void OnProcIdle(Processor *proc);
+
+ private:
+  static const u32 kFlagMask  = 3u << 30;
+  static const u32 kFlagBlock = 1u << 30;
+  static const u32 kFlagSync  = 2u << 30;
+  typedef DenseSlabAlloc<MBlock, 1<<16, 1<<12> BlockAlloc;
+  typedef DenseSlabAlloc<SyncVar, 1<<16, 1<<10> SyncAlloc;
+  BlockAlloc block_alloc_;
+  SyncAlloc sync_alloc_;
+  atomic_uint64_t uid_gen_;
+
+  SyncVar* GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,
+                      bool create);
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_SYNC_H
diff --git a/lib/tsan/tsan_trace.h b/lib/tsan/tsan_trace.h
new file mode 100644
index 0000000000..fbd0f72db6
--- /dev/null
+++ b/lib/tsan/tsan_trace.h
@@ -0,0 +1,75 @@
+//===-- tsan_trace.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_TRACE_H
+#define TSAN_TRACE_H
+
+#include "tsan_defs.h"
+#include "tsan_mutex.h"
+#include "tsan_stack_trace.h"
+#include "tsan_mutexset.h"
+
+namespace __tsan {
+
+const int kTracePartSizeBits = 13;
+const int kTracePartSize = 1 << kTracePartSizeBits;
+const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;
+const int kTraceSize = kTracePartSize * kTraceParts;
+
+// Must fit into 3 bits.
+enum EventType {
+  EventTypeMop,
+  EventTypeFuncEnter,
+  EventTypeFuncExit,
+  EventTypeLock,
+  EventTypeUnlock,
+  EventTypeRLock,
+  EventTypeRUnlock
+};
+
+// Represents a thread event (from most significant bit):
+// u64 typ  : 3;   // EventType.
+// u64 addr : 61;  // Associated pc.
+typedef u64 Event;
+
+const uptr kEventPCBits = 61;
+
+struct TraceHeader {
+#if !SANITIZER_GO
+  BufferedStackTrace stack0;  // Start stack for the trace.
+#else
+  VarSizeStackTrace stack0;
+#endif
+  u64        epoch0;  // Start epoch for the trace.
+  MutexSet   mset0;
+
+  TraceHeader() : stack0(), epoch0() {}
+};
+
+struct Trace {
+  Mutex mtx;
+#if !SANITIZER_GO
+  // Must be last to catch overflow as paging fault.
+  // Go shadow stack is dynamically allocated.
+  uptr shadow_stack[kShadowStackSize];
+#endif
+  // Must be the last field, because we unmap the unused part in
+  // CreateThreadContext.
+  TraceHeader headers[kTraceParts];
+
+  Trace()
+    : mtx(MutexTypeTrace, StatMtxTrace) {
+  }
+};
+
+}  // namespace __tsan
+
+#endif  // TSAN_TRACE_H
diff --git a/lib/tsan/tsan_update_shadow_word_inl.h b/lib/tsan/tsan_update_shadow_word_inl.h
new file mode 100644
index 0000000000..056c3aa203
--- /dev/null
+++ b/lib/tsan/tsan_update_shadow_word_inl.h
@@ -0,0 +1,69 @@
+//===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Body of the hottest inner loop.
+// If we wrap this body into a function, compilers (both gcc and clang)
+// produce sligtly less efficient code.
+//===----------------------------------------------------------------------===//
+do {
+  StatInc(thr, StatShadowProcessed);
+  const unsigned kAccessSize = 1 << kAccessSizeLog;
+  u64 *sp = &shadow_mem[idx];
+  old = LoadShadow(sp);
+  if (LIKELY(old.IsZero())) {
+    StatInc(thr, StatShadowZero);
+    if (!stored) {
+      StoreIfNotYetStored(sp, &store_word);
+      stored = true;
+    }
+    break;
+  }
+  // is the memory access equal to the previous?
+  if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
+    StatInc(thr, StatShadowSameSize);
+    // same thread?
+    if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
+      StatInc(thr, StatShadowSameThread);
+      if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
+        StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
+      break;
+    }
+    StatInc(thr, StatShadowAnotherThread);
+    if (HappensBefore(old, thr)) {
+      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
+        StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
+      break;
+    }
+    if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
+      break;
+    goto RACE;
+  }
+  // Do the memory access intersect?
+  if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
+    StatInc(thr, StatShadowIntersect);
+    if (Shadow::TidsAreEqual(old, cur)) {
+      StatInc(thr, StatShadowSameThread);
+      break;
+    }
+    StatInc(thr, StatShadowAnotherThread);
+    if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
+      break;
+    if (LIKELY(HappensBefore(old, thr)))
+      break;
+    goto RACE;
+  }
+  // The accesses do not intersect.
+  StatInc(thr, StatShadowNotIntersect);
+  break;
+} while (0);
diff --git a/lib/tsan/ubsan/ubsan_flags.h b/lib/tsan/ubsan/ubsan_flags.h
new file mode 100644
index 0000000000..daa0d7c701
--- /dev/null
+++ b/lib/tsan/ubsan/ubsan_flags.h
@@ -0,0 +1,48 @@
+//===-- ubsan_flags.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Runtime flags for UndefinedBehaviorSanitizer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef UBSAN_FLAGS_H
+#define UBSAN_FLAGS_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+class FlagParser;
+}
+
+namespace __ubsan {
+
+struct Flags {
+#define UBSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "ubsan_flags.inc"
+#undef UBSAN_FLAG
+
+  void SetDefaults();
+};
+
+extern Flags ubsan_flags;
+inline Flags *flags() { return &ubsan_flags; }
+
+void InitializeFlags();
+void RegisterUbsanFlags(FlagParser *parser, Flags *f);
+
+const char *MaybeCallUbsanDefaultOptions();
+
+}  // namespace __ubsan
+
+extern "C" {
+// Users may provide their own implementation of __ubsan_default_options to
+// override the default flag values.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char *__ubsan_default_options();
+}  // extern "C"
+
+#endif  // UBSAN_FLAGS_H
diff --git a/lib/tsan/ubsan/ubsan_flags.inc b/lib/tsan/ubsan/ubsan_flags.inc
new file mode 100644
index 0000000000..a4d0e6109e
--- /dev/null
+++ b/lib/tsan/ubsan/ubsan_flags.inc
@@ -0,0 +1,28 @@
+//===-- ubsan_flags.inc -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UBSan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef UBSAN_FLAG
+# error "Define UBSAN_FLAG prior to including this file!"
+#endif
+
+// UBSAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+UBSAN_FLAG(bool, halt_on_error, false,
+           "Crash the program after printing the first error report")
+UBSAN_FLAG(bool, print_stacktrace, false,
+           "Include full stacktrace into an error report")
+UBSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
+UBSAN_FLAG(bool, report_error_type, false,
+        "Print specific error type instead of 'undefined-behavior' in summary.")
+UBSAN_FLAG(bool, silence_unsigned_overflow, false,
+        "Do not print non-fatal error reports for unsigned integer overflow. "
+        "Used to provide fuzzing signal without blowing up logs.")
diff --git a/lib/tsan/ubsan/ubsan_init.h b/lib/tsan/ubsan/ubsan_init.h
new file mode 100644
index 0000000000..0510385b13
--- /dev/null
+++ b/lib/tsan/ubsan/ubsan_init.h
@@ -0,0 +1,33 @@
+//===-- ubsan_init.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Initialization function for UBSan runtime.
+//
+//===----------------------------------------------------------------------===//
+#ifndef UBSAN_INIT_H
+#define UBSAN_INIT_H
+
+namespace __ubsan {
+
+// Get the full tool name for UBSan.
+const char *GetSanititizerToolName();
+
+// Initialize UBSan as a standalone tool. Typically should be called early
+// during initialization.
+void InitAsStandalone();
+
+// Initialize UBSan as a standalone tool, if it hasn't been initialized before.
+void InitAsStandaloneIfNecessary();
+
+// Initializes UBSan as a plugin tool. This function should be called once
+// from "parent tool" (e.g. ASan) initialization.
+void InitAsPlugin();
+
+}  // namespace __ubsan
+
+#endif  // UBSAN_INIT_H
diff --git a/lib/tsan/ubsan/ubsan_platform.h b/lib/tsan/ubsan/ubsan_platform.h
new file mode 100644
index 0000000000..71d7fb18c9
--- /dev/null
+++ b/lib/tsan/ubsan/ubsan_platform.h
@@ -0,0 +1,25 @@
+//===-- ubsan_platform.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the platforms which UBSan is supported at.
+//
+//===----------------------------------------------------------------------===//
+#ifndef UBSAN_PLATFORM_H
+#define UBSAN_PLATFORM_H
+
+// Other platforms should be easy to add, and probably work as-is.
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) ||        \
+    defined(__NetBSD__) || defined(__OpenBSD__) || \
+    (defined(__sun__) && defined(__svr4__)) || \
+    defined(_WIN32) || defined(__Fuchsia__) || defined(__rtems__)
+# define CAN_SANITIZE_UB 1
+#else
+# define CAN_SANITIZE_UB 0
+#endif
+
+#endif
diff --git a/src/Compilation.zig b/src/Compilation.zig
index d172cbadcc..ad4dd4fcca 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -30,6 +30,7 @@ const c_codegen = @import("codegen/c.zig");
 const c_link = @import("link/C.zig");
 const ThreadPool = @import("ThreadPool.zig");
 const WaitGroup = @import("WaitGroup.zig");
+const libtsan = @import("libtsan.zig");
 
 /// General-purpose allocator. Used for both temporary and long-term storage.
 gpa: *Allocator,
@@ -94,6 +95,9 @@ libcxxabi_static_lib: ?CRTFile = null,
 /// Populated when we build the libunwind static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
 libunwind_static_lib: ?CRTFile = null,
+/// Populated when we build the TSAN static library. A Job to build this is placed in the queue
+/// and resolved before calling linker.flush().
+tsan_static_lib: ?CRTFile = null,
 /// Populated when we build the libssp static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
 libssp_static_lib: ?CRTFile = null,
@@ -179,6 +183,7 @@ const Job = union(enum) {
     libunwind: void,
     libcxx: void,
     libcxxabi: void,
+    libtsan: void,
     libssp: void,
     compiler_rt_lib: void,
     compiler_rt_obj: void,
@@ -388,6 +393,7 @@ pub const InitOptions = struct {
     want_sanitize_c: ?bool = null,
     want_stack_check: ?bool = null,
     want_valgrind: ?bool = null,
+    want_tsan: ?bool = null,
     want_compiler_rt: ?bool = null,
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
@@ -644,13 +650,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             options.libc_installation,
         );
 
+        const tsan = options.want_tsan orelse false;
+
         const must_pie = target_util.requiresPIE(options.target);
-        const pie = if (options.want_pie) |explicit| pie: {
+        const pie: bool = if (options.want_pie) |explicit| pie: {
             if (!explicit and must_pie) {
                 return error.TargetRequiresPIE;
             }
             break :pie explicit;
-        } else must_pie;
+        } else must_pie or tsan;
 
         const must_pic: bool = b: {
             if (target_util.requiresPIC(options.target, link_libc))
@@ -751,6 +759,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(ofmt);
         cache.hash.add(pic);
         cache.hash.add(pie);
+        cache.hash.add(tsan);
         cache.hash.add(stack_check);
         cache.hash.add(link_mode);
         cache.hash.add(options.function_sections);
@@ -953,6 +962,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .pic = pic,
             .pie = pie,
             .valgrind = valgrind,
+            .tsan = tsan,
             .stack_check = stack_check,
             .single_threaded = single_threaded,
             .verbose_link = options.verbose_link,
@@ -1093,6 +1103,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             try comp.work_queue.writeItem(.libcxx);
             try comp.work_queue.writeItem(.libcxxabi);
         }
+        if (build_options.have_llvm and comp.bin_file.options.tsan) {
+            try comp.work_queue.writeItem(.libtsan);
+        }
 
         // The `is_stage1` condition is here only because stage2 cannot yet build compiler-rt.
         // Once it is capable this condition should be removed.
@@ -1569,6 +1582,12 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 fatal("unable to build libcxxabi: {}", .{@errorName(err)});
             };
         },
+        .libtsan => {
+            libtsan.buildTsan(self) catch |err| {
+                // TODO Expose this as a normal compile error rather than crashing here.
+                fatal("unable to build TSAN library: {}", .{@errorName(err)});
+            };
+        },
         .compiler_rt_lib => {
             self.buildOutputFromZig("compiler_rt.zig", .Lib, &self.compiler_rt_static_lib) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
@@ -2052,6 +2071,10 @@ pub fn addCCArgs(
         try argv.append("-ffunction-sections");
     }
 
+    if (comp.bin_file.options.tsan) {
+        try argv.append("-fsanitize=thread");
+    }
+
     try argv.ensureCapacity(argv.items.len + comp.bin_file.options.framework_dirs.len * 2);
     for (comp.bin_file.options.framework_dirs) |framework_dir| {
         argv.appendAssumeCapacity("-iframework");
@@ -2887,6 +2910,7 @@ fn buildOutputFromZig(
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
@@ -3115,6 +3139,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .dll_export_fns = comp.bin_file.options.dll_export_fns,
         .link_mode_dynamic = comp.bin_file.options.link_mode == .Dynamic,
         .valgrind_enabled = comp.bin_file.options.valgrind,
+        .tsan_enabled = comp.bin_file.options.tsan,
         .function_sections = comp.bin_file.options.function_sections,
         .enable_stack_probing = comp.bin_file.options.stack_check,
         .enable_time_report = comp.time_report,
@@ -3262,6 +3287,7 @@ pub fn build_crt_file(
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
diff --git a/src/glibc.zig b/src/glibc.zig
index 413e38637b..c228d8d2ef 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -943,6 +943,7 @@ fn buildSharedLib(
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = false,
         .emit_h = null,
         .strip = comp.bin_file.options.strip,
         .is_native_os = false,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index 142d014b2f..d81562bc7e 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -139,7 +139,6 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         try cflags.append(cxxabi_include_path);
 
         try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         if (target_util.supports_fpic(target)) {
             try cflags.append("-fPIC");
         }
@@ -170,6 +169,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
@@ -289,6 +289,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
diff --git a/src/libtsan.zig b/src/libtsan.zig
new file mode 100644
index 0000000000..3e2c70328b
--- /dev/null
+++ b/src/libtsan.zig
@@ -0,0 +1,219 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const Compilation = @import("Compilation.zig");
+const build_options = @import("build_options");
+const trace = @import("tracy.zig").trace;
+
+pub fn buildTsan(comp: *Compilation) !void {
+    if (!build_options.have_llvm) {
+        return error.ZigCompilerNotBuiltWithLLVMExtensions;
+    }
+
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
+    defer arena_allocator.deinit();
+    const arena = &arena_allocator.allocator;
+
+    const root_name = "tsan";
+    const output_mode = .Lib;
+    const link_mode = .Static;
+    const target = comp.getTarget();
+    const basename = try std.zig.binNameAlloc(arena, .{
+        .root_name = root_name,
+        .target = target,
+        .output_mode = output_mode,
+        .link_mode = link_mode,
+    });
+
+    const emit_bin = Compilation.EmitLoc{
+        .directory = null, // Put it in the cache directory.
+        .basename = basename,
+    };
+
+    var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
+    try c_source_files.ensureCapacity(tsan_sources.len + sanitizer_common_sources.len);
+
+    const tsan_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"tsan"});
+    for (tsan_sources) |tsan_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    const sanitizer_common_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+        "tsan", "sanitizer_common",
+    });
+    for (sanitizer_common_sources) |common_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(sanitizer_common_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                "tsan", "sanitizer_common", common_src,
+            }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    const sub_compilation = try Compilation.create(comp.gpa, .{
+        .local_cache_directory = comp.global_cache_directory,
+        .global_cache_directory = comp.global_cache_directory,
+        .zig_lib_directory = comp.zig_lib_directory,
+        .target = target,
+        .root_name = root_name,
+        .root_pkg = null,
+        .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
+        .libc_installation = comp.bin_file.options.libc_installation,
+        .emit_bin = emit_bin,
+        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .link_mode = link_mode,
+        .want_sanitize_c = false,
+        .want_stack_check = false,
+        .want_valgrind = false,
+        .want_tsan = false,
+        .want_pic = true,
+        .want_pie = true,
+        .emit_h = null,
+        .strip = comp.bin_file.options.strip,
+        .is_native_os = comp.bin_file.options.is_native_os,
+        .is_native_abi = comp.bin_file.options.is_native_abi,
+        .self_exe_path = comp.self_exe_path,
+        .c_source_files = c_source_files.items,
+        .verbose_cc = comp.verbose_cc,
+        .verbose_link = comp.bin_file.options.verbose_link,
+        .verbose_tokenize = comp.verbose_tokenize,
+        .verbose_ast = comp.verbose_ast,
+        .verbose_ir = comp.verbose_ir,
+        .verbose_llvm_ir = comp.verbose_llvm_ir,
+        .verbose_cimport = comp.verbose_cimport,
+        .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
+        .clang_passthrough_mode = comp.clang_passthrough_mode,
+        .link_libc = true,
+    });
+    defer sub_compilation.destroy();
+
+    try sub_compilation.updateSubCompilation();
+
+    assert(comp.tsan_static_lib == null);
+    comp.tsan_static_lib = Compilation.CRTFile{
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(
+            comp.gpa,
+            &[_][]const u8{basename},
+        ),
+        .lock = sub_compilation.bin_file.toOwnedLock(),
+    };
+}
+
+const tsan_sources = [_][]const u8{
+    "tsan_clock.cpp",
+    "tsan_debugging.cpp",
+    "tsan_external.cpp",
+    "tsan_fd.cpp",
+    "tsan_flags.cpp",
+    "tsan_ignoreset.cpp",
+    "tsan_interceptors_posix.cpp",
+    "tsan_interface.cpp",
+    "tsan_interface_ann.cpp",
+    "tsan_interface_atomic.cpp",
+    "tsan_interface_java.cpp",
+    "tsan_malloc_mac.cpp",
+    "tsan_md5.cpp",
+    "tsan_mman.cpp",
+    "tsan_mutex.cpp",
+    "tsan_mutexset.cpp",
+    "tsan_preinit.cpp",
+    "tsan_report.cpp",
+    "tsan_rtl.cpp",
+    "tsan_rtl_mutex.cpp",
+    "tsan_rtl_proc.cpp",
+    "tsan_rtl_report.cpp",
+    "tsan_rtl_thread.cpp",
+    "tsan_stack_trace.cpp",
+    "tsan_stat.cpp",
+    "tsan_suppressions.cpp",
+    "tsan_symbolize.cpp",
+    "tsan_sync.cpp",
+};
+
+const sanitizer_common_sources = [_][]const u8{
+    "sanitizer_allocator.cpp",
+    "sanitizer_common.cpp",
+    "sanitizer_deadlock_detector1.cpp",
+    "sanitizer_deadlock_detector2.cpp",
+    "sanitizer_errno.cpp",
+    "sanitizer_file.cpp",
+    "sanitizer_flags.cpp",
+    "sanitizer_flag_parser.cpp",
+    "sanitizer_fuchsia.cpp",
+    "sanitizer_libc.cpp",
+    "sanitizer_libignore.cpp",
+    "sanitizer_linux.cpp",
+    "sanitizer_linux_s390.cpp",
+    "sanitizer_mac.cpp",
+    "sanitizer_netbsd.cpp",
+    "sanitizer_openbsd.cpp",
+    "sanitizer_persistent_allocator.cpp",
+    "sanitizer_platform_limits_freebsd.cpp",
+    "sanitizer_platform_limits_linux.cpp",
+    "sanitizer_platform_limits_netbsd.cpp",
+    "sanitizer_platform_limits_openbsd.cpp",
+    "sanitizer_platform_limits_posix.cpp",
+    "sanitizer_platform_limits_solaris.cpp",
+    "sanitizer_posix.cpp",
+    "sanitizer_printf.cpp",
+    "sanitizer_procmaps_common.cpp",
+    "sanitizer_procmaps_bsd.cpp",
+    "sanitizer_procmaps_fuchsia.cpp",
+    "sanitizer_procmaps_linux.cpp",
+    "sanitizer_procmaps_mac.cpp",
+    "sanitizer_procmaps_solaris.cpp",
+    "sanitizer_rtems.cpp",
+    "sanitizer_solaris.cpp",
+    "sanitizer_stoptheworld_fuchsia.cpp",
+    "sanitizer_stoptheworld_mac.cpp",
+    "sanitizer_suppressions.cpp",
+    "sanitizer_termination.cpp",
+    "sanitizer_tls_get_addr.cpp",
+    "sanitizer_thread_registry.cpp",
+    "sanitizer_type_traits.cpp",
+    "sanitizer_win.cpp",
+};
+
+// TODO This is next up
+//const sanitizer_nolibc_sources = [_][]const u8{
+//    "sanitizer_common_nolibc.cpp",
+//};
+//
+//const sanitizer_libcdep_sources = [_][]const u8{
+//    "sanitizer_common_libcdep.cpp",
+//    "sanitizer_allocator_checks.cpp",
+//    "sanitizer_linux_libcdep.cpp",
+//    "sanitizer_mac_libcdep.cpp",
+//    "sanitizer_posix_libcdep.cpp",
+//    "sanitizer_stoptheworld_linux_libcdep.cpp",
+//    "sanitizer_stoptheworld_netbsd_libcdep.cpp",
+//};
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 13a4fdf7c7..a0da7ee1fb 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -103,6 +103,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
+        .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
diff --git a/src/link.zig b/src/link.zig
index c0e78f445f..edeb24dc9a 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -75,6 +75,7 @@ pub const Options = struct {
     pic: bool,
     pie: bool,
     valgrind: bool,
+    tsan: bool,
     stack_check: bool,
     single_threaded: bool,
     verbose_link: bool,
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 180ad95c00..a3b58c1509 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1327,6 +1327,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         man.hash.addStringSet(self.base.options.system_libs);
         man.hash.add(allow_shlib_undefined);
         man.hash.add(self.base.options.bind_global_refs_locally);
+        man.hash.add(self.base.options.tsan);
 
         // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
         _ = try man.hit();
@@ -1545,6 +1546,11 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         try argv.append(p);
     }
 
+    // TSAN
+    if (self.base.options.tsan) {
+        try argv.append(comp.tsan_static_lib.?.full_object_path);
+    }
+
     // libc
     if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc and !self.base.options.link_libc) {
         try argv.append(comp.libc_static_lib.?.full_object_path);
diff --git a/src/main.zig b/src/main.zig
index 3bcae364d0..99d0dcd5f9 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -285,6 +285,8 @@ const usage_build_generic =
     \\  -fno-sanitize-c           Disable C undefined behavior detection in safe builds
     \\  -fvalgrind                Include valgrind client requests in release builds
     \\  -fno-valgrind             Omit valgrind client requests in debug builds
+    \\  -fsanitize-thread         Enable Thread Sanitizer
+    \\  -fno-sanitize-thread      Disable Thread Sanitizer
     \\  -fdll-export-fns          Mark exported functions as DLL exports (Windows)
     \\  -fno-dll-export-fns       Force-disable marking exported functions as DLL exports
     \\  -fLLVM                    Force using LLVM as the codegen backend
@@ -503,6 +505,7 @@ fn buildOutputType(
     var want_sanitize_c: ?bool = null;
     var want_stack_check: ?bool = null;
     var want_valgrind: ?bool = null;
+    var want_tsan: ?bool = null;
     var want_compiler_rt: ?bool = null;
     var rdynamic: bool = false;
     var linker_script: ?[]const u8 = null;
@@ -847,6 +850,10 @@ fn buildOutputType(
                         want_valgrind = true;
                     } else if (mem.eql(u8, arg, "-fno-valgrind")) {
                         want_valgrind = false;
+                    } else if (mem.eql(u8, arg, "-fsanitize-thread")) {
+                        want_tsan = true;
+                    } else if (mem.eql(u8, arg, "-fno-sanitize-thread")) {
+                        want_tsan = false;
                     } else if (mem.eql(u8, arg, "-fLLVM")) {
                         use_llvm = true;
                     } else if (mem.eql(u8, arg, "-fno-LLVM")) {
@@ -1106,6 +1113,8 @@ fn buildOutputType(
                     .sanitize => {
                         if (mem.eql(u8, it.only_arg, "undefined")) {
                             want_sanitize_c = true;
+                        } else if (mem.eql(u8, it.only_arg, "thread")) {
+                            want_tsan = true;
                         } else {
                             try clang_argv.appendSlice(it.other_args);
                         }
@@ -1712,6 +1721,7 @@ fn buildOutputType(
         .want_sanitize_c = want_sanitize_c,
         .want_stack_check = want_stack_check,
         .want_valgrind = want_valgrind,
+        .want_tsan = want_tsan,
         .want_compiler_rt = want_compiler_rt,
         .use_llvm = use_llvm,
         .use_lld = use_lld,
diff --git a/src/musl.zig b/src/musl.zig
index 80943ab1f7..080dadd3fd 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -207,6 +207,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .want_sanitize_c = false,
                 .want_stack_check = false,
                 .want_valgrind = false,
+                .want_tsan = false,
                 .emit_h = null,
                 .strip = comp.bin_file.options.strip,
                 .is_native_os = false,
diff --git a/src/stage1.zig b/src/stage1.zig
index 7f2d3d732d..1d50a71ad1 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -116,6 +116,7 @@ pub const Module = extern struct {
     dll_export_fns: bool,
     link_mode_dynamic: bool,
     valgrind_enabled: bool,
+    tsan_enabled: bool,
     function_sections: bool,
     enable_stack_probing: bool,
     enable_time_report: bool,
diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index 39eec65628..e39da4034b 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -2199,6 +2199,7 @@ struct CodeGen {
     bool function_sections;
     bool test_is_evented;
     bool valgrind_enabled;
+    bool tsan_enabled;
 
     Buf *root_out_name;
     Buf *test_filter;
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index c7f429ef49..3f14307d3c 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -472,6 +472,10 @@ static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) {
         ZigLLVMFunctionSetCallingConv(llvm_fn, get_llvm_cc(g, cc));
     }
 
+    if (g->tsan_enabled) {
+        addLLVMFnAttr(llvm_fn, "sanitize_thread");
+    }
+
     bool want_cold = fn->is_cold;
     if (want_cold) {
         ZigLLVMAddFunctionAttrCold(llvm_fn);
diff --git a/src/stage1/stage1.cpp b/src/stage1/stage1.cpp
index fec6760623..1195a83931 100644
--- a/src/stage1/stage1.cpp
+++ b/src/stage1/stage1.cpp
@@ -93,6 +93,7 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) {
     g->have_stack_probing = stage1->enable_stack_probing;
     g->is_single_threaded = stage1->is_single_threaded;
     g->valgrind_enabled = stage1->valgrind_enabled;
+    g->tsan_enabled = stage1->tsan_enabled;
     g->link_libc = stage1->link_libc;
     g->link_libcpp = stage1->link_libcpp;
     g->function_sections = stage1->function_sections;
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index 0552aa09e0..dbf6491699 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -185,6 +185,7 @@ struct ZigStage1 {
     bool dll_export_fns;
     bool link_mode_dynamic;
     bool valgrind_enabled;
+    bool tsan_enabled;
     bool function_sections;
     bool enable_stack_probing;
     bool enable_time_report;
-- 
cgit v1.2.3


From 8219d92987c7d9641d6b24c4d5be29e3a80fd6b9 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 22 Dec 2020 19:25:24 -0700
Subject: stage2: fix Cache deadlock and build more of TSAN

 * rename is_compiler_rt_or_libc to skip_linker_dependencies
   and set it to `true` for all sub-Compilations. I believe
   this resolves the deadlock we were experiencing on Drone
   CI and on some users' computers. I will remove the CI workaround in
   a follow-up commit.
 * enabling TSAN automatically causes the Compilation to link against
   libc++ even if not requested, because TSAN depends on libc++.
 * add -fno-rtti flags where appropriate when building TSAN objects.
   Thanks Firefox317 for pointing this out.
 * TSAN support: resolve all the undefined symbols. We are still seeing
   a dependency on __gcc_personality_v0 but will resolve this one in a
   follow-up commit.
 * static libs do not try to build libc++ or libc++abi.
---
 lib/tsan/interception/interception_linux.cpp       |   83 ++
 lib/tsan/interception/interception_mac.cpp         |   18 +
 lib/tsan/interception/interception_type_test.cpp   |   39 +
 lib/tsan/interception/interception_win.cpp         | 1022 ++++++++++++++++++++
 .../sanitizer_allocator_checks.cpp                 |   22 +
 .../sanitizer_allocator_report.cpp                 |  137 +++
 .../sanitizer_common/sanitizer_common_libcdep.cpp  |  149 +++
 .../sanitizer_common/sanitizer_common_nolibc.cpp   |   34 +
 .../sanitizer_common/sanitizer_linux_libcdep.cpp   |  846 ++++++++++++++++
 .../sanitizer_common/sanitizer_mac_libcdep.cpp     |   29 +
 .../sanitizer_common/sanitizer_posix_libcdep.cpp   |  509 ++++++++++
 lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp |  149 +++
 lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp |  133 +++
 .../sanitizer_stacktrace_libcdep.cpp               |  159 +++
 .../sanitizer_stacktrace_printer.cpp               |  263 +++++
 .../sanitizer_stacktrace_sparc.cpp                 |   85 ++
 .../sanitizer_stoptheworld_linux_libcdep.cpp       |  573 +++++++++++
 .../sanitizer_stoptheworld_netbsd_libcdep.cpp      |  364 +++++++
 lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp |  135 +++
 .../sanitizer_symbolizer_libbacktrace.cpp          |  209 ++++
 .../sanitizer_symbolizer_libcdep.cpp               |  554 +++++++++++
 .../sanitizer_common/sanitizer_symbolizer_mac.cpp  |  249 +++++
 .../sanitizer_symbolizer_markup.cpp                |  146 +++
 .../sanitizer_symbolizer_posix_libcdep.cpp         |  492 ++++++++++
 .../sanitizer_symbolizer_report.cpp                |  293 ++++++
 .../sanitizer_common/sanitizer_symbolizer_win.cpp  |  318 ++++++
 .../sanitizer_unwind_linux_libcdep.cpp             |  180 ++++
 lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp |   75 ++
 lib/tsan/tsan_interceptors_mac.cpp                 |  519 ++++++++++
 lib/tsan/tsan_interceptors_mach_vm.cpp             |   52 +
 lib/tsan/tsan_platform_linux.cpp                   |  517 ++++++++++
 lib/tsan/tsan_platform_mac.cpp                     |  324 +++++++
 lib/tsan/tsan_platform_posix.cpp                   |  167 ++++
 lib/tsan/tsan_rtl_aarch64.S                        |  245 +++++
 lib/tsan/tsan_rtl_amd64.S                          |  366 +++++++
 lib/tsan/tsan_rtl_mips64.S                         |  214 ++++
 lib/tsan/tsan_rtl_ppc64.S                          |  288 ++++++
 src/Compilation.zig                                |   33 +-
 src/glibc.zig                                      |    2 +-
 src/libcxx.zig                                     |    2 +
 src/libtsan.zig                                    |  198 +++-
 src/libunwind.zig                                  |    1 +
 src/link.zig                                       |    2 +-
 src/link/Coff.zig                                  |    4 +-
 src/link/Elf.zig                                   |    6 +-
 src/link/MachO.zig                                 |    4 +-
 src/link/Wasm.zig                                  |    2 +-
 src/musl.zig                                       |    2 +-
 48 files changed, 10172 insertions(+), 41 deletions(-)
 create mode 100644 lib/tsan/interception/interception_linux.cpp
 create mode 100644 lib/tsan/interception/interception_mac.cpp
 create mode 100644 lib/tsan/interception/interception_type_test.cpp
 create mode 100644 lib/tsan/interception/interception_win.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
 create mode 100644 lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp
 create mode 100644 lib/tsan/tsan_interceptors_mac.cpp
 create mode 100644 lib/tsan/tsan_interceptors_mach_vm.cpp
 create mode 100644 lib/tsan/tsan_platform_linux.cpp
 create mode 100644 lib/tsan/tsan_platform_mac.cpp
 create mode 100644 lib/tsan/tsan_platform_posix.cpp
 create mode 100644 lib/tsan/tsan_rtl_aarch64.S
 create mode 100644 lib/tsan/tsan_rtl_amd64.S
 create mode 100644 lib/tsan/tsan_rtl_mips64.S
 create mode 100644 lib/tsan/tsan_rtl_ppc64.S

(limited to 'src/Compilation.zig')

diff --git a/lib/tsan/interception/interception_linux.cpp b/lib/tsan/interception/interception_linux.cpp
new file mode 100644
index 0000000000..950cd51265
--- /dev/null
+++ b/lib/tsan/interception/interception_linux.cpp
@@ -0,0 +1,83 @@
+//===-- interception_linux.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Linux-specific interception methods.
+//===----------------------------------------------------------------------===//
+
+#include "interception.h"
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+#include <dlfcn.h>   // for dlsym() and dlvsym()
+
+namespace __interception {
+
+#if SANITIZER_NETBSD
+static int StrCmp(const char *s1, const char *s2) {
+  while (true) {
+    if (*s1 != *s2)
+      return false;
+    if (*s1 == 0)
+      return true;
+    s1++;
+    s2++;
+  }
+}
+#endif
+
+static void *GetFuncAddr(const char *name, uptr wrapper_addr) {
+#if SANITIZER_NETBSD
+  // FIXME: Find a better way to handle renames
+  if (StrCmp(name, "sigaction"))
+    name = "__sigaction14";
+#endif
+  void *addr = dlsym(RTLD_NEXT, name);
+  if (!addr) {
+    // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
+    // later in the library search order than the DSO that we are trying to
+    // intercept, which means that we cannot intercept this function. We still
+    // want the address of the real definition, though, so look it up using
+    // RTLD_DEFAULT.
+    addr = dlsym(RTLD_DEFAULT, name);
+
+    // In case `name' is not loaded, dlsym ends up finding the actual wrapper.
+    // We don't want to intercept the wrapper and have it point to itself.
+    if ((uptr)addr == wrapper_addr)
+      addr = nullptr;
+  }
+  return addr;
+}
+
+bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func,
+                       uptr wrapper) {
+  void *addr = GetFuncAddr(name, wrapper);
+  *ptr_to_real = (uptr)addr;
+  return addr && (func == wrapper);
+}
+
+// Android and Solaris do not have dlvsym
+#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
+static void *GetFuncAddr(const char *name, const char *ver) {
+  return dlvsym(RTLD_NEXT, name, ver);
+}
+
+bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
+                       uptr func, uptr wrapper) {
+  void *addr = GetFuncAddr(name, ver);
+  *ptr_to_real = (uptr)addr;
+  return addr && (func == wrapper);
+}
+#endif  // !SANITIZER_ANDROID
+
+}  // namespace __interception
+
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+        // SANITIZER_OPENBSD || SANITIZER_SOLARIS
diff --git a/lib/tsan/interception/interception_mac.cpp b/lib/tsan/interception/interception_mac.cpp
new file mode 100644
index 0000000000..fb6eadcff5
--- /dev/null
+++ b/lib/tsan/interception/interception_mac.cpp
@@ -0,0 +1,18 @@
+//===-- interception_mac.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Mac-specific interception methods.
+//===----------------------------------------------------------------------===//
+
+#include "interception.h"
+
+#if SANITIZER_MAC
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/interception/interception_type_test.cpp b/lib/tsan/interception/interception_type_test.cpp
new file mode 100644
index 0000000000..a611604a70
--- /dev/null
+++ b/lib/tsan/interception/interception_type_test.cpp
@@ -0,0 +1,39 @@
+//===-- interception_type_test.cpp ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Compile-time tests of the internal type definitions.
+//===----------------------------------------------------------------------===//
+
+#include "interception.h"
+
+#if SANITIZER_LINUX || SANITIZER_MAC
+
+#include <sys/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t));
+COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
+COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
+COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
+
+#if !SANITIZER_MAC
+COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
+#endif
+
+// The following are the cases when pread (and friends) is used instead of
+// pread64. In those cases we need OFF_T to match off_t. We don't care about the
+// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
+# if SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \
+  _FILE_OFFSET_BITS != 64
+COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
+# endif
+
+#endif
diff --git a/lib/tsan/interception/interception_win.cpp b/lib/tsan/interception/interception_win.cpp
new file mode 100644
index 0000000000..1a1c327e61
--- /dev/null
+++ b/lib/tsan/interception/interception_win.cpp
@@ -0,0 +1,1022 @@
+//===-- interception_linux.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Windows-specific interception methods.
+//
+// This file is implementing several hooking techniques to intercept calls
+// to functions. The hooks are dynamically installed by modifying the assembly
+// code.
+//
+// The hooking techniques are making assumptions on the way the code is
+// generated and are safe under these assumptions.
+//
+// On 64-bit architecture, there is no direct 64-bit jump instruction. To allow
+// arbitrary branching on the whole memory space, the notion of trampoline
+// region is used. A trampoline region is a memory space withing 2G boundary
+// where it is safe to add custom assembly code to build 64-bit jumps.
+//
+// Hooking techniques
+// ==================
+//
+// 1) Detour
+//
+//    The Detour hooking technique is assuming the presence of an header with
+//    padding and an overridable 2-bytes nop instruction (mov edi, edi). The
+//    nop instruction can safely be replaced by a 2-bytes jump without any need
+//    to save the instruction. A jump to the target is encoded in the function
+//    header and the nop instruction is replaced by a short jump to the header.
+//
+//        head:  5 x nop                 head:  jmp <hook>
+//        func:  mov edi, edi    -->     func:  jmp short <head>
+//               [...]                   real:  [...]
+//
+//    This technique is only implemented on 32-bit architecture.
+//    Most of the time, Windows API are hookable with the detour technique.
+//
+// 2) Redirect Jump
+//
+//    The redirect jump is applicable when the first instruction is a direct
+//    jump. The instruction is replaced by jump to the hook.
+//
+//        func:  jmp <label>     -->     func:  jmp <hook>
+//
+//    On an 64-bit architecture, a trampoline is inserted.
+//
+//        func:  jmp <label>     -->     func:  jmp <tramp>
+//                                              [...]
+//
+//                                   [trampoline]
+//                                      tramp:  jmp QWORD [addr]
+//                                       addr:  .bytes <hook>
+//
+//    Note: <real> is equilavent to <label>.
+//
+// 3) HotPatch
+//
+//    The HotPatch hooking is assuming the presence of an header with padding
+//    and a first instruction with at least 2-bytes.
+//
+//    The reason to enforce the 2-bytes limitation is to provide the minimal
+//    space to encode a short jump. HotPatch technique is only rewriting one
+//    instruction to avoid breaking a sequence of instructions containing a
+//    branching target.
+//
+//    Assumptions are enforced by MSVC compiler by using the /HOTPATCH flag.
+//      see: https://msdn.microsoft.com/en-us/library/ms173507.aspx
+//    Default padding length is 5 bytes in 32-bits and 6 bytes in 64-bits.
+//
+//        head:   5 x nop                head:  jmp <hook>
+//        func:   <instr>        -->     func:  jmp short <head>
+//                [...]                  body:  [...]
+//
+//                                   [trampoline]
+//                                       real:  <instr>
+//                                              jmp <body>
+//
+//    On an 64-bit architecture:
+//
+//        head:   6 x nop                head:  jmp QWORD [addr1]
+//        func:   <instr>        -->     func:  jmp short <head>
+//                [...]                  body:  [...]
+//
+//                                   [trampoline]
+//                                      addr1:  .bytes <hook>
+//                                       real:  <instr>
+//                                              jmp QWORD [addr2]
+//                                      addr2:  .bytes <body>
+//
+// 4) Trampoline
+//
+//    The Trampoline hooking technique is the most aggressive one. It is
+//    assuming that there is a sequence of instructions that can be safely
+//    replaced by a jump (enough room and no incoming branches).
+//
+//    Unfortunately, these assumptions can't be safely presumed and code may
+//    be broken after hooking.
+//
+//        func:   <instr>        -->     func:  jmp <hook>
+//                <instr>
+//                [...]                  body:  [...]
+//
+//                                   [trampoline]
+//                                       real:  <instr>
+//                                              <instr>
+//                                              jmp <body>
+//
+//    On an 64-bit architecture:
+//
+//        func:   <instr>        -->     func:  jmp QWORD [addr1]
+//                <instr>
+//                [...]                  body:  [...]
+//
+//                                   [trampoline]
+//                                      addr1:  .bytes <hook>
+//                                       real:  <instr>
+//                                              <instr>
+//                                              jmp QWORD [addr2]
+//                                      addr2:  .bytes <body>
+//===----------------------------------------------------------------------===//
+
+#include "interception.h"
+
+#if SANITIZER_WINDOWS
+#include "sanitizer_common/sanitizer_platform.h"
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+namespace __interception {
+
+static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
+static const int kJumpInstructionLength = 5;
+static const int kShortJumpInstructionLength = 2;
+static const int kIndirectJumpInstructionLength = 6;
+static const int kBranchLength =
+    FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
+static const int kDirectBranchLength = kBranchLength + kAddressLength;
+
+static void InterceptionFailed() {
+  // Do we have a good way to abort with an error message here?
+  __debugbreak();
+}
+
+static bool DistanceIsWithin2Gig(uptr from, uptr target) {
+#if SANITIZER_WINDOWS64
+  if (from < target)
+    return target - from <= (uptr)0x7FFFFFFFU;
+  else
+    return from - target <= (uptr)0x80000000U;
+#else
+  // In a 32-bit address space, the address calculation will wrap, so this check
+  // is unnecessary.
+  return true;
+#endif
+}
+
+static uptr GetMmapGranularity() {
+  SYSTEM_INFO si;
+  GetSystemInfo(&si);
+  return si.dwAllocationGranularity;
+}
+
+static uptr RoundUpTo(uptr size, uptr boundary) {
+  return (size + boundary - 1) & ~(boundary - 1);
+}
+
+// FIXME: internal_str* and internal_mem* functions should be moved from the
+// ASan sources into interception/.
+
+static size_t _strlen(const char *str) {
+  const char* p = str;
+  while (*p != '\0') ++p;
+  return p - str;
+}
+
+static char* _strchr(char* str, char c) {
+  while (*str) {
+    if (*str == c)
+      return str;
+    ++str;
+  }
+  return nullptr;
+}
+
+static void _memset(void *p, int value, size_t sz) {
+  for (size_t i = 0; i < sz; ++i)
+    ((char*)p)[i] = (char)value;
+}
+
+static void _memcpy(void *dst, void *src, size_t sz) {
+  char *dst_c = (char*)dst,
+       *src_c = (char*)src;
+  for (size_t i = 0; i < sz; ++i)
+    dst_c[i] = src_c[i];
+}
+
+static bool ChangeMemoryProtection(
+    uptr address, uptr size, DWORD *old_protection) {
+  return ::VirtualProtect((void*)address, size,
+                          PAGE_EXECUTE_READWRITE,
+                          old_protection) != FALSE;
+}
+
+static bool RestoreMemoryProtection(
+    uptr address, uptr size, DWORD old_protection) {
+  DWORD unused;
+  return ::VirtualProtect((void*)address, size,
+                          old_protection,
+                          &unused) != FALSE;
+}
+
+static bool IsMemoryPadding(uptr address, uptr size) {
+  u8* function = (u8*)address;
+  for (size_t i = 0; i < size; ++i)
+    if (function[i] != 0x90 && function[i] != 0xCC)
+      return false;
+  return true;
+}
+
+static const u8 kHintNop8Bytes[] = {
+  0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+template<class T>
+static bool FunctionHasPrefix(uptr address, const T &pattern) {
+  u8* function = (u8*)address - sizeof(pattern);
+  for (size_t i = 0; i < sizeof(pattern); ++i)
+    if (function[i] != pattern[i])
+      return false;
+  return true;
+}
+
+static bool FunctionHasPadding(uptr address, uptr size) {
+  if (IsMemoryPadding(address - size, size))
+    return true;
+  if (size <= sizeof(kHintNop8Bytes) &&
+      FunctionHasPrefix(address, kHintNop8Bytes))
+    return true;
+  return false;
+}
+
+static void WritePadding(uptr from, uptr size) {
+  _memset((void*)from, 0xCC, (size_t)size);
+}
+
+static void WriteJumpInstruction(uptr from, uptr target) {
+  if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
+    InterceptionFailed();
+  ptrdiff_t offset = target - from - kJumpInstructionLength;
+  *(u8*)from = 0xE9;
+  *(u32*)(from + 1) = offset;
+}
+
+static void WriteShortJumpInstruction(uptr from, uptr target) {
+  sptr offset = target - from - kShortJumpInstructionLength;
+  if (offset < -128 || offset > 127)
+    InterceptionFailed();
+  *(u8*)from = 0xEB;
+  *(u8*)(from + 1) = (u8)offset;
+}
+
+#if SANITIZER_WINDOWS64
+static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
+  // jmp [rip + <offset>] = FF 25 <offset> where <offset> is a relative
+  // offset.
+  // The offset is the distance from then end of the jump instruction to the
+  // memory location containing the targeted address. The displacement is still
+  // 32-bit in x64, so indirect_target must be located within +/- 2GB range.
+  int offset = indirect_target - from - kIndirectJumpInstructionLength;
+  if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
+                            indirect_target)) {
+    InterceptionFailed();
+  }
+  *(u16*)from = 0x25FF;
+  *(u32*)(from + 2) = offset;
+}
+#endif
+
+static void WriteBranch(
+    uptr from, uptr indirect_target, uptr target) {
+#if SANITIZER_WINDOWS64
+  WriteIndirectJumpInstruction(from, indirect_target);
+  *(u64*)indirect_target = target;
+#else
+  (void)indirect_target;
+  WriteJumpInstruction(from, target);
+#endif
+}
+
+static void WriteDirectBranch(uptr from, uptr target) {
+#if SANITIZER_WINDOWS64
+  // Emit an indirect jump through immediately following bytes:
+  //   jmp [rip + kBranchLength]
+  //   .quad <target>
+  WriteBranch(from, from + kBranchLength, target);
+#else
+  WriteJumpInstruction(from, target);
+#endif
+}
+
+struct TrampolineMemoryRegion {
+  uptr content;
+  uptr allocated_size;
+  uptr max_size;
+};
+
+static const uptr kTrampolineScanLimitRange = 1 << 31;  // 2 gig
+static const int kMaxTrampolineRegion = 1024;
+static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
+
+static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) {
+#if SANITIZER_WINDOWS64
+  uptr address = image_address;
+  uptr scanned = 0;
+  while (scanned < kTrampolineScanLimitRange) {
+    MEMORY_BASIC_INFORMATION info;
+    if (!::VirtualQuery((void*)address, &info, sizeof(info)))
+      return nullptr;
+
+    // Check whether a region can be allocated at |address|.
+    if (info.State == MEM_FREE && info.RegionSize >= granularity) {
+      void *page = ::VirtualAlloc((void*)RoundUpTo(address, granularity),
+                                  granularity,
+                                  MEM_RESERVE | MEM_COMMIT,
+                                  PAGE_EXECUTE_READWRITE);
+      return page;
+    }
+
+    // Move to the next region.
+    address = (uptr)info.BaseAddress + info.RegionSize;
+    scanned += info.RegionSize;
+  }
+  return nullptr;
+#else
+  return ::VirtualAlloc(nullptr,
+                        granularity,
+                        MEM_RESERVE | MEM_COMMIT,
+                        PAGE_EXECUTE_READWRITE);
+#endif
+}
+
+// Used by unittests to release mapped memory space.
+void TestOnlyReleaseTrampolineRegions() {
+  for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
+    TrampolineMemoryRegion *current = &TrampolineRegions[bucket];
+    if (current->content == 0)
+      return;
+    ::VirtualFree((void*)current->content, 0, MEM_RELEASE);
+    current->content = 0;
+  }
+}
+
+static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
+  // Find a region within 2G with enough space to allocate |size| bytes.
+  TrampolineMemoryRegion *region = nullptr;
+  for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
+    TrampolineMemoryRegion* current = &TrampolineRegions[bucket];
+    if (current->content == 0) {
+      // No valid region found, allocate a new region.
+      size_t bucket_size = GetMmapGranularity();
+      void *content = AllocateTrampolineRegion(image_address, bucket_size);
+      if (content == nullptr)
+        return 0U;
+
+      current->content = (uptr)content;
+      current->allocated_size = 0;
+      current->max_size = bucket_size;
+      region = current;
+      break;
+    } else if (current->max_size - current->allocated_size > size) {
+#if SANITIZER_WINDOWS64
+        // In 64-bits, the memory space must be allocated within 2G boundary.
+        uptr next_address = current->content + current->allocated_size;
+        if (next_address < image_address ||
+            next_address - image_address >= 0x7FFF0000)
+          continue;
+#endif
+      // The space can be allocated in the current region.
+      region = current;
+      break;
+    }
+  }
+
+  // Failed to find a region.
+  if (region == nullptr)
+    return 0U;
+
+  // Allocate the space in the current region.
+  uptr allocated_space = region->content + region->allocated_size;
+  region->allocated_size += size;
+  WritePadding(allocated_space, size);
+
+  return allocated_space;
+}
+
+// Returns 0 on error.
+static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
+  switch (*(u64*)address) {
+    case 0x90909090909006EB:  // stub: jmp over 6 x nop.
+      return 8;
+  }
+
+  switch (*(u8*)address) {
+    case 0x90:  // 90 : nop
+      return 1;
+
+    case 0x50:  // push eax / rax
+    case 0x51:  // push ecx / rcx
+    case 0x52:  // push edx / rdx
+    case 0x53:  // push ebx / rbx
+    case 0x54:  // push esp / rsp
+    case 0x55:  // push ebp / rbp
+    case 0x56:  // push esi / rsi
+    case 0x57:  // push edi / rdi
+    case 0x5D:  // pop ebp / rbp
+      return 1;
+
+    case 0x6A:  // 6A XX = push XX
+      return 2;
+
+    case 0xb8:  // b8 XX XX XX XX : mov eax, XX XX XX XX
+    case 0xB9:  // b9 XX XX XX XX : mov ecx, XX XX XX XX
+      return 5;
+
+    // Cannot overwrite control-instruction. Return 0 to indicate failure.
+    case 0xE9:  // E9 XX XX XX XX : jmp <label>
+    case 0xE8:  // E8 XX XX XX XX : call <func>
+    case 0xC3:  // C3 : ret
+    case 0xEB:  // EB XX : jmp XX (short jump)
+    case 0x70:  // 7Y YY : jy XX (short conditional jump)
+    case 0x71:
+    case 0x72:
+    case 0x73:
+    case 0x74:
+    case 0x75:
+    case 0x76:
+    case 0x77:
+    case 0x78:
+    case 0x79:
+    case 0x7A:
+    case 0x7B:
+    case 0x7C:
+    case 0x7D:
+    case 0x7E:
+    case 0x7F:
+      return 0;
+  }
+
+  switch (*(u16*)(address)) {
+    case 0x018A:  // 8A 01 : mov al, byte ptr [ecx]
+    case 0xFF8B:  // 8B FF : mov edi, edi
+    case 0xEC8B:  // 8B EC : mov ebp, esp
+    case 0xc889:  // 89 C8 : mov eax, ecx
+    case 0xC18B:  // 8B C1 : mov eax, ecx
+    case 0xC033:  // 33 C0 : xor eax, eax
+    case 0xC933:  // 33 C9 : xor ecx, ecx
+    case 0xD233:  // 33 D2 : xor edx, edx
+      return 2;
+
+    // Cannot overwrite control-instruction. Return 0 to indicate failure.
+    case 0x25FF:  // FF 25 XX XX XX XX : jmp [XXXXXXXX]
+      return 0;
+  }
+
+  switch (0x00FFFFFF & *(u32*)address) {
+    case 0x24A48D:  // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]
+      return 7;
+  }
+
+#if SANITIZER_WINDOWS64
+  switch (*(u8*)address) {
+    case 0xA1:  // A1 XX XX XX XX XX XX XX XX :
+                //   movabs eax, dword ptr ds:[XXXXXXXX]
+      return 9;
+  }
+
+  switch (*(u16*)address) {
+    case 0x5040:  // push rax
+    case 0x5140:  // push rcx
+    case 0x5240:  // push rdx
+    case 0x5340:  // push rbx
+    case 0x5440:  // push rsp
+    case 0x5540:  // push rbp
+    case 0x5640:  // push rsi
+    case 0x5740:  // push rdi
+    case 0x5441:  // push r12
+    case 0x5541:  // push r13
+    case 0x5641:  // push r14
+    case 0x5741:  // push r15
+    case 0x9066:  // Two-byte NOP
+      return 2;
+
+    case 0x058B:  // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
+      if (rel_offset)
+        *rel_offset = 2;
+      return 6;
+  }
+
+  switch (0x00FFFFFF & *(u32*)address) {
+    case 0xe58948:    // 48 8b c4 : mov rbp, rsp
+    case 0xc18b48:    // 48 8b c1 : mov rax, rcx
+    case 0xc48b48:    // 48 8b c4 : mov rax, rsp
+    case 0xd9f748:    // 48 f7 d9 : neg rcx
+    case 0xd12b48:    // 48 2b d1 : sub rdx, rcx
+    case 0x07c1f6:    // f6 c1 07 : test cl, 0x7
+    case 0xc98548:    // 48 85 C9 : test rcx, rcx
+    case 0xc0854d:    // 4d 85 c0 : test r8, r8
+    case 0xc2b60f:    // 0f b6 c2 : movzx eax, dl
+    case 0xc03345:    // 45 33 c0 : xor r8d, r8d
+    case 0xc93345:    // 45 33 c9 : xor r9d, r9d
+    case 0xdb3345:    // 45 33 DB : xor r11d, r11d
+    case 0xd98b4c:    // 4c 8b d9 : mov r11, rcx
+    case 0xd28b4c:    // 4c 8b d2 : mov r10, rdx
+    case 0xc98b4c:    // 4C 8B C9 : mov r9, rcx
+    case 0xc18b4c:    // 4C 8B C1 : mov r8, rcx
+    case 0xd2b60f:    // 0f b6 d2 : movzx edx, dl
+    case 0xca2b48:    // 48 2b ca : sub rcx, rdx
+    case 0x10b70f:    // 0f b7 10 : movzx edx, WORD PTR [rax]
+    case 0xc00b4d:    // 3d 0b c0 : or r8, r8
+    case 0xd18b48:    // 48 8b d1 : mov rdx, rcx
+    case 0xdc8b4c:    // 4c 8b dc : mov r11, rsp
+    case 0xd18b4c:    // 4c 8b d1 : mov r10, rcx
+    case 0xE0E483:    // 83 E4 E0 : and esp, 0xFFFFFFE0
+      return 3;
+
+    case 0xec8348:    // 48 83 ec XX : sub rsp, XX
+    case 0xf88349:    // 49 83 f8 XX : cmp r8, XX
+    case 0x588948:    // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
+      return 4;
+
+    case 0xec8148:    // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
+      return 7;
+
+    case 0x058b48:    // 48 8b 05 XX XX XX XX :
+                      //   mov rax, QWORD PTR [rip + XXXXXXXX]
+    case 0x25ff48:    // 48 ff 25 XX XX XX XX :
+                      //   rex.W jmp QWORD PTR [rip + XXXXXXXX]
+
+      // Instructions having offset relative to 'rip' need offset adjustment.
+      if (rel_offset)
+        *rel_offset = 3;
+      return 7;
+
+    case 0x2444c7:    // C7 44 24 XX YY YY YY YY
+                      //   mov dword ptr [rsp + XX], YYYYYYYY
+      return 8;
+  }
+
+  switch (*(u32*)(address)) {
+    case 0x24448b48:  // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]
+    case 0x246c8948:  // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
+    case 0x245c8948:  // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
+    case 0x24748948:  // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
+    case 0x244C8948:  // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
+    case 0x24548948:  // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
+    case 0x244c894c:  // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
+    case 0x2444894c:  // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8
+      return 5;
+    case 0x24648348:  // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY
+      return 6;
+  }
+
+#else
+
+  switch (*(u8*)address) {
+    case 0xA1:  // A1 XX XX XX XX :  mov eax, dword ptr ds:[XXXXXXXX]
+      return 5;
+  }
+  switch (*(u16*)address) {
+    case 0x458B:  // 8B 45 XX : mov eax, dword ptr [ebp + XX]
+    case 0x5D8B:  // 8B 5D XX : mov ebx, dword ptr [ebp + XX]
+    case 0x7D8B:  // 8B 7D XX : mov edi, dword ptr [ebp + XX]
+    case 0xEC83:  // 83 EC XX : sub esp, XX
+    case 0x75FF:  // FF 75 XX : push dword ptr [ebp + XX]
+      return 3;
+    case 0xC1F7:  // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
+    case 0x25FF:  // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
+      return 6;
+    case 0x3D83:  // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
+      return 7;
+    case 0x7D83:  // 83 7D XX YY : cmp dword ptr [ebp + XX], YY
+      return 4;
+  }
+
+  switch (0x00FFFFFF & *(u32*)address) {
+    case 0x24448A:  // 8A 44 24 XX : mov eal, dword ptr [esp + XX]
+    case 0x24448B:  // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
+    case 0x244C8B:  // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
+    case 0x24548B:  // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
+    case 0x24748B:  // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
+    case 0x247C8B:  // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
+      return 4;
+  }
+
+  switch (*(u32*)address) {
+    case 0x2444B60F:  // 0F B6 44 24 XX : movzx eax, byte ptr [esp + XX]
+      return 5;
+  }
+#endif
+
+  // Unknown instruction!
+  // FIXME: Unknown instruction failures might happen when we add a new
+  // interceptor or a new compiler version. In either case, they should result
+  // in visible and readable error messages. However, merely calling abort()
+  // leads to an infinite recursion in CheckFailed.
+  InterceptionFailed();
+  return 0;
+}
+
+// Returns 0 on error.
+static size_t RoundUpToInstrBoundary(size_t size, uptr address) {
+  size_t cursor = 0;
+  while (cursor < size) {
+    size_t instruction_size = GetInstructionSize(address + cursor);
+    if (!instruction_size)
+      return 0;
+    cursor += instruction_size;
+  }
+  return cursor;
+}
+
+static bool CopyInstructions(uptr to, uptr from, size_t size) {
+  size_t cursor = 0;
+  while (cursor != size) {
+    size_t rel_offset = 0;
+    size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
+    _memcpy((void*)(to + cursor), (void*)(from + cursor),
+            (size_t)instruction_size);
+    if (rel_offset) {
+      uptr delta = to - from;
+      uptr relocated_offset = *(u32*)(to + cursor + rel_offset) - delta;
+#if SANITIZER_WINDOWS64
+      if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU)
+        return false;
+#endif
+      *(u32*)(to + cursor + rel_offset) = relocated_offset;
+    }
+    cursor += instruction_size;
+  }
+  return true;
+}
+
+
+#if !SANITIZER_WINDOWS64
+bool OverrideFunctionWithDetour(
+    uptr old_func, uptr new_func, uptr *orig_old_func) {
+  const int kDetourHeaderLen = 5;
+  const u16 kDetourInstruction = 0xFF8B;
+
+  uptr header = (uptr)old_func - kDetourHeaderLen;
+  uptr patch_length = kDetourHeaderLen + kShortJumpInstructionLength;
+
+  // Validate that the function is hookable.
+  if (*(u16*)old_func != kDetourInstruction ||
+      !IsMemoryPadding(header, kDetourHeaderLen))
+    return false;
+
+  // Change memory protection to writable.
+  DWORD protection = 0;
+  if (!ChangeMemoryProtection(header, patch_length, &protection))
+    return false;
+
+  // Write a relative jump to the redirected function.
+  WriteJumpInstruction(header, new_func);
+
+  // Write the short jump to the function prefix.
+  WriteShortJumpInstruction(old_func, header);
+
+  // Restore previous memory protection.
+  if (!RestoreMemoryProtection(header, patch_length, protection))
+    return false;
+
+  if (orig_old_func)
+    *orig_old_func = old_func + kShortJumpInstructionLength;
+
+  return true;
+}
+#endif
+
+bool OverrideFunctionWithRedirectJump(
+    uptr old_func, uptr new_func, uptr *orig_old_func) {
+  // Check whether the first instruction is a relative jump.
+  if (*(u8*)old_func != 0xE9)
+    return false;
+
+  if (orig_old_func) {
+    uptr relative_offset = *(u32*)(old_func + 1);
+    uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
+    *orig_old_func = absolute_target;
+  }
+
+#if SANITIZER_WINDOWS64
+  // If needed, get memory space for a trampoline jump.
+  uptr trampoline = AllocateMemoryForTrampoline(old_func, kDirectBranchLength);
+  if (!trampoline)
+    return false;
+  WriteDirectBranch(trampoline, new_func);
+#endif
+
+  // Change memory protection to writable.
+  DWORD protection = 0;
+  if (!ChangeMemoryProtection(old_func, kJumpInstructionLength, &protection))
+    return false;
+
+  // Write a relative jump to the redirected function.
+  WriteJumpInstruction(old_func, FIRST_32_SECOND_64(new_func, trampoline));
+
+  // Restore previous memory protection.
+  if (!RestoreMemoryProtection(old_func, kJumpInstructionLength, protection))
+    return false;
+
+  return true;
+}
+
+bool OverrideFunctionWithHotPatch(
+    uptr old_func, uptr new_func, uptr *orig_old_func) {
+  const int kHotPatchHeaderLen = kBranchLength;
+
+  uptr header = (uptr)old_func - kHotPatchHeaderLen;
+  uptr patch_length = kHotPatchHeaderLen + kShortJumpInstructionLength;
+
+  // Validate that the function is hot patchable.
+  size_t instruction_size = GetInstructionSize(old_func);
+  if (instruction_size < kShortJumpInstructionLength ||
+      !FunctionHasPadding(old_func, kHotPatchHeaderLen))
+    return false;
+
+  if (orig_old_func) {
+    // Put the needed instructions into the trampoline bytes.
+    uptr trampoline_length = instruction_size + kDirectBranchLength;
+    uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
+    if (!trampoline)
+      return false;
+    if (!CopyInstructions(trampoline, old_func, instruction_size))
+      return false;
+    WriteDirectBranch(trampoline + instruction_size,
+                      old_func + instruction_size);
+    *orig_old_func = trampoline;
+  }
+
+  // If needed, get memory space for indirect address.
+  uptr indirect_address = 0;
+#if SANITIZER_WINDOWS64
+  indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
+  if (!indirect_address)
+    return false;
+#endif
+
+  // Change memory protection to writable.
+  DWORD protection = 0;
+  if (!ChangeMemoryProtection(header, patch_length, &protection))
+    return false;
+
+  // Write jumps to the redirected function.
+  WriteBranch(header, indirect_address, new_func);
+  WriteShortJumpInstruction(old_func, header);
+
+  // Restore previous memory protection.
+  if (!RestoreMemoryProtection(header, patch_length, protection))
+    return false;
+
+  return true;
+}
+
+bool OverrideFunctionWithTrampoline(
+    uptr old_func, uptr new_func, uptr *orig_old_func) {
+
+  size_t instructions_length = kBranchLength;
+  size_t padding_length = 0;
+  uptr indirect_address = 0;
+
+  if (orig_old_func) {
+    // Find out the number of bytes of the instructions we need to copy
+    // to the trampoline.
+    instructions_length = RoundUpToInstrBoundary(kBranchLength, old_func);
+    if (!instructions_length)
+      return false;
+
+    // Put the needed instructions into the trampoline bytes.
+    uptr trampoline_length = instructions_length + kDirectBranchLength;
+    uptr trampoline = AllocateMemoryForTrampoline(old_func, trampoline_length);
+    if (!trampoline)
+      return false;
+    if (!CopyInstructions(trampoline, old_func, instructions_length))
+      return false;
+    WriteDirectBranch(trampoline + instructions_length,
+                      old_func + instructions_length);
+    *orig_old_func = trampoline;
+  }
+
+#if SANITIZER_WINDOWS64
+  // Check if the targeted address can be encoded in the function padding.
+  // Otherwise, allocate it in the trampoline region.
+  if (IsMemoryPadding(old_func - kAddressLength, kAddressLength)) {
+    indirect_address = old_func - kAddressLength;
+    padding_length = kAddressLength;
+  } else {
+    indirect_address = AllocateMemoryForTrampoline(old_func, kAddressLength);
+    if (!indirect_address)
+      return false;
+  }
+#endif
+
+  // Change memory protection to writable.
+  uptr patch_address = old_func - padding_length;
+  uptr patch_length = instructions_length + padding_length;
+  DWORD protection = 0;
+  if (!ChangeMemoryProtection(patch_address, patch_length, &protection))
+    return false;
+
+  // Patch the original function.
+  WriteBranch(old_func, indirect_address, new_func);
+
+  // Restore previous memory protection.
+  if (!RestoreMemoryProtection(patch_address, patch_length, protection))
+    return false;
+
+  return true;
+}
+
+bool OverrideFunction(
+    uptr old_func, uptr new_func, uptr *orig_old_func) {
+#if !SANITIZER_WINDOWS64
+  if (OverrideFunctionWithDetour(old_func, new_func, orig_old_func))
+    return true;
+#endif
+  if (OverrideFunctionWithRedirectJump(old_func, new_func, orig_old_func))
+    return true;
+  if (OverrideFunctionWithHotPatch(old_func, new_func, orig_old_func))
+    return true;
+  if (OverrideFunctionWithTrampoline(old_func, new_func, orig_old_func))
+    return true;
+  return false;
+}
+
+static void **InterestingDLLsAvailable() {
+  static const char *InterestingDLLs[] = {
+      "kernel32.dll",
+      "msvcr100.dll",      // VS2010
+      "msvcr110.dll",      // VS2012
+      "msvcr120.dll",      // VS2013
+      "vcruntime140.dll",  // VS2015
+      "ucrtbase.dll",      // Universal CRT
+      // NTDLL should go last as it exports some functions that we should
+      // override in the CRT [presumably only used internally].
+      "ntdll.dll", NULL};
+  static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 };
+  if (!result[0]) {
+    for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) {
+      if (HMODULE h = GetModuleHandleA(InterestingDLLs[i]))
+        result[j++] = (void *)h;
+    }
+  }
+  return &result[0];
+}
+
+namespace {
+// Utility for reading loaded PE images.
+template <typename T> class RVAPtr {
+ public:
+  RVAPtr(void *module, uptr rva)
+      : ptr_(reinterpret_cast<T *>(reinterpret_cast<char *>(module) + rva)) {}
+  operator T *() { return ptr_; }
+  T *operator->() { return ptr_; }
+  T *operator++() { return ++ptr_; }
+
+ private:
+  T *ptr_;
+};
+} // namespace
+
+// Internal implementation of GetProcAddress. At least since Windows 8,
+// GetProcAddress appears to initialize DLLs before returning function pointers
+// into them. This is problematic for the sanitizers, because they typically
+// want to intercept malloc *before* MSVCRT initializes. Our internal
+// implementation walks the export list manually without doing initialization.
+uptr InternalGetProcAddress(void *module, const char *func_name) {
+  // Check that the module header is full and present.
+  RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
+  RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
+  if (!module || dos_stub->e_magic != IMAGE_DOS_SIGNATURE ||  // "MZ"
+      headers->Signature != IMAGE_NT_SIGNATURE ||             // "PE\0\0"
+      headers->FileHeader.SizeOfOptionalHeader <
+          sizeof(IMAGE_OPTIONAL_HEADER)) {
+    return 0;
+  }
+
+  IMAGE_DATA_DIRECTORY *export_directory =
+      &headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT];
+  if (export_directory->Size == 0)
+    return 0;
+  RVAPtr<IMAGE_EXPORT_DIRECTORY> exports(module,
+                                         export_directory->VirtualAddress);
+  RVAPtr<DWORD> functions(module, exports->AddressOfFunctions);
+  RVAPtr<DWORD> names(module, exports->AddressOfNames);
+  RVAPtr<WORD> ordinals(module, exports->AddressOfNameOrdinals);
+
+  for (DWORD i = 0; i < exports->NumberOfNames; i++) {
+    RVAPtr<char> name(module, names[i]);
+    if (!strcmp(func_name, name)) {
+      DWORD index = ordinals[i];
+      RVAPtr<char> func(module, functions[index]);
+
+      // Handle forwarded functions.
+      DWORD offset = functions[index];
+      if (offset >= export_directory->VirtualAddress &&
+          offset < export_directory->VirtualAddress + export_directory->Size) {
+        // An entry for a forwarded function is a string with the following
+        // format: "<module> . <function_name>" that is stored into the
+        // exported directory.
+        char function_name[256];
+        size_t funtion_name_length = _strlen(func);
+        if (funtion_name_length >= sizeof(function_name) - 1)
+          InterceptionFailed();
+
+        _memcpy(function_name, func, funtion_name_length);
+        function_name[funtion_name_length] = '\0';
+        char* separator = _strchr(function_name, '.');
+        if (!separator)
+          InterceptionFailed();
+        *separator = '\0';
+
+        void* redirected_module = GetModuleHandleA(function_name);
+        if (!redirected_module)
+          InterceptionFailed();
+        return InternalGetProcAddress(redirected_module, separator + 1);
+      }
+
+      return (uptr)(char *)func;
+    }
+  }
+
+  return 0;
+}
+
+bool OverrideFunction(
+    const char *func_name, uptr new_func, uptr *orig_old_func) {
+  bool hooked = false;
+  void **DLLs = InterestingDLLsAvailable();
+  for (size_t i = 0; DLLs[i]; ++i) {
+    uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);
+    if (func_addr &&
+        OverrideFunction(func_addr, new_func, orig_old_func)) {
+      hooked = true;
+    }
+  }
+  return hooked;
+}
+
+bool OverrideImportedFunction(const char *module_to_patch,
+                              const char *imported_module,
+                              const char *function_name, uptr new_function,
+                              uptr *orig_old_func) {
+  HMODULE module = GetModuleHandleA(module_to_patch);
+  if (!module)
+    return false;
+
+  // Check that the module header is full and present.
+  RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0);
+  RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew);
+  if (!module || dos_stub->e_magic != IMAGE_DOS_SIGNATURE ||  // "MZ"
+      headers->Signature != IMAGE_NT_SIGNATURE ||             // "PE\0\0"
+      headers->FileHeader.SizeOfOptionalHeader <
+          sizeof(IMAGE_OPTIONAL_HEADER)) {
+    return false;
+  }
+
+  IMAGE_DATA_DIRECTORY *import_directory =
+      &headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT];
+
+  // Iterate the list of imported DLLs. FirstThunk will be null for the last
+  // entry.
+  RVAPtr<IMAGE_IMPORT_DESCRIPTOR> imports(module,
+                                          import_directory->VirtualAddress);
+  for (; imports->FirstThunk != 0; ++imports) {
+    RVAPtr<const char> modname(module, imports->Name);
+    if (_stricmp(&*modname, imported_module) == 0)
+      break;
+  }
+  if (imports->FirstThunk == 0)
+    return false;
+
+  // We have two parallel arrays: the import address table (IAT) and the table
+  // of names. They start out containing the same data, but the loader rewrites
+  // the IAT to hold imported addresses and leaves the name table in
+  // OriginalFirstThunk alone.
+  RVAPtr<IMAGE_THUNK_DATA> name_table(module, imports->OriginalFirstThunk);
+  RVAPtr<IMAGE_THUNK_DATA> iat(module, imports->FirstThunk);
+  for (; name_table->u1.Ordinal != 0; ++name_table, ++iat) {
+    if (!IMAGE_SNAP_BY_ORDINAL(name_table->u1.Ordinal)) {
+      RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(
+          module, name_table->u1.ForwarderString);
+      const char *funcname = &import_by_name->Name[0];
+      if (strcmp(funcname, function_name) == 0)
+        break;
+    }
+  }
+  if (name_table->u1.Ordinal == 0)
+    return false;
+
+  // Now we have the correct IAT entry. Do the swap. We have to make the page
+  // read/write first.
+  if (orig_old_func)
+    *orig_old_func = iat->u1.AddressOfData;
+  DWORD old_prot, unused_prot;
+  if (!VirtualProtect(&iat->u1.AddressOfData, 4, PAGE_EXECUTE_READWRITE,
+                      &old_prot))
+    return false;
+  iat->u1.AddressOfData = new_function;
+  if (!VirtualProtect(&iat->u1.AddressOfData, 4, old_prot, &unused_prot))
+    return false;  // Not clear if this failure bothers us.
+  return true;
+}
+
+}  // namespace __interception
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp b/lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp
new file mode 100644
index 0000000000..9d67f679b5
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_checks.cpp
@@ -0,0 +1,22 @@
+//===-- sanitizer_allocator_checks.cpp --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
+// allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno.h"
+
+namespace __sanitizer {
+
+void SetErrnoToENOMEM() {
+  errno = errno_ENOMEM;
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp b/lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp
new file mode 100644
index 0000000000..d74e08010d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp
@@ -0,0 +1,137 @@
+//===-- sanitizer_allocator_report.cpp --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Shared allocator error reporting for ThreadSanitizer, MemorySanitizer, etc.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator.h"
+#include "sanitizer_allocator_report.h"
+#include "sanitizer_common.h"
+#include "sanitizer_report_decorator.h"
+
+namespace __sanitizer {
+
+class ScopedAllocatorErrorReport {
+ public:
+  ScopedAllocatorErrorReport(const char *error_summary_,
+                             const StackTrace *stack_)
+      : error_summary(error_summary_),
+        stack(stack_) {
+    Printf("%s", d.Error());
+  }
+  ~ScopedAllocatorErrorReport() {
+    Printf("%s", d.Default());
+    stack->Print();
+    PrintHintAllocatorCannotReturnNull();
+    ReportErrorSummary(error_summary, stack);
+  }
+
+ private:
+  ScopedErrorReportLock lock;
+  const char *error_summary;
+  const StackTrace* const stack;
+  const SanitizerCommonDecorator d;
+};
+
+void NORETURN ReportCallocOverflow(uptr count, uptr size,
+                                   const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("calloc-overflow", stack);
+    Report("ERROR: %s: calloc parameters overflow: count * size (%zd * %zd) "
+           "cannot be represented in type size_t\n", SanitizerToolName, count,
+           size);
+  }
+  Die();
+}
+
+void NORETURN ReportReallocArrayOverflow(uptr count, uptr size,
+                                         const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("reallocarray-overflow", stack);
+    Report(
+        "ERROR: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
+        "cannot be represented in type size_t\n",
+        SanitizerToolName, count, size);
+  }
+  Die();
+}
+
+void NORETURN ReportPvallocOverflow(uptr size, const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("pvalloc-overflow", stack);
+    Report("ERROR: %s: pvalloc parameters overflow: size 0x%zx rounded up to "
+           "system page size 0x%zx cannot be represented in type size_t\n",
+           SanitizerToolName, size, GetPageSizeCached());
+  }
+  Die();
+}
+
+void NORETURN ReportInvalidAllocationAlignment(uptr alignment,
+                                               const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("invalid-allocation-alignment", stack);
+    Report("ERROR: %s: invalid allocation alignment: %zd, alignment must be a "
+           "power of two\n", SanitizerToolName, alignment);
+  }
+  Die();
+}
+
+void NORETURN ReportInvalidAlignedAllocAlignment(uptr size, uptr alignment,
+                                                 const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("invalid-aligned-alloc-alignment", stack);
+#if SANITIZER_POSIX
+    Report("ERROR: %s: invalid alignment requested in "
+           "aligned_alloc: %zd, alignment must be a power of two and the "
+           "requested size 0x%zx must be a multiple of alignment\n",
+           SanitizerToolName, alignment, size);
+#else
+    Report("ERROR: %s: invalid alignment requested in aligned_alloc: %zd, "
+           "the requested size 0x%zx must be a multiple of alignment\n",
+           SanitizerToolName, alignment, size);
+#endif
+  }
+  Die();
+}
+
+void NORETURN ReportInvalidPosixMemalignAlignment(uptr alignment,
+                                                  const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("invalid-posix-memalign-alignment",
+                                      stack);
+    Report(
+        "ERROR: %s: invalid alignment requested in "
+        "posix_memalign: %zd, alignment must be a power of two and a "
+        "multiple of sizeof(void*) == %zd\n",
+        SanitizerToolName, alignment, sizeof(void *));
+  }
+  Die();
+}
+
+void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
+                                         const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("allocation-size-too-big", stack);
+    Report("ERROR: %s: requested allocation size 0x%zx exceeds maximum "
+           "supported size of 0x%zx\n", SanitizerToolName, user_size, max_size);
+  }
+  Die();
+}
+
+void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
+  {
+    ScopedAllocatorErrorReport report("out-of-memory", stack);
+    Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "
+           "bytes\n", SanitizerToolName, requested_size);
+  }
+  Die();
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp
new file mode 100644
index 0000000000..0c918ebb4a
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -0,0 +1,149 @@
+//===-- sanitizer_common_libcdep.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator_interface.h"
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_procmaps.h"
+
+
+namespace __sanitizer {
+
+static void (*SoftRssLimitExceededCallback)(bool exceeded);
+void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
+  CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
+  SoftRssLimitExceededCallback = Callback;
+}
+
+#if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO
+// Weak default implementation for when sanitizer_stackdepot is not linked in.
+SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
+  return nullptr;
+}
+
+void *BackgroundThread(void *arg) {
+  const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
+  const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
+  const bool heap_profile = common_flags()->heap_profile;
+  uptr prev_reported_rss = 0;
+  uptr prev_reported_stack_depot_size = 0;
+  bool reached_soft_rss_limit = false;
+  uptr rss_during_last_reported_profile = 0;
+  while (true) {
+    SleepForMillis(100);
+    const uptr current_rss_mb = GetRSS() >> 20;
+    if (Verbosity()) {
+      // If RSS has grown 10% since last time, print some information.
+      if (prev_reported_rss * 11 / 10 < current_rss_mb) {
+        Printf("%s: RSS: %zdMb\n", SanitizerToolName, current_rss_mb);
+        prev_reported_rss = current_rss_mb;
+      }
+      // If stack depot has grown 10% since last time, print it too.
+      StackDepotStats *stack_depot_stats = StackDepotGetStats();
+      if (stack_depot_stats) {
+        if (prev_reported_stack_depot_size * 11 / 10 <
+            stack_depot_stats->allocated) {
+          Printf("%s: StackDepot: %zd ids; %zdM allocated\n",
+                 SanitizerToolName,
+                 stack_depot_stats->n_uniq_ids,
+                 stack_depot_stats->allocated >> 20);
+          prev_reported_stack_depot_size = stack_depot_stats->allocated;
+        }
+      }
+    }
+    // Check RSS against the limit.
+    if (hard_rss_limit_mb && hard_rss_limit_mb < current_rss_mb) {
+      Report("%s: hard rss limit exhausted (%zdMb vs %zdMb)\n",
+             SanitizerToolName, hard_rss_limit_mb, current_rss_mb);
+      DumpProcessMap();
+      Die();
+    }
+    if (soft_rss_limit_mb) {
+      if (soft_rss_limit_mb < current_rss_mb && !reached_soft_rss_limit) {
+        reached_soft_rss_limit = true;
+        Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
+               SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
+        if (SoftRssLimitExceededCallback)
+          SoftRssLimitExceededCallback(true);
+      } else if (soft_rss_limit_mb >= current_rss_mb &&
+                 reached_soft_rss_limit) {
+        reached_soft_rss_limit = false;
+        if (SoftRssLimitExceededCallback)
+          SoftRssLimitExceededCallback(false);
+      }
+    }
+    if (heap_profile &&
+        current_rss_mb > rss_during_last_reported_profile * 1.1) {
+      Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb);
+      __sanitizer_print_memory_profile(90, 20);
+      rss_during_last_reported_profile = current_rss_mb;
+    }
+  }
+}
+#endif
+
+void WriteToSyslog(const char *msg) {
+  InternalScopedString msg_copy(kErrorMessageBufferSize);
+  msg_copy.append("%s", msg);
+  char *p = msg_copy.data();
+  char *q;
+
+  // Print one line at a time.
+  // syslog, at least on Android, has an implicit message length limit.
+  while ((q = internal_strchr(p, '\n'))) {
+    *q = '\0';
+    WriteOneLineToSyslog(p);
+    p = q + 1;
+  }
+  // Print remaining characters, if there are any.
+  // Note that this will add an extra newline at the end.
+  // FIXME: buffer extra output. This would need a thread-local buffer, which
+  // on Android requires plugging into the tools (ex. ASan's) Thread class.
+  if (*p)
+    WriteOneLineToSyslog(p);
+}
+
+void MaybeStartBackgroudThread() {
+#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \
+    !SANITIZER_GO  // Need to implement/test on other platforms.
+  // Start the background thread if one of the rss limits is given.
+  if (!common_flags()->hard_rss_limit_mb &&
+      !common_flags()->soft_rss_limit_mb &&
+      !common_flags()->heap_profile) return;
+  if (!&real_pthread_create) return;  // Can't spawn the thread anyway.
+  internal_start_thread(BackgroundThread, nullptr);
+#endif
+}
+
+static void (*sandboxing_callback)();
+void SetSandboxingCallback(void (*f)()) {
+  sandboxing_callback = f;
+}
+
+uptr ReservedAddressRange::InitAligned(uptr size, uptr align,
+                                       const char *name) {
+  CHECK(IsPowerOfTwo(align));
+  if (align <= GetPageSizeCached())
+    return Init(size, name);
+  uptr start = Init(size + align, name);
+  start += align - (start & (align - 1));
+  return start;
+}
+
+}  // namespace __sanitizer
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
+                             __sanitizer_sandbox_arguments *args) {
+  __sanitizer::PlatformPrepareForSandboxing(args);
+  if (__sanitizer::sandboxing_callback)
+    __sanitizer::sandboxing_callback();
+}
diff --git a/lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp b/lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp
new file mode 100644
index 0000000000..3b278e017e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp
@@ -0,0 +1,34 @@
+//===-- sanitizer_common_nolibc.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains stubs for libc function to facilitate optional use of
+// libc in no-libcdep sources.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+
+namespace __sanitizer {
+
+// The Windows implementations of these functions use the win32 API directly,
+// bypassing libc.
+#if !SANITIZER_WINDOWS
+#if SANITIZER_LINUX
+void LogMessageOnPrintf(const char *str) {}
+#endif
+void WriteToSyslog(const char *buffer) {}
+void Abort() { internal__exit(1); }
+void SleepForSeconds(int seconds) { internal_sleep(seconds); }
+#endif // !SANITIZER_WINDOWS
+
+#if !SANITIZER_WINDOWS && !SANITIZER_MAC
+void ListOfModules::init() {}
+#endif
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp
new file mode 100644
index 0000000000..4d17c9686e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -0,0 +1,846 @@
+//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements linux-specific functions from
+// sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||                \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_freebsd.h"
+#include "sanitizer_getauxval.h"
+#include "sanitizer_glibc_version.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_procmaps.h"
+
+#include <dlfcn.h>  // for dlsym()
+#include <link.h>
+#include <pthread.h>
+#include <signal.h>
+#include <sys/resource.h>
+#include <syslog.h>
+
+#if !defined(ElfW)
+#define ElfW(type) Elf_##type
+#endif
+
+#if SANITIZER_FREEBSD
+#include <pthread_np.h>
+#include <osreldate.h>
+#include <sys/sysctl.h>
+#define pthread_getattr_np pthread_attr_get_np
+#endif
+
+#if SANITIZER_OPENBSD
+#include <pthread_np.h>
+#include <sys/sysctl.h>
+#endif
+
+#if SANITIZER_NETBSD
+#include <sys/sysctl.h>
+#include <sys/tls.h>
+#include <lwp.h>
+#endif
+
+#if SANITIZER_SOLARIS
+#include <stdlib.h>
+#include <thread.h>
+#endif
+
+#if SANITIZER_ANDROID
+#include <android/api-level.h>
+#if !defined(CPU_COUNT) && !defined(__aarch64__)
+#include <dirent.h>
+#include <fcntl.h>
+struct __sanitizer::linux_dirent {
+  long           d_ino;
+  off_t          d_off;
+  unsigned short d_reclen;
+  char           d_name[];
+};
+#endif
+#endif
+
+#if !SANITIZER_ANDROID
+#include <elf.h>
+#include <unistd.h>
+#endif
+
+namespace __sanitizer {
+
+SANITIZER_WEAK_ATTRIBUTE int
+real_sigaction(int signum, const void *act, void *oldact);
+
+int internal_sigaction(int signum, const void *act, void *oldact) {
+#if !SANITIZER_GO
+  if (&real_sigaction)
+    return real_sigaction(signum, act, oldact);
+#endif
+  return sigaction(signum, (const struct sigaction *)act,
+                   (struct sigaction *)oldact);
+}
+
+void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
+                                uptr *stack_bottom) {
+  CHECK(stack_top);
+  CHECK(stack_bottom);
+  if (at_initialization) {
+    // This is the main thread. Libpthread may not be initialized yet.
+    struct rlimit rl;
+    CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
+
+    // Find the mapping that contains a stack variable.
+    MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+    if (proc_maps.Error()) {
+      *stack_top = *stack_bottom = 0;
+      return;
+    }
+    MemoryMappedSegment segment;
+    uptr prev_end = 0;
+    while (proc_maps.Next(&segment)) {
+      if ((uptr)&rl < segment.end) break;
+      prev_end = segment.end;
+    }
+    CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
+
+    // Get stacksize from rlimit, but clip it so that it does not overlap
+    // with other mappings.
+    uptr stacksize = rl.rlim_cur;
+    if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
+    // When running with unlimited stack size, we still want to set some limit.
+    // The unlimited stack size is caused by 'ulimit -s unlimited'.
+    // Also, for some reason, GNU make spawns subprocesses with unlimited stack.
+    if (stacksize > kMaxThreadStackSize)
+      stacksize = kMaxThreadStackSize;
+    *stack_top = segment.end;
+    *stack_bottom = segment.end - stacksize;
+    return;
+  }
+  uptr stacksize = 0;
+  void *stackaddr = nullptr;
+#if SANITIZER_SOLARIS
+  stack_t ss;
+  CHECK_EQ(thr_stksegment(&ss), 0);
+  stacksize = ss.ss_size;
+  stackaddr = (char *)ss.ss_sp - stacksize;
+#elif SANITIZER_OPENBSD
+  stack_t sattr;
+  CHECK_EQ(pthread_stackseg_np(pthread_self(), &sattr), 0);
+  stackaddr = sattr.ss_sp;
+  stacksize = sattr.ss_size;
+#else  // !SANITIZER_SOLARIS
+  pthread_attr_t attr;
+  pthread_attr_init(&attr);
+  CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
+  my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
+  pthread_attr_destroy(&attr);
+#endif // SANITIZER_SOLARIS
+
+  *stack_top = (uptr)stackaddr + stacksize;
+  *stack_bottom = (uptr)stackaddr;
+}
+
+#if !SANITIZER_GO
+bool SetEnv(const char *name, const char *value) {
+  void *f = dlsym(RTLD_NEXT, "setenv");
+  if (!f)
+    return false;
+  typedef int(*setenv_ft)(const char *name, const char *value, int overwrite);
+  setenv_ft setenv_f;
+  CHECK_EQ(sizeof(setenv_f), sizeof(f));
+  internal_memcpy(&setenv_f, &f, sizeof(f));
+  return setenv_f(name, value, 1) == 0;
+}
+#endif
+
+__attribute__((unused)) static bool GetLibcVersion(int *major, int *minor,
+                                                   int *patch) {
+#ifdef _CS_GNU_LIBC_VERSION
+  char buf[64];
+  uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
+  if (len >= sizeof(buf))
+    return false;
+  buf[len] = 0;
+  static const char kGLibC[] = "glibc ";
+  if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
+    return false;
+  const char *p = buf + sizeof(kGLibC) - 1;
+  *major = internal_simple_strtoll(p, &p, 10);
+  *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
+  *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
+  return true;
+#else
+  return false;
+#endif
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&               \
+    !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_SOLARIS
+static uptr g_tls_size;
+
+#ifdef __i386__
+# define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
+#else
+# define CHECK_GET_TLS_STATIC_INFO_VERSION 0
+#endif
+
+#if CHECK_GET_TLS_STATIC_INFO_VERSION
+# define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
+#else
+# define DL_INTERNAL_FUNCTION
+#endif
+
+namespace {
+struct GetTlsStaticInfoCall {
+  typedef void (*get_tls_func)(size_t*, size_t*);
+};
+struct GetTlsStaticInfoRegparmCall {
+  typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION;
+};
+
+template <typename T>
+void CallGetTls(void* ptr, size_t* size, size_t* align) {
+  typename T::get_tls_func get_tls;
+  CHECK_EQ(sizeof(get_tls), sizeof(ptr));
+  internal_memcpy(&get_tls, &ptr, sizeof(ptr));
+  CHECK_NE(get_tls, 0);
+  get_tls(size, align);
+}
+
+bool CmpLibcVersion(int major, int minor, int patch) {
+  int ma;
+  int mi;
+  int pa;
+  if (!GetLibcVersion(&ma, &mi, &pa))
+    return false;
+  if (ma > major)
+    return true;
+  if (ma < major)
+    return false;
+  if (mi > minor)
+    return true;
+  if (mi < minor)
+    return false;
+  return pa >= patch;
+}
+
+}  // namespace
+
+void InitTlsSize() {
+  // all current supported platforms have 16 bytes stack alignment
+  const size_t kStackAlign = 16;
+  void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
+  size_t tls_size = 0;
+  size_t tls_align = 0;
+  // On i?86, _dl_get_tls_static_info used to be internal_function, i.e.
+  // __attribute__((regparm(3), stdcall)) before glibc 2.27 and is normal
+  // function in 2.27 and later.
+  if (CHECK_GET_TLS_STATIC_INFO_VERSION && !CmpLibcVersion(2, 27, 0))
+    CallGetTls<GetTlsStaticInfoRegparmCall>(get_tls_static_info_ptr,
+                                            &tls_size, &tls_align);
+  else
+    CallGetTls<GetTlsStaticInfoCall>(get_tls_static_info_ptr,
+                                     &tls_size, &tls_align);
+  if (tls_align < kStackAlign)
+    tls_align = kStackAlign;
+  g_tls_size = RoundUpTo(tls_size, tls_align);
+}
+#else
+void InitTlsSize() { }
+#endif  // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
+        // !SANITIZER_NETBSD && !SANITIZER_SOLARIS
+
+#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) ||          \
+     defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) ||    \
+     defined(__arm__)) &&                                                      \
+    SANITIZER_LINUX && !SANITIZER_ANDROID
+// sizeof(struct pthread) from glibc.
+static atomic_uintptr_t thread_descriptor_size;
+
+uptr ThreadDescriptorSize() {
+  uptr val = atomic_load_relaxed(&thread_descriptor_size);
+  if (val)
+    return val;
+#if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
+  int major;
+  int minor;
+  int patch;
+  if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
+    /* sizeof(struct pthread) values from various glibc versions.  */
+    if (SANITIZER_X32)
+      val = 1728; // Assume only one particular version for x32.
+    // For ARM sizeof(struct pthread) changed in Glibc 2.23.
+    else if (SANITIZER_ARM)
+      val = minor <= 22 ? 1120 : 1216;
+    else if (minor <= 3)
+      val = FIRST_32_SECOND_64(1104, 1696);
+    else if (minor == 4)
+      val = FIRST_32_SECOND_64(1120, 1728);
+    else if (minor == 5)
+      val = FIRST_32_SECOND_64(1136, 1728);
+    else if (minor <= 9)
+      val = FIRST_32_SECOND_64(1136, 1712);
+    else if (minor == 10)
+      val = FIRST_32_SECOND_64(1168, 1776);
+    else if (minor == 11 || (minor == 12 && patch == 1))
+      val = FIRST_32_SECOND_64(1168, 2288);
+    else if (minor <= 14)
+      val = FIRST_32_SECOND_64(1168, 2304);
+    else
+      val = FIRST_32_SECOND_64(1216, 2304);
+  }
+#elif defined(__mips__)
+  // TODO(sagarthakur): add more values as per different glibc versions.
+  val = FIRST_32_SECOND_64(1152, 1776);
+#elif defined(__aarch64__)
+  // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
+  val = 1776;
+#elif defined(__powerpc64__)
+  val = 1776; // from glibc.ppc64le 2.20-8.fc21
+#elif defined(__s390__)
+  val = FIRST_32_SECOND_64(1152, 1776); // valid for glibc 2.22
+#endif
+  if (val)
+    atomic_store_relaxed(&thread_descriptor_size, val);
+  return val;
+}
+
+// The offset at which pointer to self is located in the thread descriptor.
+const uptr kThreadSelfOffset = FIRST_32_SECOND_64(8, 16);
+
+uptr ThreadSelfOffset() {
+  return kThreadSelfOffset;
+}
+
+#if defined(__mips__) || defined(__powerpc64__)
+// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
+// head structure. It lies before the static tls blocks.
+static uptr TlsPreTcbSize() {
+# if defined(__mips__)
+  const uptr kTcbHead = 16; // sizeof (tcbhead_t)
+# elif defined(__powerpc64__)
+  const uptr kTcbHead = 88; // sizeof (tcbhead_t)
+# endif
+  const uptr kTlsAlign = 16;
+  const uptr kTlsPreTcbSize =
+      RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
+  return kTlsPreTcbSize;
+}
+#endif
+
+uptr ThreadSelf() {
+  uptr descr_addr;
+# if defined(__i386__)
+  asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
+# elif defined(__x86_64__)
+  asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
+# elif defined(__mips__)
+  // MIPS uses TLS variant I. The thread pointer (in hardware register $29)
+  // points to the end of the TCB + 0x7000. The pthread_descr structure is
+  // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
+  // TCB and the size of pthread_descr.
+  const uptr kTlsTcbOffset = 0x7000;
+  uptr thread_pointer;
+  asm volatile(".set push;\
+                .set mips64r2;\
+                rdhwr %0,$29;\
+                .set pop" : "=r" (thread_pointer));
+  descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
+# elif defined(__aarch64__) || defined(__arm__)
+  descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
+                                      ThreadDescriptorSize();
+# elif defined(__s390__)
+  descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer());
+# elif defined(__powerpc64__)
+  // PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
+  // points to the end of the TCB + 0x7000. The pthread_descr structure is
+  // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
+  // TCB and the size of pthread_descr.
+  const uptr kTlsTcbOffset = 0x7000;
+  uptr thread_pointer;
+  asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
+  descr_addr = thread_pointer - TlsPreTcbSize();
+# else
+#  error "unsupported CPU arch"
+# endif
+  return descr_addr;
+}
+#endif  // (x86_64 || i386 || MIPS) && SANITIZER_LINUX
+
+#if SANITIZER_FREEBSD
+static void **ThreadSelfSegbase() {
+  void **segbase = 0;
+# if defined(__i386__)
+  // sysarch(I386_GET_GSBASE, segbase);
+  __asm __volatile("mov %%gs:0, %0" : "=r" (segbase));
+# elif defined(__x86_64__)
+  // sysarch(AMD64_GET_FSBASE, segbase);
+  __asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
+# else
+#  error "unsupported CPU arch"
+# endif
+  return segbase;
+}
+
+uptr ThreadSelf() {
+  return (uptr)ThreadSelfSegbase()[2];
+}
+#endif  // SANITIZER_FREEBSD
+
+#if SANITIZER_NETBSD
+static struct tls_tcb * ThreadSelfTlsTcb() {
+  return (struct tls_tcb *)_lwp_getprivate();
+}
+
+uptr ThreadSelf() {
+  return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
+}
+
+int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {
+  const Elf_Phdr *hdr = info->dlpi_phdr;
+  const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
+
+  for (; hdr != last_hdr; ++hdr) {
+    if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
+      *(uptr*)data = hdr->p_memsz;
+      break;
+    }
+  }
+  return 0;
+}
+#endif  // SANITIZER_NETBSD
+
+#if !SANITIZER_GO
+static void GetTls(uptr *addr, uptr *size) {
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+# if defined(__x86_64__) || defined(__i386__) || defined(__s390__)
+  *addr = ThreadSelf();
+  *size = GetTlsSize();
+  *addr -= *size;
+  *addr += ThreadDescriptorSize();
+# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) \
+    || defined(__arm__)
+  *addr = ThreadSelf();
+  *size = GetTlsSize();
+# else
+  *addr = 0;
+  *size = 0;
+# endif
+#elif SANITIZER_FREEBSD
+  void** segbase = ThreadSelfSegbase();
+  *addr = 0;
+  *size = 0;
+  if (segbase != 0) {
+    // tcbalign = 16
+    // tls_size = round(tls_static_space, tcbalign);
+    // dtv = segbase[1];
+    // dtv[2] = segbase - tls_static_space;
+    void **dtv = (void**) segbase[1];
+    *addr = (uptr) dtv[2];
+    *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
+  }
+#elif SANITIZER_NETBSD
+  struct tls_tcb * const tcb = ThreadSelfTlsTcb();
+  *addr = 0;
+  *size = 0;
+  if (tcb != 0) {
+    // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
+    // ld.elf_so hardcodes the index 1.
+    dl_iterate_phdr(GetSizeFromHdr, size);
+
+    if (*size != 0) {
+      // The block has been found and tcb_dtv[1] contains the base address
+      *addr = (uptr)tcb->tcb_dtv[1];
+    }
+  }
+#elif SANITIZER_OPENBSD
+  *addr = 0;
+  *size = 0;
+#elif SANITIZER_ANDROID
+  *addr = 0;
+  *size = 0;
+#elif SANITIZER_SOLARIS
+  // FIXME
+  *addr = 0;
+  *size = 0;
+#else
+# error "Unknown OS"
+#endif
+}
+#endif
+
+#if !SANITIZER_GO
+uptr GetTlsSize() {
+#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD ||              \
+    SANITIZER_OPENBSD || SANITIZER_SOLARIS
+  uptr addr, size;
+  GetTls(&addr, &size);
+  return size;
+#elif defined(__mips__) || defined(__powerpc64__)
+  return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
+#else
+  return g_tls_size;
+#endif
+}
+#endif
+
+void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+                          uptr *tls_addr, uptr *tls_size) {
+#if SANITIZER_GO
+  // Stub implementation for Go.
+  *stk_addr = *stk_size = *tls_addr = *tls_size = 0;
+#else
+  GetTls(tls_addr, tls_size);
+
+  uptr stack_top, stack_bottom;
+  GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
+  *stk_addr = stack_bottom;
+  *stk_size = stack_top - stack_bottom;
+
+  if (!main) {
+    // If stack and tls intersect, make them non-intersecting.
+    if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) {
+      CHECK_GT(*tls_addr + *tls_size, *stk_addr);
+      CHECK_LE(*tls_addr + *tls_size, *stk_addr + *stk_size);
+      *stk_size -= *tls_size;
+      *tls_addr = *stk_addr + *stk_size;
+    }
+  }
+#endif
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+typedef ElfW(Phdr) Elf_Phdr;
+#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
+#define Elf_Phdr XElf32_Phdr
+#define dl_phdr_info xdl_phdr_info
+#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
+#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+
+struct DlIteratePhdrData {
+  InternalMmapVectorNoCtor<LoadedModule> *modules;
+  bool first;
+};
+
+static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
+  DlIteratePhdrData *data = (DlIteratePhdrData*)arg;
+  InternalScopedString module_name(kMaxPathLength);
+  if (data->first) {
+    data->first = false;
+    // First module is the binary itself.
+    ReadBinaryNameCached(module_name.data(), module_name.size());
+  } else if (info->dlpi_name) {
+    module_name.append("%s", info->dlpi_name);
+  }
+  if (module_name[0] == '\0')
+    return 0;
+  LoadedModule cur_module;
+  cur_module.set(module_name.data(), info->dlpi_addr);
+  for (int i = 0; i < (int)info->dlpi_phnum; i++) {
+    const Elf_Phdr *phdr = &info->dlpi_phdr[i];
+    if (phdr->p_type == PT_LOAD) {
+      uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
+      uptr cur_end = cur_beg + phdr->p_memsz;
+      bool executable = phdr->p_flags & PF_X;
+      bool writable = phdr->p_flags & PF_W;
+      cur_module.addAddressRange(cur_beg, cur_end, executable,
+                                 writable);
+    }
+  }
+  data->modules->push_back(cur_module);
+  return 0;
+}
+
+#if SANITIZER_ANDROID && __ANDROID_API__ < 21
+extern "C" __attribute__((weak)) int dl_iterate_phdr(
+    int (*)(struct dl_phdr_info *, size_t, void *), void *);
+#endif
+
+static bool requiresProcmaps() {
+#if SANITIZER_ANDROID && __ANDROID_API__ <= 22
+  // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
+  // The runtime check allows the same library to work with
+  // both K and L (and future) Android releases.
+  return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
+#else
+  return false;
+#endif
+}
+
+static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
+  MemoryMappingLayout memory_mapping(/*cache_enabled*/true);
+  memory_mapping.DumpListOfModules(modules);
+}
+
+void ListOfModules::init() {
+  clearOrInit();
+  if (requiresProcmaps()) {
+    procmapsInit(&modules_);
+  } else {
+    DlIteratePhdrData data = {&modules_, true};
+    dl_iterate_phdr(dl_iterate_phdr_cb, &data);
+  }
+}
+
+// When a custom loader is used, dl_iterate_phdr may not contain the full
+// list of modules. Allow callers to fall back to using procmaps.
+void ListOfModules::fallbackInit() {
+  if (!requiresProcmaps()) {
+    clearOrInit();
+    procmapsInit(&modules_);
+  } else {
+    clear();
+  }
+}
+
+// getrusage does not give us the current RSS, only the max RSS.
+// Still, this is better than nothing if /proc/self/statm is not available
+// for some reason, e.g. due to a sandbox.
+static uptr GetRSSFromGetrusage() {
+  struct rusage usage;
+  if (getrusage(RUSAGE_SELF, &usage))  // Failed, probably due to a sandbox.
+    return 0;
+  return usage.ru_maxrss << 10;  // ru_maxrss is in Kb.
+}
+
+uptr GetRSS() {
+  if (!common_flags()->can_use_proc_maps_statm)
+    return GetRSSFromGetrusage();
+  fd_t fd = OpenFile("/proc/self/statm", RdOnly);
+  if (fd == kInvalidFd)
+    return GetRSSFromGetrusage();
+  char buf[64];
+  uptr len = internal_read(fd, buf, sizeof(buf) - 1);
+  internal_close(fd);
+  if ((sptr)len <= 0)
+    return 0;
+  buf[len] = 0;
+  // The format of the file is:
+  // 1084 89 69 11 0 79 0
+  // We need the second number which is RSS in pages.
+  char *pos = buf;
+  // Skip the first number.
+  while (*pos >= '0' && *pos <= '9')
+    pos++;
+  // Skip whitespaces.
+  while (!(*pos >= '0' && *pos <= '9') && *pos != 0)
+    pos++;
+  // Read the number.
+  uptr rss = 0;
+  while (*pos >= '0' && *pos <= '9')
+    rss = rss * 10 + *pos++ - '0';
+  return rss * GetPageSizeCached();
+}
+
+// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
+// they allocate memory.
+u32 GetNumberOfCPUs() {
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+  u32 ncpu;
+  int req[2];
+  uptr len = sizeof(ncpu);
+  req[0] = CTL_HW;
+  req[1] = HW_NCPU;
+  CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
+  return ncpu;
+#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
+  // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
+  // exist in sched.h. That is the case for toolchains generated with older
+  // NDKs.
+  // This code doesn't work on AArch64 because internal_getdents makes use of
+  // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
+  uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY);
+  if (internal_iserror(fd))
+    return 0;
+  InternalMmapVector<u8> buffer(4096);
+  uptr bytes_read = buffer.size();
+  uptr n_cpus = 0;
+  u8 *d_type;
+  struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read];
+  while (true) {
+    if ((u8 *)entry >= &buffer[bytes_read]) {
+      bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
+                                     buffer.size());
+      if (internal_iserror(bytes_read) || !bytes_read)
+        break;
+      entry = (struct linux_dirent *)buffer.data();
+    }
+    d_type = (u8 *)entry + entry->d_reclen - 1;
+    if (d_type >= &buffer[bytes_read] ||
+        (u8 *)&entry->d_name[3] >= &buffer[bytes_read])
+      break;
+    if (entry->d_ino != 0 && *d_type == DT_DIR) {
+      if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
+          entry->d_name[2] == 'u' &&
+          entry->d_name[3] >= '0' && entry->d_name[3] <= '9')
+        n_cpus++;
+    }
+    entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen);
+  }
+  internal_close(fd);
+  return n_cpus;
+#elif SANITIZER_SOLARIS
+  return sysconf(_SC_NPROCESSORS_ONLN);
+#else
+  cpu_set_t CPUs;
+  CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
+  return CPU_COUNT(&CPUs);
+#endif
+}
+
+#if SANITIZER_LINUX
+
+# if SANITIZER_ANDROID
+static atomic_uint8_t android_log_initialized;
+
+void AndroidLogInit() {
+  openlog(GetProcessName(), 0, LOG_USER);
+  atomic_store(&android_log_initialized, 1, memory_order_release);
+}
+
+static bool ShouldLogAfterPrintf() {
+  return atomic_load(&android_log_initialized, memory_order_acquire);
+}
+
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int async_safe_write_log(int pri, const char* tag, const char* msg);
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int __android_log_write(int prio, const char* tag, const char* msg);
+
+// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
+#define SANITIZER_ANDROID_LOG_INFO 4
+
+// async_safe_write_log is a new public version of __libc_write_log that is
+// used behind syslog. It is preferable to syslog as it will not do any dynamic
+// memory allocation or formatting.
+// If the function is not available, syslog is preferred for L+ (it was broken
+// pre-L) as __android_log_write triggers a racey behavior with the strncpy
+// interceptor. Fallback to __android_log_write pre-L.
+void WriteOneLineToSyslog(const char *s) {
+  if (&async_safe_write_log) {
+    async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
+  } else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
+    syslog(LOG_INFO, "%s", s);
+  } else {
+    CHECK(&__android_log_write);
+    __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
+  }
+}
+
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+void android_set_abort_message(const char *);
+
+void SetAbortMessage(const char *str) {
+  if (&android_set_abort_message)
+    android_set_abort_message(str);
+}
+# else
+void AndroidLogInit() {}
+
+static bool ShouldLogAfterPrintf() { return true; }
+
+void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
+
+void SetAbortMessage(const char *str) {}
+# endif  // SANITIZER_ANDROID
+
+void LogMessageOnPrintf(const char *str) {
+  if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
+    WriteToSyslog(str);
+}
+
+#endif  // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_GO
+// glibc crashes when using clock_gettime from a preinit_array function as the
+// vDSO function pointers haven't been initialized yet. __progname is
+// initialized after the vDSO function pointers, so if it exists, is not null
+// and is not empty, we can use clock_gettime.
+extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
+INLINE bool CanUseVDSO() {
+  // Bionic is safe, it checks for the vDSO function pointers to be initialized.
+  if (SANITIZER_ANDROID)
+    return true;
+  if (&__progname && __progname && *__progname)
+    return true;
+  return false;
+}
+
+// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
+// clock_gettime. real_clock_gettime only exists if clock_gettime is
+// intercepted, so define it weakly and use it if available.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int real_clock_gettime(u32 clk_id, void *tp);
+u64 MonotonicNanoTime() {
+  timespec ts;
+  if (CanUseVDSO()) {
+    if (&real_clock_gettime)
+      real_clock_gettime(CLOCK_MONOTONIC, &ts);
+    else
+      clock_gettime(CLOCK_MONOTONIC, &ts);
+  } else {
+    internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+  }
+  return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#else
+// Non-Linux & Go always use the syscall.
+u64 MonotonicNanoTime() {
+  timespec ts;
+  internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+  return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#endif  // SANITIZER_LINUX && !SANITIZER_GO
+
+#if !SANITIZER_OPENBSD
+void ReExec() {
+  const char *pathname = "/proc/self/exe";
+
+#if SANITIZER_NETBSD
+  static const int name[] = {
+      CTL_KERN,
+      KERN_PROC_ARGS,
+      -1,
+      KERN_PROC_PATHNAME,
+  };
+  char path[400];
+  uptr len;
+
+  len = sizeof(path);
+  if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
+    pathname = path;
+#elif SANITIZER_SOLARIS
+  pathname = getexecname();
+  CHECK_NE(pathname, NULL);
+#elif SANITIZER_USE_GETAUXVAL
+  // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
+  // rely on that will fail to load shared libraries. Query AT_EXECFN instead.
+  pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
+#endif
+
+  uptr rv = internal_execve(pathname, GetArgv(), GetEnviron());
+  int rverrno;
+  CHECK_EQ(internal_iserror(rv, &rverrno), true);
+  Printf("execve failed, errno %d\n", rverrno);
+  Die();
+}
+#endif  // !SANITIZER_OPENBSD
+
+} // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp
new file mode 100644
index 0000000000..ac7e328946
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp
@@ -0,0 +1,29 @@
+//===-- sanitizer_mac_libcdep.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements OSX-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_mac.h"
+
+#include <sys/mman.h>
+
+namespace __sanitizer {
+
+void RestrictMemoryToMaxAddress(uptr max_address) {
+  uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
+  void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
+  CHECK(res != MAP_FAILED);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp
new file mode 100644
index 0000000000..f920172c06
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -0,0 +1,509 @@
+//===-- sanitizer_posix_libcdep.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries and implements libc-dependent POSIX-specific functions
+// from sanitizer_libc.h.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_POSIX
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_platform_limits_openbsd.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
+#include "sanitizer_posix.h"
+#include "sanitizer_procmaps.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#if SANITIZER_FREEBSD
+// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
+// that, it was never implemented.  So just define it to zero.
+#undef MAP_NORESERVE
+#define MAP_NORESERVE 0
+#endif
+
+typedef void (*sa_sigaction_t)(int, siginfo_t *, void *);
+
+namespace __sanitizer {
+
+u32 GetUid() {
+  return getuid();
+}
+
+uptr GetThreadSelf() {
+  return (uptr)pthread_self();
+}
+
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
+  uptr page_size = GetPageSizeCached();
+  uptr beg_aligned = RoundUpTo(beg, page_size);
+  uptr end_aligned = RoundDownTo(end, page_size);
+  if (beg_aligned < end_aligned)
+    // In the default Solaris compilation environment, madvise() is declared
+    // to take a caddr_t arg; casting it to void * results in an invalid
+    // conversion error, so use char * instead.
+    madvise((char *)beg_aligned, end_aligned - beg_aligned,
+            SANITIZER_MADVISE_DONTNEED);
+}
+
+void SetShadowRegionHugePageMode(uptr addr, uptr size) {
+#ifdef MADV_NOHUGEPAGE  // May not be defined on old systems.
+  if (common_flags()->no_huge_pages_for_shadow)
+    madvise((char *)addr, size, MADV_NOHUGEPAGE);
+  else
+    madvise((char *)addr, size, MADV_HUGEPAGE);
+#endif  // MADV_NOHUGEPAGE
+}
+
+bool DontDumpShadowMemory(uptr addr, uptr length) {
+#if defined(MADV_DONTDUMP)
+  return madvise((char *)addr, length, MADV_DONTDUMP) == 0;
+#elif defined(MADV_NOCORE)
+  return madvise((char *)addr, length, MADV_NOCORE) == 0;
+#else
+  return true;
+#endif  // MADV_DONTDUMP
+}
+
+static rlim_t getlim(int res) {
+  rlimit rlim;
+  CHECK_EQ(0, getrlimit(res, &rlim));
+  return rlim.rlim_cur;
+}
+
+static void setlim(int res, rlim_t lim) {
+  struct rlimit rlim;
+  if (getrlimit(res, const_cast<struct rlimit *>(&rlim))) {
+    Report("ERROR: %s getrlimit() failed %d\n", SanitizerToolName, errno);
+    Die();
+  }
+  rlim.rlim_cur = lim;
+  if (setrlimit(res, const_cast<struct rlimit *>(&rlim))) {
+    Report("ERROR: %s setrlimit() failed %d\n", SanitizerToolName, errno);
+    Die();
+  }
+}
+
+void DisableCoreDumperIfNecessary() {
+  if (common_flags()->disable_coredump) {
+    setlim(RLIMIT_CORE, 0);
+  }
+}
+
+bool StackSizeIsUnlimited() {
+  rlim_t stack_size = getlim(RLIMIT_STACK);
+  return (stack_size == RLIM_INFINITY);
+}
+
+void SetStackSizeLimitInBytes(uptr limit) {
+  setlim(RLIMIT_STACK, (rlim_t)limit);
+  CHECK(!StackSizeIsUnlimited());
+}
+
+bool AddressSpaceIsUnlimited() {
+  rlim_t as_size = getlim(RLIMIT_AS);
+  return (as_size == RLIM_INFINITY);
+}
+
+void SetAddressSpaceUnlimited() {
+  setlim(RLIMIT_AS, RLIM_INFINITY);
+  CHECK(AddressSpaceIsUnlimited());
+}
+
+void SleepForSeconds(int seconds) {
+  sleep(seconds);
+}
+
+void SleepForMillis(int millis) {
+  usleep(millis * 1000);
+}
+
+void Abort() {
+#if !SANITIZER_GO
+  // If we are handling SIGABRT, unhandle it first.
+  // TODO(vitalybuka): Check if handler belongs to sanitizer.
+  if (GetHandleSignalMode(SIGABRT) != kHandleSignalNo) {
+    struct sigaction sigact;
+    internal_memset(&sigact, 0, sizeof(sigact));
+    sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL;
+    internal_sigaction(SIGABRT, &sigact, nullptr);
+  }
+#endif
+
+  abort();
+}
+
+int Atexit(void (*function)(void)) {
+#if !SANITIZER_GO
+  return atexit(function);
+#else
+  return 0;
+#endif
+}
+
+bool SupportsColoredOutput(fd_t fd) {
+  return isatty(fd) != 0;
+}
+
+#if !SANITIZER_GO
+// TODO(glider): different tools may require different altstack size.
+static const uptr kAltStackSize = SIGSTKSZ * 4;  // SIGSTKSZ is not enough.
+
+void SetAlternateSignalStack() {
+  stack_t altstack, oldstack;
+  CHECK_EQ(0, sigaltstack(nullptr, &oldstack));
+  // If the alternate stack is already in place, do nothing.
+  // Android always sets an alternate stack, but it's too small for us.
+  if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return;
+  // TODO(glider): the mapped stack should have the MAP_STACK flag in the
+  // future. It is not required by man 2 sigaltstack now (they're using
+  // malloc()).
+  void* base = MmapOrDie(kAltStackSize, __func__);
+  altstack.ss_sp = (char*) base;
+  altstack.ss_flags = 0;
+  altstack.ss_size = kAltStackSize;
+  CHECK_EQ(0, sigaltstack(&altstack, nullptr));
+}
+
+void UnsetAlternateSignalStack() {
+  stack_t altstack, oldstack;
+  altstack.ss_sp = nullptr;
+  altstack.ss_flags = SS_DISABLE;
+  altstack.ss_size = kAltStackSize;  // Some sane value required on Darwin.
+  CHECK_EQ(0, sigaltstack(&altstack, &oldstack));
+  UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
+}
+
+static void MaybeInstallSigaction(int signum,
+                                  SignalHandlerType handler) {
+  if (GetHandleSignalMode(signum) == kHandleSignalNo) return;
+
+  struct sigaction sigact;
+  internal_memset(&sigact, 0, sizeof(sigact));
+  sigact.sa_sigaction = (sa_sigaction_t)handler;
+  // Do not block the signal from being received in that signal's handler.
+  // Clients are responsible for handling this correctly.
+  sigact.sa_flags = SA_SIGINFO | SA_NODEFER;
+  if (common_flags()->use_sigaltstack) sigact.sa_flags |= SA_ONSTACK;
+  CHECK_EQ(0, internal_sigaction(signum, &sigact, nullptr));
+  VReport(1, "Installed the sigaction for signal %d\n", signum);
+}
+
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {
+  // Set the alternate signal stack for the main thread.
+  // This will cause SetAlternateSignalStack to be called twice, but the stack
+  // will be actually set only once.
+  if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
+  MaybeInstallSigaction(SIGSEGV, handler);
+  MaybeInstallSigaction(SIGBUS, handler);
+  MaybeInstallSigaction(SIGABRT, handler);
+  MaybeInstallSigaction(SIGFPE, handler);
+  MaybeInstallSigaction(SIGILL, handler);
+  MaybeInstallSigaction(SIGTRAP, handler);
+}
+
+bool SignalContext::IsStackOverflow() const {
+  // Access at a reasonable offset above SP, or slightly below it (to account
+  // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
+  // probably a stack overflow.
+#ifdef __s390__
+  // On s390, the fault address in siginfo points to start of the page, not
+  // to the precise word that was accessed.  Mask off the low bits of sp to
+  // take it into account.
+  bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF;
+#else
+  // Let's accept up to a page size away from top of stack. Things like stack
+  // probing can trigger accesses with such large offsets.
+  bool IsStackAccess = addr + GetPageSizeCached() > sp && addr < sp + 0xFFFF;
+#endif
+
+#if __powerpc__
+  // Large stack frames can be allocated with e.g.
+  //   lis r0,-10000
+  //   stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
+  // If the store faults then sp will not have been updated, so test above
+  // will not work, because the fault address will be more than just "slightly"
+  // below sp.
+  if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) {
+    u32 inst = *(unsigned *)pc;
+    u32 ra = (inst >> 16) & 0x1F;
+    u32 opcd = inst >> 26;
+    u32 xo = (inst >> 1) & 0x3FF;
+    // Check for store-with-update to sp. The instructions we accept are:
+    //   stbu rs,d(ra)          stbux rs,ra,rb
+    //   sthu rs,d(ra)          sthux rs,ra,rb
+    //   stwu rs,d(ra)          stwux rs,ra,rb
+    //   stdu rs,ds(ra)         stdux rs,ra,rb
+    // where ra is r1 (the stack pointer).
+    if (ra == 1 &&
+        (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 ||
+         (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181))))
+      IsStackAccess = true;
+  }
+#endif  // __powerpc__
+
+  // We also check si_code to filter out SEGV caused by something else other
+  // then hitting the guard page or unmapped memory, like, for example,
+  // unaligned memory access.
+  auto si = static_cast<const siginfo_t *>(siginfo);
+  return IsStackAccess &&
+         (si->si_code == si_SEGV_MAPERR || si->si_code == si_SEGV_ACCERR);
+}
+
+#endif  // SANITIZER_GO
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+  uptr page_size = GetPageSizeCached();
+  // Checking too large memory ranges is slow.
+  CHECK_LT(size, page_size * 10);
+  int sock_pair[2];
+  if (pipe(sock_pair))
+    return false;
+  uptr bytes_written =
+      internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size);
+  int write_errno;
+  bool result;
+  if (internal_iserror(bytes_written, &write_errno)) {
+    CHECK_EQ(EFAULT, write_errno);
+    result = false;
+  } else {
+    result = (bytes_written == size);
+  }
+  internal_close(sock_pair[0]);
+  internal_close(sock_pair[1]);
+  return result;
+}
+
+void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
+  // Some kinds of sandboxes may forbid filesystem access, so we won't be able
+  // to read the file mappings from /proc/self/maps. Luckily, neither the
+  // process will be able to load additional libraries, so it's fine to use the
+  // cached mappings.
+  MemoryMappingLayout::CacheMemoryMappings();
+}
+
+static bool MmapFixed(uptr fixed_addr, uptr size, int additional_flags,
+                      const char *name) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  fixed_addr = RoundDownTo(fixed_addr, GetPageSizeCached());
+  uptr p =
+      MmapNamed((void *)fixed_addr, size, PROT_READ | PROT_WRITE,
+                MAP_PRIVATE | MAP_FIXED | additional_flags | MAP_ANON, name);
+  int reserrno;
+  if (internal_iserror(p, &reserrno)) {
+    Report("ERROR: %s failed to "
+           "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
+           SanitizerToolName, size, size, fixed_addr, reserrno);
+    return false;
+  }
+  IncreaseTotalMmap(size);
+  return true;
+}
+
+bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
+  return MmapFixed(fixed_addr, size, MAP_NORESERVE, name);
+}
+
+bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size, const char *name) {
+#if SANITIZER_FREEBSD
+  if (common_flags()->no_huge_pages_for_shadow)
+    return MmapFixedNoReserve(fixed_addr, size, name);
+  // MAP_NORESERVE is implicit with FreeBSD
+  return MmapFixed(fixed_addr, size, MAP_ALIGNED_SUPER, name);
+#else
+  bool r = MmapFixedNoReserve(fixed_addr, size, name);
+  if (r)
+    SetShadowRegionHugePageMode(fixed_addr, size);
+  return r;
+#endif
+}
+
+uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
+  base_ = fixed_addr ? MmapFixedNoAccess(fixed_addr, size, name)
+                     : MmapNoAccess(size);
+  size_ = size;
+  name_ = name;
+  (void)os_handle_;  // unsupported
+  return reinterpret_cast<uptr>(base_);
+}
+
+// Uses fixed_addr for now.
+// Will use offset instead once we've implemented this function for real.
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size, const char *name) {
+  return reinterpret_cast<uptr>(
+      MmapFixedOrDieOnFatalError(fixed_addr, size, name));
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size,
+                                    const char *name) {
+  return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size, name));
+}
+
+void ReservedAddressRange::Unmap(uptr addr, uptr size) {
+  CHECK_LE(size, size_);
+  if (addr == reinterpret_cast<uptr>(base_))
+    // If we unmap the whole range, just null out the base.
+    base_ = (size == size_) ? nullptr : reinterpret_cast<void*>(addr + size);
+  else
+    CHECK_EQ(addr + size, reinterpret_cast<uptr>(base_) + size_);
+  size_ -= size;
+  UnmapOrDie(reinterpret_cast<void*>(addr), size);
+}
+
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  return (void *)MmapNamed((void *)fixed_addr, size, PROT_NONE,
+                           MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANON,
+                           name);
+}
+
+void *MmapNoAccess(uptr size) {
+  unsigned flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
+  return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
+}
+
+// This function is defined elsewhere if we intercepted pthread_attr_getstack.
+extern "C" {
+SANITIZER_WEAK_ATTRIBUTE int
+real_pthread_attr_getstack(void *attr, void **addr, size_t *size);
+} // extern "C"
+
+int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) {
+#if !SANITIZER_GO && !SANITIZER_MAC
+  if (&real_pthread_attr_getstack)
+    return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
+                                      (size_t *)size);
+#endif
+  return pthread_attr_getstack((pthread_attr_t *)attr, addr, (size_t *)size);
+}
+
+#if !SANITIZER_GO
+void AdjustStackSize(void *attr_) {
+  pthread_attr_t *attr = (pthread_attr_t *)attr_;
+  uptr stackaddr = 0;
+  uptr stacksize = 0;
+  my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);
+  // GLibC will return (0 - stacksize) as the stack address in the case when
+  // stacksize is set, but stackaddr is not.
+  bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);
+  // We place a lot of tool data into TLS, account for that.
+  const uptr minstacksize = GetTlsSize() + 128*1024;
+  if (stacksize < minstacksize) {
+    if (!stack_set) {
+      if (stacksize != 0) {
+        VPrintf(1, "Sanitizer: increasing stacksize %zu->%zu\n", stacksize,
+                minstacksize);
+        pthread_attr_setstacksize(attr, minstacksize);
+      }
+    } else {
+      Printf("Sanitizer: pre-allocated stack size is insufficient: "
+             "%zu < %zu\n", stacksize, minstacksize);
+      Printf("Sanitizer: pthread_create is likely to fail.\n");
+    }
+  }
+}
+#endif // !SANITIZER_GO
+
+pid_t StartSubprocess(const char *program, const char *const argv[],
+                      const char *const envp[], fd_t stdin_fd, fd_t stdout_fd,
+                      fd_t stderr_fd) {
+  auto file_closer = at_scope_exit([&] {
+    if (stdin_fd != kInvalidFd) {
+      internal_close(stdin_fd);
+    }
+    if (stdout_fd != kInvalidFd) {
+      internal_close(stdout_fd);
+    }
+    if (stderr_fd != kInvalidFd) {
+      internal_close(stderr_fd);
+    }
+  });
+
+  int pid = internal_fork();
+
+  if (pid < 0) {
+    int rverrno;
+    if (internal_iserror(pid, &rverrno)) {
+      Report("WARNING: failed to fork (errno %d)\n", rverrno);
+    }
+    return pid;
+  }
+
+  if (pid == 0) {
+    // Child subprocess
+    if (stdin_fd != kInvalidFd) {
+      internal_close(STDIN_FILENO);
+      internal_dup2(stdin_fd, STDIN_FILENO);
+      internal_close(stdin_fd);
+    }
+    if (stdout_fd != kInvalidFd) {
+      internal_close(STDOUT_FILENO);
+      internal_dup2(stdout_fd, STDOUT_FILENO);
+      internal_close(stdout_fd);
+    }
+    if (stderr_fd != kInvalidFd) {
+      internal_close(STDERR_FILENO);
+      internal_dup2(stderr_fd, STDERR_FILENO);
+      internal_close(stderr_fd);
+    }
+
+    for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
+
+    internal_execve(program, const_cast<char **>(&argv[0]),
+                    const_cast<char *const *>(envp));
+    internal__exit(1);
+  }
+
+  return pid;
+}
+
+bool IsProcessRunning(pid_t pid) {
+  int process_status;
+  uptr waitpid_status = internal_waitpid(pid, &process_status, WNOHANG);
+  int local_errno;
+  if (internal_iserror(waitpid_status, &local_errno)) {
+    VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
+    return false;
+  }
+  return waitpid_status == 0;
+}
+
+int WaitForProcess(pid_t pid) {
+  int process_status;
+  uptr waitpid_status = internal_waitpid(pid, &process_status, 0);
+  int local_errno;
+  if (internal_iserror(waitpid_status, &local_errno)) {
+    VReport(1, "Waiting on the process failed (errno %d).\n", local_errno);
+    return -1;
+  }
+  return process_status;
+}
+
+bool IsStateDetached(int state) {
+  return state == PTHREAD_CREATE_DETACHED;
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_POSIX
diff --git a/lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp b/lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp
new file mode 100644
index 0000000000..30073a96ce
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp
@@ -0,0 +1,149 @@
+//===-- sanitizer_stackdepot.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_stackdepot.h"
+
+#include "sanitizer_common.h"
+#include "sanitizer_hash.h"
+#include "sanitizer_stackdepotbase.h"
+
+namespace __sanitizer {
+
+struct StackDepotNode {
+  StackDepotNode *link;
+  u32 id;
+  atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;
+  u32 size;
+  u32 tag;
+  uptr stack[1];  // [size]
+
+  static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
+  // Lower kTabSizeLog bits are equal for all items in one bucket.
+  // We use these bits to store the per-stack use counter.
+  static const u32 kUseCountBits = kTabSizeLog;
+  static const u32 kMaxUseCount = 1 << kUseCountBits;
+  static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
+  static const u32 kHashMask = ~kUseCountMask;
+
+  typedef StackTrace args_type;
+  bool eq(u32 hash, const args_type &args) const {
+    u32 hash_bits =
+        atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
+    if ((hash & kHashMask) != hash_bits || args.size != size || args.tag != tag)
+      return false;
+    uptr i = 0;
+    for (; i < size; i++) {
+      if (stack[i] != args.trace[i]) return false;
+    }
+    return true;
+  }
+  static uptr storage_size(const args_type &args) {
+    return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
+  }
+  static u32 hash(const args_type &args) {
+    MurMur2HashBuilder H(args.size * sizeof(uptr));
+    for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
+    return H.get();
+  }
+  static bool is_valid(const args_type &args) {
+    return args.size > 0 && args.trace;
+  }
+  void store(const args_type &args, u32 hash) {
+    atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);
+    size = args.size;
+    tag = args.tag;
+    internal_memcpy(stack, args.trace, size * sizeof(uptr));
+  }
+  args_type load() const {
+    return args_type(&stack[0], size, tag);
+  }
+  StackDepotHandle get_handle() { return StackDepotHandle(this); }
+
+  typedef StackDepotHandle handle_type;
+};
+
+COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);
+
+u32 StackDepotHandle::id() { return node_->id; }
+int StackDepotHandle::use_count() {
+  return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
+         StackDepotNode::kUseCountMask;
+}
+void StackDepotHandle::inc_use_count_unsafe() {
+  u32 prev =
+      atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
+      StackDepotNode::kUseCountMask;
+  CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
+}
+
+// FIXME(dvyukov): this single reserved bit is used in TSan.
+typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
+    StackDepot;
+static StackDepot theDepot;
+
+StackDepotStats *StackDepotGetStats() {
+  return theDepot.GetStats();
+}
+
+u32 StackDepotPut(StackTrace stack) {
+  StackDepotHandle h = theDepot.Put(stack);
+  return h.valid() ? h.id() : 0;
+}
+
+StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
+  return theDepot.Put(stack);
+}
+
+StackTrace StackDepotGet(u32 id) {
+  return theDepot.Get(id);
+}
+
+void StackDepotLockAll() {
+  theDepot.LockAll();
+}
+
+void StackDepotUnlockAll() {
+  theDepot.UnlockAll();
+}
+
+bool StackDepotReverseMap::IdDescPair::IdComparator(
+    const StackDepotReverseMap::IdDescPair &a,
+    const StackDepotReverseMap::IdDescPair &b) {
+  return a.id < b.id;
+}
+
+StackDepotReverseMap::StackDepotReverseMap() {
+  map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);
+  for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
+    atomic_uintptr_t *p = &theDepot.tab[idx];
+    uptr v = atomic_load(p, memory_order_consume);
+    StackDepotNode *s = (StackDepotNode*)(v & ~1);
+    for (; s; s = s->link) {
+      IdDescPair pair = {s->id, s};
+      map_.push_back(pair);
+    }
+  }
+  Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
+}
+
+StackTrace StackDepotReverseMap::Get(u32 id) {
+  if (!map_.size())
+    return StackTrace();
+  IdDescPair pair = {id, nullptr};
+  uptr idx =
+      InternalLowerBound(map_, 0, map_.size(), pair, IdDescPair::IdComparator);
+  if (idx > map_.size() || map_[idx].id != id)
+    return StackTrace();
+  return map_[idx].desc->load();
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp b/lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp
new file mode 100644
index 0000000000..ef14fb704e
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp
@@ -0,0 +1,133 @@
+//===-- sanitizer_stacktrace.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+uptr StackTrace::GetNextInstructionPc(uptr pc) {
+#if defined(__sparc__) || defined(__mips__)
+  return pc + 8;
+#elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__)
+  return pc + 4;
+#else
+  return pc + 1;
+#endif
+}
+
+uptr StackTrace::GetCurrentPc() {
+  return GET_CALLER_PC();
+}
+
+void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
+  size = cnt + !!extra_top_pc;
+  CHECK_LE(size, kStackTraceMax);
+  internal_memcpy(trace_buffer, pcs, cnt * sizeof(trace_buffer[0]));
+  if (extra_top_pc)
+    trace_buffer[cnt] = extra_top_pc;
+  top_frame_bp = 0;
+}
+
+// Sparc implemention is in its own file.
+#if !defined(__sparc__)
+
+// In GCC on ARM bp points to saved lr, not fp, so we should check the next
+// cell in stack to be a saved frame pointer. GetCanonicFrame returns the
+// pointer to saved frame pointer in any case.
+static inline uhwptr *GetCanonicFrame(uptr bp,
+                                      uptr stack_top,
+                                      uptr stack_bottom) {
+  CHECK_GT(stack_top, stack_bottom);
+#ifdef __arm__
+  if (!IsValidFrame(bp, stack_top, stack_bottom)) return 0;
+  uhwptr *bp_prev = (uhwptr *)bp;
+  if (IsValidFrame((uptr)bp_prev[0], stack_top, stack_bottom)) return bp_prev;
+  // The next frame pointer does not look right. This could be a GCC frame, step
+  // back by 1 word and try again.
+  if (IsValidFrame((uptr)bp_prev[-1], stack_top, stack_bottom))
+    return bp_prev - 1;
+  // Nope, this does not look right either. This means the frame after next does
+  // not have a valid frame pointer, but we can still extract the caller PC.
+  // Unfortunately, there is no way to decide between GCC and LLVM frame
+  // layouts. Assume LLVM.
+  return bp_prev;
+#else
+  return (uhwptr*)bp;
+#endif
+}
+
+void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
+                                    uptr stack_bottom, u32 max_depth) {
+  // TODO(yln): add arg sanity check for stack_top/stack_bottom
+  CHECK_GE(max_depth, 2);
+  const uptr kPageSize = GetPageSizeCached();
+  trace_buffer[0] = pc;
+  size = 1;
+  if (stack_top < 4096) return;  // Sanity check for stack top.
+  uhwptr *frame = GetCanonicFrame(bp, stack_top, stack_bottom);
+  // Lowest possible address that makes sense as the next frame pointer.
+  // Goes up as we walk the stack.
+  uptr bottom = stack_bottom;
+  // Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
+  while (IsValidFrame((uptr)frame, stack_top, bottom) &&
+         IsAligned((uptr)frame, sizeof(*frame)) &&
+         size < max_depth) {
+#ifdef __powerpc__
+    // PowerPC ABIs specify that the return address is saved at offset
+    // 16 of the *caller's* stack frame.  Thus we must dereference the
+    // back chain to find the caller frame before extracting it.
+    uhwptr *caller_frame = (uhwptr*)frame[0];
+    if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) ||
+        !IsAligned((uptr)caller_frame, sizeof(uhwptr)))
+      break;
+    uhwptr pc1 = caller_frame[2];
+#elif defined(__s390__)
+    uhwptr pc1 = frame[14];
+#else
+    uhwptr pc1 = frame[1];
+#endif
+    // Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
+    // x86_64) is invalid and stop unwinding here.  If we're adding support for
+    // a platform where this isn't true, we need to reconsider this check.
+    if (pc1 < kPageSize)
+      break;
+    if (pc1 != pc) {
+      trace_buffer[size++] = (uptr) pc1;
+    }
+    bottom = (uptr)frame;
+    frame = GetCanonicFrame((uptr)frame[0], stack_top, bottom);
+  }
+}
+
+#endif  // !defined(__sparc__)
+
+void BufferedStackTrace::PopStackFrames(uptr count) {
+  CHECK_LT(count, size);
+  size -= count;
+  for (uptr i = 0; i < size; ++i) {
+    trace_buffer[i] = trace_buffer[i + count];
+  }
+}
+
+static uptr Distance(uptr a, uptr b) { return a < b ? b - a : a - b; }
+
+uptr BufferedStackTrace::LocatePcInTrace(uptr pc) {
+  uptr best = 0;
+  for (uptr i = 1; i < size; ++i) {
+    if (Distance(trace[i], pc) < Distance(trace[best], pc)) best = i;
+  }
+  return best;
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
new file mode 100644
index 0000000000..4ef305cf17
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp
@@ -0,0 +1,159 @@
+//===-- sanitizer_stacktrace_libcdep.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_stacktrace.h"
+#include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_symbolizer.h"
+
+namespace __sanitizer {
+
+void StackTrace::Print() const {
+  if (trace == nullptr || size == 0) {
+    Printf("    <empty stack>\n\n");
+    return;
+  }
+  InternalScopedString frame_desc(GetPageSizeCached() * 2);
+  InternalScopedString dedup_token(GetPageSizeCached());
+  int dedup_frames = common_flags()->dedup_token_length;
+  uptr frame_num = 0;
+  for (uptr i = 0; i < size && trace[i]; i++) {
+    // PCs in stack traces are actually the return addresses, that is,
+    // addresses of the next instructions after the call.
+    uptr pc = GetPreviousInstructionPc(trace[i]);
+    SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(pc);
+    CHECK(frames);
+    for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+      frame_desc.clear();
+      RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++,
+                  cur->info, common_flags()->symbolize_vs_style,
+                  common_flags()->strip_path_prefix);
+      Printf("%s\n", frame_desc.data());
+      if (dedup_frames-- > 0) {
+        if (dedup_token.length())
+          dedup_token.append("--");
+        if (cur->info.function != nullptr)
+          dedup_token.append(cur->info.function);
+      }
+    }
+    frames->ClearAll();
+  }
+  // Always print a trailing empty line after stack trace.
+  Printf("\n");
+  if (dedup_token.length())
+    Printf("DEDUP_TOKEN: %s\n", dedup_token.data());
+}
+
+void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
+                                uptr stack_top, uptr stack_bottom,
+                                bool request_fast_unwind) {
+  // Ensures all call sites get what they requested.
+  CHECK_EQ(request_fast_unwind, WillUseFastUnwind(request_fast_unwind));
+  top_frame_bp = (max_depth > 0) ? bp : 0;
+  // Avoid doing any work for small max_depth.
+  if (max_depth == 0) {
+    size = 0;
+    return;
+  }
+  if (max_depth == 1) {
+    size = 1;
+    trace_buffer[0] = pc;
+    return;
+  }
+  if (!WillUseFastUnwind(request_fast_unwind)) {
+#if SANITIZER_CAN_SLOW_UNWIND
+    if (context)
+      UnwindSlow(pc, context, max_depth);
+    else
+      UnwindSlow(pc, max_depth);
+#else
+    UNREACHABLE("slow unwind requested but not available");
+#endif
+  } else {
+    UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
+  }
+}
+
+static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
+                                   uptr module_name_len, uptr *pc_offset) {
+  const char *found_module_name = nullptr;
+  bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
+      pc, &found_module_name, pc_offset);
+
+  if (!ok) return false;
+
+  if (module_name && module_name_len) {
+    internal_strncpy(module_name, found_module_name, module_name_len);
+    module_name[module_name_len - 1] = '\x00';
+  }
+  return true;
+}
+
+}  // namespace __sanitizer
+using namespace __sanitizer;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf,
+                              uptr out_buf_size) {
+  if (!out_buf_size) return;
+  pc = StackTrace::GetPreviousInstructionPc(pc);
+  SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
+  if (!frame) {
+    internal_strncpy(out_buf, "<can't symbolize>", out_buf_size);
+    out_buf[out_buf_size - 1] = 0;
+    return;
+  }
+  InternalScopedString frame_desc(GetPageSizeCached());
+  uptr frame_num = 0;
+  // Reserve one byte for the final 0.
+  char *out_end = out_buf + out_buf_size - 1;
+  for (SymbolizedStack *cur = frame; cur && out_buf < out_end;
+       cur = cur->next) {
+    frame_desc.clear();
+    RenderFrame(&frame_desc, fmt, frame_num++, cur->info,
+                common_flags()->symbolize_vs_style,
+                common_flags()->strip_path_prefix);
+    if (!frame_desc.length())
+      continue;
+    // Reserve one byte for the terminating 0.
+    uptr n = out_end - out_buf - 1;
+    internal_strncpy(out_buf, frame_desc.data(), n);
+    out_buf += __sanitizer::Min<uptr>(n, frame_desc.length());
+    *out_buf++ = 0;
+  }
+  CHECK(out_buf <= out_end);
+  *out_buf = 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
+                                  char *out_buf, uptr out_buf_size) {
+  if (!out_buf_size) return;
+  out_buf[0] = 0;
+  DataInfo DI;
+  if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
+  InternalScopedString data_desc(GetPageSizeCached());
+  RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
+  internal_strncpy(out_buf, data_desc.data(), out_buf_size);
+  out_buf[out_buf_size - 1] = 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,
+                                             uptr module_name_len,
+                                             uptr *pc_offset) {
+  return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
+                                              pc_offset);
+}
+}  // extern "C"
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp b/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp
new file mode 100644
index 0000000000..150ff47531
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -0,0 +1,263 @@
+//===-- sanitizer_common.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers' run-time libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_file.h"
+#include "sanitizer_fuchsia.h"
+
+namespace __sanitizer {
+
+// sanitizer_symbolizer_markup.cpp implements these differently.
+#if !SANITIZER_SYMBOLIZER_MARKUP
+
+static const char *StripFunctionName(const char *function, const char *prefix) {
+  if (!function) return nullptr;
+  if (!prefix) return function;
+  uptr prefix_len = internal_strlen(prefix);
+  if (0 == internal_strncmp(function, prefix, prefix_len))
+    return function + prefix_len;
+  return function;
+}
+
+static const char *DemangleFunctionName(const char *function) {
+  if (!function) return nullptr;
+
+  // NetBSD uses indirection for old threading functions for historical reasons
+  // The mangled names are internal implementation detail and should not be
+  // exposed even in backtraces.
+#if SANITIZER_NETBSD
+  if (!internal_strcmp(function, "__libc_mutex_init"))
+    return "pthread_mutex_init";
+  if (!internal_strcmp(function, "__libc_mutex_lock"))
+    return "pthread_mutex_lock";
+  if (!internal_strcmp(function, "__libc_mutex_trylock"))
+    return "pthread_mutex_trylock";
+  if (!internal_strcmp(function, "__libc_mutex_unlock"))
+    return "pthread_mutex_unlock";
+  if (!internal_strcmp(function, "__libc_mutex_destroy"))
+    return "pthread_mutex_destroy";
+  if (!internal_strcmp(function, "__libc_mutexattr_init"))
+    return "pthread_mutexattr_init";
+  if (!internal_strcmp(function, "__libc_mutexattr_settype"))
+    return "pthread_mutexattr_settype";
+  if (!internal_strcmp(function, "__libc_mutexattr_destroy"))
+    return "pthread_mutexattr_destroy";
+  if (!internal_strcmp(function, "__libc_cond_init"))
+    return "pthread_cond_init";
+  if (!internal_strcmp(function, "__libc_cond_signal"))
+    return "pthread_cond_signal";
+  if (!internal_strcmp(function, "__libc_cond_broadcast"))
+    return "pthread_cond_broadcast";
+  if (!internal_strcmp(function, "__libc_cond_wait"))
+    return "pthread_cond_wait";
+  if (!internal_strcmp(function, "__libc_cond_timedwait"))
+    return "pthread_cond_timedwait";
+  if (!internal_strcmp(function, "__libc_cond_destroy"))
+    return "pthread_cond_destroy";
+  if (!internal_strcmp(function, "__libc_rwlock_init"))
+    return "pthread_rwlock_init";
+  if (!internal_strcmp(function, "__libc_rwlock_rdlock"))
+    return "pthread_rwlock_rdlock";
+  if (!internal_strcmp(function, "__libc_rwlock_wrlock"))
+    return "pthread_rwlock_wrlock";
+  if (!internal_strcmp(function, "__libc_rwlock_tryrdlock"))
+    return "pthread_rwlock_tryrdlock";
+  if (!internal_strcmp(function, "__libc_rwlock_trywrlock"))
+    return "pthread_rwlock_trywrlock";
+  if (!internal_strcmp(function, "__libc_rwlock_unlock"))
+    return "pthread_rwlock_unlock";
+  if (!internal_strcmp(function, "__libc_rwlock_destroy"))
+    return "pthread_rwlock_destroy";
+  if (!internal_strcmp(function, "__libc_thr_keycreate"))
+    return "pthread_key_create";
+  if (!internal_strcmp(function, "__libc_thr_setspecific"))
+    return "pthread_setspecific";
+  if (!internal_strcmp(function, "__libc_thr_getspecific"))
+    return "pthread_getspecific";
+  if (!internal_strcmp(function, "__libc_thr_keydelete"))
+    return "pthread_key_delete";
+  if (!internal_strcmp(function, "__libc_thr_once"))
+    return "pthread_once";
+  if (!internal_strcmp(function, "__libc_thr_self"))
+    return "pthread_self";
+  if (!internal_strcmp(function, "__libc_thr_exit"))
+    return "pthread_exit";
+  if (!internal_strcmp(function, "__libc_thr_setcancelstate"))
+    return "pthread_setcancelstate";
+  if (!internal_strcmp(function, "__libc_thr_equal"))
+    return "pthread_equal";
+  if (!internal_strcmp(function, "__libc_thr_curcpu"))
+    return "pthread_curcpu_np";
+  if (!internal_strcmp(function, "__libc_thr_sigsetmask"))
+    return "pthread_sigmask";
+#endif
+
+  return function;
+}
+
+static const char kDefaultFormat[] = "    #%n %p %F %L";
+
+void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+                 const AddressInfo &info, bool vs_style,
+                 const char *strip_path_prefix, const char *strip_func_prefix) {
+  if (0 == internal_strcmp(format, "DEFAULT"))
+    format = kDefaultFormat;
+  for (const char *p = format; *p != '\0'; p++) {
+    if (*p != '%') {
+      buffer->append("%c", *p);
+      continue;
+    }
+    p++;
+    switch (*p) {
+    case '%':
+      buffer->append("%%");
+      break;
+    // Frame number and all fields of AddressInfo structure.
+    case 'n':
+      buffer->append("%zu", frame_no);
+      break;
+    case 'p':
+      buffer->append("0x%zx", info.address);
+      break;
+    case 'm':
+      buffer->append("%s", StripPathPrefix(info.module, strip_path_prefix));
+      break;
+    case 'o':
+      buffer->append("0x%zx", info.module_offset);
+      break;
+    case 'f':
+      buffer->append("%s",
+        DemangleFunctionName(
+          StripFunctionName(info.function, strip_func_prefix)));
+      break;
+    case 'q':
+      buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown
+                                  ? info.function_offset
+                                  : 0x0);
+      break;
+    case 's':
+      buffer->append("%s", StripPathPrefix(info.file, strip_path_prefix));
+      break;
+    case 'l':
+      buffer->append("%d", info.line);
+      break;
+    case 'c':
+      buffer->append("%d", info.column);
+      break;
+    // Smarter special cases.
+    case 'F':
+      // Function name and offset, if file is unknown.
+      if (info.function) {
+        buffer->append("in %s",
+                       DemangleFunctionName(
+                         StripFunctionName(info.function, strip_func_prefix)));
+        if (!info.file && info.function_offset != AddressInfo::kUnknown)
+          buffer->append("+0x%zx", info.function_offset);
+      }
+      break;
+    case 'S':
+      // File/line information.
+      RenderSourceLocation(buffer, info.file, info.line, info.column, vs_style,
+                           strip_path_prefix);
+      break;
+    case 'L':
+      // Source location, or module location.
+      if (info.file) {
+        RenderSourceLocation(buffer, info.file, info.line, info.column,
+                             vs_style, strip_path_prefix);
+      } else if (info.module) {
+        RenderModuleLocation(buffer, info.module, info.module_offset,
+                             info.module_arch, strip_path_prefix);
+      } else {
+        buffer->append("(<unknown module>)");
+      }
+      break;
+    case 'M':
+      // Module basename and offset, or PC.
+      if (info.address & kExternalPCBit)
+        {} // There PCs are not meaningful.
+      else if (info.module)
+        // Always strip the module name for %M.
+        RenderModuleLocation(buffer, StripModuleName(info.module),
+                             info.module_offset, info.module_arch, "");
+      else
+        buffer->append("(%p)", (void *)info.address);
+      break;
+    default:
+      Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
+             *p);
+      Die();
+    }
+  }
+}
+
+void RenderData(InternalScopedString *buffer, const char *format,
+                const DataInfo *DI, const char *strip_path_prefix) {
+  for (const char *p = format; *p != '\0'; p++) {
+    if (*p != '%') {
+      buffer->append("%c", *p);
+      continue;
+    }
+    p++;
+    switch (*p) {
+      case '%':
+        buffer->append("%%");
+        break;
+      case 's':
+        buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
+        break;
+      case 'l':
+        buffer->append("%d", DI->line);
+        break;
+      case 'g':
+        buffer->append("%s", DI->name);
+        break;
+      default:
+        Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
+               *p);
+        Die();
+    }
+  }
+}
+
+#endif  // !SANITIZER_SYMBOLIZER_MARKUP
+
+void RenderSourceLocation(InternalScopedString *buffer, const char *file,
+                          int line, int column, bool vs_style,
+                          const char *strip_path_prefix) {
+  if (vs_style && line > 0) {
+    buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line);
+    if (column > 0)
+      buffer->append(",%d", column);
+    buffer->append(")");
+    return;
+  }
+
+  buffer->append("%s", StripPathPrefix(file, strip_path_prefix));
+  if (line > 0) {
+    buffer->append(":%d", line);
+    if (column > 0)
+      buffer->append(":%d", column);
+  }
+}
+
+void RenderModuleLocation(InternalScopedString *buffer, const char *module,
+                          uptr offset, ModuleArch arch,
+                          const char *strip_path_prefix) {
+  buffer->append("(%s", StripPathPrefix(module, strip_path_prefix));
+  if (arch != kModuleArchUnknown) {
+    buffer->append(":%s", ModuleArchToString(arch));
+  }
+  buffer->append("+0x%zx)", offset);
+}
+
+} // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp b/lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp
new file mode 100644
index 0000000000..34190fb1bb
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp
@@ -0,0 +1,85 @@
+//===-- sanitizer_stacktrace_sparc.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//
+// Implemention of fast stack unwinding for Sparc.
+//===----------------------------------------------------------------------===//
+
+#if defined(__sparc__)
+
+#if defined(__arch64__) || defined(__sparcv9)
+#define STACK_BIAS 2047
+#else
+#define STACK_BIAS 0
+#endif
+
+#include "sanitizer_common.h"
+#include "sanitizer_stacktrace.h"
+
+namespace __sanitizer {
+
+void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
+                                    uptr stack_bottom, u32 max_depth) {
+  // TODO(yln): add arg sanity check for stack_top/stack_bottom
+  CHECK_GE(max_depth, 2);
+  const uptr kPageSize = GetPageSizeCached();
+#if defined(__GNUC__)
+  // __builtin_return_address returns the address of the call instruction
+  // on the SPARC and not the return address, so we need to compensate.
+  trace_buffer[0] = GetNextInstructionPc(pc);
+#else
+  trace_buffer[0] = pc;
+#endif
+  size = 1;
+  if (stack_top < 4096) return;  // Sanity check for stack top.
+  // Flush register windows to memory
+#if defined(__sparc_v9__) || defined(__sparcv9__) || defined(__sparcv9)
+  asm volatile("flushw" ::: "memory");
+#else
+  asm volatile("ta 3" ::: "memory");
+#endif
+  // On the SPARC, the return address is not in the frame, it is in a
+  // register.  There is no way to access it off of the current frame
+  // pointer, but it can be accessed off the previous frame pointer by
+  // reading the value from the register window save area.
+  uptr prev_bp = GET_CURRENT_FRAME();
+  uptr next_bp = prev_bp;
+  unsigned int i = 0;
+  while (next_bp != bp && IsAligned(next_bp, sizeof(uhwptr)) && i++ < 8) {
+    prev_bp = next_bp;
+    next_bp = (uptr)((uhwptr *)next_bp)[14] + STACK_BIAS;
+  }
+  if (next_bp == bp)
+    bp = prev_bp;
+  // Lowest possible address that makes sense as the next frame pointer.
+  // Goes up as we walk the stack.
+  uptr bottom = stack_bottom;
+  // Avoid infinite loop when frame == frame[0] by using frame > prev_frame.
+  while (IsValidFrame(bp, stack_top, bottom) && IsAligned(bp, sizeof(uhwptr)) &&
+         size < max_depth) {
+    uhwptr pc1 = ((uhwptr *)bp)[15];
+    // Let's assume that any pointer in the 0th page is invalid and
+    // stop unwinding here.  If we're adding support for a platform
+    // where this isn't true, we need to reconsider this check.
+    if (pc1 < kPageSize)
+      break;
+    if (pc1 != pc) {
+      // %o7 contains the address of the call instruction and not the
+      // return address, so we need to compensate.
+      trace_buffer[size++] = GetNextInstructionPc((uptr)pc1);
+    }
+    bottom = bp;
+    bp = (uptr)((uhwptr *)bp)[14] + STACK_BIAS;
+  }
+}
+
+}  // namespace __sanitizer
+
+#endif  // !defined(__sparc__)
diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
new file mode 100644
index 0000000000..651d5056dd
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -0,0 +1,573 @@
+//===-- sanitizer_stoptheworld_linux_libcdep.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) || \
+                        defined(__aarch64__) || defined(__powerpc64__) || \
+                        defined(__s390__) || defined(__i386__) || \
+                        defined(__arm__))
+
+#include "sanitizer_stoptheworld.h"
+
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_atomic.h"
+
+#include <errno.h>
+#include <sched.h> // for CLONE_* definitions
+#include <stddef.h>
+#include <sys/prctl.h> // for PR_* definitions
+#include <sys/ptrace.h> // for PTRACE_* definitions
+#include <sys/types.h> // for pid_t
+#include <sys/uio.h> // for iovec
+#include <elf.h> // for NT_PRSTATUS
+#if defined(__aarch64__) && !SANITIZER_ANDROID
+// GLIBC 2.20+ sys/user does not include asm/ptrace.h
+# include <asm/ptrace.h>
+#endif
+#include <sys/user.h>  // for user_regs_struct
+#if SANITIZER_ANDROID && SANITIZER_MIPS
+# include <asm/reg.h>  // for mips SP register in sys/user.h
+#endif
+#include <sys/wait.h> // for signal-related stuff
+
+#ifdef sa_handler
+# undef sa_handler
+#endif
+
+#ifdef sa_sigaction
+# undef sa_sigaction
+#endif
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_placement_new.h"
+
+// Sufficiently old kernel headers don't provide this value, but we can still
+// call prctl with it. If the runtime kernel is new enough, the prctl call will
+// have the desired effect; if the kernel is too old, the call will error and we
+// can ignore said error.
+#ifndef PR_SET_PTRACER
+#define PR_SET_PTRACER 0x59616d61
+#endif
+
+// This module works by spawning a Linux task which then attaches to every
+// thread in the caller process with ptrace. This suspends the threads, and
+// PTRACE_GETREGS can then be used to obtain their register state. The callback
+// supplied to StopTheWorld() is run in the tracer task while the threads are
+// suspended.
+// The tracer task must be placed in a different thread group for ptrace to
+// work, so it cannot be spawned as a pthread. Instead, we use the low-level
+// clone() interface (we want to share the address space with the caller
+// process, so we prefer clone() over fork()).
+//
+// We don't use any libc functions, relying instead on direct syscalls. There
+// are two reasons for this:
+// 1. calling a library function while threads are suspended could cause a
+// deadlock, if one of the treads happens to be holding a libc lock;
+// 2. it's generally not safe to call libc functions from the tracer task,
+// because clone() does not set up a thread-local storage for it. Any
+// thread-local variables used by libc will be shared between the tracer task
+// and the thread which spawned it.
+
+namespace __sanitizer {
+
+class SuspendedThreadsListLinux : public SuspendedThreadsList {
+ public:
+  SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
+
+  tid_t GetThreadID(uptr index) const;
+  uptr ThreadCount() const;
+  bool ContainsTid(tid_t thread_id) const;
+  void Append(tid_t tid);
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                          uptr *sp) const;
+  uptr RegisterCount() const;
+
+ private:
+  InternalMmapVector<tid_t> thread_ids_;
+};
+
+// Structure for passing arguments into the tracer thread.
+struct TracerThreadArgument {
+  StopTheWorldCallback callback;
+  void *callback_argument;
+  // The tracer thread waits on this mutex while the parent finishes its
+  // preparations.
+  BlockingMutex mutex;
+  // Tracer thread signals its completion by setting done.
+  atomic_uintptr_t done;
+  uptr parent_pid;
+};
+
+// This class handles thread suspending/unsuspending in the tracer thread.
+class ThreadSuspender {
+ public:
+  explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
+    : arg(arg)
+    , pid_(pid) {
+      CHECK_GE(pid, 0);
+    }
+  bool SuspendAllThreads();
+  void ResumeAllThreads();
+  void KillAllThreads();
+  SuspendedThreadsListLinux &suspended_threads_list() {
+    return suspended_threads_list_;
+  }
+  TracerThreadArgument *arg;
+ private:
+  SuspendedThreadsListLinux suspended_threads_list_;
+  pid_t pid_;
+  bool SuspendThread(tid_t thread_id);
+};
+
+bool ThreadSuspender::SuspendThread(tid_t tid) {
+  // Are we already attached to this thread?
+  // Currently this check takes linear time, however the number of threads is
+  // usually small.
+  if (suspended_threads_list_.ContainsTid(tid)) return false;
+  int pterrno;
+  if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
+                       &pterrno)) {
+    // Either the thread is dead, or something prevented us from attaching.
+    // Log this event and move on.
+    VReport(1, "Could not attach to thread %zu (errno %d).\n", (uptr)tid,
+            pterrno);
+    return false;
+  } else {
+    VReport(2, "Attached to thread %zu.\n", (uptr)tid);
+    // The thread is not guaranteed to stop before ptrace returns, so we must
+    // wait on it. Note: if the thread receives a signal concurrently,
+    // we can get notification about the signal before notification about stop.
+    // In such case we need to forward the signal to the thread, otherwise
+    // the signal will be missed (as we do PTRACE_DETACH with arg=0) and
+    // any logic relying on signals will break. After forwarding we need to
+    // continue to wait for stopping, because the thread is not stopped yet.
+    // We do ignore delivery of SIGSTOP, because we want to make stop-the-world
+    // as invisible as possible.
+    for (;;) {
+      int status;
+      uptr waitpid_status;
+      HANDLE_EINTR(waitpid_status, internal_waitpid(tid, &status, __WALL));
+      int wperrno;
+      if (internal_iserror(waitpid_status, &wperrno)) {
+        // Got a ECHILD error. I don't think this situation is possible, but it
+        // doesn't hurt to report it.
+        VReport(1, "Waiting on thread %zu failed, detaching (errno %d).\n",
+                (uptr)tid, wperrno);
+        internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr);
+        return false;
+      }
+      if (WIFSTOPPED(status) && WSTOPSIG(status) != SIGSTOP) {
+        internal_ptrace(PTRACE_CONT, tid, nullptr,
+                        (void*)(uptr)WSTOPSIG(status));
+        continue;
+      }
+      break;
+    }
+    suspended_threads_list_.Append(tid);
+    return true;
+  }
+}
+
+void ThreadSuspender::ResumeAllThreads() {
+  for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++) {
+    pid_t tid = suspended_threads_list_.GetThreadID(i);
+    int pterrno;
+    if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr),
+                          &pterrno)) {
+      VReport(2, "Detached from thread %d.\n", tid);
+    } else {
+      // Either the thread is dead, or we are already detached.
+      // The latter case is possible, for instance, if this function was called
+      // from a signal handler.
+      VReport(1, "Could not detach from thread %d (errno %d).\n", tid, pterrno);
+    }
+  }
+}
+
+void ThreadSuspender::KillAllThreads() {
+  for (uptr i = 0; i < suspended_threads_list_.ThreadCount(); i++)
+    internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i),
+                    nullptr, nullptr);
+}
+
+bool ThreadSuspender::SuspendAllThreads() {
+  ThreadLister thread_lister(pid_);
+  bool retry = true;
+  InternalMmapVector<tid_t> threads;
+  threads.reserve(128);
+  for (int i = 0; i < 30 && retry; ++i) {
+    retry = false;
+    switch (thread_lister.ListThreads(&threads)) {
+      case ThreadLister::Error:
+        ResumeAllThreads();
+        return false;
+      case ThreadLister::Incomplete:
+        retry = true;
+        break;
+      case ThreadLister::Ok:
+        break;
+    }
+    for (tid_t tid : threads) {
+      if (SuspendThread(tid))
+        retry = true;
+    }
+  }
+  return suspended_threads_list_.ThreadCount();
+}
+
+// Pointer to the ThreadSuspender instance for use in signal handler.
+static ThreadSuspender *thread_suspender_instance = nullptr;
+
+// Synchronous signals that should not be blocked.
+static const int kSyncSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS,
+                                    SIGXCPU, SIGXFSZ };
+
+static void TracerThreadDieCallback() {
+  // Generally a call to Die() in the tracer thread should be fatal to the
+  // parent process as well, because they share the address space.
+  // This really only works correctly if all the threads are suspended at this
+  // point. So we correctly handle calls to Die() from within the callback, but
+  // not those that happen before or after the callback. Hopefully there aren't
+  // a lot of opportunities for that to happen...
+  ThreadSuspender *inst = thread_suspender_instance;
+  if (inst && stoptheworld_tracer_pid == internal_getpid()) {
+    inst->KillAllThreads();
+    thread_suspender_instance = nullptr;
+  }
+}
+
+// Signal handler to wake up suspended threads when the tracer thread dies.
+static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
+                                      void *uctx) {
+  SignalContext ctx(siginfo, uctx);
+  Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
+         ctx.addr, ctx.pc, ctx.sp);
+  ThreadSuspender *inst = thread_suspender_instance;
+  if (inst) {
+    if (signum == SIGABRT)
+      inst->KillAllThreads();
+    else
+      inst->ResumeAllThreads();
+    RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
+    thread_suspender_instance = nullptr;
+    atomic_store(&inst->arg->done, 1, memory_order_relaxed);
+  }
+  internal__exit((signum == SIGABRT) ? 1 : 2);
+}
+
+// Size of alternative stack for signal handlers in the tracer thread.
+static const int kHandlerStackSize = 8192;
+
+// This function will be run as a cloned task.
+static int TracerThread(void* argument) {
+  TracerThreadArgument *tracer_thread_argument =
+      (TracerThreadArgument *)argument;
+
+  internal_prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
+  // Check if parent is already dead.
+  if (internal_getppid() != tracer_thread_argument->parent_pid)
+    internal__exit(4);
+
+  // Wait for the parent thread to finish preparations.
+  tracer_thread_argument->mutex.Lock();
+  tracer_thread_argument->mutex.Unlock();
+
+  RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
+
+  ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
+  // Global pointer for the signal handler.
+  thread_suspender_instance = &thread_suspender;
+
+  // Alternate stack for signal handling.
+  InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
+  stack_t handler_stack;
+  internal_memset(&handler_stack, 0, sizeof(handler_stack));
+  handler_stack.ss_sp = handler_stack_memory.data();
+  handler_stack.ss_size = kHandlerStackSize;
+  internal_sigaltstack(&handler_stack, nullptr);
+
+  // Install our handler for synchronous signals. Other signals should be
+  // blocked by the mask we inherited from the parent thread.
+  for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
+    __sanitizer_sigaction act;
+    internal_memset(&act, 0, sizeof(act));
+    act.sigaction = TracerThreadSignalHandler;
+    act.sa_flags = SA_ONSTACK | SA_SIGINFO;
+    internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
+  }
+
+  int exit_code = 0;
+  if (!thread_suspender.SuspendAllThreads()) {
+    VReport(1, "Failed suspending threads.\n");
+    exit_code = 3;
+  } else {
+    tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
+                                     tracer_thread_argument->callback_argument);
+    thread_suspender.ResumeAllThreads();
+    exit_code = 0;
+  }
+  RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
+  thread_suspender_instance = nullptr;
+  atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
+  return exit_code;
+}
+
+class ScopedStackSpaceWithGuard {
+ public:
+  explicit ScopedStackSpaceWithGuard(uptr stack_size) {
+    stack_size_ = stack_size;
+    guard_size_ = GetPageSizeCached();
+    // FIXME: Omitting MAP_STACK here works in current kernels but might break
+    // in the future.
+    guard_start_ = (uptr)MmapOrDie(stack_size_ + guard_size_,
+                                   "ScopedStackWithGuard");
+    CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
+  }
+  ~ScopedStackSpaceWithGuard() {
+    UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
+  }
+  void *Bottom() const {
+    return (void *)(guard_start_ + stack_size_ + guard_size_);
+  }
+
+ private:
+  uptr stack_size_;
+  uptr guard_size_;
+  uptr guard_start_;
+};
+
+// We have a limitation on the stack frame size, so some stuff had to be moved
+// into globals.
+static __sanitizer_sigset_t blocked_sigset;
+static __sanitizer_sigset_t old_sigset;
+
+class StopTheWorldScope {
+ public:
+  StopTheWorldScope() {
+    // Make this process dumpable. Processes that are not dumpable cannot be
+    // attached to.
+    process_was_dumpable_ = internal_prctl(PR_GET_DUMPABLE, 0, 0, 0, 0);
+    if (!process_was_dumpable_)
+      internal_prctl(PR_SET_DUMPABLE, 1, 0, 0, 0);
+  }
+
+  ~StopTheWorldScope() {
+    // Restore the dumpable flag.
+    if (!process_was_dumpable_)
+      internal_prctl(PR_SET_DUMPABLE, 0, 0, 0, 0);
+  }
+
+ private:
+  int process_was_dumpable_;
+};
+
+// When sanitizer output is being redirected to file (i.e. by using log_path),
+// the tracer should write to the parent's log instead of trying to open a new
+// file. Alert the logging code to the fact that we have a tracer.
+struct ScopedSetTracerPID {
+  explicit ScopedSetTracerPID(uptr tracer_pid) {
+    stoptheworld_tracer_pid = tracer_pid;
+    stoptheworld_tracer_ppid = internal_getpid();
+  }
+  ~ScopedSetTracerPID() {
+    stoptheworld_tracer_pid = 0;
+    stoptheworld_tracer_ppid = 0;
+  }
+};
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  StopTheWorldScope in_stoptheworld;
+  // Prepare the arguments for TracerThread.
+  struct TracerThreadArgument tracer_thread_argument;
+  tracer_thread_argument.callback = callback;
+  tracer_thread_argument.callback_argument = argument;
+  tracer_thread_argument.parent_pid = internal_getpid();
+  atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
+  const uptr kTracerStackSize = 2 * 1024 * 1024;
+  ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
+  // Block the execution of TracerThread until after we have set ptrace
+  // permissions.
+  tracer_thread_argument.mutex.Lock();
+  // Signal handling story.
+  // We don't want async signals to be delivered to the tracer thread,
+  // so we block all async signals before creating the thread. An async signal
+  // handler can temporary modify errno, which is shared with this thread.
+  // We ought to use pthread_sigmask here, because sigprocmask has undefined
+  // behavior in multithreaded programs. However, on linux sigprocmask is
+  // equivalent to pthread_sigmask with the exception that pthread_sigmask
+  // does not allow to block some signals used internally in pthread
+  // implementation. We are fine with blocking them here, we are really not
+  // going to pthread_cancel the thread.
+  // The tracer thread should not raise any synchronous signals. But in case it
+  // does, we setup a special handler for sync signals that properly kills the
+  // parent as well. Note: we don't pass CLONE_SIGHAND to clone, so handlers
+  // in the tracer thread won't interfere with user program. Double note: if a
+  // user does something along the lines of 'kill -11 pid', that can kill the
+  // process even if user setup own handler for SEGV.
+  // Thing to watch out for: this code should not change behavior of user code
+  // in any observable way. In particular it should not override user signal
+  // handlers.
+  internal_sigfillset(&blocked_sigset);
+  for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
+    internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
+  int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
+  CHECK_EQ(rv, 0);
+  uptr tracer_pid = internal_clone(
+      TracerThread, tracer_stack.Bottom(),
+      CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED,
+      &tracer_thread_argument, nullptr /* parent_tidptr */,
+      nullptr /* newtls */, nullptr /* child_tidptr */);
+  internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
+  int local_errno = 0;
+  if (internal_iserror(tracer_pid, &local_errno)) {
+    VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
+    tracer_thread_argument.mutex.Unlock();
+  } else {
+    ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
+    // On some systems we have to explicitly declare that we want to be traced
+    // by the tracer thread.
+    internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
+    // Allow the tracer thread to start.
+    tracer_thread_argument.mutex.Unlock();
+    // NOTE: errno is shared between this thread and the tracer thread.
+    // internal_waitpid() may call syscall() which can access/spoil errno,
+    // so we can't call it now. Instead we for the tracer thread to finish using
+    // the spin loop below. Man page for sched_yield() says "In the Linux
+    // implementation, sched_yield() always succeeds", so let's hope it does not
+    // spoil errno. Note that this spin loop runs only for brief periods before
+    // the tracer thread has suspended us and when it starts unblocking threads.
+    while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
+      sched_yield();
+    // Now the tracer thread is about to exit and does not touch errno,
+    // wait for it.
+    for (;;) {
+      uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
+      if (!internal_iserror(waitpid_status, &local_errno))
+        break;
+      if (local_errno == EINTR)
+        continue;
+      VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
+              local_errno);
+      break;
+    }
+  }
+}
+
+// Platform-specific methods from SuspendedThreadsList.
+#if SANITIZER_ANDROID && defined(__arm__)
+typedef pt_regs regs_struct;
+#define REG_SP ARM_sp
+
+#elif SANITIZER_LINUX && defined(__arm__)
+typedef user_regs regs_struct;
+#define REG_SP uregs[13]
+
+#elif defined(__i386__) || defined(__x86_64__)
+typedef user_regs_struct regs_struct;
+#if defined(__i386__)
+#define REG_SP esp
+#else
+#define REG_SP rsp
+#endif
+
+#elif defined(__powerpc__) || defined(__powerpc64__)
+typedef pt_regs regs_struct;
+#define REG_SP gpr[PT_R1]
+
+#elif defined(__mips__)
+typedef struct user regs_struct;
+# if SANITIZER_ANDROID
+#  define REG_SP regs[EF_R29]
+# else
+#  define REG_SP regs[EF_REG29]
+# endif
+
+#elif defined(__aarch64__)
+typedef struct user_pt_regs regs_struct;
+#define REG_SP sp
+#define ARCH_IOVEC_FOR_GETREGSET
+
+#elif defined(__s390__)
+typedef _user_regs_struct regs_struct;
+#define REG_SP gprs[15]
+#define ARCH_IOVEC_FOR_GETREGSET
+
+#else
+#error "Unsupported architecture"
+#endif // SANITIZER_ANDROID && defined(__arm__)
+
+tid_t SuspendedThreadsListLinux::GetThreadID(uptr index) const {
+  CHECK_LT(index, thread_ids_.size());
+  return thread_ids_[index];
+}
+
+uptr SuspendedThreadsListLinux::ThreadCount() const {
+  return thread_ids_.size();
+}
+
+bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id) const {
+  for (uptr i = 0; i < thread_ids_.size(); i++) {
+    if (thread_ids_[i] == thread_id) return true;
+  }
+  return false;
+}
+
+void SuspendedThreadsListLinux::Append(tid_t tid) {
+  thread_ids_.push_back(tid);
+}
+
+PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
+    uptr index, uptr *buffer, uptr *sp) const {
+  pid_t tid = GetThreadID(index);
+  regs_struct regs;
+  int pterrno;
+#ifdef ARCH_IOVEC_FOR_GETREGSET
+  struct iovec regset_io;
+  regset_io.iov_base = &regs;
+  regset_io.iov_len = sizeof(regs_struct);
+  bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid,
+                                (void*)NT_PRSTATUS, (void*)&regset_io),
+                                &pterrno);
+#else
+  bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, nullptr,
+                                &regs), &pterrno);
+#endif
+  if (isErr) {
+    VReport(1, "Could not get registers from thread %d (errno %d).\n", tid,
+            pterrno);
+    // ESRCH means that the given thread is not suspended or already dead.
+    // Therefore it's unsafe to inspect its data (e.g. walk through stack) and
+    // we should notify caller about this.
+    return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
+                            : REGISTERS_UNAVAILABLE;
+  }
+
+  *sp = regs.REG_SP;
+  internal_memcpy(buffer, &regs, sizeof(regs));
+  return REGISTERS_AVAILABLE;
+}
+
+uptr SuspendedThreadsListLinux::RegisterCount() const {
+  return sizeof(regs_struct) / sizeof(uptr);
+}
+} // namespace __sanitizer
+
+#endif  // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)
+        // || defined(__aarch64__) || defined(__powerpc64__)
+        // || defined(__s390__) || defined(__i386__) || defined(__arm__)
diff --git a/lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
new file mode 100644
index 0000000000..1ed2134325
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
@@ -0,0 +1,364 @@
+//===-- sanitizer_stoptheworld_netbsd_libcdep.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See sanitizer_stoptheworld.h for details.
+// This implementation was inspired by Markus Gutschke's linuxthreads.cc.
+//
+// This is a NetBSD variation of Linux stoptheworld implementation
+// See sanitizer_stoptheworld_linux_libcdep.cpp for code comments.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_stoptheworld.h"
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_platform_limits_posix.h"
+
+#include <sys/types.h>
+
+#include <sys/ptrace.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+
+#include <machine/reg.h>
+
+#include <elf.h>
+#include <errno.h>
+#include <sched.h>
+#include <signal.h>
+#include <stddef.h>
+
+#define internal_sigaction_norestorer internal_sigaction
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_placement_new.h"
+
+namespace __sanitizer {
+
+class SuspendedThreadsListNetBSD : public SuspendedThreadsList {
+ public:
+  SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); }
+
+  tid_t GetThreadID(uptr index) const;
+  uptr ThreadCount() const;
+  bool ContainsTid(tid_t thread_id) const;
+  void Append(tid_t tid);
+
+  PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+                                          uptr *sp) const;
+  uptr RegisterCount() const;
+
+ private:
+  InternalMmapVector<tid_t> thread_ids_;
+};
+
+struct TracerThreadArgument {
+  StopTheWorldCallback callback;
+  void *callback_argument;
+  BlockingMutex mutex;
+  atomic_uintptr_t done;
+  uptr parent_pid;
+};
+
+class ThreadSuspender {
+ public:
+  explicit ThreadSuspender(pid_t pid, TracerThreadArgument *arg)
+      : arg(arg), pid_(pid) {
+    CHECK_GE(pid, 0);
+  }
+  bool SuspendAllThreads();
+  void ResumeAllThreads();
+  void KillAllThreads();
+  SuspendedThreadsListNetBSD &suspended_threads_list() {
+    return suspended_threads_list_;
+  }
+  TracerThreadArgument *arg;
+
+ private:
+  SuspendedThreadsListNetBSD suspended_threads_list_;
+  pid_t pid_;
+};
+
+void ThreadSuspender::ResumeAllThreads() {
+  int pterrno;
+  if (!internal_iserror(internal_ptrace(PT_DETACH, pid_, (void *)(uptr)1, 0),
+                        &pterrno)) {
+    VReport(2, "Detached from process %d.\n", pid_);
+  } else {
+    VReport(1, "Could not detach from process %d (errno %d).\n", pid_, pterrno);
+  }
+}
+
+void ThreadSuspender::KillAllThreads() {
+  internal_ptrace(PT_KILL, pid_, nullptr, 0);
+}
+
+bool ThreadSuspender::SuspendAllThreads() {
+  int pterrno;
+  if (internal_iserror(internal_ptrace(PT_ATTACH, pid_, nullptr, 0),
+                       &pterrno)) {
+    Printf("Could not attach to process %d (errno %d).\n", pid_, pterrno);
+    return false;
+  }
+
+  int status;
+  uptr waitpid_status;
+  HANDLE_EINTR(waitpid_status, internal_waitpid(pid_, &status, 0));
+
+  VReport(2, "Attached to process %d.\n", pid_);
+
+#ifdef PT_LWPNEXT
+  struct ptrace_lwpstatus pl;
+  int op = PT_LWPNEXT;
+#else
+  struct ptrace_lwpinfo pl;
+  int op = PT_LWPINFO;
+#endif
+
+  pl.pl_lwpid = 0;
+
+  int val;
+  while ((val = ptrace(op, pid_, (void *)&pl, sizeof(pl))) != -1 &&
+         pl.pl_lwpid != 0) {
+    suspended_threads_list_.Append(pl.pl_lwpid);
+    VReport(2, "Appended thread %d in process %d.\n", pl.pl_lwpid, pid_);
+  }
+  return true;
+}
+
+// Pointer to the ThreadSuspender instance for use in signal handler.
+static ThreadSuspender *thread_suspender_instance = nullptr;
+
+// Synchronous signals that should not be blocked.
+static const int kSyncSignals[] = {SIGABRT, SIGILL,  SIGFPE, SIGSEGV,
+                                   SIGBUS,  SIGXCPU, SIGXFSZ};
+
+static void TracerThreadDieCallback() {
+  ThreadSuspender *inst = thread_suspender_instance;
+  if (inst && stoptheworld_tracer_pid == internal_getpid()) {
+    inst->KillAllThreads();
+    thread_suspender_instance = nullptr;
+  }
+}
+
+// Signal handler to wake up suspended threads when the tracer thread dies.
+static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
+                                      void *uctx) {
+  SignalContext ctx(siginfo, uctx);
+  Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
+         ctx.addr, ctx.pc, ctx.sp);
+  ThreadSuspender *inst = thread_suspender_instance;
+  if (inst) {
+    if (signum == SIGABRT)
+      inst->KillAllThreads();
+    else
+      inst->ResumeAllThreads();
+    RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
+    thread_suspender_instance = nullptr;
+    atomic_store(&inst->arg->done, 1, memory_order_relaxed);
+  }
+  internal__exit((signum == SIGABRT) ? 1 : 2);
+}
+
+// Size of alternative stack for signal handlers in the tracer thread.
+static const int kHandlerStackSize = 8192;
+
+// This function will be run as a cloned task.
+static int TracerThread(void *argument) {
+  TracerThreadArgument *tracer_thread_argument =
+      (TracerThreadArgument *)argument;
+
+  // Check if parent is already dead.
+  if (internal_getppid() != tracer_thread_argument->parent_pid)
+    internal__exit(4);
+
+  // Wait for the parent thread to finish preparations.
+  tracer_thread_argument->mutex.Lock();
+  tracer_thread_argument->mutex.Unlock();
+
+  RAW_CHECK(AddDieCallback(TracerThreadDieCallback));
+
+  ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument);
+  // Global pointer for the signal handler.
+  thread_suspender_instance = &thread_suspender;
+
+  // Alternate stack for signal handling.
+  InternalMmapVector<char> handler_stack_memory(kHandlerStackSize);
+  stack_t handler_stack;
+  internal_memset(&handler_stack, 0, sizeof(handler_stack));
+  handler_stack.ss_sp = handler_stack_memory.data();
+  handler_stack.ss_size = kHandlerStackSize;
+  internal_sigaltstack(&handler_stack, nullptr);
+
+  // Install our handler for synchronous signals. Other signals should be
+  // blocked by the mask we inherited from the parent thread.
+  for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++) {
+    __sanitizer_sigaction act;
+    internal_memset(&act, 0, sizeof(act));
+    act.sigaction = TracerThreadSignalHandler;
+    act.sa_flags = SA_ONSTACK | SA_SIGINFO;
+    internal_sigaction_norestorer(kSyncSignals[i], &act, 0);
+  }
+
+  int exit_code = 0;
+  if (!thread_suspender.SuspendAllThreads()) {
+    VReport(1, "Failed suspending threads.\n");
+    exit_code = 3;
+  } else {
+    tracer_thread_argument->callback(thread_suspender.suspended_threads_list(),
+                                     tracer_thread_argument->callback_argument);
+    thread_suspender.ResumeAllThreads();
+    exit_code = 0;
+  }
+  RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback));
+  thread_suspender_instance = nullptr;
+  atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed);
+  return exit_code;
+}
+
+class ScopedStackSpaceWithGuard {
+ public:
+  explicit ScopedStackSpaceWithGuard(uptr stack_size) {
+    stack_size_ = stack_size;
+    guard_size_ = GetPageSizeCached();
+    // FIXME: Omitting MAP_STACK here works in current kernels but might break
+    // in the future.
+    guard_start_ =
+        (uptr)MmapOrDie(stack_size_ + guard_size_, "ScopedStackWithGuard");
+    CHECK(MprotectNoAccess((uptr)guard_start_, guard_size_));
+  }
+  ~ScopedStackSpaceWithGuard() {
+    UnmapOrDie((void *)guard_start_, stack_size_ + guard_size_);
+  }
+  void *Bottom() const {
+    return (void *)(guard_start_ + stack_size_ + guard_size_);
+  }
+
+ private:
+  uptr stack_size_;
+  uptr guard_size_;
+  uptr guard_start_;
+};
+
+static __sanitizer_sigset_t blocked_sigset;
+static __sanitizer_sigset_t old_sigset;
+
+struct ScopedSetTracerPID {
+  explicit ScopedSetTracerPID(uptr tracer_pid) {
+    stoptheworld_tracer_pid = tracer_pid;
+    stoptheworld_tracer_ppid = internal_getpid();
+  }
+  ~ScopedSetTracerPID() {
+    stoptheworld_tracer_pid = 0;
+    stoptheworld_tracer_ppid = 0;
+  }
+};
+
+void StopTheWorld(StopTheWorldCallback callback, void *argument) {
+  // Prepare the arguments for TracerThread.
+  struct TracerThreadArgument tracer_thread_argument;
+  tracer_thread_argument.callback = callback;
+  tracer_thread_argument.callback_argument = argument;
+  tracer_thread_argument.parent_pid = internal_getpid();
+  atomic_store(&tracer_thread_argument.done, 0, memory_order_relaxed);
+  const uptr kTracerStackSize = 2 * 1024 * 1024;
+  ScopedStackSpaceWithGuard tracer_stack(kTracerStackSize);
+
+  tracer_thread_argument.mutex.Lock();
+
+  internal_sigfillset(&blocked_sigset);
+  for (uptr i = 0; i < ARRAY_SIZE(kSyncSignals); i++)
+    internal_sigdelset(&blocked_sigset, kSyncSignals[i]);
+  int rv = internal_sigprocmask(SIG_BLOCK, &blocked_sigset, &old_sigset);
+  CHECK_EQ(rv, 0);
+  uptr tracer_pid = internal_clone(TracerThread, tracer_stack.Bottom(),
+                                   CLONE_VM | CLONE_FS | CLONE_FILES,
+                                   &tracer_thread_argument);
+  internal_sigprocmask(SIG_SETMASK, &old_sigset, 0);
+  int local_errno = 0;
+  if (internal_iserror(tracer_pid, &local_errno)) {
+    VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno);
+    tracer_thread_argument.mutex.Unlock();
+  } else {
+    ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
+
+    tracer_thread_argument.mutex.Unlock();
+
+    while (atomic_load(&tracer_thread_argument.done, memory_order_relaxed) == 0)
+      sched_yield();
+
+    for (;;) {
+      uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL);
+      if (!internal_iserror(waitpid_status, &local_errno))
+        break;
+      if (local_errno == EINTR)
+        continue;
+      VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
+              local_errno);
+      break;
+    }
+  }
+}
+
+tid_t SuspendedThreadsListNetBSD::GetThreadID(uptr index) const {
+  CHECK_LT(index, thread_ids_.size());
+  return thread_ids_[index];
+}
+
+uptr SuspendedThreadsListNetBSD::ThreadCount() const {
+  return thread_ids_.size();
+}
+
+bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const {
+  for (uptr i = 0; i < thread_ids_.size(); i++) {
+    if (thread_ids_[i] == thread_id)
+      return true;
+  }
+  return false;
+}
+
+void SuspendedThreadsListNetBSD::Append(tid_t tid) {
+  thread_ids_.push_back(tid);
+}
+
+PtraceRegistersStatus SuspendedThreadsListNetBSD::GetRegistersAndSP(
+    uptr index, uptr *buffer, uptr *sp) const {
+  lwpid_t tid = GetThreadID(index);
+  pid_t ppid = internal_getppid();
+  struct reg regs;
+  int pterrno;
+  bool isErr =
+      internal_iserror(internal_ptrace(PT_GETREGS, ppid, &regs, tid), &pterrno);
+  if (isErr) {
+    VReport(1,
+            "Could not get registers from process %d thread %d (errno %d).\n",
+            ppid, tid, pterrno);
+    return pterrno == ESRCH ? REGISTERS_UNAVAILABLE_FATAL
+                            : REGISTERS_UNAVAILABLE;
+  }
+
+  *sp = PTRACE_REG_SP(&regs);
+  internal_memcpy(buffer, &regs, sizeof(regs));
+
+  return REGISTERS_AVAILABLE;
+}
+
+uptr SuspendedThreadsListNetBSD::RegisterCount() const {
+  return sizeof(struct reg) / sizeof(uptr);
+}
+}  // namespace __sanitizer
+
+#endif
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp
new file mode 100644
index 0000000000..0c4b84c767
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp
@@ -0,0 +1,135 @@
+//===-- sanitizer_symbolizer.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_platform.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_symbolizer_internal.h"
+
+namespace __sanitizer {
+
+AddressInfo::AddressInfo() {
+  internal_memset(this, 0, sizeof(AddressInfo));
+  function_offset = kUnknown;
+}
+
+void AddressInfo::Clear() {
+  InternalFree(module);
+  InternalFree(function);
+  InternalFree(file);
+  internal_memset(this, 0, sizeof(AddressInfo));
+  function_offset = kUnknown;
+}
+
+void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
+                                 ModuleArch mod_arch) {
+  module = internal_strdup(mod_name);
+  module_offset = mod_offset;
+  module_arch = mod_arch;
+}
+
+SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
+
+SymbolizedStack *SymbolizedStack::New(uptr addr) {
+  void *mem = InternalAlloc(sizeof(SymbolizedStack));
+  SymbolizedStack *res = new(mem) SymbolizedStack();
+  res->info.address = addr;
+  return res;
+}
+
+void SymbolizedStack::ClearAll() {
+  info.Clear();
+  if (next)
+    next->ClearAll();
+  InternalFree(this);
+}
+
+DataInfo::DataInfo() {
+  internal_memset(this, 0, sizeof(DataInfo));
+}
+
+void DataInfo::Clear() {
+  InternalFree(module);
+  InternalFree(file);
+  InternalFree(name);
+  internal_memset(this, 0, sizeof(DataInfo));
+}
+
+void FrameInfo::Clear() {
+  InternalFree(module);
+  for (LocalInfo &local : locals) {
+    InternalFree(local.function_name);
+    InternalFree(local.name);
+    InternalFree(local.decl_file);
+  }
+  locals.clear();
+}
+
+Symbolizer *Symbolizer::symbolizer_;
+StaticSpinMutex Symbolizer::init_mu_;
+LowLevelAllocator Symbolizer::symbolizer_allocator_;
+
+void Symbolizer::InvalidateModuleList() {
+  modules_fresh_ = false;
+}
+
+void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook,
+                          Symbolizer::EndSymbolizationHook end_hook) {
+  CHECK(start_hook_ == 0 && end_hook_ == 0);
+  start_hook_ = start_hook;
+  end_hook_ = end_hook;
+}
+
+const char *Symbolizer::ModuleNameOwner::GetOwnedCopy(const char *str) {
+  mu_->CheckLocked();
+
+  // 'str' will be the same string multiple times in a row, optimize this case.
+  if (last_match_ && !internal_strcmp(last_match_, str))
+    return last_match_;
+
+  // FIXME: this is linear search.
+  // We should optimize this further if this turns out to be a bottleneck later.
+  for (uptr i = 0; i < storage_.size(); ++i) {
+    if (!internal_strcmp(storage_[i], str)) {
+      last_match_ = storage_[i];
+      return last_match_;
+    }
+  }
+  last_match_ = internal_strdup(str);
+  storage_.push_back(last_match_);
+  return last_match_;
+}
+
+Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools)
+    : module_names_(&mu_), modules_(), modules_fresh_(false), tools_(tools),
+      start_hook_(0), end_hook_(0) {}
+
+Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym)
+    : sym_(sym) {
+  if (sym_->start_hook_)
+    sym_->start_hook_();
+}
+
+Symbolizer::SymbolizerScope::~SymbolizerScope() {
+  if (sym_->end_hook_)
+    sym_->end_hook_();
+}
+
+void Symbolizer::LateInitializeTools() {
+  for (auto &tool : tools_) {
+    tool.LateInitialize();
+  }
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
new file mode 100644
index 0000000000..27ed222745
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
@@ -0,0 +1,209 @@
+//===-- sanitizer_symbolizer_libbacktrace.cpp -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// Libbacktrace implementation of symbolizer parts.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_symbolizer.h"
+#include "sanitizer_symbolizer_libbacktrace.h"
+
+#if SANITIZER_LIBBACKTRACE
+# include "backtrace-supported.h"
+# if SANITIZER_POSIX && BACKTRACE_SUPPORTED && !BACKTRACE_USES_MALLOC
+#  include "backtrace.h"
+#  if SANITIZER_CP_DEMANGLE
+#   undef ARRAY_SIZE
+#   include "demangle.h"
+#  endif
+# else
+#  define SANITIZER_LIBBACKTRACE 0
+# endif
+#endif
+
+namespace __sanitizer {
+
+static char *DemangleAlloc(const char *name, bool always_alloc);
+
+#if SANITIZER_LIBBACKTRACE
+
+namespace {
+
+# if SANITIZER_CP_DEMANGLE
+struct CplusV3DemangleData {
+  char *buf;
+  uptr size, allocated;
+};
+
+extern "C" {
+static void CplusV3DemangleCallback(const char *s, size_t l, void *vdata) {
+  CplusV3DemangleData *data = (CplusV3DemangleData *)vdata;
+  uptr needed = data->size + l + 1;
+  if (needed > data->allocated) {
+    data->allocated *= 2;
+    if (needed > data->allocated)
+      data->allocated = needed;
+    char *buf = (char *)InternalAlloc(data->allocated);
+    if (data->buf) {
+      internal_memcpy(buf, data->buf, data->size);
+      InternalFree(data->buf);
+    }
+    data->buf = buf;
+  }
+  internal_memcpy(data->buf + data->size, s, l);
+  data->buf[data->size + l] = '\0';
+  data->size += l;
+}
+}  // extern "C"
+
+char *CplusV3Demangle(const char *name) {
+  CplusV3DemangleData data;
+  data.buf = 0;
+  data.size = 0;
+  data.allocated = 0;
+  if (cplus_demangle_v3_callback(name, DMGL_PARAMS | DMGL_ANSI,
+                                 CplusV3DemangleCallback, &data)) {
+    if (data.size + 64 > data.allocated)
+      return data.buf;
+    char *buf = internal_strdup(data.buf);
+    InternalFree(data.buf);
+    return buf;
+  }
+  if (data.buf)
+    InternalFree(data.buf);
+  return 0;
+}
+# endif  // SANITIZER_CP_DEMANGLE
+
+struct SymbolizeCodeCallbackArg {
+  SymbolizedStack *first;
+  SymbolizedStack *last;
+  uptr frames_symbolized;
+
+  AddressInfo *get_new_frame(uintptr_t addr) {
+    CHECK(last);
+    if (frames_symbolized > 0) {
+      SymbolizedStack *cur = SymbolizedStack::New(addr);
+      AddressInfo *info = &cur->info;
+      info->FillModuleInfo(first->info.module, first->info.module_offset,
+                           first->info.module_arch);
+      last->next = cur;
+      last = cur;
+    }
+    CHECK_EQ(addr, first->info.address);
+    CHECK_EQ(addr, last->info.address);
+    return &last->info;
+  }
+};
+
+extern "C" {
+static int SymbolizeCodePCInfoCallback(void *vdata, uintptr_t addr,
+                                       const char *filename, int lineno,
+                                       const char *function) {
+  SymbolizeCodeCallbackArg *cdata = (SymbolizeCodeCallbackArg *)vdata;
+  if (function) {
+    AddressInfo *info = cdata->get_new_frame(addr);
+    info->function = DemangleAlloc(function, /*always_alloc*/ true);
+    if (filename)
+      info->file = internal_strdup(filename);
+    info->line = lineno;
+    cdata->frames_symbolized++;
+  }
+  return 0;
+}
+
+static void SymbolizeCodeCallback(void *vdata, uintptr_t addr,
+                                  const char *symname, uintptr_t, uintptr_t) {
+  SymbolizeCodeCallbackArg *cdata = (SymbolizeCodeCallbackArg *)vdata;
+  if (symname) {
+    AddressInfo *info = cdata->get_new_frame(addr);
+    info->function = DemangleAlloc(symname, /*always_alloc*/ true);
+    cdata->frames_symbolized++;
+  }
+}
+
+static void SymbolizeDataCallback(void *vdata, uintptr_t, const char *symname,
+                                  uintptr_t symval, uintptr_t symsize) {
+  DataInfo *info = (DataInfo *)vdata;
+  if (symname && symval) {
+    info->name = DemangleAlloc(symname, /*always_alloc*/ true);
+    info->start = symval;
+    info->size = symsize;
+  }
+}
+
+static void ErrorCallback(void *, const char *, int) {}
+}  // extern "C"
+
+}  // namespace
+
+LibbacktraceSymbolizer *LibbacktraceSymbolizer::get(LowLevelAllocator *alloc) {
+  // State created in backtrace_create_state is leaked.
+  void *state = (void *)(backtrace_create_state("/proc/self/exe", 0,
+                                                ErrorCallback, NULL));
+  if (!state)
+    return 0;
+  return new(*alloc) LibbacktraceSymbolizer(state);
+}
+
+bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
+  SymbolizeCodeCallbackArg data;
+  data.first = stack;
+  data.last = stack;
+  data.frames_symbolized = 0;
+  backtrace_pcinfo((backtrace_state *)state_, addr, SymbolizeCodePCInfoCallback,
+                   ErrorCallback, &data);
+  if (data.frames_symbolized > 0)
+    return true;
+  backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeCodeCallback,
+                    ErrorCallback, &data);
+  return (data.frames_symbolized > 0);
+}
+
+bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  backtrace_syminfo((backtrace_state *)state_, addr, SymbolizeDataCallback,
+                    ErrorCallback, info);
+  return true;
+}
+
+#else  // SANITIZER_LIBBACKTRACE
+
+LibbacktraceSymbolizer *LibbacktraceSymbolizer::get(LowLevelAllocator *alloc) {
+  return 0;
+}
+
+bool LibbacktraceSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
+  (void)state_;
+  return false;
+}
+
+bool LibbacktraceSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  return false;
+}
+
+#endif  // SANITIZER_LIBBACKTRACE
+
+static char *DemangleAlloc(const char *name, bool always_alloc) {
+#if SANITIZER_LIBBACKTRACE && SANITIZER_CP_DEMANGLE
+  if (char *demangled = CplusV3Demangle(name))
+    return demangled;
+#endif
+  if (always_alloc)
+    return internal_strdup(name);
+  return 0;
+}
+
+const char *LibbacktraceSymbolizer::Demangle(const char *name) {
+  return DemangleAlloc(name, /*always_alloc*/ false);
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
new file mode 100644
index 0000000000..490c6fe89b
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
@@ -0,0 +1,554 @@
+//===-- sanitizer_symbolizer_libcdep.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_symbolizer_internal.h"
+
+namespace __sanitizer {
+
+Symbolizer *Symbolizer::GetOrInit() {
+  SpinMutexLock l(&init_mu_);
+  if (symbolizer_)
+    return symbolizer_;
+  symbolizer_ = PlatformInit();
+  CHECK(symbolizer_);
+  return symbolizer_;
+}
+
+// See sanitizer_symbolizer_markup.cpp.
+#if !SANITIZER_SYMBOLIZER_MARKUP
+
+const char *ExtractToken(const char *str, const char *delims, char **result) {
+  uptr prefix_len = internal_strcspn(str, delims);
+  *result = (char*)InternalAlloc(prefix_len + 1);
+  internal_memcpy(*result, str, prefix_len);
+  (*result)[prefix_len] = '\0';
+  const char *prefix_end = str + prefix_len;
+  if (*prefix_end != '\0') prefix_end++;
+  return prefix_end;
+}
+
+const char *ExtractInt(const char *str, const char *delims, int *result) {
+  char *buff = nullptr;
+  const char *ret = ExtractToken(str, delims, &buff);
+  if (buff) {
+    *result = (int)internal_atoll(buff);
+  }
+  InternalFree(buff);
+  return ret;
+}
+
+const char *ExtractUptr(const char *str, const char *delims, uptr *result) {
+  char *buff = nullptr;
+  const char *ret = ExtractToken(str, delims, &buff);
+  if (buff) {
+    *result = (uptr)internal_atoll(buff);
+  }
+  InternalFree(buff);
+  return ret;
+}
+
+const char *ExtractSptr(const char *str, const char *delims, sptr *result) {
+  char *buff = nullptr;
+  const char *ret = ExtractToken(str, delims, &buff);
+  if (buff) {
+    *result = (sptr)internal_atoll(buff);
+  }
+  InternalFree(buff);
+  return ret;
+}
+
+const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter,
+                                      char **result) {
+  const char *found_delimiter = internal_strstr(str, delimiter);
+  uptr prefix_len =
+      found_delimiter ? found_delimiter - str : internal_strlen(str);
+  *result = (char *)InternalAlloc(prefix_len + 1);
+  internal_memcpy(*result, str, prefix_len);
+  (*result)[prefix_len] = '\0';
+  const char *prefix_end = str + prefix_len;
+  if (*prefix_end != '\0') prefix_end += internal_strlen(delimiter);
+  return prefix_end;
+}
+
+SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
+  BlockingMutexLock l(&mu_);
+  const char *module_name = nullptr;
+  uptr module_offset;
+  ModuleArch arch;
+  SymbolizedStack *res = SymbolizedStack::New(addr);
+  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
+                                         &arch))
+    return res;
+  // Always fill data about module name and offset.
+  res->info.FillModuleInfo(module_name, module_offset, arch);
+  for (auto &tool : tools_) {
+    SymbolizerScope sym_scope(this);
+    if (tool.SymbolizePC(addr, res)) {
+      return res;
+    }
+  }
+  return res;
+}
+
+bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  BlockingMutexLock l(&mu_);
+  const char *module_name = nullptr;
+  uptr module_offset;
+  ModuleArch arch;
+  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
+                                         &arch))
+    return false;
+  info->Clear();
+  info->module = internal_strdup(module_name);
+  info->module_offset = module_offset;
+  info->module_arch = arch;
+  for (auto &tool : tools_) {
+    SymbolizerScope sym_scope(this);
+    if (tool.SymbolizeData(addr, info)) {
+      return true;
+    }
+  }
+  return true;
+}
+
+bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
+  BlockingMutexLock l(&mu_);
+  const char *module_name = nullptr;
+  if (!FindModuleNameAndOffsetForAddress(
+          addr, &module_name, &info->module_offset, &info->module_arch))
+    return false;
+  info->module = internal_strdup(module_name);
+  for (auto &tool : tools_) {
+    SymbolizerScope sym_scope(this);
+    if (tool.SymbolizeFrame(addr, info)) {
+      return true;
+    }
+  }
+  return true;
+}
+
+bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
+                                             uptr *module_address) {
+  BlockingMutexLock l(&mu_);
+  const char *internal_module_name = nullptr;
+  ModuleArch arch;
+  if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
+                                         module_address, &arch))
+    return false;
+
+  if (module_name)
+    *module_name = module_names_.GetOwnedCopy(internal_module_name);
+  return true;
+}
+
+void Symbolizer::Flush() {
+  BlockingMutexLock l(&mu_);
+  for (auto &tool : tools_) {
+    SymbolizerScope sym_scope(this);
+    tool.Flush();
+  }
+}
+
+const char *Symbolizer::Demangle(const char *name) {
+  BlockingMutexLock l(&mu_);
+  for (auto &tool : tools_) {
+    SymbolizerScope sym_scope(this);
+    if (const char *demangled = tool.Demangle(name))
+      return demangled;
+  }
+  return PlatformDemangle(name);
+}
+
+bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
+                                                   const char **module_name,
+                                                   uptr *module_offset,
+                                                   ModuleArch *module_arch) {
+  const LoadedModule *module = FindModuleForAddress(address);
+  if (!module)
+    return false;
+  *module_name = module->full_name();
+  *module_offset = address - module->base_address();
+  *module_arch = module->arch();
+  return true;
+}
+
+void Symbolizer::RefreshModules() {
+  modules_.init();
+  fallback_modules_.fallbackInit();
+  RAW_CHECK(modules_.size() > 0);
+  modules_fresh_ = true;
+}
+
+static const LoadedModule *SearchForModule(const ListOfModules &modules,
+                                           uptr address) {
+  for (uptr i = 0; i < modules.size(); i++) {
+    if (modules[i].containsAddress(address)) {
+      return &modules[i];
+    }
+  }
+  return nullptr;
+}
+
+const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
+  bool modules_were_reloaded = false;
+  if (!modules_fresh_) {
+    RefreshModules();
+    modules_were_reloaded = true;
+  }
+  const LoadedModule *module = SearchForModule(modules_, address);
+  if (module) return module;
+
+  // dlopen/dlclose interceptors invalidate the module list, but when
+  // interception is disabled, we need to retry if the lookup fails in
+  // case the module list changed.
+#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
+  if (!modules_were_reloaded) {
+    RefreshModules();
+    module = SearchForModule(modules_, address);
+    if (module) return module;
+  }
+#endif
+
+  if (fallback_modules_.size()) {
+    module = SearchForModule(fallback_modules_, address);
+  }
+  return module;
+}
+
+// For now we assume the following protocol:
+// For each request of the form
+//   <module_name> <module_offset>
+// passed to STDIN, external symbolizer prints to STDOUT response:
+//   <function_name>
+//   <file_name>:<line_number>:<column_number>
+//   <function_name>
+//   <file_name>:<line_number>:<column_number>
+//   ...
+//   <empty line>
+class LLVMSymbolizerProcess : public SymbolizerProcess {
+ public:
+  explicit LLVMSymbolizerProcess(const char *path)
+      : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_MAC) {}
+
+ private:
+  bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
+    // Empty line marks the end of llvm-symbolizer output.
+    return length >= 2 && buffer[length - 1] == '\n' &&
+           buffer[length - 2] == '\n';
+  }
+
+  // When adding a new architecture, don't forget to also update
+  // script/asan_symbolize.py and sanitizer_common.h.
+  void GetArgV(const char *path_to_binary,
+               const char *(&argv)[kArgVMax]) const override {
+#if defined(__x86_64h__)
+    const char* const kSymbolizerArch = "--default-arch=x86_64h";
+#elif defined(__x86_64__)
+    const char* const kSymbolizerArch = "--default-arch=x86_64";
+#elif defined(__i386__)
+    const char* const kSymbolizerArch = "--default-arch=i386";
+#elif defined(__aarch64__)
+    const char* const kSymbolizerArch = "--default-arch=arm64";
+#elif defined(__arm__)
+    const char* const kSymbolizerArch = "--default-arch=arm";
+#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    const char* const kSymbolizerArch = "--default-arch=powerpc64";
+#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    const char* const kSymbolizerArch = "--default-arch=powerpc64le";
+#elif defined(__s390x__)
+    const char* const kSymbolizerArch = "--default-arch=s390x";
+#elif defined(__s390__)
+    const char* const kSymbolizerArch = "--default-arch=s390";
+#else
+    const char* const kSymbolizerArch = "--default-arch=unknown";
+#endif
+
+    const char *const inline_flag = common_flags()->symbolize_inline_frames
+                                        ? "--inlining=true"
+                                        : "--inlining=false";
+    int i = 0;
+    argv[i++] = path_to_binary;
+    argv[i++] = inline_flag;
+    argv[i++] = kSymbolizerArch;
+    argv[i++] = nullptr;
+  }
+};
+
+LLVMSymbolizer::LLVMSymbolizer(const char *path, LowLevelAllocator *allocator)
+    : symbolizer_process_(new(*allocator) LLVMSymbolizerProcess(path)) {}
+
+// Parse a <file>:<line>[:<column>] buffer. The file path may contain colons on
+// Windows, so extract tokens from the right hand side first. The column info is
+// also optional.
+static const char *ParseFileLineInfo(AddressInfo *info, const char *str) {
+  char *file_line_info = nullptr;
+  str = ExtractToken(str, "\n", &file_line_info);
+  CHECK(file_line_info);
+
+  if (uptr size = internal_strlen(file_line_info)) {
+    char *back = file_line_info + size - 1;
+    for (int i = 0; i < 2; ++i) {
+      while (back > file_line_info && IsDigit(*back)) --back;
+      if (*back != ':' || !IsDigit(back[1])) break;
+      info->column = info->line;
+      info->line = internal_atoll(back + 1);
+      // Truncate the string at the colon to keep only filename.
+      *back = '\0';
+      --back;
+    }
+    ExtractToken(file_line_info, "", &info->file);
+  }
+
+  InternalFree(file_line_info);
+  return str;
+}
+
+// Parses one or more two-line strings in the following format:
+//   <function_name>
+//   <file_name>:<line_number>[:<column_number>]
+// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of
+// them use the same output format.
+void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) {
+  bool top_frame = true;
+  SymbolizedStack *last = res;
+  while (true) {
+    char *function_name = nullptr;
+    str = ExtractToken(str, "\n", &function_name);
+    CHECK(function_name);
+    if (function_name[0] == '\0') {
+      // There are no more frames.
+      InternalFree(function_name);
+      break;
+    }
+    SymbolizedStack *cur;
+    if (top_frame) {
+      cur = res;
+      top_frame = false;
+    } else {
+      cur = SymbolizedStack::New(res->info.address);
+      cur->info.FillModuleInfo(res->info.module, res->info.module_offset,
+                               res->info.module_arch);
+      last->next = cur;
+      last = cur;
+    }
+
+    AddressInfo *info = &cur->info;
+    info->function = function_name;
+    str = ParseFileLineInfo(info, str);
+
+    // Functions and filenames can be "??", in which case we write 0
+    // to address info to mark that names are unknown.
+    if (0 == internal_strcmp(info->function, "??")) {
+      InternalFree(info->function);
+      info->function = 0;
+    }
+    if (0 == internal_strcmp(info->file, "??")) {
+      InternalFree(info->file);
+      info->file = 0;
+    }
+  }
+}
+
+// Parses a two-line string in the following format:
+//   <symbol_name>
+//   <start_address> <size>
+// Used by LLVMSymbolizer and InternalSymbolizer.
+void ParseSymbolizeDataOutput(const char *str, DataInfo *info) {
+  str = ExtractToken(str, "\n", &info->name);
+  str = ExtractUptr(str, " ", &info->start);
+  str = ExtractUptr(str, "\n", &info->size);
+}
+
+static void ParseSymbolizeFrameOutput(const char *str,
+                                      InternalMmapVector<LocalInfo> *locals) {
+  if (internal_strncmp(str, "??", 2) == 0)
+    return;
+
+  while (*str) {
+    LocalInfo local;
+    str = ExtractToken(str, "\n", &local.function_name);
+    str = ExtractToken(str, "\n", &local.name);
+
+    AddressInfo addr;
+    str = ParseFileLineInfo(&addr, str);
+    local.decl_file = addr.file;
+    local.decl_line = addr.line;
+
+    local.has_frame_offset = internal_strncmp(str, "??", 2) != 0;
+    str = ExtractSptr(str, " ", &local.frame_offset);
+
+    local.has_size = internal_strncmp(str, "??", 2) != 0;
+    str = ExtractUptr(str, " ", &local.size);
+
+    local.has_tag_offset = internal_strncmp(str, "??", 2) != 0;
+    str = ExtractUptr(str, "\n", &local.tag_offset);
+
+    locals->push_back(local);
+  }
+}
+
+bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
+  AddressInfo *info = &stack->info;
+  const char *buf = FormatAndSendCommand(
+      "CODE", info->module, info->module_offset, info->module_arch);
+  if (!buf)
+    return false;
+  ParseSymbolizePCOutput(buf, stack);
+  return true;
+}
+
+bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  const char *buf = FormatAndSendCommand(
+      "DATA", info->module, info->module_offset, info->module_arch);
+  if (!buf)
+    return false;
+  ParseSymbolizeDataOutput(buf, info);
+  info->start += (addr - info->module_offset); // Add the base address.
+  return true;
+}
+
+bool LLVMSymbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
+  const char *buf = FormatAndSendCommand(
+      "FRAME", info->module, info->module_offset, info->module_arch);
+  if (!buf)
+    return false;
+  ParseSymbolizeFrameOutput(buf, &info->locals);
+  return true;
+}
+
+const char *LLVMSymbolizer::FormatAndSendCommand(const char *command_prefix,
+                                                 const char *module_name,
+                                                 uptr module_offset,
+                                                 ModuleArch arch) {
+  CHECK(module_name);
+  int size_needed = 0;
+  if (arch == kModuleArchUnknown)
+    size_needed = internal_snprintf(buffer_, kBufferSize, "%s \"%s\" 0x%zx\n",
+                                    command_prefix, module_name, module_offset);
+  else
+    size_needed = internal_snprintf(buffer_, kBufferSize,
+                                    "%s \"%s:%s\" 0x%zx\n", command_prefix,
+                                    module_name, ModuleArchToString(arch),
+                                    module_offset);
+
+  if (size_needed >= static_cast<int>(kBufferSize)) {
+    Report("WARNING: Command buffer too small");
+    return nullptr;
+  }
+
+  return symbolizer_process_->SendCommand(buffer_);
+}
+
+SymbolizerProcess::SymbolizerProcess(const char *path, bool use_posix_spawn)
+    : path_(path),
+      input_fd_(kInvalidFd),
+      output_fd_(kInvalidFd),
+      times_restarted_(0),
+      failed_to_start_(false),
+      reported_invalid_path_(false),
+      use_posix_spawn_(use_posix_spawn) {
+  CHECK(path_);
+  CHECK_NE(path_[0], '\0');
+}
+
+static bool IsSameModule(const char* path) {
+  if (const char* ProcessName = GetProcessName()) {
+    if (const char* SymbolizerName = StripModuleName(path)) {
+      return !internal_strcmp(ProcessName, SymbolizerName);
+    }
+  }
+  return false;
+}
+
+const char *SymbolizerProcess::SendCommand(const char *command) {
+  if (failed_to_start_)
+    return nullptr;
+  if (IsSameModule(path_)) {
+    Report("WARNING: Symbolizer was blocked from starting itself!\n");
+    failed_to_start_ = true;
+    return nullptr;
+  }
+  for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) {
+    // Start or restart symbolizer if we failed to send command to it.
+    if (const char *res = SendCommandImpl(command))
+      return res;
+    Restart();
+  }
+  if (!failed_to_start_) {
+    Report("WARNING: Failed to use and restart external symbolizer!\n");
+    failed_to_start_ = true;
+  }
+  return nullptr;
+}
+
+const char *SymbolizerProcess::SendCommandImpl(const char *command) {
+  if (input_fd_ == kInvalidFd || output_fd_ == kInvalidFd)
+      return nullptr;
+  if (!WriteToSymbolizer(command, internal_strlen(command)))
+      return nullptr;
+  if (!ReadFromSymbolizer(buffer_, kBufferSize))
+      return nullptr;
+  return buffer_;
+}
+
+bool SymbolizerProcess::Restart() {
+  if (input_fd_ != kInvalidFd)
+    CloseFile(input_fd_);
+  if (output_fd_ != kInvalidFd)
+    CloseFile(output_fd_);
+  return StartSymbolizerSubprocess();
+}
+
+bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
+  if (max_length == 0)
+    return true;
+  uptr read_len = 0;
+  while (true) {
+    uptr just_read = 0;
+    bool success = ReadFromFile(input_fd_, buffer + read_len,
+                                max_length - read_len - 1, &just_read);
+    // We can't read 0 bytes, as we don't expect external symbolizer to close
+    // its stdout.
+    if (!success || just_read == 0) {
+      Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
+      return false;
+    }
+    read_len += just_read;
+    if (ReachedEndOfOutput(buffer, read_len))
+      break;
+    if (read_len + 1 == max_length) {
+      Report("WARNING: Symbolizer buffer too small\n");
+      read_len = 0;
+      break;
+    }
+  }
+  buffer[read_len] = '\0';
+  return true;
+}
+
+bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
+  if (length == 0)
+    return true;
+  uptr write_len = 0;
+  bool success = WriteToFile(output_fd_, buffer, length, &write_len);
+  if (!success || write_len != length) {
+    Report("WARNING: Can't write to symbolizer at fd %d\n", output_fd_);
+    return false;
+  }
+  return true;
+}
+
+#endif  // !SANITIZER_SYMBOLIZER_MARKUP
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp
new file mode 100644
index 0000000000..29cbf62acd
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp
@@ -0,0 +1,249 @@
+//===-- sanitizer_symbolizer_mac.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Implementation of Mac-specific "atos" symbolizer.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_mac.h"
+#include "sanitizer_symbolizer_mac.h"
+
+#include <dlfcn.h>
+#include <errno.h>
+#include <mach/mach.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <util.h>
+
+namespace __sanitizer {
+
+bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
+  Dl_info info;
+  int result = dladdr((const void *)addr, &info);
+  if (!result) return false;
+
+  CHECK(addr >= reinterpret_cast<uptr>(info.dli_saddr));
+  stack->info.function_offset = addr - reinterpret_cast<uptr>(info.dli_saddr);
+  const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
+  if (!demangled) return false;
+  stack->info.function = internal_strdup(demangled);
+  return true;
+}
+
+bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
+  Dl_info info;
+  int result = dladdr((const void *)addr, &info);
+  if (!result) return false;
+  const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
+  datainfo->name = internal_strdup(demangled);
+  datainfo->start = (uptr)info.dli_saddr;
+  return true;
+}
+
+#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
+
+// This cannot live in `AtosSymbolizerProcess` because instances of that object
+// are allocated by the internal allocator which under ASan is poisoned with
+// kAsanInternalHeapMagic.
+static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
+
+class AtosSymbolizerProcess : public SymbolizerProcess {
+ public:
+  explicit AtosSymbolizerProcess(const char *path)
+      : SymbolizerProcess(path, /*use_posix_spawn*/ true) {
+    pid_str_[0] = '\0';
+  }
+
+  void LateInitialize() {
+    if (SANITIZER_IOSSIM) {
+      // `putenv()` may call malloc/realloc so it is only safe to do this
+      // during LateInitialize() or later (i.e. we can't do this in the
+      // constructor).  We also can't do this in `StartSymbolizerSubprocess()`
+      // because in TSan we switch allocators when we're symbolizing.
+      // We use `putenv()` rather than `setenv()` so that we can later directly
+      // write into the storage without LibC getting involved to change what the
+      // variable is set to
+      int result = putenv(kAtosMachPortEnvEntry);
+      CHECK_EQ(result, 0);
+    }
+  }
+
+ private:
+  bool StartSymbolizerSubprocess() override {
+    // Configure sandbox before starting atos process.
+
+    // Put the string command line argument in the object so that it outlives
+    // the call to GetArgV.
+    internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
+
+    if (SANITIZER_IOSSIM) {
+      // `atos` in the simulator is restricted in its ability to retrieve the
+      // task port for the target process (us) so we need to do extra work
+      // to pass our task port to it.
+      mach_port_t ports[]{mach_task_self()};
+      kern_return_t ret =
+          mach_ports_register(mach_task_self(), ports, /*count=*/1);
+      CHECK_EQ(ret, KERN_SUCCESS);
+
+      // Set environment variable that signals to `atos` that it should look
+      // for our task port. We can't call `setenv()` here because it might call
+      // malloc/realloc. To avoid that we instead update the
+      // `mach_port_env_var_entry_` variable with our current PID.
+      uptr count = internal_snprintf(kAtosMachPortEnvEntry,
+                                     sizeof(kAtosMachPortEnvEntry),
+                                     K_ATOS_ENV_VAR "=%s", pid_str_);
+      CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
+      // Document our assumption but without calling `getenv()` in normal
+      // builds.
+      DCHECK(getenv(K_ATOS_ENV_VAR));
+      DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
+    }
+
+    return SymbolizerProcess::StartSymbolizerSubprocess();
+  }
+
+  bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
+    return (length >= 1 && buffer[length - 1] == '\n');
+  }
+
+  void GetArgV(const char *path_to_binary,
+               const char *(&argv)[kArgVMax]) const override {
+    int i = 0;
+    argv[i++] = path_to_binary;
+    argv[i++] = "-p";
+    argv[i++] = &pid_str_[0];
+    if (GetMacosAlignedVersion() == MacosVersion(10, 9)) {
+      // On Mavericks atos prints a deprecation warning which we suppress by
+      // passing -d. The warning isn't present on other OSX versions, even the
+      // newer ones.
+      argv[i++] = "-d";
+    }
+    argv[i++] = nullptr;
+  }
+
+  char pid_str_[16];
+  // Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
+  static_assert(sizeof(kAtosMachPortEnvEntry) ==
+                    (sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
+                "sizes should match");
+};
+
+#undef K_ATOS_ENV_VAR
+
+static bool ParseCommandOutput(const char *str, uptr addr, char **out_name,
+                               char **out_module, char **out_file, uptr *line,
+                               uptr *start_address) {
+  // Trim ending newlines.
+  char *trim;
+  ExtractTokenUpToDelimiter(str, "\n", &trim);
+
+  // The line from `atos` is in one of these formats:
+  //   myfunction (in library.dylib) (sourcefile.c:17)
+  //   myfunction (in library.dylib) + 0x1fe
+  //   myfunction (in library.dylib) + 15
+  //   0xdeadbeef (in library.dylib) + 0x1fe
+  //   0xdeadbeef (in library.dylib) + 15
+  //   0xdeadbeef (in library.dylib)
+  //   0xdeadbeef
+
+  const char *rest = trim;
+  char *symbol_name;
+  rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name);
+  if (rest[0] == '\0') {
+    InternalFree(symbol_name);
+    InternalFree(trim);
+    return false;
+  }
+
+  if (internal_strncmp(symbol_name, "0x", 2) != 0)
+    *out_name = symbol_name;
+  else
+    InternalFree(symbol_name);
+  rest = ExtractTokenUpToDelimiter(rest, ") ", out_module);
+
+  if (rest[0] == '(') {
+    if (out_file) {
+      rest++;
+      rest = ExtractTokenUpToDelimiter(rest, ":", out_file);
+      char *extracted_line_number;
+      rest = ExtractTokenUpToDelimiter(rest, ")", &extracted_line_number);
+      if (line) *line = (uptr)internal_atoll(extracted_line_number);
+      InternalFree(extracted_line_number);
+    }
+  } else if (rest[0] == '+') {
+    rest += 2;
+    uptr offset = internal_atoll(rest);
+    if (start_address) *start_address = addr - offset;
+  }
+
+  InternalFree(trim);
+  return true;
+}
+
+AtosSymbolizer::AtosSymbolizer(const char *path, LowLevelAllocator *allocator)
+    : process_(new (*allocator) AtosSymbolizerProcess(path)) {}
+
+bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
+  if (!process_) return false;
+  if (addr == 0) return false;
+  char command[32];
+  internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
+  const char *buf = process_->SendCommand(command);
+  if (!buf) return false;
+  uptr line;
+  uptr start_address = AddressInfo::kUnknown;
+  if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
+                          &stack->info.file, &line, &start_address)) {
+    process_ = nullptr;
+    return false;
+  }
+  stack->info.line = (int)line;
+
+  if (start_address == AddressInfo::kUnknown) {
+    // Fallback to dladdr() to get function start address if atos doesn't report
+    // it.
+    Dl_info info;
+    int result = dladdr((const void *)addr, &info);
+    if (result)
+      start_address = reinterpret_cast<uptr>(info.dli_saddr);
+  }
+
+  // Only assig to `function_offset` if we were able to get the function's
+  // start address.
+  if (start_address != AddressInfo::kUnknown) {
+    CHECK(addr >= start_address);
+    stack->info.function_offset = addr - start_address;
+  }
+  return true;
+}
+
+bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  if (!process_) return false;
+  char command[32];
+  internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
+  const char *buf = process_->SendCommand(command);
+  if (!buf) return false;
+  if (!ParseCommandOutput(buf, addr, &info->name, &info->module, nullptr,
+                          nullptr, &info->start)) {
+    process_ = nullptr;
+    return false;
+  }
+  return true;
+}
+
+void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp
new file mode 100644
index 0000000000..2963af9536
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp
@@ -0,0 +1,146 @@
+//===-- sanitizer_symbolizer_markup.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Implementation of offline markup symbolizer.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_SYMBOLIZER_MARKUP
+
+#if SANITIZER_FUCHSIA
+#include "sanitizer_symbolizer_fuchsia.h"
+#elif SANITIZER_RTEMS
+#include "sanitizer_symbolizer_rtems.h"
+#endif
+#include "sanitizer_stacktrace.h"
+#include "sanitizer_symbolizer.h"
+
+#include <limits.h>
+#include <unwind.h>
+
+namespace __sanitizer {
+
+// This generic support for offline symbolizing is based on the
+// Fuchsia port.  We don't do any actual symbolization per se.
+// Instead, we emit text containing raw addresses and raw linkage
+// symbol names, embedded in Fuchsia's symbolization markup format.
+// Fuchsia's logging infrastructure emits enough information about
+// process memory layout that a post-processing filter can do the
+// symbolization and pretty-print the markup.  See the spec at:
+// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+// It's expected to return a static buffer that will be reused on each call.
+const char *Symbolizer::Demangle(const char *name) {
+  static char buffer[kFormatDemangleMax];
+  internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
+  return buffer;
+}
+
+// This is used mostly for suppression matching.  Making it work
+// would enable "interceptor_via_lib" suppressions.  It's also used
+// once in UBSan to say "in module ..." in a message that also
+// includes an address in the module, so post-processing can already
+// pretty-print that so as to indicate the module.
+bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
+                                             uptr *module_address) {
+  return false;
+}
+
+// This is used in some places for suppression checking, which we
+// don't really support for Fuchsia.  It's also used in UBSan to
+// identify a PC location to a function name, so we always fill in
+// the function member with a string containing markup around the PC
+// value.
+// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
+// to render stack frames, but that should be changed to use
+// RenderStackFrame.
+SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
+  SymbolizedStack *s = SymbolizedStack::New(addr);
+  char buffer[kFormatFunctionMax];
+  internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
+  s->info.function = internal_strdup(buffer);
+  return s;
+}
+
+// Always claim we succeeded, so that RenderDataInfo will be called.
+bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+  info->Clear();
+  info->start = addr;
+  return true;
+}
+
+// We ignore the format argument to __sanitizer_symbolize_global.
+void RenderData(InternalScopedString *buffer, const char *format,
+                const DataInfo *DI, const char *strip_path_prefix) {
+  buffer->append(kFormatData, DI->start);
+}
+
+// We don't support the stack_trace_format flag at all.
+void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+                 const AddressInfo &info, bool vs_style,
+                 const char *strip_path_prefix, const char *strip_func_prefix) {
+  buffer->append(kFormatFrame, frame_no, info.address);
+}
+
+Symbolizer *Symbolizer::PlatformInit() {
+  return new (symbolizer_allocator_) Symbolizer({});
+}
+
+void Symbolizer::LateInitialize() {
+  Symbolizer::GetOrInit()->LateInitializeTools();
+}
+
+void StartReportDeadlySignal() {}
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {}
+
+#if SANITIZER_CAN_SLOW_UNWIND
+struct UnwindTraceArg {
+  BufferedStackTrace *stack;
+  u32 max_depth;
+};
+
+_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+  UnwindTraceArg *arg = static_cast<UnwindTraceArg *>(param);
+  CHECK_LT(arg->stack->size, arg->max_depth);
+  uptr pc = _Unwind_GetIP(ctx);
+  if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
+  arg->stack->trace_buffer[arg->stack->size++] = pc;
+  return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
+                                             : _URC_NO_REASON);
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
+  CHECK_GE(max_depth, 2);
+  size = 0;
+  UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
+  _Unwind_Backtrace(Unwind_Trace, &arg);
+  CHECK_GT(size, 0);
+  // We need to pop a few frames so that pc is on top.
+  uptr to_pop = LocatePcInTrace(pc);
+  // trace_buffer[0] belongs to the current function so we always pop it,
+  // unless there is only 1 frame in the stack trace (1 frame is always better
+  // than 0!).
+  PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
+  trace_buffer[0] = pc;
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
+  CHECK(context);
+  CHECK_GE(max_depth, 2);
+  UNREACHABLE("signal context doesn't exist");
+}
+#endif  // SANITIZER_CAN_SLOW_UNWIND
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_SYMBOLIZER_MARKUP
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
new file mode 100644
index 0000000000..3c379a8480
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
@@ -0,0 +1,492 @@
+//===-- sanitizer_symbolizer_posix_libcdep.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// POSIX-specific implementation of symbolizer parts.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_POSIX
+#include "sanitizer_allocator_internal.h"
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_posix.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_symbolizer_internal.h"
+#include "sanitizer_symbolizer_libbacktrace.h"
+#include "sanitizer_symbolizer_mac.h"
+
+#include <dlfcn.h>   // for dlsym()
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+// C++ demangling function, as required by Itanium C++ ABI. This is weak,
+// because we do not require a C++ ABI library to be linked to a program
+// using sanitizers; if it's not present, we'll just use the mangled name.
+namespace __cxxabiv1 {
+  extern "C" SANITIZER_WEAK_ATTRIBUTE
+  char *__cxa_demangle(const char *mangled, char *buffer,
+                                  size_t *length, int *status);
+}
+
+namespace __sanitizer {
+
+// Attempts to demangle the name via __cxa_demangle from __cxxabiv1.
+const char *DemangleCXXABI(const char *name) {
+  // FIXME: __cxa_demangle aggressively insists on allocating memory.
+  // There's not much we can do about that, short of providing our
+  // own demangler (libc++abi's implementation could be adapted so that
+  // it does not allocate). For now, we just call it anyway, and we leak
+  // the returned value.
+  if (&__cxxabiv1::__cxa_demangle)
+    if (const char *demangled_name =
+          __cxxabiv1::__cxa_demangle(name, 0, 0, 0))
+      return demangled_name;
+
+  return name;
+}
+
+// As of now, there are no headers for the Swift runtime. Once they are
+// present, we will weakly link since we do not require Swift runtime to be
+// linked.
+typedef char *(*swift_demangle_ft)(const char *mangledName,
+                                   size_t mangledNameLength, char *outputBuffer,
+                                   size_t *outputBufferSize, uint32_t flags);
+static swift_demangle_ft swift_demangle_f;
+
+// This must not happen lazily at symbolication time, because dlsym uses
+// malloc and thread-local storage, which is not a good thing to do during
+// symbolication.
+static void InitializeSwiftDemangler() {
+  swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
+  (void)dlerror(); // Cleanup error message in case of failure
+}
+
+// Attempts to demangle a Swift name. The demangler will return nullptr if a
+// non-Swift name is passed in.
+const char *DemangleSwift(const char *name) {
+  if (swift_demangle_f)
+    return swift_demangle_f(name, internal_strlen(name), 0, 0, 0);
+
+  return nullptr;
+}
+
+const char *DemangleSwiftAndCXX(const char *name) {
+  if (!name) return nullptr;
+  if (const char *swift_demangled_name = DemangleSwift(name))
+    return swift_demangled_name;
+  return DemangleCXXABI(name);
+}
+
+static bool CreateTwoHighNumberedPipes(int *infd_, int *outfd_) {
+  int *infd = NULL;
+  int *outfd = NULL;
+  // The client program may close its stdin and/or stdout and/or stderr
+  // thus allowing socketpair to reuse file descriptors 0, 1 or 2.
+  // In this case the communication between the forked processes may be
+  // broken if either the parent or the child tries to close or duplicate
+  // these descriptors. The loop below produces two pairs of file
+  // descriptors, each greater than 2 (stderr).
+  int sock_pair[5][2];
+  for (int i = 0; i < 5; i++) {
+    if (pipe(sock_pair[i]) == -1) {
+      for (int j = 0; j < i; j++) {
+        internal_close(sock_pair[j][0]);
+        internal_close(sock_pair[j][1]);
+      }
+      return false;
+    } else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) {
+      if (infd == NULL) {
+        infd = sock_pair[i];
+      } else {
+        outfd = sock_pair[i];
+        for (int j = 0; j < i; j++) {
+          if (sock_pair[j] == infd) continue;
+          internal_close(sock_pair[j][0]);
+          internal_close(sock_pair[j][1]);
+        }
+        break;
+      }
+    }
+  }
+  CHECK(infd);
+  CHECK(outfd);
+  infd_[0] = infd[0];
+  infd_[1] = infd[1];
+  outfd_[0] = outfd[0];
+  outfd_[1] = outfd[1];
+  return true;
+}
+
+bool SymbolizerProcess::StartSymbolizerSubprocess() {
+  if (!FileExists(path_)) {
+    if (!reported_invalid_path_) {
+      Report("WARNING: invalid path to external symbolizer!\n");
+      reported_invalid_path_ = true;
+    }
+    return false;
+  }
+
+  const char *argv[kArgVMax];
+  GetArgV(path_, argv);
+  pid_t pid;
+
+  // Report how symbolizer is being launched for debugging purposes.
+  if (Verbosity() >= 3) {
+    // Only use `Report` for first line so subsequent prints don't get prefixed
+    // with current PID.
+    Report("Launching Symbolizer process: ");
+    for (unsigned index = 0; index < kArgVMax && argv[index]; ++index)
+      Printf("%s ", argv[index]);
+    Printf("\n");
+  }
+
+  if (use_posix_spawn_) {
+#if SANITIZER_MAC
+    fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
+    if (fd == kInvalidFd) {
+      Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
+             errno);
+      return false;
+    }
+
+    input_fd_ = fd;
+    output_fd_ = fd;
+#else  // SANITIZER_MAC
+    UNIMPLEMENTED();
+#endif  // SANITIZER_MAC
+  } else {
+    fd_t infd[2] = {}, outfd[2] = {};
+    if (!CreateTwoHighNumberedPipes(infd, outfd)) {
+      Report("WARNING: Can't create a socket pair to start "
+             "external symbolizer (errno: %d)\n", errno);
+      return false;
+    }
+
+    pid = StartSubprocess(path_, argv, GetEnvP(), /* stdin */ outfd[0],
+                          /* stdout */ infd[1]);
+    if (pid < 0) {
+      internal_close(infd[0]);
+      internal_close(outfd[1]);
+      return false;
+    }
+
+    input_fd_ = infd[0];
+    output_fd_ = outfd[1];
+  }
+
+  CHECK_GT(pid, 0);
+
+  // Check that symbolizer subprocess started successfully.
+  SleepForMillis(kSymbolizerStartupTimeMillis);
+  if (!IsProcessRunning(pid)) {
+    // Either waitpid failed, or child has already exited.
+    Report("WARNING: external symbolizer didn't start up correctly!\n");
+    return false;
+  }
+
+  return true;
+}
+
+class Addr2LineProcess : public SymbolizerProcess {
+ public:
+  Addr2LineProcess(const char *path, const char *module_name)
+      : SymbolizerProcess(path), module_name_(internal_strdup(module_name)) {}
+
+  const char *module_name() const { return module_name_; }
+
+ private:
+  void GetArgV(const char *path_to_binary,
+               const char *(&argv)[kArgVMax]) const override {
+    int i = 0;
+    argv[i++] = path_to_binary;
+    argv[i++] = "-iCfe";
+    argv[i++] = module_name_;
+    argv[i++] = nullptr;
+  }
+
+  bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
+
+  bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
+    if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
+      return false;
+    // The returned buffer is empty when output is valid, but exceeds
+    // max_length.
+    if (*buffer == '\0')
+      return true;
+    // We should cut out output_terminator_ at the end of given buffer,
+    // appended by addr2line to mark the end of its meaningful output.
+    // We cannot scan buffer from it's beginning, because it is legal for it
+    // to start with output_terminator_ in case given offset is invalid. So,
+    // scanning from second character.
+    char *garbage = internal_strstr(buffer + 1, output_terminator_);
+    // This should never be NULL since buffer must end up with
+    // output_terminator_.
+    CHECK(garbage);
+    // Trim the buffer.
+    garbage[0] = '\0';
+    return true;
+  }
+
+  const char *module_name_;  // Owned, leaked.
+  static const char output_terminator_[];
+};
+
+const char Addr2LineProcess::output_terminator_[] = "??\n??:0\n";
+
+bool Addr2LineProcess::ReachedEndOfOutput(const char *buffer,
+                                          uptr length) const {
+  const size_t kTerminatorLen = sizeof(output_terminator_) - 1;
+  // Skip, if we read just kTerminatorLen bytes, because Addr2Line output
+  // should consist at least of two pairs of lines:
+  // 1. First one, corresponding to given offset to be symbolized
+  // (may be equal to output_terminator_, if offset is not valid).
+  // 2. Second one for output_terminator_, itself to mark the end of output.
+  if (length <= kTerminatorLen) return false;
+  // Addr2Line output should end up with output_terminator_.
+  return !internal_memcmp(buffer + length - kTerminatorLen,
+                          output_terminator_, kTerminatorLen);
+}
+
+class Addr2LinePool : public SymbolizerTool {
+ public:
+  explicit Addr2LinePool(const char *addr2line_path,
+                         LowLevelAllocator *allocator)
+      : addr2line_path_(addr2line_path), allocator_(allocator) {
+    addr2line_pool_.reserve(16);
+  }
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
+    if (const char *buf =
+            SendCommand(stack->info.module, stack->info.module_offset)) {
+      ParseSymbolizePCOutput(buf, stack);
+      return true;
+    }
+    return false;
+  }
+
+  bool SymbolizeData(uptr addr, DataInfo *info) override {
+    return false;
+  }
+
+ private:
+  const char *SendCommand(const char *module_name, uptr module_offset) {
+    Addr2LineProcess *addr2line = 0;
+    for (uptr i = 0; i < addr2line_pool_.size(); ++i) {
+      if (0 ==
+          internal_strcmp(module_name, addr2line_pool_[i]->module_name())) {
+        addr2line = addr2line_pool_[i];
+        break;
+      }
+    }
+    if (!addr2line) {
+      addr2line =
+          new(*allocator_) Addr2LineProcess(addr2line_path_, module_name);
+      addr2line_pool_.push_back(addr2line);
+    }
+    CHECK_EQ(0, internal_strcmp(module_name, addr2line->module_name()));
+    char buffer[kBufferSize];
+    internal_snprintf(buffer, kBufferSize, "0x%zx\n0x%zx\n",
+                      module_offset, dummy_address_);
+    return addr2line->SendCommand(buffer);
+  }
+
+  static const uptr kBufferSize = 64;
+  const char *addr2line_path_;
+  LowLevelAllocator *allocator_;
+  InternalMmapVector<Addr2LineProcess*> addr2line_pool_;
+  static const uptr dummy_address_ =
+      FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
+};
+
+#if SANITIZER_SUPPORTS_WEAK_HOOKS
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
+__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
+                           char *Buffer, int MaxLength,
+                           bool SymbolizeInlineFrames);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
+                                char *Buffer, int MaxLength);
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+void __sanitizer_symbolize_flush();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+int __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
+                                   int MaxLength);
+}  // extern "C"
+
+class InternalSymbolizer : public SymbolizerTool {
+ public:
+  static InternalSymbolizer *get(LowLevelAllocator *alloc) {
+    if (__sanitizer_symbolize_code != 0 &&
+        __sanitizer_symbolize_data != 0) {
+      return new(*alloc) InternalSymbolizer();
+    }
+    return 0;
+  }
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
+    bool result = __sanitizer_symbolize_code(
+        stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
+        common_flags()->symbolize_inline_frames);
+    if (result) ParseSymbolizePCOutput(buffer_, stack);
+    return result;
+  }
+
+  bool SymbolizeData(uptr addr, DataInfo *info) override {
+    bool result = __sanitizer_symbolize_data(info->module, info->module_offset,
+                                             buffer_, kBufferSize);
+    if (result) {
+      ParseSymbolizeDataOutput(buffer_, info);
+      info->start += (addr - info->module_offset);  // Add the base address.
+    }
+    return result;
+  }
+
+  void Flush() override {
+    if (__sanitizer_symbolize_flush)
+      __sanitizer_symbolize_flush();
+  }
+
+  const char *Demangle(const char *name) override {
+    if (__sanitizer_symbolize_demangle) {
+      for (uptr res_length = 1024;
+           res_length <= InternalSizeClassMap::kMaxSize;) {
+        char *res_buff = static_cast<char*>(InternalAlloc(res_length));
+        uptr req_length =
+            __sanitizer_symbolize_demangle(name, res_buff, res_length);
+        if (req_length > res_length) {
+          res_length = req_length + 1;
+          InternalFree(res_buff);
+          continue;
+        }
+        return res_buff;
+      }
+    }
+    return name;
+  }
+
+ private:
+  InternalSymbolizer() { }
+
+  static const int kBufferSize = 16 * 1024;
+  char buffer_[kBufferSize];
+};
+#else  // SANITIZER_SUPPORTS_WEAK_HOOKS
+
+class InternalSymbolizer : public SymbolizerTool {
+ public:
+  static InternalSymbolizer *get(LowLevelAllocator *alloc) { return 0; }
+};
+
+#endif  // SANITIZER_SUPPORTS_WEAK_HOOKS
+
+const char *Symbolizer::PlatformDemangle(const char *name) {
+  return DemangleSwiftAndCXX(name);
+}
+
+static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) {
+  const char *path = common_flags()->external_symbolizer_path;
+  const char *binary_name = path ? StripModuleName(path) : "";
+  if (path && path[0] == '\0') {
+    VReport(2, "External symbolizer is explicitly disabled.\n");
+    return nullptr;
+  } else if (!internal_strcmp(binary_name, "llvm-symbolizer")) {
+    VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
+    return new(*allocator) LLVMSymbolizer(path, allocator);
+  } else if (!internal_strcmp(binary_name, "atos")) {
+#if SANITIZER_MAC
+    VReport(2, "Using atos at user-specified path: %s\n", path);
+    return new(*allocator) AtosSymbolizer(path, allocator);
+#else  // SANITIZER_MAC
+    Report("ERROR: Using `atos` is only supported on Darwin.\n");
+    Die();
+#endif  // SANITIZER_MAC
+  } else if (!internal_strcmp(binary_name, "addr2line")) {
+    VReport(2, "Using addr2line at user-specified path: %s\n", path);
+    return new(*allocator) Addr2LinePool(path, allocator);
+  } else if (path) {
+    Report("ERROR: External symbolizer path is set to '%s' which isn't "
+           "a known symbolizer. Please set the path to the llvm-symbolizer "
+           "binary or other known tool.\n", path);
+    Die();
+  }
+
+  // Otherwise symbolizer program is unknown, let's search $PATH
+  CHECK(path == nullptr);
+#if SANITIZER_MAC
+  if (const char *found_path = FindPathToBinary("atos")) {
+    VReport(2, "Using atos found at: %s\n", found_path);
+    return new(*allocator) AtosSymbolizer(found_path, allocator);
+  }
+#endif  // SANITIZER_MAC
+  if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
+    VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
+    return new(*allocator) LLVMSymbolizer(found_path, allocator);
+  }
+  if (common_flags()->allow_addr2line) {
+    if (const char *found_path = FindPathToBinary("addr2line")) {
+      VReport(2, "Using addr2line found at: %s\n", found_path);
+      return new(*allocator) Addr2LinePool(found_path, allocator);
+    }
+  }
+  return nullptr;
+}
+
+static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
+                                  LowLevelAllocator *allocator) {
+  if (!common_flags()->symbolize) {
+    VReport(2, "Symbolizer is disabled.\n");
+    return;
+  }
+  if (IsAllocatorOutOfMemory()) {
+    VReport(2, "Cannot use internal symbolizer: out of memory\n");
+  } else if (SymbolizerTool *tool = InternalSymbolizer::get(allocator)) {
+    VReport(2, "Using internal symbolizer.\n");
+    list->push_back(tool);
+    return;
+  }
+  if (SymbolizerTool *tool = LibbacktraceSymbolizer::get(allocator)) {
+    VReport(2, "Using libbacktrace symbolizer.\n");
+    list->push_back(tool);
+    return;
+  }
+
+  if (SymbolizerTool *tool = ChooseExternalSymbolizer(allocator)) {
+    list->push_back(tool);
+  }
+
+#if SANITIZER_MAC
+  VReport(2, "Using dladdr symbolizer.\n");
+  list->push_back(new(*allocator) DlAddrSymbolizer());
+#endif  // SANITIZER_MAC
+}
+
+Symbolizer *Symbolizer::PlatformInit() {
+  IntrusiveList<SymbolizerTool> list;
+  list.clear();
+  ChooseSymbolizerTools(&list, &symbolizer_allocator_);
+  return new(symbolizer_allocator_) Symbolizer(list);
+}
+
+void Symbolizer::LateInitialize() {
+  Symbolizer::GetOrInit()->LateInitializeTools();
+  InitializeSwiftDemangler();
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_POSIX
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp
new file mode 100644
index 0000000000..c26724ceb7
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp
@@ -0,0 +1,293 @@
+//===-- sanitizer_symbolizer_report.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file is shared between AddressSanitizer and other sanitizer run-time
+/// libraries and implements symbolized reports related functions.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_report_decorator.h"
+#include "sanitizer_stacktrace.h"
+#include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_symbolizer.h"
+
+#if SANITIZER_POSIX
+# include "sanitizer_posix.h"
+# include <sys/mman.h>
+#endif
+
+namespace __sanitizer {
+
+#if !SANITIZER_GO
+void ReportErrorSummary(const char *error_type, const AddressInfo &info,
+                        const char *alt_tool_name) {
+  if (!common_flags()->print_summary) return;
+  InternalScopedString buff(kMaxSummaryLength);
+  buff.append("%s ", error_type);
+  RenderFrame(&buff, "%L %F", 0, info, common_flags()->symbolize_vs_style,
+              common_flags()->strip_path_prefix);
+  ReportErrorSummary(buff.data(), alt_tool_name);
+}
+#endif
+
+#if !SANITIZER_FUCHSIA
+
+bool ReportFile::SupportsColors() {
+  SpinMutexLock l(mu);
+  ReopenIfNecessary();
+  return SupportsColoredOutput(fd);
+}
+
+static INLINE bool ReportSupportsColors() {
+  return report_file.SupportsColors();
+}
+
+#else  // SANITIZER_FUCHSIA
+
+// Fuchsia's logs always go through post-processing that handles colorization.
+static INLINE bool ReportSupportsColors() { return true; }
+
+#endif  // !SANITIZER_FUCHSIA
+
+bool ColorizeReports() {
+  // FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color
+  // printing on Windows.
+  if (SANITIZER_WINDOWS)
+    return false;
+
+  const char *flag = common_flags()->color;
+  return internal_strcmp(flag, "always") == 0 ||
+         (internal_strcmp(flag, "auto") == 0 && ReportSupportsColors());
+}
+
+void ReportErrorSummary(const char *error_type, const StackTrace *stack,
+                        const char *alt_tool_name) {
+#if !SANITIZER_GO
+  if (!common_flags()->print_summary)
+    return;
+  if (stack->size == 0) {
+    ReportErrorSummary(error_type);
+    return;
+  }
+  // Currently, we include the first stack frame into the report summary.
+  // Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc).
+  uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]);
+  SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
+  ReportErrorSummary(error_type, frame->info, alt_tool_name);
+  frame->ClearAll();
+#endif
+}
+
+void ReportMmapWriteExec(int prot) {
+#if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
+  if ((prot & (PROT_WRITE | PROT_EXEC)) != (PROT_WRITE | PROT_EXEC))
+    return;
+
+  ScopedErrorReportLock l;
+  SanitizerCommonDecorator d;
+
+  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  uptr top = 0;
+  uptr bottom = 0;
+  GET_CALLER_PC_BP_SP;
+  (void)sp;
+  bool fast = common_flags()->fast_unwind_on_fatal;
+  if (StackTrace::WillUseFastUnwind(fast)) {
+    GetThreadStackTopAndBottom(false, &top, &bottom);
+    stack->Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom, true);
+  } else {
+    stack->Unwind(kStackTraceMax, pc, 0, nullptr, 0, 0, false);
+  }
+
+  Printf("%s", d.Warning());
+  Report("WARNING: %s: writable-executable page usage\n", SanitizerToolName);
+  Printf("%s", d.Default());
+
+  stack->Print();
+  ReportErrorSummary("w-and-x-usage", stack);
+#endif
+}
+
+#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO
+void StartReportDeadlySignal() {
+  // Write the first message using fd=2, just in case.
+  // It may actually fail to write in case stderr is closed.
+  CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName));
+  static const char kDeadlySignal[] = ":DEADLYSIGNAL\n";
+  CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1);
+}
+
+static void MaybeReportNonExecRegion(uptr pc) {
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+  MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
+      Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
+  }
+#endif
+}
+
+static void PrintMemoryByte(InternalScopedString *str, const char *before,
+                            u8 byte) {
+  SanitizerCommonDecorator d;
+  str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
+              d.Default());
+}
+
+static void MaybeDumpInstructionBytes(uptr pc) {
+  if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached()))
+    return;
+  InternalScopedString str(1024);
+  str.append("First 16 instruction bytes at pc: ");
+  if (IsAccessibleMemoryRange(pc, 16)) {
+    for (int i = 0; i < 16; ++i) {
+      PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
+    }
+    str.append("\n");
+  } else {
+    str.append("unaccessible\n");
+  }
+  Report("%s", str.data());
+}
+
+static void MaybeDumpRegisters(void *context) {
+  if (!common_flags()->dump_registers) return;
+  SignalContext::DumpAllRegisters(context);
+}
+
+static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
+                                    UnwindSignalStackCallbackType unwind,
+                                    const void *unwind_context) {
+  SanitizerCommonDecorator d;
+  Printf("%s", d.Warning());
+  static const char kDescription[] = "stack-overflow";
+  Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n",
+         SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc,
+         (void *)sig.bp, (void *)sig.sp, tid);
+  Printf("%s", d.Default());
+  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  unwind(sig, unwind_context, stack);
+  stack->Print();
+  ReportErrorSummary(kDescription, stack);
+}
+
+static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
+                                   UnwindSignalStackCallbackType unwind,
+                                   const void *unwind_context) {
+  SanitizerCommonDecorator d;
+  Printf("%s", d.Warning());
+  const char *description = sig.Describe();
+  if (sig.is_memory_access && !sig.is_true_faulting_addr)
+    Report("ERROR: %s: %s on unknown address (pc %p bp %p sp %p T%d)\n",
+           SanitizerToolName, description, (void *)sig.pc, (void *)sig.bp,
+           (void *)sig.sp, tid);
+  else
+    Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n",
+           SanitizerToolName, description, (void *)sig.addr, (void *)sig.pc,
+           (void *)sig.bp, (void *)sig.sp, tid);
+  Printf("%s", d.Default());
+  if (sig.pc < GetPageSizeCached())
+    Report("Hint: pc points to the zero page.\n");
+  if (sig.is_memory_access) {
+    const char *access_type =
+        sig.write_flag == SignalContext::WRITE
+            ? "WRITE"
+            : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
+    Report("The signal is caused by a %s memory access.\n", access_type);
+    if (!sig.is_true_faulting_addr)
+      Report("Hint: this fault was caused by a dereference of a high value "
+             "address (see register values below).  Dissassemble the provided "
+             "pc to learn which register was used.\n");
+    else if (sig.addr < GetPageSizeCached())
+      Report("Hint: address points to the zero page.\n");
+  }
+  MaybeReportNonExecRegion(sig.pc);
+  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  unwind(sig, unwind_context, stack);
+  stack->Print();
+  MaybeDumpInstructionBytes(sig.pc);
+  MaybeDumpRegisters(sig.context);
+  Printf("%s can not provide additional info.\n", SanitizerToolName);
+  ReportErrorSummary(description, stack);
+}
+
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {
+  if (sig.IsStackOverflow())
+    ReportStackOverflowImpl(sig, tid, unwind, unwind_context);
+  else
+    ReportDeadlySignalImpl(sig, tid, unwind, unwind_context);
+}
+
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+                        UnwindSignalStackCallbackType unwind,
+                        const void *unwind_context) {
+  StartReportDeadlySignal();
+  ScopedErrorReportLock rl;
+  SignalContext sig(siginfo, context);
+  ReportDeadlySignal(sig, tid, unwind, unwind_context);
+  Report("ABORTING\n");
+  Die();
+}
+
+#endif  // !SANITIZER_FUCHSIA && !SANITIZER_GO
+
+static atomic_uintptr_t reporting_thread = {0};
+static StaticSpinMutex CommonSanitizerReportMutex;
+
+ScopedErrorReportLock::ScopedErrorReportLock() {
+  uptr current = GetThreadSelf();
+  for (;;) {
+    uptr expected = 0;
+    if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
+                                       memory_order_relaxed)) {
+      // We've claimed reporting_thread so proceed.
+      CommonSanitizerReportMutex.Lock();
+      return;
+    }
+
+    if (expected == current) {
+      // This is either asynch signal or nested error during error reporting.
+      // Fail simple to avoid deadlocks in Report().
+
+      // Can't use Report() here because of potential deadlocks in nested
+      // signal handlers.
+      CatastrophicErrorWrite(SanitizerToolName,
+                             internal_strlen(SanitizerToolName));
+      static const char msg[] = ": nested bug in the same thread, aborting.\n";
+      CatastrophicErrorWrite(msg, sizeof(msg) - 1);
+
+      internal__exit(common_flags()->exitcode);
+    }
+
+    internal_sched_yield();
+  }
+}
+
+ScopedErrorReportLock::~ScopedErrorReportLock() {
+  CommonSanitizerReportMutex.Unlock();
+  atomic_store_relaxed(&reporting_thread, 0);
+}
+
+void ScopedErrorReportLock::CheckLocked() {
+  CommonSanitizerReportMutex.CheckLocked();
+}
+
+}  // namespace __sanitizer
diff --git a/lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp b/lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp
new file mode 100644
index 0000000000..373437e7ee
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp
@@ -0,0 +1,318 @@
+//===-- sanitizer_symbolizer_win.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+// Windows-specific implementation of symbolizer parts.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#include "sanitizer_dbghelp.h"
+#include "sanitizer_symbolizer_internal.h"
+
+namespace __sanitizer {
+
+decltype(::StackWalk64) *StackWalk64;
+decltype(::SymCleanup) *SymCleanup;
+decltype(::SymFromAddr) *SymFromAddr;
+decltype(::SymFunctionTableAccess64) *SymFunctionTableAccess64;
+decltype(::SymGetLineFromAddr64) *SymGetLineFromAddr64;
+decltype(::SymGetModuleBase64) *SymGetModuleBase64;
+decltype(::SymGetSearchPathW) *SymGetSearchPathW;
+decltype(::SymInitialize) *SymInitialize;
+decltype(::SymSetOptions) *SymSetOptions;
+decltype(::SymSetSearchPathW) *SymSetSearchPathW;
+decltype(::UnDecorateSymbolName) *UnDecorateSymbolName;
+
+namespace {
+
+class WinSymbolizerTool : public SymbolizerTool {
+ public:
+  // The constructor is provided to avoid synthesized memsets.
+  WinSymbolizerTool() {}
+
+  bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
+  bool SymbolizeData(uptr addr, DataInfo *info) override {
+    return false;
+  }
+  const char *Demangle(const char *name) override;
+};
+
+bool is_dbghelp_initialized = false;
+
+bool TrySymInitialize() {
+  SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME | SYMOPT_LOAD_LINES);
+  return SymInitialize(GetCurrentProcess(), 0, TRUE);
+  // FIXME: We don't call SymCleanup() on exit yet - should we?
+}
+
+}  // namespace
+
+// Initializes DbgHelp library, if it's not yet initialized. Calls to this
+// function should be synchronized with respect to other calls to DbgHelp API
+// (e.g. from WinSymbolizerTool).
+void InitializeDbgHelpIfNeeded() {
+  if (is_dbghelp_initialized)
+    return;
+
+  HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
+  CHECK(dbghelp && "failed to load dbghelp.dll");
+
+#define DBGHELP_IMPORT(name)                                                  \
+  do {                                                                        \
+    name =                                                                    \
+        reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \
+    CHECK(name != nullptr);                                                   \
+  } while (0)
+  DBGHELP_IMPORT(StackWalk64);
+  DBGHELP_IMPORT(SymCleanup);
+  DBGHELP_IMPORT(SymFromAddr);
+  DBGHELP_IMPORT(SymFunctionTableAccess64);
+  DBGHELP_IMPORT(SymGetLineFromAddr64);
+  DBGHELP_IMPORT(SymGetModuleBase64);
+  DBGHELP_IMPORT(SymGetSearchPathW);
+  DBGHELP_IMPORT(SymInitialize);
+  DBGHELP_IMPORT(SymSetOptions);
+  DBGHELP_IMPORT(SymSetSearchPathW);
+  DBGHELP_IMPORT(UnDecorateSymbolName);
+#undef DBGHELP_IMPORT
+
+  if (!TrySymInitialize()) {
+    // OK, maybe the client app has called SymInitialize already.
+    // That's a bit unfortunate for us as all the DbgHelp functions are
+    // single-threaded and we can't coordinate with the app.
+    // FIXME: Can we stop the other threads at this point?
+    // Anyways, we have to reconfigure stuff to make sure that SymInitialize
+    // has all the appropriate options set.
+    // Cross our fingers and reinitialize DbgHelp.
+    Report("*** WARNING: Failed to initialize DbgHelp!              ***\n");
+    Report("*** Most likely this means that the app is already      ***\n");
+    Report("*** using DbgHelp, possibly with incompatible flags.    ***\n");
+    Report("*** Due to technical reasons, symbolization might crash ***\n");
+    Report("*** or produce wrong results.                           ***\n");
+    SymCleanup(GetCurrentProcess());
+    TrySymInitialize();
+  }
+  is_dbghelp_initialized = true;
+
+  // When an executable is run from a location different from the one where it
+  // was originally built, we may not see the nearby PDB files.
+  // To work around this, let's append the directory of the main module
+  // to the symbol search path.  All the failures below are not fatal.
+  const size_t kSymPathSize = 2048;
+  static wchar_t path_buffer[kSymPathSize + 1 + MAX_PATH];
+  if (!SymGetSearchPathW(GetCurrentProcess(), path_buffer, kSymPathSize)) {
+    Report("*** WARNING: Failed to SymGetSearchPathW ***\n");
+    return;
+  }
+  size_t sz = wcslen(path_buffer);
+  if (sz) {
+    CHECK_EQ(0, wcscat_s(path_buffer, L";"));
+    sz++;
+  }
+  DWORD res = GetModuleFileNameW(NULL, path_buffer + sz, MAX_PATH);
+  if (res == 0 || res == MAX_PATH) {
+    Report("*** WARNING: Failed to getting the EXE directory ***\n");
+    return;
+  }
+  // Write the zero character in place of the last backslash to get the
+  // directory of the main module at the end of path_buffer.
+  wchar_t *last_bslash = wcsrchr(path_buffer + sz, L'\\');
+  CHECK_NE(last_bslash, 0);
+  *last_bslash = L'\0';
+  if (!SymSetSearchPathW(GetCurrentProcess(), path_buffer)) {
+    Report("*** WARNING: Failed to SymSetSearchPathW\n");
+    return;
+  }
+}
+
+bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
+  InitializeDbgHelpIfNeeded();
+
+  // See http://msdn.microsoft.com/en-us/library/ms680578(VS.85).aspx
+  char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)];
+  PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer;
+  symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+  symbol->MaxNameLen = MAX_SYM_NAME;
+  DWORD64 offset = 0;
+  BOOL got_objname = SymFromAddr(GetCurrentProcess(),
+                                 (DWORD64)addr, &offset, symbol);
+  if (!got_objname)
+    return false;
+
+  DWORD unused;
+  IMAGEHLP_LINE64 line_info;
+  line_info.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+  BOOL got_fileline = SymGetLineFromAddr64(GetCurrentProcess(), (DWORD64)addr,
+                                           &unused, &line_info);
+  frame->info.function = internal_strdup(symbol->Name);
+  frame->info.function_offset = (uptr)offset;
+  if (got_fileline) {
+    frame->info.file = internal_strdup(line_info.FileName);
+    frame->info.line = line_info.LineNumber;
+  }
+  // Only consider this a successful symbolization attempt if we got file info.
+  // Otherwise, try llvm-symbolizer.
+  return got_fileline;
+}
+
+const char *WinSymbolizerTool::Demangle(const char *name) {
+  CHECK(is_dbghelp_initialized);
+  static char demangle_buffer[1000];
+  if (name[0] == '\01' &&
+      UnDecorateSymbolName(name + 1, demangle_buffer, sizeof(demangle_buffer),
+                           UNDNAME_NAME_ONLY))
+    return demangle_buffer;
+  else
+    return name;
+}
+
+const char *Symbolizer::PlatformDemangle(const char *name) {
+  return name;
+}
+
+namespace {
+struct ScopedHandle {
+  ScopedHandle() : h_(nullptr) {}
+  explicit ScopedHandle(HANDLE h) : h_(h) {}
+  ~ScopedHandle() {
+    if (h_)
+      ::CloseHandle(h_);
+  }
+  HANDLE get() { return h_; }
+  HANDLE *receive() { return &h_; }
+  HANDLE release() {
+    HANDLE h = h_;
+    h_ = nullptr;
+    return h;
+  }
+  HANDLE h_;
+};
+} // namespace
+
+bool SymbolizerProcess::StartSymbolizerSubprocess() {
+  // Create inherited pipes for stdin and stdout.
+  ScopedHandle stdin_read, stdin_write;
+  ScopedHandle stdout_read, stdout_write;
+  SECURITY_ATTRIBUTES attrs;
+  attrs.nLength = sizeof(SECURITY_ATTRIBUTES);
+  attrs.bInheritHandle = TRUE;
+  attrs.lpSecurityDescriptor = nullptr;
+  if (!::CreatePipe(stdin_read.receive(), stdin_write.receive(), &attrs, 0) ||
+      !::CreatePipe(stdout_read.receive(), stdout_write.receive(), &attrs, 0)) {
+    VReport(2, "WARNING: %s CreatePipe failed (error code: %d)\n",
+            SanitizerToolName, path_, GetLastError());
+    return false;
+  }
+
+  // Don't inherit the writing end of stdin or the reading end of stdout.
+  if (!SetHandleInformation(stdin_write.get(), HANDLE_FLAG_INHERIT, 0) ||
+      !SetHandleInformation(stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) {
+    VReport(2, "WARNING: %s SetHandleInformation failed (error code: %d)\n",
+            SanitizerToolName, path_, GetLastError());
+    return false;
+  }
+
+  // Compute the command line. Wrap double quotes around everything.
+  const char *argv[kArgVMax];
+  GetArgV(path_, argv);
+  InternalScopedString command_line(kMaxPathLength * 3);
+  for (int i = 0; argv[i]; i++) {
+    const char *arg = argv[i];
+    int arglen = internal_strlen(arg);
+    // Check that tool command lines are simple and that complete escaping is
+    // unnecessary.
+    CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
+    CHECK(!internal_strstr(arg, "\\\\") &&
+          "double backslashes in args unsupported");
+    CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
+          "args ending in backslash and empty args unsupported");
+    command_line.append("\"%s\" ", arg);
+  }
+  VReport(3, "Launching symbolizer command: %s\n", command_line.data());
+
+  // Launch llvm-symbolizer with stdin and stdout redirected.
+  STARTUPINFOA si;
+  memset(&si, 0, sizeof(si));
+  si.cb = sizeof(si);
+  si.dwFlags |= STARTF_USESTDHANDLES;
+  si.hStdInput = stdin_read.get();
+  si.hStdOutput = stdout_write.get();
+  PROCESS_INFORMATION pi;
+  memset(&pi, 0, sizeof(pi));
+  if (!CreateProcessA(path_,               // Executable
+                      command_line.data(), // Command line
+                      nullptr,             // Process handle not inheritable
+                      nullptr,             // Thread handle not inheritable
+                      TRUE,                // Set handle inheritance to TRUE
+                      0,                   // Creation flags
+                      nullptr,             // Use parent's environment block
+                      nullptr,             // Use parent's starting directory
+                      &si, &pi)) {
+    VReport(2, "WARNING: %s failed to create process for %s (error code: %d)\n",
+            SanitizerToolName, path_, GetLastError());
+    return false;
+  }
+
+  // Process creation succeeded, so transfer handle ownership into the fields.
+  input_fd_ = stdout_read.release();
+  output_fd_ = stdin_write.release();
+
+  // The llvm-symbolizer process is responsible for quitting itself when the
+  // stdin pipe is closed, so we don't need these handles. Close them to prevent
+  // leaks. If we ever want to try to kill the symbolizer process from the
+  // parent, we'll want to hang on to these handles.
+  CloseHandle(pi.hProcess);
+  CloseHandle(pi.hThread);
+  return true;
+}
+
+static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
+                                  LowLevelAllocator *allocator) {
+  if (!common_flags()->symbolize) {
+    VReport(2, "Symbolizer is disabled.\n");
+    return;
+  }
+
+  // Add llvm-symbolizer in case the binary has dwarf.
+  const char *user_path = common_flags()->external_symbolizer_path;
+  const char *path =
+      user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
+  if (path) {
+    VReport(2, "Using llvm-symbolizer at %spath: %s\n",
+            user_path ? "user-specified " : "", path);
+    list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
+  } else {
+    if (user_path && user_path[0] == '\0') {
+      VReport(2, "External symbolizer is explicitly disabled.\n");
+    } else {
+      VReport(2, "External symbolizer is not present.\n");
+    }
+  }
+
+  // Add the dbghelp based symbolizer.
+  list->push_back(new(*allocator) WinSymbolizerTool());
+}
+
+Symbolizer *Symbolizer::PlatformInit() {
+  IntrusiveList<SymbolizerTool> list;
+  list.clear();
+  ChooseSymbolizerTools(&list, &symbolizer_allocator_);
+
+  return new(symbolizer_allocator_) Symbolizer(list);
+}
+
+void Symbolizer::LateInitialize() {
+  Symbolizer::GetOrInit()->LateInitializeTools();
+}
+
+}  // namespace __sanitizer
+
+#endif  // _WIN32
diff --git a/lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
new file mode 100644
index 0000000000..b2628dcc4d
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
@@ -0,0 +1,180 @@
+//===-- sanitizer_unwind_linux_libcdep.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the unwind.h-based (aka "slow") stack unwinding routines
+// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+    SANITIZER_SOLARIS
+#include "sanitizer_common.h"
+#include "sanitizer_stacktrace.h"
+
+#if SANITIZER_ANDROID
+#include <dlfcn.h>  // for dlopen()
+#endif
+
+#if SANITIZER_FREEBSD
+#define _GNU_SOURCE  // to declare _Unwind_Backtrace() from <unwind.h>
+#endif
+#include <unwind.h>
+
+namespace __sanitizer {
+
+namespace {
+
+//---------------------------- UnwindSlow --------------------------------------
+
+typedef struct {
+  uptr absolute_pc;
+  uptr stack_top;
+  uptr stack_size;
+} backtrace_frame_t;
+
+extern "C" {
+typedef void *(*acquire_my_map_info_list_func)();
+typedef void (*release_my_map_info_list_func)(void *map);
+typedef sptr (*unwind_backtrace_signal_arch_func)(
+    void *siginfo, void *sigcontext, void *map_info_list,
+    backtrace_frame_t *backtrace, uptr ignore_depth, uptr max_depth);
+acquire_my_map_info_list_func acquire_my_map_info_list;
+release_my_map_info_list_func release_my_map_info_list;
+unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
+} // extern "C"
+
+#if defined(__arm__) && !SANITIZER_NETBSD
+// NetBSD uses dwarf EH
+#define UNWIND_STOP _URC_END_OF_STACK
+#define UNWIND_CONTINUE _URC_NO_REASON
+#else
+#define UNWIND_STOP _URC_NORMAL_STOP
+#define UNWIND_CONTINUE _URC_NO_REASON
+#endif
+
+uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
+#if defined(__arm__) && !SANITIZER_MAC
+  uptr val;
+  _Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
+      15 /* r15 = PC */, _UVRSD_UINT32, &val);
+  CHECK(res == _UVRSR_OK && "_Unwind_VRS_Get failed");
+  // Clear the Thumb bit.
+  return val & ~(uptr)1;
+#else
+  return (uptr)_Unwind_GetIP(ctx);
+#endif
+}
+
+struct UnwindTraceArg {
+  BufferedStackTrace *stack;
+  u32 max_depth;
+};
+
+_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+  UnwindTraceArg *arg = (UnwindTraceArg*)param;
+  CHECK_LT(arg->stack->size, arg->max_depth);
+  uptr pc = Unwind_GetIP(ctx);
+  const uptr kPageSize = GetPageSizeCached();
+  // Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
+  // x86_64) is invalid and stop unwinding here.  If we're adding support for
+  // a platform where this isn't true, we need to reconsider this check.
+  if (pc < kPageSize) return UNWIND_STOP;
+  arg->stack->trace_buffer[arg->stack->size++] = pc;
+  if (arg->stack->size == arg->max_depth) return UNWIND_STOP;
+  return UNWIND_CONTINUE;
+}
+
+}  // namespace
+
+#if SANITIZER_ANDROID
+void SanitizerInitializeUnwinder() {
+  if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
+
+  // Pre-lollipop Android can not unwind through signal handler frames with
+  // libgcc unwinder, but it has a libcorkscrew.so library with the necessary
+  // workarounds.
+  void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
+  if (!p) {
+    VReport(1,
+            "Failed to open libcorkscrew.so. You may see broken stack traces "
+            "in SEGV reports.");
+    return;
+  }
+  acquire_my_map_info_list =
+      (acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
+  release_my_map_info_list =
+      (release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
+  unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
+      p, "unwind_backtrace_signal_arch");
+  if (!acquire_my_map_info_list || !release_my_map_info_list ||
+      !unwind_backtrace_signal_arch) {
+    VReport(1,
+            "Failed to find one of the required symbols in libcorkscrew.so. "
+            "You may see broken stack traces in SEGV reports.");
+    acquire_my_map_info_list = 0;
+    unwind_backtrace_signal_arch = 0;
+    release_my_map_info_list = 0;
+  }
+}
+#endif
+
+void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
+  CHECK_GE(max_depth, 2);
+  size = 0;
+  UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
+  _Unwind_Backtrace(Unwind_Trace, &arg);
+  // We need to pop a few frames so that pc is on top.
+  uptr to_pop = LocatePcInTrace(pc);
+  // trace_buffer[0] belongs to the current function so we always pop it,
+  // unless there is only 1 frame in the stack trace (1 frame is always better
+  // than 0!).
+  // 1-frame stacks don't normally happen, but this depends on the actual
+  // unwinder implementation (libgcc, libunwind, etc) which is outside of our
+  // control.
+  if (to_pop == 0 && size > 1)
+    to_pop = 1;
+  PopStackFrames(to_pop);
+#if defined(__GNUC__) && defined(__sparc__)
+  // __builtin_return_address returns the address of the call instruction
+  // on the SPARC and not the return address, so we need to compensate.
+  trace_buffer[0] = GetNextInstructionPc(pc);
+#else
+  trace_buffer[0] = pc;
+#endif
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
+  CHECK(context);
+  CHECK_GE(max_depth, 2);
+  if (!unwind_backtrace_signal_arch) {
+    UnwindSlow(pc, max_depth);
+    return;
+  }
+
+  void *map = acquire_my_map_info_list();
+  CHECK(map);
+  InternalMmapVector<backtrace_frame_t> frames(kStackTraceMax);
+  // siginfo argument appears to be unused.
+  sptr res = unwind_backtrace_signal_arch(/* siginfo */ 0, context, map,
+                                          frames.data(),
+                                          /* ignore_depth */ 0, max_depth);
+  release_my_map_info_list(map);
+  if (res < 0) return;
+  CHECK_LE((uptr)res, kStackTraceMax);
+
+  size = 0;
+  // +2 compensate for libcorkscrew unwinder returning addresses of call
+  // instructions instead of raw return addresses.
+  for (sptr i = 0; i < res; ++i)
+    trace_buffer[size++] = frames[i].absolute_pc + 2;
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+        // SANITIZER_SOLARIS
diff --git a/lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp b/lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp
new file mode 100644
index 0000000000..8e06940685
--- /dev/null
+++ b/lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp
@@ -0,0 +1,75 @@
+//===-- sanitizer_unwind_win.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Sanitizer unwind Windows specific functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_WINDOWS
+
+#define WIN32_LEAN_AND_MEAN
+#define NOGDI
+#include <windows.h>
+
+#include "sanitizer_dbghelp.h"  // for StackWalk64
+#include "sanitizer_stacktrace.h"
+#include "sanitizer_symbolizer.h"  // for InitializeDbgHelpIfNeeded
+
+using namespace __sanitizer;
+
+#if !SANITIZER_GO
+void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
+  CHECK_GE(max_depth, 2);
+  // FIXME: CaptureStackBackTrace might be too slow for us.
+  // FIXME: Compare with StackWalk64.
+  // FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc
+  size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax),
+    (void **)&trace_buffer[0], 0);
+  if (size == 0)
+    return;
+
+  // Skip the RTL frames by searching for the PC in the stacktrace.
+  uptr pc_location = LocatePcInTrace(pc);
+  PopStackFrames(pc_location);
+}
+
+void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
+  CHECK(context);
+  CHECK_GE(max_depth, 2);
+  CONTEXT ctx = *(CONTEXT *)context;
+  STACKFRAME64 stack_frame;
+  memset(&stack_frame, 0, sizeof(stack_frame));
+
+  InitializeDbgHelpIfNeeded();
+
+  size = 0;
+#if defined(_WIN64)
+  int machine_type = IMAGE_FILE_MACHINE_AMD64;
+  stack_frame.AddrPC.Offset = ctx.Rip;
+  stack_frame.AddrFrame.Offset = ctx.Rbp;
+  stack_frame.AddrStack.Offset = ctx.Rsp;
+#else
+  int machine_type = IMAGE_FILE_MACHINE_I386;
+  stack_frame.AddrPC.Offset = ctx.Eip;
+  stack_frame.AddrFrame.Offset = ctx.Ebp;
+  stack_frame.AddrStack.Offset = ctx.Esp;
+#endif
+  stack_frame.AddrPC.Mode = AddrModeFlat;
+  stack_frame.AddrFrame.Mode = AddrModeFlat;
+  stack_frame.AddrStack.Mode = AddrModeFlat;
+  while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
+    &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
+    SymGetModuleBase64, NULL) &&
+    size < Min(max_depth, kStackTraceMax)) {
+    trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
+  }
+}
+#endif  // #if !SANITIZER_GO
+
+#endif  // SANITIZER_WINDOWS
diff --git a/lib/tsan/tsan_interceptors_mac.cpp b/lib/tsan/tsan_interceptors_mac.cpp
new file mode 100644
index 0000000000..aa29536d86
--- /dev/null
+++ b/lib/tsan/tsan_interceptors_mac.cpp
@@ -0,0 +1,519 @@
+//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific interceptors.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_interface.h"
+#include "tsan_interface_ann.h"
+#include "sanitizer_common/sanitizer_addrhashmap.h"
+
+#include <errno.h>
+#include <libkern/OSAtomic.h>
+#include <objc/objc-sync.h>
+#include <os/lock.h>
+#include <sys/ucontext.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+#include <xpc/xpc.h>
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+typedef long long_t;
+
+extern "C" {
+int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
+int setcontext(const ucontext_t *ucp);
+}
+
+namespace __tsan {
+
+// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
+// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
+// actually aliases of each other, and we cannot have different interceptors for
+// them, because they're actually the same function.  Thus, we have to stay
+// conservative and treat the non-barrier versions as mo_acq_rel.
+static const morder kMacOrderBarrier = mo_acq_rel;
+static const morder kMacOrderNonBarrier = mo_acq_rel;
+
+#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                 \
+    SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                 \
+    return tsan_atomic_f((volatile tsan_t *)ptr, x, mo);                \
+  }
+
+#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                        \
+    SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                        \
+    return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x;                   \
+  }
+
+#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
+  TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                             \
+    SCOPED_TSAN_INTERCEPTOR(f, ptr);                                           \
+    return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1;                   \
+  }
+
+#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
+                                     mo)                                    \
+  TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                          \
+    SCOPED_TSAN_INTERCEPTOR(f, ptr);                                        \
+    return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1;                \
+  }
+
+#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m)                  \
+  m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,             \
+    kMacOrderNonBarrier)                                                       \
+  m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,    \
+    kMacOrderBarrier)                                                          \
+  m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f,             \
+    kMacOrderNonBarrier)                                                       \
+  m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f,    \
+    kMacOrderBarrier)
+
+#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig)             \
+  m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,            \
+    kMacOrderNonBarrier)                                                       \
+  m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,   \
+    kMacOrderBarrier)                                                          \
+  m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
+    kMacOrderNonBarrier)                                                       \
+  m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier,                           \
+    __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
+
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
+                                 OSATOMIC_INTERCEPTOR_PLUS_X)
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
+                                 OSATOMIC_INTERCEPTOR_PLUS_1)
+OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
+                                 OSATOMIC_INTERCEPTOR_MINUS_1)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
+                              OSATOMIC_INTERCEPTOR)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
+                              OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
+OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
+                              OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
+
+#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t)              \
+  TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) {    \
+    SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr);                  \
+    return tsan_atomic_f##_compare_exchange_strong(                         \
+        (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,    \
+        kMacOrderNonBarrier, kMacOrderNonBarrier);                          \
+  }                                                                         \
+                                                                            \
+  TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value,              \
+                   t volatile *ptr) {                                       \
+    SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr);         \
+    return tsan_atomic_f##_compare_exchange_strong(                         \
+        (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,    \
+        kMacOrderBarrier, kMacOrderNonBarrier);                             \
+  }
+
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
+                          long_t)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
+                          void *)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
+                          int32_t)
+OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
+                          int64_t)
+
+#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo)             \
+  TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) {    \
+    SCOPED_TSAN_INTERCEPTOR(f, n, ptr);                          \
+    volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
+    char bit = 0x80u >> (n & 7);                                 \
+    char mask = clear ? ~bit : bit;                              \
+    char orig_byte = op((volatile a8 *)byte_ptr, mask, mo);      \
+    return orig_byte & bit;                                      \
+  }
+
+#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear)               \
+  OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
+  OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
+
+OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
+OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
+                            true)
+
+TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
+  __tsan_release(item);
+  REAL(OSAtomicEnqueue)(list, item, offset);
+}
+
+TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
+  void *item = REAL(OSAtomicDequeue)(list, offset);
+  if (item) __tsan_acquire(item);
+  return item;
+}
+
+// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
+#if !SANITIZER_IOS
+
+TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
+  __tsan_release(item);
+  REAL(OSAtomicFifoEnqueue)(list, item, offset);
+}
+
+TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
+                 size_t offset) {
+  SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
+  void *item = REAL(OSAtomicFifoDequeue)(list, offset);
+  if (item) __tsan_acquire(item);
+  return item;
+}
+
+#endif
+
+TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockLock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
+  REAL(OSSpinLockLock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockTry)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
+  bool result = REAL(OSSpinLockTry)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(OSSpinLockUnlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(OSSpinLockUnlock)(lock);
+}
+
+TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_lock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
+  REAL(os_lock_lock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_trylock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
+  bool result = REAL(os_lock_trylock)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
+  CHECK(!cur_thread()->is_dead);
+  if (!cur_thread()->is_inited) {
+    return REAL(os_lock_unlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(os_lock_unlock)(lock);
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_lock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
+  REAL(os_unfair_lock_lock)(lock);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
+                 u32 options) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_lock_with_options)(lock, options);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
+  REAL(os_unfair_lock_lock_with_options)(lock, options);
+  Acquire(thr, pc, (uptr)lock);
+}
+
+TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_trylock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
+  bool result = REAL(os_unfair_lock_trylock)(lock);
+  if (result)
+    Acquire(thr, pc, (uptr)lock);
+  return result;
+}
+
+TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
+  if (!cur_thread()->is_inited || cur_thread()->is_dead) {
+    return REAL(os_unfair_lock_unlock)(lock);
+  }
+  SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
+  Release(thr, pc, (uptr)lock);
+  REAL(os_unfair_lock_unlock)(lock);
+}
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
+                 xpc_connection_t connection, xpc_handler_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
+                          handler);
+  Release(thr, pc, (uptr)connection);
+  xpc_handler_t new_handler = ^(xpc_object_t object) {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    handler(object);
+  };
+  REAL(xpc_connection_set_event_handler)(connection, new_handler);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
+                 dispatch_block_t barrier) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
+  Release(thr, pc, (uptr)connection);
+  dispatch_block_t new_barrier = ^() {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    barrier();
+  };
+  REAL(xpc_connection_send_barrier)(connection, new_barrier);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
+                 xpc_connection_t connection, xpc_object_t message,
+                 dispatch_queue_t replyq, xpc_handler_t handler) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
+                          message, replyq, handler);
+  Release(thr, pc, (uptr)connection);
+  xpc_handler_t new_handler = ^(xpc_object_t object) {
+    {
+      SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
+      Acquire(thr, pc, (uptr)connection);
+    }
+    handler(object);
+  };
+  REAL(xpc_connection_send_message_with_reply)
+  (connection, message, replyq, new_handler);
+}
+
+TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
+  SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
+  Release(thr, pc, (uptr)connection);
+  REAL(xpc_connection_cancel)(connection);
+}
+
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
+// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
+// pointers encode the object data directly in their pointer bits and do not
+// have an associated memory allocation. The Obj-C runtime uses tagged pointers
+// to transparently optimize small objects.
+static bool IsTaggedObjCPointer(id obj) {
+  const uptr kPossibleTaggedBits = 0x8000000000000001ull;
+  return ((uptr)obj & kPossibleTaggedBits) != 0;
+}
+
+// Returns an address which can be used to inform TSan about synchronization
+// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
+// address in the process space. We do a small allocation here to obtain a
+// stable address (the array backing the hash map can change). The memory is
+// never free'd (leaked) and allocation and locking are slow, but this code only
+// runs for @synchronized with tagged pointers, which is very rare.
+static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
+  typedef AddrHashMap<uptr, 5> Map;
+  static Map Addresses;
+  Map::Handle h(&Addresses, addr);
+  if (h.created()) {
+    ThreadIgnoreBegin(thr, pc);
+    *h = (uptr) user_alloc(thr, pc, /*size=*/1);
+    ThreadIgnoreEnd(thr, pc);
+  }
+  return *h;
+}
+
+// Returns an address on which we can synchronize given an Obj-C object pointer.
+// For normal object pointers, this is just the address of the object in memory.
+// Tagged pointers are not backed by an actual memory allocation, so we need to
+// synthesize a valid address.
+static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
+  if (IsTaggedObjCPointer(obj))
+    return GetOrCreateSyncAddress((uptr)obj, thr, pc);
+  return (uptr)obj;
+}
+
+TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
+  SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
+  if (!obj) return REAL(objc_sync_enter)(obj);
+  uptr addr = SyncAddressForObjCObject(obj, thr, pc);
+  MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
+  int result = REAL(objc_sync_enter)(obj);
+  CHECK_EQ(result, OBJC_SYNC_SUCCESS);
+  MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
+  return result;
+}
+
+TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
+  SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
+  if (!obj) return REAL(objc_sync_exit)(obj);
+  uptr addr = SyncAddressForObjCObject(obj, thr, pc);
+  MutexUnlock(thr, pc, addr);
+  int result = REAL(objc_sync_exit)(obj);
+  if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
+  return result;
+}
+
+TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {
+  {
+    SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
+  }
+  // Bacause of swapcontext() semantics we have no option but to copy its
+  // impementation here
+  if (!oucp || !ucp) {
+    errno = EINVAL;
+    return -1;
+  }
+  ThreadState *thr = cur_thread();
+  const int UCF_SWAPPED = 0x80000000;
+  oucp->uc_onstack &= ~UCF_SWAPPED;
+  thr->ignore_interceptors++;
+  int ret = getcontext(oucp);
+  if (!(oucp->uc_onstack & UCF_SWAPPED)) {
+    thr->ignore_interceptors--;
+    if (!ret) {
+      oucp->uc_onstack |= UCF_SWAPPED;
+      ret = setcontext(ucp);
+    }
+  }
+  return ret;
+}
+
+// On macOS, libc++ is always linked dynamically, so intercepting works the
+// usual way.
+#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
+
+namespace {
+struct fake_shared_weak_count {
+  volatile a64 shared_owners;
+  volatile a64 shared_weak_owners;
+  virtual void _unused_0x0() = 0;
+  virtual void _unused_0x8() = 0;
+  virtual void on_zero_shared() = 0;
+  virtual void _unused_0x18() = 0;
+  virtual void on_zero_shared_weak() = 0;
+};
+}  // namespace
+
+// The following code adds libc++ interceptors for:
+//     void __shared_weak_count::__release_shared() _NOEXCEPT;
+//     bool __shared_count::__release_shared() _NOEXCEPT;
+// Shared and weak pointers in C++ maintain reference counts via atomics in
+// libc++.dylib, which are TSan-invisible, and this leads to false positives in
+// destructor code. These interceptors re-implements the whole functions so that
+// the mo_acq_rel semantics of the atomic decrement are visible.
+//
+// Unfortunately, the interceptors cannot simply Acquire/Release some sync
+// object and call the original function, because it would have a race between
+// the sync and the destruction of the object.  Calling both under a lock will
+// not work because the destructor can invoke this interceptor again (and even
+// in a different thread, so recursive locks don't help).
+
+STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
+                   fake_shared_weak_count *o) {
+  if (!flags()->shared_ptr_interceptor)
+    return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
+
+  SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
+                          o);
+  if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
+    Acquire(thr, pc, (uptr)&o->shared_owners);
+    o->on_zero_shared();
+    if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
+        0) {
+      Acquire(thr, pc, (uptr)&o->shared_weak_owners);
+      o->on_zero_shared_weak();
+    }
+  }
+}
+
+STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
+                   fake_shared_weak_count *o) {
+  if (!flags()->shared_ptr_interceptor)
+    return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
+
+  SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
+  if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
+    Acquire(thr, pc, (uptr)&o->shared_owners);
+    o->on_zero_shared();
+    return true;
+  }
+  return false;
+}
+
+namespace {
+struct call_once_callback_args {
+  void (*orig_func)(void *arg);
+  void *orig_arg;
+  void *flag;
+};
+
+void call_once_callback_wrapper(void *arg) {
+  call_once_callback_args *new_args = (call_once_callback_args *)arg;
+  new_args->orig_func(new_args->orig_arg);
+  __tsan_release(new_args->flag);
+}
+}  // namespace
+
+// This adds a libc++ interceptor for:
+//     void __call_once(volatile unsigned long&, void*, void(*)(void*));
+// C++11 call_once is implemented via an internal function __call_once which is
+// inside libc++.dylib, and the atomic release store inside it is thus
+// TSan-invisible. To avoid false positives, this interceptor wraps the callback
+// function and performs an explicit Release after the user code has run.
+STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
+                   void *arg, void (*func)(void *arg)) {
+  call_once_callback_args new_args = {func, arg, flag};
+  REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
+                                            call_once_callback_wrapper);
+}
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/tsan_interceptors_mach_vm.cpp b/lib/tsan/tsan_interceptors_mach_vm.cpp
new file mode 100644
index 0000000000..cd318f8af9
--- /dev/null
+++ b/lib/tsan/tsan_interceptors_mach_vm.cpp
@@ -0,0 +1,52 @@
+//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interceptors for mach_vm_* user space memory routines on Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_platform.h"
+
+#include <mach/mach.h>
+
+namespace __tsan {
+
+static bool intersects_with_shadow(mach_vm_address_t *address,
+                                   mach_vm_size_t size, int flags) {
+  // VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
+  if (flags & VM_FLAGS_ANYWHERE) return false;
+  uptr ptr = *address;
+  return !IsAppMem(ptr) || !IsAppMem(ptr + size - 1);
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
+                 mach_vm_address_t *address, mach_vm_size_t size, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
+  if (target != mach_task_self())
+    return REAL(mach_vm_allocate)(target, address, size, flags);
+  if (intersects_with_shadow(address, size, flags))
+    return KERN_NO_SPACE;
+  kern_return_t res = REAL(mach_vm_allocate)(target, address, size, flags);
+  if (res == KERN_SUCCESS)
+    MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
+  return res;
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
+                 mach_vm_address_t address, mach_vm_size_t size) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
+  if (target != mach_task_self())
+    return REAL(mach_vm_deallocate)(target, address, size);
+  UnmapShadow(thr, address, size);
+  return REAL(mach_vm_deallocate)(target, address, size);
+}
+
+}  // namespace __tsan
diff --git a/lib/tsan/tsan_platform_linux.cpp b/lib/tsan/tsan_platform_linux.cpp
new file mode 100644
index 0000000000..645152a06c
--- /dev/null
+++ b/lib/tsan/tsan_platform_linux.cpp
@@ -0,0 +1,517 @@
+//===-- tsan_platform_linux.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Linux- and BSD-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+    SANITIZER_OPENBSD
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_openbsd.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stoptheworld.h"
+#include "tsan_flags.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <sys/mman.h>
+#if SANITIZER_LINUX
+#include <sys/personality.h>
+#include <setjmp.h>
+#endif
+#include <sys/syscall.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sched.h>
+#include <dlfcn.h>
+#if SANITIZER_LINUX
+#define __need_res_state
+#include <resolv.h>
+#endif
+
+#ifdef sa_handler
+# undef sa_handler
+#endif
+
+#ifdef sa_sigaction
+# undef sa_sigaction
+#endif
+
+#if SANITIZER_FREEBSD
+extern "C" void *__libc_stack_end;
+void *__libc_stack_end = 0;
+#endif
+
+#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
+# define INIT_LONGJMP_XOR_KEY 1
+#else
+# define INIT_LONGJMP_XOR_KEY 0
+#endif
+
+#if INIT_LONGJMP_XOR_KEY
+#include "interception/interception.h"
+// Must be declared outside of other namespaces.
+DECLARE_REAL(int, _setjmp, void *env)
+#endif
+
+namespace __tsan {
+
+#if INIT_LONGJMP_XOR_KEY
+static void InitializeLongjmpXorKey();
+static uptr longjmp_xor_key;
+#endif
+
+#ifdef TSAN_RUNTIME_VMA
+// Runtime detected VMA size.
+uptr vmaSize;
+#endif
+
+enum {
+  MemTotal  = 0,
+  MemShadow = 1,
+  MemMeta   = 2,
+  MemFile   = 3,
+  MemMmap   = 4,
+  MemTrace  = 5,
+  MemHeap   = 6,
+  MemOther  = 7,
+  MemCount  = 8,
+};
+
+void FillProfileCallback(uptr p, uptr rss, bool file,
+                         uptr *mem, uptr stats_size) {
+  mem[MemTotal] += rss;
+  if (p >= ShadowBeg() && p < ShadowEnd())
+    mem[MemShadow] += rss;
+  else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
+    mem[MemMeta] += rss;
+#if !SANITIZER_GO
+  else if (p >= HeapMemBeg() && p < HeapMemEnd())
+    mem[MemHeap] += rss;
+  else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
+    mem[file ? MemFile : MemMmap] += rss;
+  else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
+    mem[file ? MemFile : MemMmap] += rss;
+#else
+  else if (p >= AppMemBeg() && p < AppMemEnd())
+    mem[file ? MemFile : MemMmap] += rss;
+#endif
+  else if (p >= TraceMemBeg() && p < TraceMemEnd())
+    mem[MemTrace] += rss;
+  else
+    mem[MemOther] += rss;
+}
+
+void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+  uptr mem[MemCount];
+  internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
+  __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
+  StackDepotStats *stacks = StackDepotGetStats();
+  internal_snprintf(buf, buf_size,
+      "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
+      " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
+      mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
+      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
+      mem[MemHeap] >> 20, mem[MemOther] >> 20,
+      stacks->allocated >> 20, stacks->n_uniq_ids,
+      nlive, nthread);
+}
+
+#if SANITIZER_LINUX
+void FlushShadowMemoryCallback(
+    const SuspendedThreadsList &suspended_threads_list,
+    void *argument) {
+  ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
+}
+#endif
+
+void FlushShadowMemory() {
+#if SANITIZER_LINUX
+  StopTheWorld(FlushShadowMemoryCallback, 0);
+#endif
+}
+
+#if !SANITIZER_GO
+// Mark shadow for .rodata sections with the special kShadowRodata marker.
+// Accesses to .rodata can't race, so this saves time, memory and trace space.
+static void MapRodata() {
+  // First create temp file.
+  const char *tmpdir = GetEnv("TMPDIR");
+  if (tmpdir == 0)
+    tmpdir = GetEnv("TEST_TMPDIR");
+#ifdef P_tmpdir
+  if (tmpdir == 0)
+    tmpdir = P_tmpdir;
+#endif
+  if (tmpdir == 0)
+    return;
+  char name[256];
+  internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d",
+                    tmpdir, (int)internal_getpid());
+  uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
+  if (internal_iserror(openrv))
+    return;
+  internal_unlink(name);  // Unlink it now, so that we can reuse the buffer.
+  fd_t fd = openrv;
+  // Fill the file with kShadowRodata.
+  const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
+  InternalMmapVector<u64> marker(kMarkerSize);
+  // volatile to prevent insertion of memset
+  for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
+    *p = kShadowRodata;
+  internal_write(fd, marker.data(), marker.size() * sizeof(u64));
+  // Map the file into memory.
+  uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE,
+                            MAP_PRIVATE | MAP_ANONYMOUS, fd, 0);
+  if (internal_iserror(page)) {
+    internal_close(fd);
+    return;
+  }
+  // Map the file into shadow of .rodata sections.
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  // Reusing the buffer 'name'.
+  MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+  while (proc_maps.Next(&segment)) {
+    if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+        segment.IsReadable() && segment.IsExecutable() &&
+        !segment.IsWritable() && IsAppMem(segment.start)) {
+      // Assume it's .rodata
+      char *shadow_start = (char *)MemToShadow(segment.start);
+      char *shadow_end = (char *)MemToShadow(segment.end);
+      for (char *p = shadow_start; p < shadow_end;
+           p += marker.size() * sizeof(u64)) {
+        internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
+                      PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
+      }
+    }
+  }
+  internal_close(fd);
+}
+
+void InitializeShadowMemoryPlatform() {
+  MapRodata();
+}
+
+#endif  // #if !SANITIZER_GO
+
+void InitializePlatformEarly() {
+#ifdef TSAN_RUNTIME_VMA
+  vmaSize =
+    (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
+#if defined(__aarch64__)
+# if !SANITIZER_GO
+  if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize);
+    Die();
+  }
+#else
+  if (vmaSize != 48) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 48\n", vmaSize);
+    Die();
+  }
+#endif
+#elif defined(__powerpc64__)
+# if !SANITIZER_GO
+  if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize);
+    Die();
+  }
+# else
+  if (vmaSize != 46 && vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize);
+    Die();
+  }
+# endif
+#endif
+#endif
+}
+
+void InitializePlatform() {
+  DisableCoreDumperIfNecessary();
+
+  // Go maps shadow memory lazily and works fine with limited address space.
+  // Unlimited stack is not a problem as well, because the executable
+  // is not compiled with -pie.
+#if !SANITIZER_GO
+  {
+    bool reexec = false;
+    // TSan doesn't play well with unlimited stack size (as stack
+    // overlaps with shadow memory). If we detect unlimited stack size,
+    // we re-exec the program with limited stack size as a best effort.
+    if (StackSizeIsUnlimited()) {
+      const uptr kMaxStackSize = 32 * 1024 * 1024;
+      VReport(1, "Program is run with unlimited stack size, which wouldn't "
+                 "work with ThreadSanitizer.\n"
+                 "Re-execing with stack size limited to %zd bytes.\n",
+              kMaxStackSize);
+      SetStackSizeLimitInBytes(kMaxStackSize);
+      reexec = true;
+    }
+
+    if (!AddressSpaceIsUnlimited()) {
+      Report("WARNING: Program is run with limited virtual address space,"
+             " which wouldn't work with ThreadSanitizer.\n");
+      Report("Re-execing with unlimited virtual address space.\n");
+      SetAddressSpaceUnlimited();
+      reexec = true;
+    }
+#if SANITIZER_LINUX && defined(__aarch64__)
+    // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
+    // linux kernel, the random gap between stack and mapped area is increased
+    // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
+    // this big range, we should disable randomized virtual space on aarch64.
+    int old_personality = personality(0xffffffff);
+    if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
+      VReport(1, "WARNING: Program is run with randomized virtual address "
+              "space, which wouldn't work with ThreadSanitizer.\n"
+              "Re-execing with fixed virtual address space.\n");
+      CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
+      reexec = true;
+    }
+    // Initialize the xor key used in {sig}{set,long}jump.
+    InitializeLongjmpXorKey();
+#endif
+    if (reexec)
+      ReExec();
+  }
+
+  CheckAndProtect();
+  InitTlsSize();
+#endif  // !SANITIZER_GO
+}
+
+#if !SANITIZER_GO
+// Extract file descriptors passed to glibc internal __res_iclose function.
+// This is required to properly "close" the fds, because we do not see internal
+// closes within glibc. The code is a pure hack.
+int ExtractResolvFDs(void *state, int *fds, int nfd) {
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+  int cnt = 0;
+  struct __res_state *statp = (struct __res_state*)state;
+  for (int i = 0; i < MAXNS && cnt < nfd; i++) {
+    if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
+      fds[cnt++] = statp->_u._ext.nssocks[i];
+  }
+  return cnt;
+#else
+  return 0;
+#endif
+}
+
+// Extract file descriptors passed via UNIX domain sockets.
+// This is requried to properly handle "open" of these fds.
+// see 'man recvmsg' and 'man 3 cmsg'.
+int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
+  int res = 0;
+  msghdr *msg = (msghdr*)msgp;
+  struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
+  for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+    if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS)
+      continue;
+    int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
+    for (int i = 0; i < n; i++) {
+      fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
+      if (res == nfd)
+        return res;
+    }
+  }
+  return res;
+}
+
+// Reverse operation of libc stack pointer mangling
+static uptr UnmangleLongJmpSp(uptr mangled_sp) {
+#if defined(__x86_64__)
+# if SANITIZER_LINUX
+  // Reverse of:
+  //   xor  %fs:0x30, %rsi
+  //   rol  $0x11, %rsi
+  uptr sp;
+  asm("ror  $0x11,     %0 \n"
+      "xor  %%fs:0x30, %0 \n"
+      : "=r" (sp)
+      : "0" (mangled_sp));
+  return sp;
+# else
+  return mangled_sp;
+# endif
+#elif defined(__aarch64__)
+# if SANITIZER_LINUX
+  return mangled_sp ^ longjmp_xor_key;
+# else
+  return mangled_sp;
+# endif
+#elif defined(__powerpc64__)
+  // Reverse of:
+  //   ld   r4, -28696(r13)
+  //   xor  r4, r3, r4
+  uptr xor_key;
+  asm("ld  %0, -28696(%%r13)" : "=r" (xor_key));
+  return mangled_sp ^ xor_key;
+#elif defined(__mips__)
+  return mangled_sp;
+#else
+  #error "Unknown platform"
+#endif
+}
+
+#ifdef __powerpc__
+# define LONG_JMP_SP_ENV_SLOT 0
+#elif SANITIZER_FREEBSD
+# define LONG_JMP_SP_ENV_SLOT 2
+#elif SANITIZER_NETBSD
+# define LONG_JMP_SP_ENV_SLOT 6
+#elif SANITIZER_LINUX
+# ifdef __aarch64__
+#  define LONG_JMP_SP_ENV_SLOT 13
+# elif defined(__mips64)
+#  define LONG_JMP_SP_ENV_SLOT 1
+# else
+#  define LONG_JMP_SP_ENV_SLOT 6
+# endif
+#endif
+
+uptr ExtractLongJmpSp(uptr *env) {
+  uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
+  return UnmangleLongJmpSp(mangled_sp);
+}
+
+#if INIT_LONGJMP_XOR_KEY
+// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
+// functions) by XORing them with a random key.  For AArch64 it is a global
+// variable rather than a TCB one (as for x86_64/powerpc).  We obtain the key by
+// issuing a setjmp and XORing the SP pointer values to derive the key.
+static void InitializeLongjmpXorKey() {
+  // 1. Call REAL(setjmp), which stores the mangled SP in env.
+  jmp_buf env;
+  REAL(_setjmp)(env);
+
+  // 2. Retrieve vanilla/mangled SP.
+  uptr sp;
+  asm("mov  %0, sp" : "=r" (sp));
+  uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
+
+  // 3. xor SPs to obtain key.
+  longjmp_xor_key = mangled_sp ^ sp;
+}
+#endif
+
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  // Check that the thr object is in tls;
+  const uptr thr_beg = (uptr)thr;
+  const uptr thr_end = (uptr)thr + sizeof(*thr);
+  CHECK_GE(thr_beg, tls_addr);
+  CHECK_LE(thr_beg, tls_addr + tls_size);
+  CHECK_GE(thr_end, tls_addr);
+  CHECK_LE(thr_end, tls_addr + tls_size);
+  // Since the thr object is huge, skip it.
+  MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr);
+  MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end,
+                          tls_addr + tls_size - thr_end);
+}
+
+// Note: this function runs with async signals enabled,
+// so it must not touch any tsan state.
+int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
+    void *abstime), void *c, void *m, void *abstime,
+    void(*cleanup)(void *arg), void *arg) {
+  // pthread_cleanup_push/pop are hardcore macros mess.
+  // We can't intercept nor call them w/o including pthread.h.
+  int res;
+  pthread_cleanup_push(cleanup, arg);
+  res = fn(c, m, abstime);
+  pthread_cleanup_pop(0);
+  return res;
+}
+#endif  // !SANITIZER_GO
+
+#if !SANITIZER_GO
+void ReplaceSystemMalloc() { }
+#endif
+
+#if !SANITIZER_GO
+#if SANITIZER_ANDROID
+// On Android, one thread can call intercepted functions after
+// DestroyThreadState(), so add a fake thread state for "dead" threads.
+static ThreadState *dead_thread_state = nullptr;
+
+ThreadState *cur_thread() {
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+  if (thr == nullptr) {
+    __sanitizer_sigset_t emptyset;
+    internal_sigfillset(&emptyset);
+    __sanitizer_sigset_t oldset;
+    CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
+    thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+    if (thr == nullptr) {
+      thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
+                                                     "ThreadState"));
+      *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
+      if (dead_thread_state == nullptr) {
+        dead_thread_state = reinterpret_cast<ThreadState*>(
+            MmapOrDie(sizeof(ThreadState), "ThreadState"));
+        dead_thread_state->fast_state.SetIgnoreBit();
+        dead_thread_state->ignore_interceptors = 1;
+        dead_thread_state->is_dead = true;
+        *const_cast<int*>(&dead_thread_state->tid) = -1;
+        CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
+                                      PROT_READ));
+      }
+    }
+    CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
+  }
+  return thr;
+}
+
+void set_cur_thread(ThreadState *thr) {
+  *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
+}
+
+void cur_thread_finalize() {
+  __sanitizer_sigset_t emptyset;
+  internal_sigfillset(&emptyset);
+  __sanitizer_sigset_t oldset;
+  CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
+  ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
+  if (thr != dead_thread_state) {
+    *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
+    UnmapOrDie(thr, sizeof(ThreadState));
+  }
+  CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
+}
+#endif  // SANITIZER_ANDROID
+#endif  // if !SANITIZER_GO
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+        // SANITIZER_OPENBSD
diff --git a/lib/tsan/tsan_platform_mac.cpp b/lib/tsan/tsan_platform_mac.cpp
new file mode 100644
index 0000000000..eea52a34e9
--- /dev/null
+++ b/lib/tsan/tsan_platform_mac.cpp
@@ -0,0 +1,324 @@
+//===-- tsan_platform_mac.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Mac-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_MAC
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+#include "tsan_flags.h"
+
+#include <mach/mach.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sched.h>
+
+namespace __tsan {
+
+#if !SANITIZER_GO
+static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) {
+  atomic_uintptr_t *a = (atomic_uintptr_t *)dst;
+  void *val = (void *)atomic_load_relaxed(a);
+  atomic_signal_fence(memory_order_acquire);  // Turns the previous load into
+                                              // acquire wrt signals.
+  if (UNLIKELY(val == nullptr)) {
+    val = (void *)internal_mmap(nullptr, size, PROT_READ | PROT_WRITE,
+                                MAP_PRIVATE | MAP_ANON, -1, 0);
+    CHECK(val);
+    void *cmp = nullptr;
+    if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,
+                                        memory_order_acq_rel)) {
+      internal_munmap(val, size);
+      val = cmp;
+    }
+  }
+  return val;
+}
+
+// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is
+// problematic, because there are several places where interceptors are called
+// when TLVs are not accessible (early process startup, thread cleanup, ...).
+// The following provides a "poor man's TLV" implementation, where we use the
+// shadow memory of the pointer returned by pthread_self() to store a pointer to
+// the ThreadState object. The main thread's ThreadState is stored separately
+// in a static variable, because we need to access it even before the
+// shadow memory is set up.
+static uptr main_thread_identity = 0;
+ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
+static ThreadState *main_thread_state_loc = (ThreadState *)main_thread_state;
+
+// We cannot use pthread_self() before libpthread has been initialized.  Our
+// current heuristic for guarding this is checking `main_thread_identity` which
+// is only assigned in `__tsan::InitializePlatform`.
+static ThreadState **cur_thread_location() {
+  if (main_thread_identity == 0)
+    return &main_thread_state_loc;
+  uptr thread_identity = (uptr)pthread_self();
+  if (thread_identity == main_thread_identity)
+    return &main_thread_state_loc;
+  return (ThreadState **)MemToShadow(thread_identity);
+}
+
+ThreadState *cur_thread() {
+  return (ThreadState *)SignalSafeGetOrAllocate(
+      (uptr *)cur_thread_location(), sizeof(ThreadState));
+}
+
+void set_cur_thread(ThreadState *thr) {
+  *cur_thread_location() = thr;
+}
+
+// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
+// munmap first and then clear `fake_tls`; if we receive a signal in between,
+// handler will try to access the unmapped ThreadState.
+void cur_thread_finalize() {
+  ThreadState **thr_state_loc = cur_thread_location();
+  if (thr_state_loc == &main_thread_state_loc) {
+    // Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
+    // exit the main thread. Let's keep the main thread's ThreadState.
+    return;
+  }
+  internal_munmap(*thr_state_loc, sizeof(ThreadState));
+  *thr_state_loc = nullptr;
+}
+#endif
+
+void FlushShadowMemory() {
+}
+
+static void RegionMemUsage(uptr start, uptr end, uptr *res, uptr *dirty) {
+  vm_address_t address = start;
+  vm_address_t end_address = end;
+  uptr resident_pages = 0;
+  uptr dirty_pages = 0;
+  while (address < end_address) {
+    vm_size_t vm_region_size;
+    mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
+    vm_region_extended_info_data_t vm_region_info;
+    mach_port_t object_name;
+    kern_return_t ret = vm_region_64(
+        mach_task_self(), &address, &vm_region_size, VM_REGION_EXTENDED_INFO,
+        (vm_region_info_t)&vm_region_info, &count, &object_name);
+    if (ret != KERN_SUCCESS) break;
+
+    resident_pages += vm_region_info.pages_resident;
+    dirty_pages += vm_region_info.pages_dirtied;
+
+    address += vm_region_size;
+  }
+  *res = resident_pages * GetPageSizeCached();
+  *dirty = dirty_pages * GetPageSizeCached();
+}
+
+void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+  uptr shadow_res, shadow_dirty;
+  uptr meta_res, meta_dirty;
+  uptr trace_res, trace_dirty;
+  RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
+  RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
+  RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
+
+#if !SANITIZER_GO
+  uptr low_res, low_dirty;
+  uptr high_res, high_dirty;
+  uptr heap_res, heap_dirty;
+  RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &low_res, &low_dirty);
+  RegionMemUsage(HiAppMemBeg(), HiAppMemEnd(), &high_res, &high_dirty);
+  RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
+#else  // !SANITIZER_GO
+  uptr app_res, app_dirty;
+  RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
+#endif
+
+  StackDepotStats *stacks = StackDepotGetStats();
+  internal_snprintf(buf, buf_size,
+    "shadow   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "meta     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "traces   (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#if !SANITIZER_GO
+    "low app  (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+    "heap     (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#else  // !SANITIZER_GO
+    "app      (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
+#endif
+    "stacks: %zd unique IDs, %zd kB allocated\n"
+    "threads: %zd total, %zd live\n"
+    "------------------------------\n",
+    ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
+    MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
+    TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
+#if !SANITIZER_GO
+    LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
+    HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
+    HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
+#else  // !SANITIZER_GO
+    AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
+#endif
+    stacks->n_uniq_ids, stacks->allocated / 1024,
+    nthread, nlive);
+}
+
+#if !SANITIZER_GO
+void InitializeShadowMemoryPlatform() { }
+
+// On OS X, GCD worker threads are created without a call to pthread_create. We
+// need to properly register these threads with ThreadCreate and ThreadStart.
+// These threads don't have a parent thread, as they are created "spuriously".
+// We're using a libpthread API that notifies us about a newly created thread.
+// The `thread == pthread_self()` check indicates this is actually a worker
+// thread. If it's just a regular thread, this hook is called on the parent
+// thread.
+typedef void (*pthread_introspection_hook_t)(unsigned int event,
+                                             pthread_t thread, void *addr,
+                                             size_t size);
+extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
+    pthread_introspection_hook_t hook);
+static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
+static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
+static pthread_introspection_hook_t prev_pthread_introspection_hook;
+static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
+                                          void *addr, size_t size) {
+  if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
+    if (thread == pthread_self()) {
+      // The current thread is a newly created GCD worker thread.
+      ThreadState *thr = cur_thread();
+      Processor *proc = ProcCreate();
+      ProcWire(proc, thr);
+      ThreadState *parent_thread_state = nullptr;  // No parent.
+      int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
+      CHECK_NE(tid, 0);
+      ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
+    }
+  } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
+    if (thread == pthread_self()) {
+      ThreadState *thr = cur_thread();
+      if (thr->tctx) {
+        DestroyThreadState();
+      }
+    }
+  }
+
+  if (prev_pthread_introspection_hook != nullptr)
+    prev_pthread_introspection_hook(event, thread, addr, size);
+}
+#endif
+
+void InitializePlatformEarly() {
+#if defined(__aarch64__)
+  uptr max_vm = GetMaxUserVirtualAddress() + 1;
+  if (max_vm != Mapping::kHiAppMemEnd) {
+    Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+           max_vm, Mapping::kHiAppMemEnd);
+    Die();
+  }
+#endif
+}
+
+static uptr longjmp_xor_key = 0;
+
+void InitializePlatform() {
+  DisableCoreDumperIfNecessary();
+#if !SANITIZER_GO
+  CheckAndProtect();
+
+  CHECK_EQ(main_thread_identity, 0);
+  main_thread_identity = (uptr)pthread_self();
+
+  prev_pthread_introspection_hook =
+      pthread_introspection_hook_install(&my_pthread_introspection_hook);
+#endif
+
+  if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
+    // Libsystem currently uses a process-global key; this might change.
+    const unsigned kTLSLongjmpXorKeySlot = 0x7;
+    longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot);
+  }
+}
+
+#ifdef __aarch64__
+# define LONG_JMP_SP_ENV_SLOT \
+    ((GetMacosAlignedVersion() >= MacosVersion(10, 14)) ? 12 : 13)
+#else
+# define LONG_JMP_SP_ENV_SLOT 2
+#endif
+
+uptr ExtractLongJmpSp(uptr *env) {
+  uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
+  uptr sp = mangled_sp ^ longjmp_xor_key;
+  sp = (uptr)ptrauth_auth_data((void *)sp, ptrauth_key_asdb,
+                               ptrauth_string_discriminator("sp"));
+  return sp;
+}
+
+#if !SANITIZER_GO
+void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
+  // The pointer to the ThreadState object is stored in the shadow memory
+  // of the tls.
+  uptr tls_end = tls_addr + tls_size;
+  uptr thread_identity = (uptr)pthread_self();
+  if (thread_identity == main_thread_identity) {
+    MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, tls_size);
+  } else {
+    uptr thr_state_start = thread_identity;
+    uptr thr_state_end = thr_state_start + sizeof(uptr);
+    CHECK_GE(thr_state_start, tls_addr);
+    CHECK_LE(thr_state_start, tls_addr + tls_size);
+    CHECK_GE(thr_state_end, tls_addr);
+    CHECK_LE(thr_state_end, tls_addr + tls_size);
+    MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr,
+                            thr_state_start - tls_addr);
+    MemoryRangeImitateWrite(thr, /*pc=*/2, thr_state_end,
+                            tls_end - thr_state_end);
+  }
+}
+#endif
+
+#if !SANITIZER_GO
+// Note: this function runs with async signals enabled,
+// so it must not touch any tsan state.
+int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
+    void *abstime), void *c, void *m, void *abstime,
+    void(*cleanup)(void *arg), void *arg) {
+  // pthread_cleanup_push/pop are hardcore macros mess.
+  // We can't intercept nor call them w/o including pthread.h.
+  int res;
+  pthread_cleanup_push(cleanup, arg);
+  res = fn(c, m, abstime);
+  pthread_cleanup_pop(0);
+  return res;
+}
+#endif
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_MAC
diff --git a/lib/tsan/tsan_platform_posix.cpp b/lib/tsan/tsan_platform_posix.cpp
new file mode 100644
index 0000000000..1a0faee025
--- /dev/null
+++ b/lib/tsan/tsan_platform_posix.cpp
@@ -0,0 +1,167 @@
+//===-- tsan_platform_posix.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// POSIX-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_POSIX
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "tsan_platform.h"
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+static const char kShadowMemoryMappingWarning[] =
+    "FATAL: %s can not madvise shadow region [%zx, %zx] with %s (errno: %d)\n";
+static const char kShadowMemoryMappingHint[] =
+    "HINT: if %s is not supported in your environment, you may set "
+    "TSAN_OPTIONS=%s=0\n";
+
+static void NoHugePagesInShadow(uptr addr, uptr size) {
+  SetShadowRegionHugePageMode(addr, size);
+}
+
+static void DontDumpShadow(uptr addr, uptr size) {
+  if (common_flags()->use_madv_dontdump)
+    if (!DontDumpShadowMemory(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_DONTDUMP", errno);
+      Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump");
+      Die();
+    }
+}
+
+#if !SANITIZER_GO
+void InitializeShadowMemory() {
+  // Map memory shadow.
+  if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) {
+    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
+    Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+    Die();
+  }
+  // This memory range is used for thread stacks and large user mmaps.
+  // Frequently a thread uses only a small part of stack and similarly
+  // a program uses a small part of large mmap. On some programs
+  // we see 20% memory usage reduction without huge pages for this range.
+  // FIXME: don't use constants here.
+#if defined(__x86_64__)
+  const uptr kMadviseRangeBeg  = 0x7f0000000000ull;
+  const uptr kMadviseRangeSize = 0x010000000000ull;
+#elif defined(__mips64)
+  const uptr kMadviseRangeBeg  = 0xff00000000ull;
+  const uptr kMadviseRangeSize = 0x0100000000ull;
+#elif defined(__aarch64__) && defined(__APPLE__)
+  uptr kMadviseRangeBeg = LoAppMemBeg();
+  uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
+#elif defined(__aarch64__)
+  uptr kMadviseRangeBeg = 0;
+  uptr kMadviseRangeSize = 0;
+  if (vmaSize == 39) {
+    kMadviseRangeBeg  = 0x7d00000000ull;
+    kMadviseRangeSize = 0x0300000000ull;
+  } else if (vmaSize == 42) {
+    kMadviseRangeBeg  = 0x3f000000000ull;
+    kMadviseRangeSize = 0x01000000000ull;
+  } else {
+    DCHECK(0);
+  }
+#elif defined(__powerpc64__)
+  uptr kMadviseRangeBeg = 0;
+  uptr kMadviseRangeSize = 0;
+  if (vmaSize == 44) {
+    kMadviseRangeBeg  = 0x0f60000000ull;
+    kMadviseRangeSize = 0x0010000000ull;
+  } else if (vmaSize == 46) {
+    kMadviseRangeBeg  = 0x3f0000000000ull;
+    kMadviseRangeSize = 0x010000000000ull;
+  } else {
+    DCHECK(0);
+  }
+#endif
+  NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
+                      kMadviseRangeSize * kShadowMultiplier);
+  DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
+      ShadowBeg(), ShadowEnd(),
+      (ShadowEnd() - ShadowBeg()) >> 30);
+
+  // Map meta shadow.
+  const uptr meta = MetaShadowBeg();
+  const uptr meta_size = MetaShadowEnd() - meta;
+  if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) {
+    Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
+    Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+    Die();
+  }
+  NoHugePagesInShadow(meta, meta_size);
+  DontDumpShadow(meta, meta_size);
+  DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
+      meta, meta + meta_size, meta_size >> 30);
+
+  InitializeShadowMemoryPlatform();
+}
+
+static void ProtectRange(uptr beg, uptr end) {
+  CHECK_LE(beg, end);
+  if (beg == end)
+    return;
+  if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) {
+    Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
+    Printf("FATAL: Make sure you are not using unlimited stack\n");
+    Die();
+  }
+}
+
+void CheckAndProtect() {
+  // Ensure that the binary is indeed compiled with -pie.
+  MemoryMappingLayout proc_maps(true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (IsAppMem(segment.start)) continue;
+    if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+    if (segment.protection == 0)  // Zero page or mprotected.
+      continue;
+    if (segment.start >= VdsoBeg())  // vdso
+      break;
+    Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+           segment.start, segment.end);
+    Die();
+  }
+
+#if defined(__aarch64__) && defined(__APPLE__)
+  ProtectRange(HeapMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+  ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
+  ProtectRange(LoAppMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+#ifdef TSAN_MID_APP_RANGE
+  ProtectRange(MetaShadowEnd(), MidAppMemBeg());
+  ProtectRange(MidAppMemEnd(), TraceMemBeg());
+#else
+  ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#endif
+  // Memory for traces is mapped lazily in MapThreadTrace.
+  // Protect the whole range for now, so that user does not map something here.
+  ProtectRange(TraceMemBeg(), TraceMemEnd());
+  ProtectRange(TraceMemEnd(), HeapMemBeg());
+  ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
+}
+#endif
+
+}  // namespace __tsan
+
+#endif  // SANITIZER_POSIX
diff --git a/lib/tsan/tsan_rtl_aarch64.S b/lib/tsan/tsan_rtl_aarch64.S
new file mode 100644
index 0000000000..e0b4c71dfe
--- /dev/null
+++ b/lib/tsan/tsan_rtl_aarch64.S
@@ -0,0 +1,245 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if defined(__APPLE__)
+.align  2
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
+
+#if !defined(__APPLE__)
+.section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
+
+ASM_HIDDEN(__tsan_setjmp)
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env parameter
+  str     x0, [sp, 16]
+  CFI_OFFSET (0, -16)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ldr     x0, [sp, 16]
+  CFI_RESTORE (0)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc setjmp
+#if !defined(__APPLE__)
+  adrp    x1, :got:_ZN14__interception11real_setjmpE
+  ldr     x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
+  ldr     x1, [x1]
+#else
+  adrp    x1, _setjmp$non_lazy_ptr@page
+  add     x1, x1, _setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
+  br      x1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env parameter
+  str     x0, [sp, 16]
+  CFI_OFFSET (0, -16)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ldr     x0, [sp, 16]
+  CFI_RESTORE (0)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc setjmp
+#if !defined(__APPLE__)
+  adrp    x1, :got:_ZN14__interception12real__setjmpE
+  ldr     x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
+  ldr     x1, [x1]
+#else
+  adrp    x1, __setjmp$non_lazy_ptr@page
+  add     x1, x1, __setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
+  br      x1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env and savesigs parameter
+  stp     x0, x1, [sp, 16]
+  CFI_OFFSET (0, -16)
+  CFI_OFFSET (1, -8)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env and savesigs parameter
+  ldp     x0, x1, [sp, 16]
+  CFI_RESTORE (0)
+  CFI_RESTORE (1)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
+  adrp    x2, :got:_ZN14__interception14real_sigsetjmpE
+  ldr     x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
+  ldr     x2, [x2]
+#else
+  adrp    x2, _sigsetjmp$non_lazy_ptr@page
+  add     x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+  ldr     x2, [x2]
+#endif
+  br      x2
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+
+#if !defined(__APPLE__)
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame/link register
+  stp     x29, x30, [sp, -32]!
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (29, -32)
+  CFI_OFFSET (30, -24)
+
+  // Adjust the SP for previous frame
+  add     x29, sp, 0
+  CFI_DEF_CFA_REGISTER (29)
+
+  // Save env and savesigs parameter
+  stp     x0, x1, [sp, 16]
+  CFI_OFFSET (0, -16)
+  CFI_OFFSET (1, -8)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  add     x0, x29, 32
+
+  // call tsan interceptor
+  bl      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env and savesigs parameter
+  ldp     x0, x1, [sp, 16]
+  CFI_RESTORE (0)
+  CFI_RESTORE (1)
+
+  // Restore frame/link register
+  ldp     x29, x30, [sp], 32
+  CFI_RESTORE (29)
+  CFI_RESTORE (30)
+  CFI_DEF_CFA (31, 0)
+
+  // tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
+  adrp    x2, :got:_ZN14__interception16real___sigsetjmpE
+  ldr     x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
+  ldr     x2, [x2]
+#else
+  adrp    x2, ASM_SYMBOL(__sigsetjmp)@page
+  add     x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff
+#endif
+  br      x2
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
+
+NO_EXEC_STACK_DIRECTIVE
+
+#endif
diff --git a/lib/tsan/tsan_rtl_amd64.S b/lib/tsan/tsan_rtl_amd64.S
new file mode 100644
index 0000000000..5913aa360c
--- /dev/null
+++ b/lib/tsan/tsan_rtl_amd64.S
@@ -0,0 +1,366 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+#if !defined(__APPLE__)
+.section .text
+#else
+.section __TEXT,__text
+#endif
+
+ASM_HIDDEN(__tsan_trace_switch)
+.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
+ASM_SYMBOL(__tsan_trace_switch_thunk):
+  CFI_STARTPROC
+  # Save scratch registers.
+  push %rax
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rax, 0)
+  push %rcx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rcx, 0)
+  push %rdx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdx, 0)
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  push %r8
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r8, 0)
+  push %r9
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r9, 0)
+  push %r10
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r10, 0)
+  push %r11
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r11, 0)
+  # Align stack frame.
+  push %rbx  # non-scratch
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rbx, 0)
+  mov %rsp, %rbx  # save current rsp
+  CFI_DEF_CFA_REGISTER(%rbx)
+  shr $4, %rsp  # clear 4 lsb, align to 16
+  shl $4, %rsp
+
+  call ASM_SYMBOL(__tsan_trace_switch)
+
+  # Unalign stack frame back.
+  mov %rbx, %rsp  # restore the original rsp
+  CFI_DEF_CFA_REGISTER(%rsp)
+  pop %rbx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  # Restore scratch registers.
+  pop %r11
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r10
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r9
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r8
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rcx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rax
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rax)
+  CFI_RESTORE(%rbx)
+  CFI_RESTORE(%rcx)
+  CFI_RESTORE(%rdx)
+  CFI_RESTORE(%rsi)
+  CFI_RESTORE(%rdi)
+  CFI_RESTORE(%r8)
+  CFI_RESTORE(%r9)
+  CFI_RESTORE(%r10)
+  CFI_RESTORE(%r11)
+  ret
+  CFI_ENDPROC
+
+ASM_HIDDEN(__tsan_report_race)
+.globl ASM_SYMBOL(__tsan_report_race_thunk)
+ASM_SYMBOL(__tsan_report_race_thunk):
+  CFI_STARTPROC
+  # Save scratch registers.
+  push %rax
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rax, 0)
+  push %rcx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rcx, 0)
+  push %rdx
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdx, 0)
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  push %r8
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r8, 0)
+  push %r9
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r9, 0)
+  push %r10
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r10, 0)
+  push %r11
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%r11, 0)
+  # Align stack frame.
+  push %rbx  # non-scratch
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rbx, 0)
+  mov %rsp, %rbx  # save current rsp
+  CFI_DEF_CFA_REGISTER(%rbx)
+  shr $4, %rsp  # clear 4 lsb, align to 16
+  shl $4, %rsp
+
+  call ASM_SYMBOL(__tsan_report_race)
+
+  # Unalign stack frame back.
+  mov %rbx, %rsp  # restore the original rsp
+  CFI_DEF_CFA_REGISTER(%rsp)
+  pop %rbx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  # Restore scratch registers.
+  pop %r11
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r10
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r9
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %r8
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rdx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rcx
+  CFI_ADJUST_CFA_OFFSET(-8)
+  pop %rax
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rax)
+  CFI_RESTORE(%rbx)
+  CFI_RESTORE(%rcx)
+  CFI_RESTORE(%rdx)
+  CFI_RESTORE(%rsi)
+  CFI_RESTORE(%rdi)
+  CFI_RESTORE(%r8)
+  CFI_RESTORE(%r9)
+  CFI_RESTORE(%r10)
+  CFI_RESTORE(%r11)
+  ret
+  CFI_ENDPROC
+
+ASM_HIDDEN(__tsan_setjmp)
+#if defined(__NetBSD__)
+.comm _ZN14__interception15real___setjmp14E,8,8
+#elif !defined(__APPLE__)
+.comm _ZN14__interception11real_setjmpE,8,8
+#endif
+#if defined(__NetBSD__)
+.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+ASM_SYMBOL_INTERCEPTOR(__setjmp14):
+#else
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+#endif
+  CFI_STARTPROC
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 8(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 16(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc setjmp
+  movl $0, %eax
+#if defined(__NetBSD__)
+  movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#elif !defined(__APPLE__)
+  movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(setjmp)
+#endif
+  CFI_ENDPROC
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+#endif
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+  CFI_STARTPROC
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 8(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 16(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc setjmp
+  movl $0, %eax
+#if !defined(__APPLE__)
+  movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(_setjmp)
+#endif
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+#if defined(__NetBSD__)
+.comm _ZN14__interception18real___sigsetjmp14E,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
+#else
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+#endif
+  CFI_STARTPROC
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // save savesigs parameter
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  // align stack frame
+  sub $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(8)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__) || defined(__NetBSD__)
+  lea 24(%rsp), %rdi
+#elif defined(__linux__) || defined(__APPLE__)
+  lea 32(%rsp), %rdi
+#else
+# error "Unknown platform"
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // unalign stack frame
+  add $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(-8)
+  // restore savesigs parameter
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rsi)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc sigsetjmp
+  movl $0, %eax
+#if defined(__NetBSD__)
+  movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#elif !defined(__APPLE__)
+  movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+#else
+  jmp ASM_SYMBOL(sigsetjmp)
+#endif
+  CFI_ENDPROC
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+#endif
+
+#if !defined(__APPLE__) && !defined(__NetBSD__)
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+  CFI_STARTPROC
+  // save env parameter
+  push %rdi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rdi, 0)
+  // save savesigs parameter
+  push %rsi
+  CFI_ADJUST_CFA_OFFSET(8)
+  CFI_REL_OFFSET(%rsi, 0)
+  // align stack frame
+  sub $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(8)
+  // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
+#if defined(__FreeBSD__)
+  lea 24(%rsp), %rdi
+#else
+  lea 32(%rsp), %rdi
+#endif
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+  // unalign stack frame
+  add $8, %rsp
+  CFI_ADJUST_CFA_OFFSET(-8)
+  // restore savesigs parameter
+  pop %rsi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rsi)
+  // restore env parameter
+  pop %rdi
+  CFI_ADJUST_CFA_OFFSET(-8)
+  CFI_RESTORE(%rdi)
+  // tail jump to libc sigsetjmp
+  movl $0, %eax
+  movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
+  jmp *(%rdx)
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif  // !defined(__APPLE__) && !defined(__NetBSD__)
+
+NO_EXEC_STACK_DIRECTIVE
+
+#endif
diff --git a/lib/tsan/tsan_rtl_mips64.S b/lib/tsan/tsan_rtl_mips64.S
new file mode 100644
index 0000000000..d0f7a3f9af
--- /dev/null
+++ b/lib/tsan/tsan_rtl_mips64.S
@@ -0,0 +1,214 @@
+.section .text
+.set noreorder
+
+.hidden __tsan_setjmp
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl setjmp
+.type setjmp, @function
+setjmp:
+
+  // save env parameters
+  daddiu $sp,$sp,-40
+  sd $s0,32($sp)
+  sd $ra,24($sp)
+  sd $fp,16($sp)
+  sd $gp,8($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(setjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(setjmp)))
+  move $s0,$gp
+
+  // save jmp_buf
+  sd $a0,0($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,40
+
+  // restore jmp_buf
+  ld $a0,0($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc setjmp to t9
+  dla $t9,(_ZN14__interception11real_setjmpE) 
+
+  // restore env parameters
+  ld $gp,8($sp)
+  ld $fp,16($sp)
+  ld $ra,24($sp)
+  ld $s0,32($sp)
+  daddiu $sp,$sp,40
+
+  // tail jump to libc setjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size setjmp, .-setjmp
+
+.hidden __tsan_setjmp
+.globl _setjmp
+.comm _ZN14__interception12real__setjmpE,8,8
+.type _setjmp, @function
+_setjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-40
+  sd $s0,32($sp)
+  sd $ra,24($sp)
+  sd $fp,16($sp)
+  sd $gp,8($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(_setjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(_setjmp)))
+  move $s0,$gp
+
+  // save jmp_buf
+  sd $a0,0($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,40
+
+  // restore jmp_buf
+  ld $a0,0($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc _setjmp to t9
+  dla $t9,(_ZN14__interception12real__setjmpE)
+
+  // restore env parameters
+  ld $gp,8($sp)
+  ld $fp,16($sp)
+  ld $ra,24($sp)
+  ld $s0,32($sp)
+  daddiu $sp,$sp,40
+
+  // tail jump to libc _setjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size _setjmp, .-_setjmp
+
+.hidden __tsan_setjmp
+.globl sigsetjmp
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.type sigsetjmp, @function
+sigsetjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-48
+  sd $s0,40($sp)
+  sd $ra,32($sp)
+  sd $fp,24($sp)
+  sd $gp,16($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(sigsetjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(sigsetjmp)))
+  move $s0,$gp
+
+  // save jmp_buf and savesig
+  sd $a0,0($sp)
+  sd $a1,8($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,48
+
+  // restore jmp_buf and savesig
+  ld $a0,0($sp)
+  ld $a1,8($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer of libc sigsetjmp to t9
+  dla $t9,(_ZN14__interception14real_sigsetjmpE) 
+
+  // restore env parameters
+  ld $gp,16($sp)
+  ld $fp,24($sp)
+  ld $ra,32($sp)
+  ld $s0,40($sp)
+  daddiu $sp,$sp,48
+
+  // tail jump to libc sigsetjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size sigsetjmp, .-sigsetjmp
+
+.hidden __tsan_setjmp
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl __sigsetjmp
+.type __sigsetjmp, @function
+__sigsetjmp:
+
+  // Save env parameters
+  daddiu $sp,$sp,-48
+  sd $s0,40($sp)
+  sd $ra,32($sp)
+  sd $fp,24($sp)
+  sd $gp,16($sp)
+
+  // calculate and save pointer to GOT
+  lui $gp,%hi(%neg(%gp_rel(__sigsetjmp)))
+  daddu $gp,$gp,$t9
+  daddiu $gp,$gp,%lo(%neg(%gp_rel(__sigsetjmp)))
+  move $s0,$gp
+
+  // save jmp_buf and savesig
+  sd $a0,0($sp)
+  sd $a1,8($sp)
+
+  // obtain $sp
+  dadd $a0,$zero,$sp
+
+  // call tsan interceptor
+  jal __tsan_setjmp
+  daddiu $a1,$a0,48
+
+  // restore jmp_buf and savesig
+  ld $a0,0($sp)
+  ld $a1,8($sp)
+
+  // restore gp
+  move $gp,$s0
+
+  // load pointer to libc __sigsetjmp in t9
+  dla $t9,(_ZN14__interception16real___sigsetjmpE)
+
+  // restore env parameters
+  ld $gp,16($sp)
+  ld $fp,24($sp)
+  ld $ra,32($sp)
+  ld $s0,40($sp)
+  daddiu $sp,$sp,48
+
+  // tail jump to libc __sigsetjmp
+  ld $t9,0($t9)
+  jr $t9
+  nop
+
+.size __sigsetjmp, .-__sigsetjmp
diff --git a/lib/tsan/tsan_rtl_ppc64.S b/lib/tsan/tsan_rtl_ppc64.S
new file mode 100644
index 0000000000..8285e21aa1
--- /dev/null
+++ b/lib/tsan/tsan_rtl_ppc64.S
@@ -0,0 +1,288 @@
+#include "tsan_ppc_regs.h"
+
+        .section .text
+        .hidden __tsan_setjmp
+        .globl _setjmp
+        .type _setjmp, @function
+        .align 4
+#if _CALL_ELF == 2
+_setjmp:
+#else
+	.section ".opd","aw"
+	.align 3
+_setjmp:
+	.quad   .L._setjmp,.TOC.@tocbase,0
+	.previous
+#endif
+.L._setjmp:
+        mflr    r0
+        stdu    r1,-48(r1)
+        std     r2,24(r1)
+        std     r3,32(r1)
+        std     r0,40(r1)
+        // r3 is the original stack pointer.
+        addi    r3,r1,48
+        // r4 is the mangled stack pointer (see glibc)
+        ld      r4,-28696(r13)
+        xor     r4,r3,r4
+        // Materialize a TOC in case we were called from libc.
+        // For big-endian, we load the TOC from the OPD.  For little-
+        // endian, we use the .TOC. symbol to find it.
+        nop
+        bcl     20,31,0f
+0:
+        mflr    r2
+#if _CALL_ELF == 2
+        addis   r2,r2,.TOC.-0b@ha
+        addi    r2,r2,.TOC.-0b@l
+#else
+        addis   r2,r2,_setjmp-0b@ha
+        addi    r2,r2,_setjmp-0b@l
+        ld      r2,8(r2)
+#endif
+        // Call the interceptor.
+        bl      __tsan_setjmp
+        nop
+        // Restore regs needed for setjmp.
+        ld      r3,32(r1)
+        ld      r0,40(r1)
+        // Emulate the real setjmp function.  We do this because we can't
+        // perform a sibcall:  The real setjmp function trashes the TOC
+        // pointer, and with a sibcall we have no way to restore it.
+        // This way we can make sure our caller's stack pointer and
+        // link register are saved correctly in the jmpbuf.
+        ld      r6,-28696(r13)
+        addi    r5,r1,48  // original stack ptr of caller
+        xor     r5,r6,r5
+        std     r5,0(r3)  // mangled stack ptr of caller
+        ld      r5,24(r1)
+        std     r5,8(r3)  // caller's saved TOC pointer
+        xor     r0,r6,r0
+        std     r0,16(r3) // caller's mangled return address
+        mfcr    r0
+        // Nonvolatiles.
+        std     r14,24(r3)
+        stfd    f14,176(r3)
+        stw     r0,172(r3) // CR
+        std     r15,32(r3)
+        stfd    f15,184(r3)
+        std     r16,40(r3)
+        stfd    f16,192(r3)
+        std     r17,48(r3)
+        stfd    f17,200(r3)
+        std     r18,56(r3)
+        stfd    f18,208(r3)
+        std     r19,64(r3)
+        stfd    f19,216(r3)
+        std     r20,72(r3)
+        stfd    f20,224(r3)
+        std     r21,80(r3)
+        stfd    f21,232(r3)
+        std     r22,88(r3)
+        stfd    f22,240(r3)
+        std     r23,96(r3)
+        stfd    f23,248(r3)
+        std     r24,104(r3)
+        stfd    f24,256(r3)
+        std     r25,112(r3)
+        stfd    f25,264(r3)
+        std     r26,120(r3)
+        stfd    f26,272(r3)
+        std     r27,128(r3)
+        stfd    f27,280(r3)
+        std     r28,136(r3)
+        stfd    f28,288(r3)
+        std     r29,144(r3)
+        stfd    f29,296(r3)
+        std     r30,152(r3)
+        stfd    f30,304(r3)
+        std     r31,160(r3)
+        stfd    f31,312(r3)
+        addi    r5,r3,320
+        mfspr   r0,256
+        stw     r0,168(r3)  // VRSAVE
+        addi    r6,r5,16
+        stvx    v20,0,r5
+        addi    r5,r5,32
+        stvx    v21,0,r6
+        addi    r6,r6,32
+        stvx    v22,0,r5
+        addi    r5,r5,32
+        stvx    v23,0,r6
+        addi    r6,r6,32
+        stvx    v24,0,r5
+        addi    r5,r5,32
+        stvx    v25,0,r6
+        addi    r6,r6,32
+        stvx    v26,0,r5
+        addi    r5,r5,32
+        stvx    v27,0,r6
+        addi    r6,r6,32
+        stvx    v28,0,r5
+        addi    r5,r5,32
+        stvx    v29,0,r6
+        addi    r6,r6,32
+        stvx    v30,0,r5
+        stvx    v31,0,r6
+        // Clear the "mask-saved" slot.
+        li      r4,0
+        stw     r4,512(r3)
+        // Restore TOC, LR, and stack and return to caller.
+        ld      r2,24(r1)
+        ld      r0,40(r1)
+        addi    r1,r1,48
+        li      r3,0  // This is the setjmp return path
+        mtlr    r0
+        blr
+        .size _setjmp, .-.L._setjmp
+
+        .globl setjmp
+        .type setjmp, @function
+        .align 4
+setjmp:
+        b       _setjmp
+        .size setjmp, .-setjmp
+
+        // sigsetjmp is like setjmp, except that the mask in r4 needs
+        // to be saved at offset 512 of the jump buffer.
+        .globl __sigsetjmp
+        .type __sigsetjmp, @function
+        .align 4
+#if _CALL_ELF == 2
+__sigsetjmp:
+#else
+	.section ".opd","aw"
+	.align 3
+__sigsetjmp:
+	.quad   .L.__sigsetjmp,.TOC.@tocbase,0
+	.previous
+#endif
+.L.__sigsetjmp:
+        mflr    r0
+        stdu    r1,-64(r1)
+        std     r2,24(r1)
+        std     r3,32(r1)
+        std     r4,40(r1)
+        std     r0,48(r1)
+        // r3 is the original stack pointer.
+        addi    r3,r1,64
+        // r4 is the mangled stack pointer (see glibc)
+        ld      r4,-28696(r13)
+        xor     r4,r3,r4
+        // Materialize a TOC in case we were called from libc.
+        // For big-endian, we load the TOC from the OPD.  For little-
+        // endian, we use the .TOC. symbol to find it.
+        nop
+        bcl     20,31,1f
+1:
+        mflr    r2
+#if _CALL_ELF == 2
+        addis   r2,r2,.TOC.-1b@ha
+        addi    r2,r2,.TOC.-1b@l
+#else
+        addis   r2,r2,_setjmp-1b@ha
+        addi    r2,r2,_setjmp-1b@l
+        ld      r2,8(r2)
+#endif
+        // Call the interceptor.
+        bl      __tsan_setjmp
+        nop
+        // Restore regs needed for __sigsetjmp.
+        ld      r3,32(r1)
+        ld      r4,40(r1)
+        ld      r0,48(r1)
+        // Emulate the real sigsetjmp function.  We do this because we can't
+        // perform a sibcall:  The real sigsetjmp function trashes the TOC
+        // pointer, and with a sibcall we have no way to restore it.
+        // This way we can make sure our caller's stack pointer and
+        // link register are saved correctly in the jmpbuf.
+        ld      r6,-28696(r13)
+        addi    r5,r1,64  // original stack ptr of caller
+        xor     r5,r6,r5
+        std     r5,0(r3)  // mangled stack ptr of caller
+        ld      r5,24(r1)
+        std     r5,8(r3)  // caller's saved TOC pointer
+        xor     r0,r6,r0
+        std     r0,16(r3) // caller's mangled return address
+        mfcr    r0
+        // Nonvolatiles.
+        std     r14,24(r3)
+        stfd    f14,176(r3)
+        stw     r0,172(r3) // CR
+        std     r15,32(r3)
+        stfd    f15,184(r3)
+        std     r16,40(r3)
+        stfd    f16,192(r3)
+        std     r17,48(r3)
+        stfd    f17,200(r3)
+        std     r18,56(r3)
+        stfd    f18,208(r3)
+        std     r19,64(r3)
+        stfd    f19,216(r3)
+        std     r20,72(r3)
+        stfd    f20,224(r3)
+        std     r21,80(r3)
+        stfd    f21,232(r3)
+        std     r22,88(r3)
+        stfd    f22,240(r3)
+        std     r23,96(r3)
+        stfd    f23,248(r3)
+        std     r24,104(r3)
+        stfd    f24,256(r3)
+        std     r25,112(r3)
+        stfd    f25,264(r3)
+        std     r26,120(r3)
+        stfd    f26,272(r3)
+        std     r27,128(r3)
+        stfd    f27,280(r3)
+        std     r28,136(r3)
+        stfd    f28,288(r3)
+        std     r29,144(r3)
+        stfd    f29,296(r3)
+        std     r30,152(r3)
+        stfd    f30,304(r3)
+        std     r31,160(r3)
+        stfd    f31,312(r3)
+        addi    r5,r3,320
+        mfspr   r0,256
+        stw     r0,168(r3) // VRSAVE
+        addi    r6,r5,16
+        stvx    v20,0,r5
+        addi    r5,r5,32
+        stvx    v21,0,r6
+        addi    r6,r6,32
+        stvx    v22,0,r5
+        addi    r5,r5,32
+        stvx    v23,0,r6
+        addi    r6,r6,32
+        stvx    v24,0,r5
+        addi    r5,r5,32
+        stvx    v25,0,r6
+        addi    r6,r6,32
+        stvx    v26,0,r5
+        addi    r5,r5,32
+        stvx    v27,0,r6
+        addi    r6,r6,32
+        stvx    v28,0,r5
+        addi    r5,r5,32
+        stvx    v29,0,r6
+        addi    r6,r6,32
+        stvx    v30,0,r5
+        stvx    v31,0,r6
+        // Save into the "mask-saved" slot.
+        stw     r4,512(r3)
+        // Restore TOC, LR, and stack and return to caller.
+        ld      r2,24(r1)
+        ld      r0,48(r1)
+        addi    r1,r1,64
+        li      r3,0  // This is the sigsetjmp return path
+        mtlr    r0
+        blr
+        .size __sigsetjmp, .-.L.__sigsetjmp
+
+        .globl sigsetjmp
+        .type sigsetjmp, @function
+        .align 4
+sigsetjmp:
+        b       __sigsetjmp
+        .size sigsetjmp, .-sigsetjmp
diff --git a/src/Compilation.zig b/src/Compilation.zig
index ad4dd4fcca..6372dbff3a 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -429,7 +429,11 @@ pub const InitOptions = struct {
     verbose_llvm_cpu_features: bool = false,
     is_test: bool = false,
     test_evented_io: bool = false,
-    is_compiler_rt_or_libc: bool = false,
+    /// Normally when you create a `Compilation`, Zig will automatically build
+    /// and link in required dependencies, such as compiler-rt and libc. When
+    /// building such dependencies themselves, this flag must be set to avoid
+    /// infinite recursion.
+    skip_linker_dependencies: bool = false,
     parent_compilation_link_libc: bool = false,
     stack_size_override: ?u64 = null,
     image_base_override: ?u64 = null,
@@ -499,7 +503,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         .Lib => is_dyn_lib,
         .Exe => true,
     };
-    const needs_c_symbols = !options.is_compiler_rt_or_libc and
+    const needs_c_symbols = !options.skip_linker_dependencies and
         (is_exe_or_dyn_lib or (options.target.isWasm() and options.output_mode != .Obj));
 
     const comp: *Compilation = comp: {
@@ -677,6 +681,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             break :pic explicit;
         } else pie or must_pic;
 
+        // TSAN is implemented in C++ so it requires linking libc++.
+        const link_libcpp = options.link_libcpp or tsan;
+
         // Make a decision on whether to use Clang for translate-c and compiling C files.
         const use_clang = if (options.use_clang) |explicit| explicit else blk: {
             if (build_options.have_llvm) {
@@ -765,7 +772,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(options.function_sections);
         cache.hash.add(strip);
         cache.hash.add(link_libc);
-        cache.hash.add(options.link_libcpp);
+        cache.hash.add(link_libcpp);
         cache.hash.add(options.output_mode);
         cache.hash.add(options.machine_code_model);
         cache.hash.addOptionalEmitLoc(options.emit_bin);
@@ -793,7 +800,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             hash.add(single_threaded);
             hash.add(dll_export_fns);
             hash.add(options.is_test);
-            hash.add(options.is_compiler_rt_or_libc);
+            hash.add(options.skip_linker_dependencies);
             hash.add(options.parent_compilation_link_libc);
 
             const digest = hash.final();
@@ -930,7 +937,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .use_llvm = use_llvm,
             .system_linker_hack = darwin_options.system_linker_hack,
             .link_libc = link_libc,
-            .link_libcpp = options.link_libcpp,
+            .link_libcpp = link_libcpp,
             .objects = options.link_objects,
             .frameworks = options.frameworks,
             .framework_dirs = options.framework_dirs,
@@ -970,7 +977,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .dll_export_fns = dll_export_fns,
             .error_return_tracing = error_return_tracing,
             .llvm_cpu_features = llvm_cpu_features,
-            .is_compiler_rt_or_libc = options.is_compiler_rt_or_libc,
+            .skip_linker_dependencies = options.skip_linker_dependencies,
             .parent_compilation_link_libc = options.parent_compilation_link_libc,
             .each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,
             .disable_lld_caching = options.disable_lld_caching,
@@ -1044,7 +1051,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
     }
 
-    if (comp.bin_file.options.emit != null and !comp.bin_file.options.is_compiler_rt_or_libc) {
+    if (comp.bin_file.options.emit != null and !comp.bin_file.options.skip_linker_dependencies) {
         // If we need to build glibc for the target, add work items for it.
         // We go through the work queue so that building can be done in parallel.
         if (comp.wantBuildGLibCFromSource()) {
@@ -1097,9 +1104,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         if (comp.wantBuildLibUnwindFromSource()) {
             try comp.work_queue.writeItem(.{ .libunwind = {} });
         }
-        if (build_options.have_llvm and comp.bin_file.options.output_mode != .Obj and
-            comp.bin_file.options.link_libcpp)
-        {
+        if (build_options.have_llvm and is_exe_or_dyn_lib and comp.bin_file.options.link_libcpp) {
             try comp.work_queue.writeItem(.libcxx);
             try comp.work_queue.writeItem(.libcxxabi);
         }
@@ -2772,7 +2777,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
     // in. For example, compiler_rt will not export the __chkstk symbol if it
     // knows libc will provide it, and likewise c.zig will not export memcpy.
     const link_libc = comp.bin_file.options.link_libc or
-        (comp.bin_file.options.is_compiler_rt_or_libc and comp.bin_file.options.parent_compilation_link_libc);
+        (comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
 
     try buffer.writer().print(
         \\pub const object_format = ObjectFormat.{};
@@ -2927,7 +2932,7 @@ fn buildOutputFromZig(
         .verbose_cimport = comp.verbose_cimport,
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
-        .is_compiler_rt_or_libc = true,
+        .skip_linker_dependencies = true,
         .parent_compilation_link_libc = comp.bin_file.options.link_libc,
     });
     defer sub_compilation.destroy();
@@ -3305,7 +3310,7 @@ pub fn build_crt_file(
         .verbose_cimport = comp.verbose_cimport,
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
-        .is_compiler_rt_or_libc = true,
+        .skip_linker_dependencies = true,
         .parent_compilation_link_libc = comp.bin_file.options.link_libc,
     });
     defer sub_compilation.destroy();
@@ -3326,7 +3331,7 @@ pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {
     // Avoid deadlocking on building import libs such as kernel32.lib
     // This can happen when the user uses `build-exe foo.obj -lkernel32` and then
     // when we create a sub-Compilation for zig libc, it also tries to build kernel32.lib.
-    if (comp.bin_file.options.is_compiler_rt_or_libc) return;
+    if (comp.bin_file.options.skip_linker_dependencies) return;
 
     // This happens when an `extern "foo"` function is referenced by the stage1 backend.
     // If we haven't seen this library yet and we're targeting Windows, we need to queue up
diff --git a/src/glibc.zig b/src/glibc.zig
index c228d8d2ef..2e42cef519 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -962,7 +962,7 @@ fn buildSharedLib(
         .version_script = map_file_path,
         .soname = soname,
         .c_source_files = &c_source_files,
-        .is_compiler_rt_or_libc = true,
+        .skip_linker_dependencies = true,
     });
     defer sub_compilation.destroy();
 
diff --git a/src/libcxx.zig b/src/libcxx.zig
index d81562bc7e..47bf93dee3 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -188,6 +188,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .link_libc = true,
+        .skip_linker_dependencies = true,
     });
     defer sub_compilation.destroy();
 
@@ -308,6 +309,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .link_libc = true,
+        .skip_linker_dependencies = true,
     });
     defer sub_compilation.destroy();
 
diff --git a/src/libtsan.zig b/src/libtsan.zig
index 3e2c70328b..ecf15693e1 100644
--- a/src/libtsan.zig
+++ b/src/libtsan.zig
@@ -34,7 +34,7 @@ pub fn buildTsan(comp: *Compilation) !void {
     };
 
     var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
-    try c_source_files.ensureCapacity(tsan_sources.len + sanitizer_common_sources.len);
+    try c_source_files.ensureCapacity(c_source_files.items.len + tsan_sources.len);
 
     const tsan_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"tsan"});
     for (tsan_sources) |tsan_src| {
@@ -48,6 +48,7 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
 
         c_source_files.appendAssumeCapacity(.{
             .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
@@ -55,6 +56,51 @@ pub fn buildTsan(comp: *Compilation) !void {
         });
     }
 
+    const platform_tsan_sources = if (target.isDarwin())
+        &darwin_tsan_sources
+    else
+        &unix_tsan_sources;
+    try c_source_files.ensureCapacity(c_source_files.items.len + platform_tsan_sources.len);
+    for (platform_tsan_sources) |tsan_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", tsan_src }),
+            .extra_flags = cflags.items,
+        });
+    }
+    {
+        const asm_source = switch (target.cpu.arch) {
+            .aarch64 => "tsan_rtl_aarch64.S",
+            .x86_64 => "tsan_rtl_amd64.S",
+            .mips64 => "tsan_rtl_mips64.S",
+            .powerpc64 => "tsan_rtl_ppc64.S",
+            else => return error.TSANUnsupportedCPUArchitecture,
+        };
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
+
+        try cflags.append("-DNDEBUG");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "tsan", asm_source }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_common_sources.len);
     const sanitizer_common_include_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
         "tsan", "sanitizer_common",
     });
@@ -69,6 +115,7 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
 
         c_source_files.appendAssumeCapacity(.{
             .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
@@ -78,6 +125,88 @@ pub fn buildTsan(comp: *Compilation) !void {
         });
     }
 
+    const to_c_or_not_to_c_sources = if (comp.bin_file.options.link_libc)
+        &sanitizer_libcdep_sources
+    else
+        &sanitizer_nolibc_sources;
+    try c_source_files.ensureCapacity(c_source_files.items.len + to_c_or_not_to_c_sources.len);
+    for (to_c_or_not_to_c_sources) |c_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(sanitizer_common_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                "tsan", "sanitizer_common", c_src,
+            }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    try c_source_files.ensureCapacity(c_source_files.items.len + sanitizer_symbolizer_sources.len);
+    for (sanitizer_symbolizer_sources) |c_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                "tsan", "sanitizer_common", c_src,
+            }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    const interception_include_path = try comp.zig_lib_directory.join(
+        arena,
+        &[_][]const u8{"interception"},
+    );
+
+    try c_source_files.ensureCapacity(c_source_files.items.len + interception_sources.len);
+    for (interception_sources) |c_src| {
+        var cflags = std.ArrayList([]const u8).init(arena);
+
+        try cflags.append("-I");
+        try cflags.append(interception_include_path);
+
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
+
+        try cflags.append("-O3");
+        try cflags.append("-DNDEBUG");
+        try cflags.append("-nostdinc++");
+        try cflags.append("-fvisibility-inlines-hidden");
+        try cflags.append("-std=c++14");
+        try cflags.append("-fno-rtti");
+
+        c_source_files.appendAssumeCapacity(.{
+            .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
+                "tsan", "interception", c_src,
+            }),
+            .extra_flags = cflags.items,
+        });
+    }
+
+    const common_flags = [_][]const u8{
+        "-DTSAN_CONTAINS_UBSAN=0",
+    };
+
     const sub_compilation = try Compilation.create(comp.gpa, .{
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
@@ -113,6 +242,8 @@ pub fn buildTsan(comp: *Compilation) !void {
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .link_libc = true,
+        .skip_linker_dependencies = true,
+        .clang_argv = &common_flags,
     });
     defer sub_compilation.destroy();
 
@@ -159,6 +290,18 @@ const tsan_sources = [_][]const u8{
     "tsan_sync.cpp",
 };
 
+const darwin_tsan_sources = [_][]const u8{
+    "tsan_interceptors_mac.cpp",
+    "tsan_interceptors_mach_vm.cpp",
+    "tsan_platform_mac.cpp",
+    "tsan_platform_posix.cpp",
+};
+
+const unix_tsan_sources = [_][]const u8{
+    "tsan_platform_linux.cpp",
+    "tsan_platform_posix.cpp",
+};
+
 const sanitizer_common_sources = [_][]const u8{
     "sanitizer_allocator.cpp",
     "sanitizer_common.cpp",
@@ -203,17 +346,42 @@ const sanitizer_common_sources = [_][]const u8{
     "sanitizer_win.cpp",
 };
 
-// TODO This is next up
-//const sanitizer_nolibc_sources = [_][]const u8{
-//    "sanitizer_common_nolibc.cpp",
-//};
-//
-//const sanitizer_libcdep_sources = [_][]const u8{
-//    "sanitizer_common_libcdep.cpp",
-//    "sanitizer_allocator_checks.cpp",
-//    "sanitizer_linux_libcdep.cpp",
-//    "sanitizer_mac_libcdep.cpp",
-//    "sanitizer_posix_libcdep.cpp",
-//    "sanitizer_stoptheworld_linux_libcdep.cpp",
-//    "sanitizer_stoptheworld_netbsd_libcdep.cpp",
-//};
+const sanitizer_nolibc_sources = [_][]const u8{
+    "sanitizer_common_nolibc.cpp",
+};
+
+const sanitizer_libcdep_sources = [_][]const u8{
+    "sanitizer_common_libcdep.cpp",
+    "sanitizer_allocator_checks.cpp",
+    "sanitizer_linux_libcdep.cpp",
+    "sanitizer_mac_libcdep.cpp",
+    "sanitizer_posix_libcdep.cpp",
+    "sanitizer_stoptheworld_linux_libcdep.cpp",
+    "sanitizer_stoptheworld_netbsd_libcdep.cpp",
+};
+
+const sanitizer_symbolizer_sources = [_][]const u8{
+    "sanitizer_allocator_report.cpp",
+    "sanitizer_stackdepot.cpp",
+    "sanitizer_stacktrace.cpp",
+    "sanitizer_stacktrace_libcdep.cpp",
+    "sanitizer_stacktrace_printer.cpp",
+    "sanitizer_stacktrace_sparc.cpp",
+    "sanitizer_symbolizer.cpp",
+    "sanitizer_symbolizer_libbacktrace.cpp",
+    "sanitizer_symbolizer_libcdep.cpp",
+    "sanitizer_symbolizer_mac.cpp",
+    "sanitizer_symbolizer_markup.cpp",
+    "sanitizer_symbolizer_posix_libcdep.cpp",
+    "sanitizer_symbolizer_report.cpp",
+    "sanitizer_symbolizer_win.cpp",
+    "sanitizer_unwind_linux_libcdep.cpp",
+    "sanitizer_unwind_win.cpp",
+};
+
+const interception_sources = [_][]const u8{
+    "interception_linux.cpp",
+    "interception_mac.cpp",
+    "interception_win.cpp",
+    "interception_type_test.cpp",
+};
diff --git a/src/libunwind.zig b/src/libunwind.zig
index a0da7ee1fb..db1056b776 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -122,6 +122,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .link_libc = true,
+        .skip_linker_dependencies = true,
     });
     defer sub_compilation.destroy();
 
diff --git a/src/link.zig b/src/link.zig
index edeb24dc9a..530bcd0b25 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -81,7 +81,7 @@ pub const Options = struct {
     verbose_link: bool,
     dll_export_fns: bool,
     error_return_tracing: bool,
-    is_compiler_rt_or_libc: bool,
+    skip_linker_dependencies: bool,
     parent_compilation_link_libc: bool,
     each_lib_rpath: bool,
     disable_lld_caching: bool,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 0ba12ca45d..bea3033f88 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -835,7 +835,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         man.hash.addOptional(self.base.options.image_base_override);
         man.hash.addListOfBytes(self.base.options.extra_lld_args);
         man.hash.addListOfBytes(self.base.options.lib_dirs);
-        man.hash.add(self.base.options.is_compiler_rt_or_libc);
+        man.hash.add(self.base.options.skip_linker_dependencies);
         if (self.base.options.link_libc) {
             man.hash.add(self.base.options.libc_installation != null);
             if (self.base.options.libc_installation) |libc_installation| {
@@ -1125,7 +1125,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         }
 
         // compiler-rt, libc and libssp
-        if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+        if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
             if (!self.base.options.link_libc) {
                 try argv.append(comp.libc_static_lib.?.full_object_path);
             }
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index a3b58c1509..6232a64e4a 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1310,7 +1310,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         man.hash.addListOfBytes(self.base.options.lib_dirs);
         man.hash.addListOfBytes(self.base.options.rpath_list);
         man.hash.add(self.base.options.each_lib_rpath);
-        man.hash.add(self.base.options.is_compiler_rt_or_libc);
+        man.hash.add(self.base.options.skip_linker_dependencies);
         man.hash.add(self.base.options.z_nodelete);
         man.hash.add(self.base.options.z_defs);
         if (self.base.options.link_libc) {
@@ -1552,7 +1552,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     }
 
     // libc
-    if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc and !self.base.options.link_libc) {
+    if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies and !self.base.options.link_libc) {
         try argv.append(comp.libc_static_lib.?.full_object_path);
     }
 
@@ -1574,9 +1574,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
             const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
             argv.appendAssumeCapacity(arg);
         }
-    }
 
-    if (!is_obj) {
         // libc++ dep
         if (self.base.options.link_libcpp) {
             try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 65dbe4e5f6..17816959a3 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -438,7 +438,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         man.hash.addListOfBytes(self.base.options.framework_dirs);
         man.hash.addListOfBytes(self.base.options.frameworks);
         man.hash.addListOfBytes(self.base.options.rpath_list);
-        man.hash.add(self.base.options.is_compiler_rt_or_libc);
+        man.hash.add(self.base.options.skip_linker_dependencies);
         man.hash.add(self.base.options.z_nodelete);
         man.hash.add(self.base.options.z_defs);
         if (is_dyn_lib) {
@@ -633,7 +633,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         }
 
         // compiler_rt on darwin is missing some stuff, so we still build it and rely on LinkOnce
-        if (is_exe_or_dyn_lib and !self.base.options.is_compiler_rt_or_libc) {
+        if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies) {
             try argv.append(comp.compiler_rt_static_lib.?.full_object_path);
         }
 
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 74a6607051..374fa2230b 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -387,7 +387,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
     }
 
     if (self.base.options.output_mode != .Obj and
-        !self.base.options.is_compiler_rt_or_libc and
+        !self.base.options.skip_linker_dependencies and
         !self.base.options.link_libc)
     {
         try argv.append(comp.libc_static_lib.?.full_object_path);
diff --git a/src/musl.zig b/src/musl.zig
index 080dadd3fd..6680eabb03 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -225,7 +225,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .c_source_files = &[_]Compilation.CSourceFile{
                     .{ .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "musl", "libc.s" }) },
                 },
-                .is_compiler_rt_or_libc = true,
+                .skip_linker_dependencies = true,
                 .soname = "libc.so",
             });
             defer sub_compilation.destroy();
-- 
cgit v1.2.3


From a4b134746f61a2f07ea5450c0504f51cba8f6be3 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 22 Dec 2020 20:45:37 -0700
Subject: zig cc: support both ubsan and tsan at the same time

---
 src/Compilation.zig | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 6372dbff3a..5c38d131e6 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2076,10 +2076,6 @@ pub fn addCCArgs(
         try argv.append("-ffunction-sections");
     }
 
-    if (comp.bin_file.options.tsan) {
-        try argv.append("-fsanitize=thread");
-    }
-
     try argv.ensureCapacity(argv.items.len + comp.bin_file.options.framework_dirs.len * 2);
     for (comp.bin_file.options.framework_dirs) |framework_dir| {
         argv.appendAssumeCapacity("-iframework");
@@ -2203,9 +2199,14 @@ pub fn addCCArgs(
                 try argv.append("-fomit-frame-pointer");
             }
 
-            if (comp.sanitize_c) {
+            if (comp.sanitize_c and !comp.bin_file.options.tsan) {
                 try argv.append("-fsanitize=undefined");
                 try argv.append("-fsanitize-trap=undefined");
+            } else if (comp.sanitize_c and comp.bin_file.options.tsan) {
+                try argv.append("-fsanitize=undefined,thread");
+                try argv.append("-fsanitize-trap=undefined");
+            } else if (!comp.sanitize_c and comp.bin_file.options.tsan) {
+                try argv.append("-fsanitize=thread");
             }
 
             switch (comp.bin_file.options.optimize_mode) {
-- 
cgit v1.2.3


From ed26b3204a3028fbc7d05a03b7e995d2f0de7a75 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 22 Dec 2020 22:11:46 -0700
Subject: stage2: tsan forces linking libc

---
 src/Compilation.zig | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 5c38d131e6..010ac96b6f 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -593,7 +593,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             break :outer opts;
         } else .{};
 
-        const link_libc = options.link_libc or target_util.osRequiresLibC(options.target);
+        const tsan = options.want_tsan orelse false;
+
+        const link_libc = options.link_libc or target_util.osRequiresLibC(options.target) or tsan;
 
         const must_dynamic_link = dl: {
             if (target_util.cannotDynamicLink(options.target))
@@ -654,8 +656,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             options.libc_installation,
         );
 
-        const tsan = options.want_tsan orelse false;
-
         const must_pie = target_util.requiresPIE(options.target);
         const pie: bool = if (options.want_pie) |explicit| pie: {
             if (!explicit and must_pie) {
-- 
cgit v1.2.3


From 6ab9268a9024fe8e9076f07cb207107d2cad0876 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 24 Dec 2020 14:11:58 -0700
Subject: stage2: re-use compiler runtime libs across opt modes and strip flag

Previously Zig would need to recompile runtime libs if you changed the
values of --strip or -O. Now, unless the `debug_compiler_runtime_libs`
flag is set (which is currently not exposed to the CLI), Zig will always
choose ReleaseFast or ReleaseSmall for compiler runtime libraries.

When the main application chooses ReleaseFast or ReleaseSmall, that
value is propagated to compiler runtime libraries. Otherwise a decision
is made based on the target, which is currently ReleaseSmall for
freestanding WebAssembly and ReleaseFast for everything else.

Ultimately the purpose of this commit is to have Debug and ReleaseSafe
builds of applications still get optimized builds of, e.g. libcxx and
libunwind, as well as to spend less time unnecessarily rebuilding compiler
runtime libraries.
---
 src/Compilation.zig | 42 ++++++++++++++++++++++++++++++------------
 src/glibc.zig       | 12 ++----------
 src/libcxx.zig      | 13 ++++---------
 src/libtsan.zig     | 16 ++--------------
 src/libunwind.zig   |  4 ++--
 src/mingw.zig       |  9 ---------
 src/musl.zig        |  4 ++--
 src/target.zig      |  8 ++++++++
 8 files changed, 50 insertions(+), 58 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 010ac96b6f..885ecbb95c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -131,6 +131,7 @@ mutex: std.Mutex = .{},
 test_filter: ?[]const u8,
 test_name_prefix: ?[]const u8,
 test_evented_io: bool,
+debug_compiler_runtime_libs: bool,
 
 emit_asm: ?EmitLoc,
 emit_llvm_ir: ?EmitLoc,
@@ -429,6 +430,7 @@ pub const InitOptions = struct {
     verbose_llvm_cpu_features: bool = false,
     is_test: bool = false,
     test_evented_io: bool = false,
+    debug_compiler_runtime_libs: bool = false,
     /// Normally when you create a `Compilation`, Zig will automatically build
     /// and link in required dependencies, such as compiler-rt and libc. When
     /// building such dependencies themselves, this flag must be set to avoid
@@ -1025,6 +1027,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .test_filter = options.test_filter,
             .test_name_prefix = options.test_name_prefix,
             .test_evented_io = options.test_evented_io,
+            .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs,
             .work_queue_wait_group = undefined,
         };
         break :comp comp;
@@ -2891,14 +2894,6 @@ fn buildOutputFromZig(
         .directory = null, // Put it in the cache directory.
         .basename = bin_basename,
     };
-    const optimize_mode: std.builtin.Mode = blk: {
-        if (comp.bin_file.options.is_test)
-            break :blk comp.bin_file.options.optimize_mode;
-        switch (comp.bin_file.options.optimize_mode) {
-            .Debug, .ReleaseFast, .ReleaseSafe => break :blk .ReleaseFast,
-            .ReleaseSmall => break :blk .ReleaseSmall,
-        }
-    };
     const sub_compilation = try Compilation.create(comp.gpa, .{
         .global_cache_directory = comp.global_cache_directory,
         .local_cache_directory = comp.global_cache_directory,
@@ -2910,7 +2905,7 @@ fn buildOutputFromZig(
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .link_mode = .Static,
         .function_sections = true,
         .want_sanitize_c = false,
@@ -2920,7 +2915,7 @@ fn buildOutputFromZig(
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
@@ -3289,7 +3284,7 @@ pub fn build_crt_file(
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
@@ -3297,7 +3292,7 @@ pub fn build_crt_file(
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
@@ -3344,3 +3339,26 @@ pub fn stage1AddLinkLib(comp: *Compilation, lib_name: []const u8) !void {
         });
     }
 }
+
+/// This decides the optimization mode for all zig-provided libraries, including
+/// compiler-rt, libcxx, libc, libunwind, etc.
+pub fn compilerRtOptMode(comp: Compilation) std.builtin.Mode {
+    if (comp.debug_compiler_runtime_libs) {
+        return comp.bin_file.options.optimize_mode;
+    }
+    switch (comp.bin_file.options.optimize_mode) {
+        .Debug, .ReleaseSafe => return target_util.defaultCompilerRtOptimizeMode(comp.getTarget()),
+        .ReleaseFast => return .ReleaseFast,
+        .ReleaseSmall => return .ReleaseSmall,
+    }
+}
+
+/// This decides whether to strip debug info for all zig-provided libraries, including
+/// compiler-rt, libcxx, libc, libunwind, etc.
+pub fn compilerRtStrip(comp: Compilation) bool {
+    if (comp.debug_compiler_runtime_libs) {
+        return comp.bin_file.options.strip;
+    } else {
+        return true;
+    }
+}
diff --git a/src/glibc.zig b/src/glibc.zig
index 2e42cef519..0c27872720 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -271,7 +271,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"),
                 "-DTOP_NAMESPACE=glibc",
                 "-DASSEMBLER",
-                "-g",
                 "-Wa,--noexecstack",
             });
             return comp.build_crt_file("crti", .Obj, &[1]Compilation.CSourceFile{
@@ -291,7 +290,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 try lib_path(comp, arena, lib_libc_glibc ++ "include" ++ path.sep_str ++ "libc-symbols.h"),
                 "-DTOP_NAMESPACE=glibc",
                 "-DASSEMBLER",
-                "-g",
                 "-Wa,--noexecstack",
             });
             return comp.build_crt_file("crtn", .Obj, &[1]Compilation.CSourceFile{
@@ -317,7 +315,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                     "-DSHARED",
                     "-DTOP_NAMESPACE=glibc",
                     "-DASSEMBLER",
-                    "-g",
                     "-Wa,--noexecstack",
                 });
                 break :blk .{
@@ -337,7 +334,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                     "-DMODULE_NAME=libc",
                     "-DTOP_NAMESPACE=glibc",
                     "-DASSEMBLER",
-                    "-g",
                     "-Wa,--noexecstack",
                 });
                 break :blk .{
@@ -372,8 +368,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 try args.appendSlice(&[_][]const u8{
                     "-std=gnu11",
                     "-fgnu89-inline",
-                    "-g",
-                    "-O2",
                     "-fmerge-all-constants",
                     "-fno-stack-protector",
                     "-fmath-errno",
@@ -409,8 +403,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 try args.appendSlice(&[_][]const u8{
                     "-std=gnu11",
                     "-fgnu89-inline",
-                    "-g",
-                    "-O2",
                     "-fmerge-all-constants",
                     "-fno-stack-protector",
                     "-fmath-errno",
@@ -939,13 +931,13 @@ fn buildSharedLib(
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
         .want_valgrind = false,
         .want_tsan = false,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = false,
         .is_native_abi = false,
         .self_exe_path = comp.self_exe_path,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index 47bf93dee3..e79a0106e9 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -138,7 +138,6 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(cxxabi_include_path);
 
-        try cflags.append("-O3");
         if (target_util.supports_fpic(target)) {
             try cflags.append("-fPIC");
         }
@@ -164,7 +163,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
@@ -173,7 +172,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
@@ -256,16 +255,12 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(cxx_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         if (target_util.supports_fpic(target)) {
             try cflags.append("-fPIC");
         }
         try cflags.append("-nostdinc++");
         try cflags.append("-fstrict-aliasing");
         try cflags.append("-funwind-tables");
-        try cflags.append("-D_DEBUG");
-        try cflags.append("-UNDEBUG");
         try cflags.append("-std=c++11");
 
         c_source_files[i] = .{
@@ -285,7 +280,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
@@ -294,7 +289,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
diff --git a/src/libtsan.zig b/src/libtsan.zig
index ecf15693e1..d126d7313a 100644
--- a/src/libtsan.zig
+++ b/src/libtsan.zig
@@ -43,8 +43,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(tsan_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -67,8 +65,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(tsan_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -110,8 +106,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(sanitizer_common_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -136,8 +130,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(sanitizer_common_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -158,8 +150,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(tsan_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -188,8 +178,6 @@ pub fn buildTsan(comp: *Compilation) !void {
         try cflags.append("-I");
         try cflags.append(tsan_include_path);
 
-        try cflags.append("-O3");
-        try cflags.append("-DNDEBUG");
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
         try cflags.append("-std=c++14");
@@ -218,7 +206,7 @@ pub fn buildTsan(comp: *Compilation) !void {
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
@@ -227,7 +215,7 @@ pub fn buildTsan(comp: *Compilation) !void {
         .want_pic = true,
         .want_pie = true,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 057663a44b..7710b1aa77 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -104,7 +104,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
         .emit_bin = emit_bin,
-        .optimize_mode = comp.bin_file.options.optimize_mode,
+        .optimize_mode = comp.compilerRtOptMode(),
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
@@ -113,7 +113,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
         .emit_h = null,
-        .strip = comp.bin_file.options.strip,
+        .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
         .is_native_abi = comp.bin_file.options.is_native_abi,
         .self_exe_path = comp.self_exe_path,
diff --git a/src/mingw.zig b/src/mingw.zig
index 8c819627ba..a9e275434f 100644
--- a/src/mingw.zig
+++ b/src/mingw.zig
@@ -91,8 +91,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                     "-D_CRTBLD",
                     "-D_WIN32_WINNT=0x0f00",
                     "-D__MSVCRT_VERSION__=0x700",
-                    "-g",
-                    "-O2",
                 });
                 c_source_files[i] = .{
                     .src_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{
@@ -119,9 +117,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
 
                 "-isystem",
                 try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "any-windows-any" }),
-
-                "-g",
-                "-O2",
             });
             var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
 
@@ -167,8 +162,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 "-D_CRTBLD",
                 "-D_WIN32_WINNT=0x0f00",
                 "-D__MSVCRT_VERSION__=0x700",
-                "-g",
-                "-O2",
 
                 "-isystem",
                 try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "include", "any-windows-any" }),
@@ -233,8 +226,6 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 "-D_CRTBLD",
                 "-D_WIN32_WINNT=0x0f00",
                 "-D__MSVCRT_VERSION__=0x700",
-                "-g",
-                "-O2",
 
                 "-isystem",
                 try comp.zig_lib_directory.join(arena, &[_][]const u8{
diff --git a/src/musl.zig b/src/musl.zig
index 6680eabb03..f67fe90add 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -203,13 +203,13 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .thread_pool = comp.thread_pool,
                 .libc_installation = comp.bin_file.options.libc_installation,
                 .emit_bin = Compilation.EmitLoc{ .directory = null, .basename = "libc.so" },
-                .optimize_mode = comp.bin_file.options.optimize_mode,
+                .optimize_mode = comp.compilerRtOptMode(),
                 .want_sanitize_c = false,
                 .want_stack_check = false,
                 .want_valgrind = false,
                 .want_tsan = false,
                 .emit_h = null,
-                .strip = comp.bin_file.options.strip,
+                .strip = comp.compilerRtStrip(),
                 .is_native_os = false,
                 .is_native_abi = false,
                 .self_exe_path = comp.self_exe_path,
diff --git a/src/target.zig b/src/target.zig
index 9749675a89..b3c83c76d8 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -344,3 +344,11 @@ pub fn is_libcpp_lib_name(target: std.Target, name: []const u8) bool {
 pub fn hasDebugInfo(target: std.Target) bool {
     return !target.cpu.arch.isWasm();
 }
+
+pub fn defaultCompilerRtOptimizeMode(target: std.Target) std.builtin.Mode {
+    if (target.cpu.arch.isWasm() and target.os.tag == .freestanding) {
+        return .ReleaseSmall;
+    } else {
+        return .ReleaseFast;
+    }
+}
-- 
cgit v1.2.3


From 1634d45f1d53c8d7bfefa56ab4d2fa4cc8218b6d Mon Sep 17 00:00:00 2001
From: g-w1 <58830309+g-w1@users.noreply.github.com>
Date: Fri, 25 Dec 2020 19:40:49 -0500
Subject: stage2: add compile log statement (#7191)

---
 src/Compilation.zig  | 25 +++++++++++++------------
 src/Module.zig       | 46 +++++++++++++++++++++++++++++-----------------
 src/astgen.zig       | 13 +++++++++++++
 src/codegen/c.zig    |  2 +-
 src/link/C.zig       |  2 +-
 src/link/Coff.zig    |  2 +-
 src/link/Elf.zig     |  2 +-
 src/link/MachO.zig   |  2 +-
 src/value.zig        |  2 +-
 src/zir.zig          | 23 ++++++++++++++++++++---
 src/zir_sema.zig     | 23 +++++++++++++++++++++++
 test/stage2/test.zig | 15 +++++++++++++++
 12 files changed, 119 insertions(+), 38 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 885ecbb95c..6b5ae3abed 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1350,8 +1350,11 @@ pub fn totalErrorCount(self: *Compilation) usize {
     var total: usize = self.failed_c_objects.items().len;
 
     if (self.bin_file.options.module) |module| {
-        total += module.failed_decls.items().len +
-            module.failed_exports.items().len +
+        for (module.failed_decls.items()) |entry| {
+            assert(entry.value.items.len > 0);
+            total += entry.value.items.len;
+        }
+        total += module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
     }
@@ -1385,9 +1388,11 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
         }
         for (module.failed_decls.items()) |entry| {
             const decl = entry.key;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            const err_msg_list = entry.value;
+            for (err_msg_list.items) |err_msg| {
+                const source = try decl.scope.getSource(module);
+                try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            }
         }
         for (module.failed_exports.items()) |entry| {
             const decl = entry.key.owner_decl;
@@ -1480,7 +1485,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 }
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
-
                 self.bin_file.updateDecl(module, decl) catch |err| {
                     switch (err) {
                         error.OutOfMemory => return error.OutOfMemory,
@@ -1488,8 +1492,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             decl.analysis = .dependency_failure;
                         },
                         else => {
-                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                            try module.addDeclErr(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
                                 "unable to codegen: {}",
@@ -1508,8 +1511,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             decl.analysis = .dependency_failure;
                         },
                         else => {
-                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                            try module.addDeclErr(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
                                 "unable to generate C header: {}",
@@ -1531,8 +1533,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
         .update_line_number => |decl| {
             const module = self.bin_file.options.module.?;
             self.bin_file.updateDeclLineNumber(module, decl) catch |err| {
-                try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                try module.addDeclErr(decl, try ErrorMsg.create(
                     module.gpa,
                     decl.src(),
                     "unable to update line number: {}",
diff --git a/src/Module.zig b/src/Module.zig
index 20cb7bf195..897294069f 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -53,7 +53,7 @@ decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_has
 /// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator.
 /// Note that a Decl can succeed but the Fn it represents can fail. In this case,
 /// a Decl can have a failed_decls entry but have analysis status of success.
-failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
+failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, ArrayListUnmanaged(*Compilation.ErrorMsg)) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator.
 failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{},
@@ -845,8 +845,11 @@ pub fn deinit(self: *Module) void {
     }
     self.decl_table.deinit(gpa);
 
-    for (self.failed_decls.items()) |entry| {
-        entry.value.destroy(gpa);
+    for (self.failed_decls.items()) |*entry| {
+        for (entry.value.items) |compile_err| {
+            compile_err.destroy(gpa);
+        }
+        entry.value.deinit(gpa);
     }
     self.failed_decls.deinit(gpa);
 
@@ -942,8 +945,7 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
             error.OutOfMemory => return error.OutOfMemory,
             error.AnalysisFail => return error.AnalysisFail,
             else => {
-                try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
-                self.failed_decls.putAssumeCapacityNoClobber(decl, try Compilation.ErrorMsg.create(
+                try self.addDeclErr(decl, try Compilation.ErrorMsg.create(
                     self.gpa,
                     decl.src(),
                     "unable to analyze: {}",
@@ -1550,7 +1552,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                     decl.analysis = .sema_failure;
                     const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
-                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
+                    try self.addDeclErr(decl, err_msg);
                 } else {
                     if (!srcHashEql(decl.contents_hash, contents_hash)) {
                         try self.markOutdatedDecl(decl);
@@ -1592,7 +1594,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                     decl.analysis = .sema_failure;
                     const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
-                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
+                    try self.addDeclErr(decl, err_msg);
                 } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
                     try self.markOutdatedDecl(decl);
                     decl.contents_hash = contents_hash;
@@ -1718,8 +1720,11 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
             try self.markOutdatedDecl(dep);
         }
     }
-    if (self.failed_decls.remove(decl)) |entry| {
-        entry.value.destroy(self.gpa);
+    if (self.failed_decls.remove(decl)) |*entry| {
+        for (entry.value.items) |compile_err| {
+            compile_err.destroy(self.gpa);
+        }
+        entry.value.deinit(self.gpa);
     }
     self.deleteDeclExports(decl);
     self.comp.bin_file.freeDecl(decl);
@@ -1798,8 +1803,11 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
 fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     log.debug("mark {} outdated\n", .{decl.name});
     try self.comp.work_queue.writeItem(.{ .analyze_decl = decl });
-    if (self.failed_decls.remove(decl)) |entry| {
-        entry.value.destroy(self.gpa);
+    if (self.failed_decls.remove(decl)) |*entry| {
+        for (entry.value.items) |compile_err| {
+            compile_err.destroy(self.gpa);
+        }
+        entry.value.deinit(self.gpa);
     }
     decl.analysis = .outdated;
 }
@@ -2944,10 +2952,14 @@ pub fn failNode(
     return self.fail(scope, src, format, args);
 }
 
+pub fn addDeclErr(self: *Module, decl: *Decl, err: *Compilation.ErrorMsg) error{OutOfMemory}!void {
+    const entry = try self.failed_decls.getOrPutValue(self.gpa, decl, .{});
+    try entry.value.append(self.gpa, err);
+}
+
 fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Compilation.ErrorMsg) InnerError {
     {
         errdefer err_msg.destroy(self.gpa);
-        try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
         try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1);
     }
     switch (scope.tag) {
@@ -2955,7 +2967,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
             const decl = scope.cast(Scope.DeclAnalysis).?.decl;
             decl.analysis = .sema_failure;
             decl.generation = self.generation;
-            self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg);
+            try self.addDeclErr(decl, err_msg);
         },
         .block => {
             const block = scope.cast(Scope.Block).?;
@@ -2965,25 +2977,25 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
                 block.decl.analysis = .sema_failure;
                 block.decl.generation = self.generation;
             }
-            self.failed_decls.putAssumeCapacityNoClobber(block.decl, err_msg);
+            try self.addDeclErr(block.decl, err_msg);
         },
         .gen_zir => {
             const gen_zir = scope.cast(Scope.GenZIR).?;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
+            try self.addDeclErr(gen_zir.decl, err_msg);
         },
         .local_val => {
             const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
+            try self.addDeclErr(gen_zir.decl, err_msg);
         },
         .local_ptr => {
             const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
+            try self.addDeclErr(gen_zir.decl, err_msg);
         },
         .zir_module => {
             const zir_module = scope.cast(Scope.ZIRModule).?;
diff --git a/src/astgen.zig b/src/astgen.zig
index 622a18c443..ed5281ba42 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -2248,6 +2248,17 @@ fn import(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinCall) InnerError!*
     return addZIRUnOp(mod, scope, src, .import, target);
 }
 
+fn compileLog(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
+    const tree = scope.tree();
+    const arena = scope.arena();
+    const src = tree.token_locs[call.builtin_token].start;
+    const params = call.params();
+    var targets = try arena.alloc(*zir.Inst, params.len);
+    for (params) |param, param_i|
+        targets[param_i] = try expr(mod, scope, .none, param);
+    return addZIRInst(mod, scope, src, zir.Inst.CompileLog, .{ .to_log = targets }, .{});
+}
+
 fn typeOf(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
     const tree = scope.tree();
     const arena = scope.arena();
@@ -2291,6 +2302,8 @@ fn builtinCall(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.Built
         return rlWrap(mod, scope, rl, try addZIRNoOp(mod, scope, src, .breakpoint));
     } else if (mem.eql(u8, builtin_name, "@import")) {
         return rlWrap(mod, scope, rl, try import(mod, scope, call));
+    } else if (mem.eql(u8, builtin_name, "@compileLog")) {
+        return compileLog(mod, scope, call);
     } else {
         return mod.failTok(scope, call.builtin_token, "invalid builtin function: '{}'", .{builtin_name});
     }
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 589e2f17e0..73ac3e2f38 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -106,7 +106,7 @@ pub fn generateHeader(
             const writer = header.buf.writer();
             renderFunctionSignature(&ctx, header, writer, decl) catch |err| {
                 if (err == error.AnalysisFail) {
-                    try module.failed_decls.put(module.gpa, decl, ctx.error_msg);
+                    try module.addDeclErr(decl, ctx.error_msg);
                 }
                 return err;
             };
diff --git a/src/link/C.zig b/src/link/C.zig
index 325f2ee048..cadbcf105f 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -106,7 +106,7 @@ pub fn deinit(self: *C) void {
 pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {
     codegen.generate(self, decl) catch |err| {
         if (err == error.AnalysisFail) {
-            try module.failed_decls.put(module.gpa, decl, self.error_msg);
+            try module.addDeclErr(decl, self.error_msg);
         }
         return err;
     };
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index bea3033f88..5ec4284623 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -658,7 +658,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.failed_decls.put(module.gpa, decl, em);
+            try module.addDeclErr(decl, em);
             return;
         },
     };
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 6232a64e4a..2cd668c09f 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2248,7 +2248,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.failed_decls.put(module.gpa, decl, em);
+            try module.addDeclErr(decl, em);
             return;
         },
     };
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 17816959a3..8c3001938f 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -1062,7 +1062,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.failed_decls.put(module.gpa, decl, em);
+            try module.addDeclErr(decl, em);
             return;
         },
     };
diff --git a/src/value.zig b/src/value.zig
index 4271ae66f4..56b3434400 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -350,7 +350,7 @@ pub const Value = extern union {
                 val = elem_ptr.array_ptr;
             },
             .empty_array => return out_stream.writeAll(".{}"),
-            .enum_literal => return out_stream.print(".{z}", .{self.cast(Payload.Bytes).?.data}),
+            .enum_literal => return out_stream.print(".{z}", .{@fieldParentPtr(Payload.Bytes, "base", self.ptr_otherwise).data}),
             .bytes => return out_stream.print("\"{Z}\"", .{self.cast(Payload.Bytes).?.data}),
             .repeated => {
                 try out_stream.writeAll("(repeated) ");
diff --git a/src/zir.zig b/src/zir.zig
index f4f52cbc0f..982c98ddc9 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -122,6 +122,8 @@ pub const Inst = struct {
         coerce_to_ptr_elem,
         /// Emit an error message and fail compilation.
         compileerror,
+        /// Log compile time variables and emit an error message.
+        compilelog,
         /// Conditional branch. Splits control flow based on a boolean condition value.
         condbr,
         /// Special case, has no textual representation.
@@ -386,6 +388,7 @@ pub const Inst = struct {
                 .declval_in_module => DeclValInModule,
                 .coerce_result_block_ptr => CoerceResultBlockPtr,
                 .compileerror => CompileError,
+                .compilelog => CompileLog,
                 .loop => Loop,
                 .@"const" => Const,
                 .str => Str,
@@ -513,6 +516,7 @@ pub const Inst = struct {
                 .slice_start,
                 .import,
                 .switch_range,
+                .compilelog,
                 .typeof_peer,
                 => false,
 
@@ -707,6 +711,19 @@ pub const Inst = struct {
         kw_args: struct {},
     };
 
+    pub const CompileLog = struct {
+        pub const base_tag = Tag.compilelog;
+        base: Inst,
+
+        positionals: struct {
+            to_log: []*Inst,
+        },
+        kw_args: struct {
+            /// If we have seen it already so don't make another error
+            seen: bool = false,
+        },
+    };
+
     pub const Const = struct {
         pub const base_tag = Tag.@"const";
         base: Inst,
@@ -1925,7 +1942,7 @@ const EmitZIR = struct {
                 .sema_failure,
                 .sema_failure_retryable,
                 .dependency_failure,
-                => if (self.old_module.failed_decls.get(ir_decl)) |err_msg| {
+                => if (self.old_module.failed_decls.get(ir_decl)) |err_msg_list| {
                     const fail_inst = try self.arena.allocator.create(Inst.CompileError);
                     fail_inst.* = .{
                         .base = .{
@@ -1933,7 +1950,7 @@ const EmitZIR = struct {
                             .tag = Inst.CompileError.base_tag,
                         },
                         .positionals = .{
-                            .msg = try self.arena.allocator.dupe(u8, err_msg.msg),
+                            .msg = try self.arena.allocator.dupe(u8, err_msg_list.items[0].msg),
                         },
                         .kw_args = .{},
                     };
@@ -2055,7 +2072,7 @@ const EmitZIR = struct {
                 try self.emitBody(body, &inst_table, &instructions);
             },
             .sema_failure => {
-                const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?;
+                const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?.items[0];
                 const fail_inst = try self.arena.allocator.create(Inst.CompileError);
                 fail_inst.* = .{
                     .base = .{
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 42d4577f71..16f2297ee6 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -43,6 +43,7 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
         .coerce_result_ptr => return analyzeInstCoerceResultPtr(mod, scope, old_inst.castTag(.coerce_result_ptr).?),
         .coerce_to_ptr_elem => return analyzeInstCoerceToPtrElem(mod, scope, old_inst.castTag(.coerce_to_ptr_elem).?),
         .compileerror => return analyzeInstCompileError(mod, scope, old_inst.castTag(.compileerror).?),
+        .compilelog => return analyzeInstCompileLog(mod, scope, old_inst.castTag(.compilelog).?),
         .@"const" => return analyzeInstConst(mod, scope, old_inst.castTag(.@"const").?),
         .dbg_stmt => return analyzeInstDbgStmt(mod, scope, old_inst.castTag(.dbg_stmt).?),
         .declref => return analyzeInstDeclRef(mod, scope, old_inst.castTag(.declref).?),
@@ -489,6 +490,28 @@ fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.CompileE
     return mod.fail(scope, inst.base.src, "{}", .{inst.positionals.msg});
 }
 
+fn analyzeInstCompileLog(mod: *Module, scope: *Scope, inst: *zir.Inst.CompileLog) InnerError!*Inst {
+    std.debug.print("| ", .{});
+    for (inst.positionals.to_log) |item, i| {
+        const to_log = try resolveInst(mod, scope, item);
+        if (to_log.value()) |val| {
+            std.debug.print("{}", .{val});
+        } else {
+            std.debug.print("(runtime value)", .{});
+        }
+        if (i != inst.positionals.to_log.len - 1) std.debug.print(", ", .{});
+    }
+    std.debug.print("\n", .{});
+    if (!inst.kw_args.seen) {
+        inst.kw_args.seen = true; // so that we do not give multiple compile errors if it gets evaled twice
+        switch (mod.fail(scope, inst.base.src, "found compile log statement", .{})) {
+            error.AnalysisFail => {}, // analysis continues
+            else => |e| return e,
+        }
+    }
+    return mod.constVoid(scope, inst.base.src);
+}
+
 fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*Inst {
     const b = try mod.requireRuntimeBlock(scope, inst.base.src);
     const fn_ty = b.func.?.owner_decl.typed_value.most_recent.typed_value.ty;
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index 3da497fb9b..d17ccc7b8b 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -1171,6 +1171,21 @@ pub fn addCases(ctx: *TestContext) !void {
         \\fn entry() void {}
     , &[_][]const u8{":2:4: error: redefinition of 'entry'"});
 
+    ctx.compileError("compileLog", linux_x64,
+        \\export fn _start() noreturn {
+        \\  const b = true;
+        \\  var f: u32 = 1;
+        \\  @compileLog(b, 20, f, x, .foo);
+        \\  var y: u32 = true;
+        \\  unreachable;
+        \\}
+        \\fn x() void {}
+    , &[_][]const u8{
+        ":4:3: error: found compile log statement", ":5:16: error: expected u32, found bool",
+    });
+
+    // "| true, 20, (runtime value), (function)" // TODO if this is here it invalidates the compile error checker. Need a way to check though.
+
     {
         var case = ctx.obj("variable shadowing", linux_x64);
         case.addError(
-- 
cgit v1.2.3


From f75d4cbe56f9f8212581f00700600a57ce545ba1 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 28 Dec 2020 11:24:53 -0700
Subject: Revert "stage2: add compile log statement (#7191)"

The addition of `addDeclErr` introduced a memory leak at every call
site, and I also would like to push back on having more than 1
compilation error per `Decl`.

This reverts commit 1634d45f1d53c8d7bfefa56ab4d2fa4cc8218b6d.
---
 src/Compilation.zig  | 25 ++++++++++++-------------
 src/Module.zig       | 46 +++++++++++++++++-----------------------------
 src/astgen.zig       | 13 -------------
 src/codegen/c.zig    |  2 +-
 src/link/C.zig       |  2 +-
 src/link/Coff.zig    |  2 +-
 src/link/Elf.zig     |  2 +-
 src/link/MachO.zig   |  2 +-
 src/value.zig        |  2 +-
 src/zir.zig          | 25 ++++---------------------
 src/zir_sema.zig     | 23 -----------------------
 test/stage2/test.zig | 16 +---------------
 12 files changed, 40 insertions(+), 120 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 6b5ae3abed..885ecbb95c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1350,11 +1350,8 @@ pub fn totalErrorCount(self: *Compilation) usize {
     var total: usize = self.failed_c_objects.items().len;
 
     if (self.bin_file.options.module) |module| {
-        for (module.failed_decls.items()) |entry| {
-            assert(entry.value.items.len > 0);
-            total += entry.value.items.len;
-        }
-        total += module.failed_exports.items().len +
+        total += module.failed_decls.items().len +
+            module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
     }
@@ -1388,11 +1385,9 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
         }
         for (module.failed_decls.items()) |entry| {
             const decl = entry.key;
-            const err_msg_list = entry.value;
-            for (err_msg_list.items) |err_msg| {
-                const source = try decl.scope.getSource(module);
-                try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
-            }
+            const err_msg = entry.value;
+            const source = try decl.scope.getSource(module);
+            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
         }
         for (module.failed_exports.items()) |entry| {
             const decl = entry.key.owner_decl;
@@ -1485,6 +1480,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 }
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
+
                 self.bin_file.updateDecl(module, decl) catch |err| {
                     switch (err) {
                         error.OutOfMemory => return error.OutOfMemory,
@@ -1492,7 +1488,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             decl.analysis = .dependency_failure;
                         },
                         else => {
-                            try module.addDeclErr(decl, try ErrorMsg.create(
+                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
                                 "unable to codegen: {}",
@@ -1511,7 +1508,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             decl.analysis = .dependency_failure;
                         },
                         else => {
-                            try module.addDeclErr(decl, try ErrorMsg.create(
+                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
                                 "unable to generate C header: {}",
@@ -1533,7 +1531,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
         .update_line_number => |decl| {
             const module = self.bin_file.options.module.?;
             self.bin_file.updateDeclLineNumber(module, decl) catch |err| {
-                try module.addDeclErr(decl, try ErrorMsg.create(
+                try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                     module.gpa,
                     decl.src(),
                     "unable to update line number: {}",
diff --git a/src/Module.zig b/src/Module.zig
index 5a4a76150c..c0b7011f43 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -53,7 +53,7 @@ decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_has
 /// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator.
 /// Note that a Decl can succeed but the Fn it represents can fail. In this case,
 /// a Decl can have a failed_decls entry but have analysis status of success.
-failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, ArrayListUnmanaged(*Compilation.ErrorMsg)) = .{},
+failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator.
 failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{},
@@ -849,11 +849,8 @@ pub fn deinit(self: *Module) void {
     }
     self.decl_table.deinit(gpa);
 
-    for (self.failed_decls.items()) |*entry| {
-        for (entry.value.items) |compile_err| {
-            compile_err.destroy(gpa);
-        }
-        entry.value.deinit(gpa);
+    for (self.failed_decls.items()) |entry| {
+        entry.value.destroy(gpa);
     }
     self.failed_decls.deinit(gpa);
 
@@ -949,7 +946,8 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
             error.OutOfMemory => return error.OutOfMemory,
             error.AnalysisFail => return error.AnalysisFail,
             else => {
-                try self.addDeclErr(decl, try Compilation.ErrorMsg.create(
+                try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
+                self.failed_decls.putAssumeCapacityNoClobber(decl, try Compilation.ErrorMsg.create(
                     self.gpa,
                     decl.src(),
                     "unable to analyze: {}",
@@ -1556,7 +1554,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                     decl.analysis = .sema_failure;
                     const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
-                    try self.addDeclErr(decl, err_msg);
+                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
                 } else {
                     if (!srcHashEql(decl.contents_hash, contents_hash)) {
                         try self.markOutdatedDecl(decl);
@@ -1598,7 +1596,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                     decl.analysis = .sema_failure;
                     const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
-                    try self.addDeclErr(decl, err_msg);
+                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
                 } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
                     try self.markOutdatedDecl(decl);
                     decl.contents_hash = contents_hash;
@@ -1724,11 +1722,8 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
             try self.markOutdatedDecl(dep);
         }
     }
-    if (self.failed_decls.remove(decl)) |*entry| {
-        for (entry.value.items) |compile_err| {
-            compile_err.destroy(self.gpa);
-        }
-        entry.value.deinit(self.gpa);
+    if (self.failed_decls.remove(decl)) |entry| {
+        entry.value.destroy(self.gpa);
     }
     self.deleteDeclExports(decl);
     self.comp.bin_file.freeDecl(decl);
@@ -1807,11 +1802,8 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
 fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     log.debug("mark {} outdated\n", .{decl.name});
     try self.comp.work_queue.writeItem(.{ .analyze_decl = decl });
-    if (self.failed_decls.remove(decl)) |*entry| {
-        for (entry.value.items) |compile_err| {
-            compile_err.destroy(self.gpa);
-        }
-        entry.value.deinit(self.gpa);
+    if (self.failed_decls.remove(decl)) |entry| {
+        entry.value.destroy(self.gpa);
     }
     decl.analysis = .outdated;
 }
@@ -2956,14 +2948,10 @@ pub fn failNode(
     return self.fail(scope, src, format, args);
 }
 
-pub fn addDeclErr(self: *Module, decl: *Decl, err: *Compilation.ErrorMsg) error{OutOfMemory}!void {
-    const entry = try self.failed_decls.getOrPutValue(self.gpa, decl, .{});
-    try entry.value.append(self.gpa, err);
-}
-
 fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Compilation.ErrorMsg) InnerError {
     {
         errdefer err_msg.destroy(self.gpa);
+        try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
         try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1);
     }
     switch (scope.tag) {
@@ -2971,7 +2959,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
             const decl = scope.cast(Scope.DeclAnalysis).?.decl;
             decl.analysis = .sema_failure;
             decl.generation = self.generation;
-            try self.addDeclErr(decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg);
         },
         .block => {
             const block = scope.cast(Scope.Block).?;
@@ -2981,25 +2969,25 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
                 block.decl.analysis = .sema_failure;
                 block.decl.generation = self.generation;
             }
-            try self.addDeclErr(block.decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(block.decl, err_msg);
         },
         .gen_zir => {
             const gen_zir = scope.cast(Scope.GenZIR).?;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            try self.addDeclErr(gen_zir.decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
         },
         .local_val => {
             const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            try self.addDeclErr(gen_zir.decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
         },
         .local_ptr => {
             const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir;
             gen_zir.decl.analysis = .sema_failure;
             gen_zir.decl.generation = self.generation;
-            try self.addDeclErr(gen_zir.decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
         },
         .zir_module => {
             const zir_module = scope.cast(Scope.ZIRModule).?;
diff --git a/src/astgen.zig b/src/astgen.zig
index 100b2b9d02..672fe343e2 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -2333,17 +2333,6 @@ fn compileError(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinCall) InnerE
     return addZIRUnOp(mod, scope, src, .compileerror, target);
 }
 
-fn compileLog(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
-    const tree = scope.tree();
-    const arena = scope.arena();
-    const src = tree.token_locs[call.builtin_token].start;
-    const params = call.params();
-    var targets = try arena.alloc(*zir.Inst, params.len);
-    for (params) |param, param_i|
-        targets[param_i] = try expr(mod, scope, .none, param);
-    return addZIRInst(mod, scope, src, zir.Inst.CompileLog, .{ .to_log = targets }, .{});
-}
-
 fn typeOf(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
     const tree = scope.tree();
     const arena = scope.arena();
@@ -2389,8 +2378,6 @@ fn builtinCall(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.Built
         return rlWrap(mod, scope, rl, try import(mod, scope, call));
     } else if (mem.eql(u8, builtin_name, "@compileError")) {
         return compileError(mod, scope, call);
-    } else if (mem.eql(u8, builtin_name, "@compileLog")) {
-        return compileLog(mod, scope, call);
     } else {
         return mod.failTok(scope, call.builtin_token, "invalid builtin function: '{}'", .{builtin_name});
     }
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 73ac3e2f38..589e2f17e0 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -106,7 +106,7 @@ pub fn generateHeader(
             const writer = header.buf.writer();
             renderFunctionSignature(&ctx, header, writer, decl) catch |err| {
                 if (err == error.AnalysisFail) {
-                    try module.addDeclErr(decl, ctx.error_msg);
+                    try module.failed_decls.put(module.gpa, decl, ctx.error_msg);
                 }
                 return err;
             };
diff --git a/src/link/C.zig b/src/link/C.zig
index cadbcf105f..325f2ee048 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -106,7 +106,7 @@ pub fn deinit(self: *C) void {
 pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {
     codegen.generate(self, decl) catch |err| {
         if (err == error.AnalysisFail) {
-            try module.addDeclErr(decl, self.error_msg);
+            try module.failed_decls.put(module.gpa, decl, self.error_msg);
         }
         return err;
     };
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 5ec4284623..bea3033f88 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -658,7 +658,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.addDeclErr(decl, em);
+            try module.failed_decls.put(module.gpa, decl, em);
             return;
         },
     };
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 2cd668c09f..6232a64e4a 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2248,7 +2248,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.addDeclErr(decl, em);
+            try module.failed_decls.put(module.gpa, decl, em);
             return;
         },
     };
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 8c3001938f..17816959a3 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -1062,7 +1062,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
         .appended => code_buffer.items,
         .fail => |em| {
             decl.analysis = .codegen_failure;
-            try module.addDeclErr(decl, em);
+            try module.failed_decls.put(module.gpa, decl, em);
             return;
         },
     };
diff --git a/src/value.zig b/src/value.zig
index 56b3434400..4271ae66f4 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -350,7 +350,7 @@ pub const Value = extern union {
                 val = elem_ptr.array_ptr;
             },
             .empty_array => return out_stream.writeAll(".{}"),
-            .enum_literal => return out_stream.print(".{z}", .{@fieldParentPtr(Payload.Bytes, "base", self.ptr_otherwise).data}),
+            .enum_literal => return out_stream.print(".{z}", .{self.cast(Payload.Bytes).?.data}),
             .bytes => return out_stream.print("\"{Z}\"", .{self.cast(Payload.Bytes).?.data}),
             .repeated => {
                 try out_stream.writeAll("(repeated) ");
diff --git a/src/zir.zig b/src/zir.zig
index 10f64be609..47e1abc24b 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -126,8 +126,6 @@ pub const Inst = struct {
         coerce_to_ptr_elem,
         /// Emit an error message and fail compilation.
         compileerror,
-        /// Log compile time variables and emit an error message.
-        compilelog,
         /// Conditional branch. Splits control flow based on a boolean condition value.
         condbr,
         /// Special case, has no textual representation.
@@ -394,7 +392,6 @@ pub const Inst = struct {
                 .declval => DeclVal,
                 .declval_in_module => DeclValInModule,
                 .coerce_result_block_ptr => CoerceResultBlockPtr,
-                .compilelog => CompileLog,
                 .loop => Loop,
                 .@"const" => Const,
                 .str => Str,
@@ -524,7 +521,6 @@ pub const Inst = struct {
                 .slice_start,
                 .import,
                 .switch_range,
-                .compilelog,
                 .typeof_peer,
                 => false,
 
@@ -709,19 +705,6 @@ pub const Inst = struct {
         kw_args: struct {},
     };
 
-    pub const CompileLog = struct {
-        pub const base_tag = Tag.compilelog;
-        base: Inst,
-
-        positionals: struct {
-            to_log: []*Inst,
-        },
-        kw_args: struct {
-            /// If we have seen it already so don't make another error
-            seen: bool = false,
-        },
-    };
-
     pub const Const = struct {
         pub const base_tag = Tag.@"const";
         base: Inst,
@@ -1940,7 +1923,7 @@ const EmitZIR = struct {
                 .sema_failure,
                 .sema_failure_retryable,
                 .dependency_failure,
-                => if (self.old_module.failed_decls.get(ir_decl)) |err_msg_list| {
+                => if (self.old_module.failed_decls.get(ir_decl)) |err_msg| {
                     const fail_inst = try self.arena.allocator.create(Inst.UnOp);
                     fail_inst.* = .{
                         .base = .{
@@ -1949,7 +1932,7 @@ const EmitZIR = struct {
                         },
                         .positionals = .{
                             .operand = blk: {
-                                const msg_str = try self.arena.allocator.dupe(u8, err_msg_list.items[0].msg);
+                                const msg_str = try self.arena.allocator.dupe(u8, err_msg.msg);
 
                                 const str_inst = try self.arena.allocator.create(Inst.Str);
                                 str_inst.* = .{
@@ -1958,7 +1941,7 @@ const EmitZIR = struct {
                                         .tag = Inst.Str.base_tag,
                                     },
                                     .positionals = .{
-                                        .bytes = msg_str,
+                                        .bytes = err_msg.msg,
                                     },
                                     .kw_args = .{},
                                 };
@@ -2085,7 +2068,7 @@ const EmitZIR = struct {
                 try self.emitBody(body, &inst_table, &instructions);
             },
             .sema_failure => {
-                const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?.items[0];
+                const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?;
                 const fail_inst = try self.arena.allocator.create(Inst.UnOp);
                 fail_inst.* = .{
                     .base = .{
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index f3dae8f5b1..5700c18578 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -45,7 +45,6 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
         .coerce_result_ptr => return analyzeInstCoerceResultPtr(mod, scope, old_inst.castTag(.coerce_result_ptr).?),
         .coerce_to_ptr_elem => return analyzeInstCoerceToPtrElem(mod, scope, old_inst.castTag(.coerce_to_ptr_elem).?),
         .compileerror => return analyzeInstCompileError(mod, scope, old_inst.castTag(.compileerror).?),
-        .compilelog => return analyzeInstCompileLog(mod, scope, old_inst.castTag(.compilelog).?),
         .@"const" => return analyzeInstConst(mod, scope, old_inst.castTag(.@"const").?),
         .dbg_stmt => return analyzeInstDbgStmt(mod, scope, old_inst.castTag(.dbg_stmt).?),
         .declref => return analyzeInstDeclRef(mod, scope, old_inst.castTag(.declref).?),
@@ -503,28 +502,6 @@ fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) In
     return mod.fail(scope, inst.base.src, "{}", .{msg});
 }
 
-fn analyzeInstCompileLog(mod: *Module, scope: *Scope, inst: *zir.Inst.CompileLog) InnerError!*Inst {
-    std.debug.print("| ", .{});
-    for (inst.positionals.to_log) |item, i| {
-        const to_log = try resolveInst(mod, scope, item);
-        if (to_log.value()) |val| {
-            std.debug.print("{}", .{val});
-        } else {
-            std.debug.print("(runtime value)", .{});
-        }
-        if (i != inst.positionals.to_log.len - 1) std.debug.print(", ", .{});
-    }
-    std.debug.print("\n", .{});
-    if (!inst.kw_args.seen) {
-        inst.kw_args.seen = true; // so that we do not give multiple compile errors if it gets evaled twice
-        switch (mod.fail(scope, inst.base.src, "found compile log statement", .{})) {
-            error.AnalysisFail => {}, // analysis continues
-            else => |e| return e,
-        }
-    }
-    return mod.constVoid(scope, inst.base.src);
-}
-
 fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*Inst {
     const b = try mod.requireRuntimeBlock(scope, inst.base.src);
     const fn_ty = b.func.?.owner_decl.typed_value.most_recent.typed_value.ty;
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index 4e57bbf9f7..5a200e9cf4 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -1171,27 +1171,13 @@ pub fn addCases(ctx: *TestContext) !void {
         \\fn entry() void {}
     , &[_][]const u8{":2:4: error: redefinition of 'entry'"});
 
-    ctx.compileError("compileLog", linux_x64,
-        \\export fn _start() noreturn {
-        \\  const b = true;
-        \\  var f: u32 = 1;
-        \\  @compileLog(b, 20, f, x, .foo);
-        \\  var y: u32 = true;
-        \\  unreachable;
-        \\}
-        \\fn x() void {}
-    , &[_][]const u8{
-        ":4:3: error: found compile log statement", ":5:16: error: expected u32, found bool",
-    });
-
-    // "| true, 20, (runtime value), (function)" // TODO if this is here it invalidates the compile error checker. Need a way to check though.
-
     ctx.compileError("compileError", linux_x64,
         \\export fn _start() noreturn {
         \\  @compileError("this is an error");
         \\  unreachable;
         \\}
     , &[_][]const u8{":2:3: error: this is an error"});
+
     {
         var case = ctx.obj("variable shadowing", linux_x64);
         case.addError(
-- 
cgit v1.2.3


From 071417161d5d7ab91c32073693590ef161017dbe Mon Sep 17 00:00:00 2001
From: Timon Kruiper <timonkruiper@gmail.com>
Date: Sat, 19 Dec 2020 11:22:49 +0100
Subject: stage2: add initial impl of LLVM backend in self-hosted compiler

---
 CMakeLists.txt        |   2 +-
 src/Compilation.zig   |   2 +-
 src/codegen/llvm.zig  | 125 ---------------
 src/link/Elf.zig      |  45 +++++-
 src/llvm_backend.zig  | 410 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/llvm_bindings.zig | 358 +++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 813 insertions(+), 129 deletions(-)
 delete mode 100644 src/codegen/llvm.zig
 create mode 100644 src/llvm_backend.zig

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3e002ca5c1..a9dc38b40b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -527,7 +527,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/codegen/aarch64.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/arm.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/c.zig"
-    "${CMAKE_SOURCE_DIR}/src/codegen/llvm.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/riscv64.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/spu-mk2.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/wasm.zig"
@@ -549,6 +548,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/link/cbe.h"
     "${CMAKE_SOURCE_DIR}/src/link/msdos-stub.bin"
     "${CMAKE_SOURCE_DIR}/src/liveness.zig"
+    "${CMAKE_SOURCE_DIR}/src/llvm_backend.zig"
     "${CMAKE_SOURCE_DIR}/src/llvm_bindings.zig"
     "${CMAKE_SOURCE_DIR}/src/main.zig"
     "${CMAKE_SOURCE_DIR}/src/mingw.zig"
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 885ecbb95c..26beeccd9e 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2106,7 +2106,7 @@ pub fn addCCArgs(
         try argv.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS");
     }
 
-    const llvm_triple = try @import("codegen/llvm.zig").targetTriple(arena, target);
+    const llvm_triple = try @import("llvm_backend.zig").targetTriple(arena, target);
     try argv.appendSlice(&[_][]const u8{ "-target", llvm_triple });
 
     switch (ext) {
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
deleted file mode 100644
index f87a7a940c..0000000000
--- a/src/codegen/llvm.zig
+++ /dev/null
@@ -1,125 +0,0 @@
-const std = @import("std");
-const Allocator = std.mem.Allocator;
-
-pub fn targetTriple(allocator: *Allocator, target: std.Target) ![]u8 {
-    const llvm_arch = switch (target.cpu.arch) {
-        .arm => "arm",
-        .armeb => "armeb",
-        .aarch64 => "aarch64",
-        .aarch64_be => "aarch64_be",
-        .aarch64_32 => "aarch64_32",
-        .arc => "arc",
-        .avr => "avr",
-        .bpfel => "bpfel",
-        .bpfeb => "bpfeb",
-        .hexagon => "hexagon",
-        .mips => "mips",
-        .mipsel => "mipsel",
-        .mips64 => "mips64",
-        .mips64el => "mips64el",
-        .msp430 => "msp430",
-        .powerpc => "powerpc",
-        .powerpc64 => "powerpc64",
-        .powerpc64le => "powerpc64le",
-        .r600 => "r600",
-        .amdgcn => "amdgcn",
-        .riscv32 => "riscv32",
-        .riscv64 => "riscv64",
-        .sparc => "sparc",
-        .sparcv9 => "sparcv9",
-        .sparcel => "sparcel",
-        .s390x => "s390x",
-        .tce => "tce",
-        .tcele => "tcele",
-        .thumb => "thumb",
-        .thumbeb => "thumbeb",
-        .i386 => "i386",
-        .x86_64 => "x86_64",
-        .xcore => "xcore",
-        .nvptx => "nvptx",
-        .nvptx64 => "nvptx64",
-        .le32 => "le32",
-        .le64 => "le64",
-        .amdil => "amdil",
-        .amdil64 => "amdil64",
-        .hsail => "hsail",
-        .hsail64 => "hsail64",
-        .spir => "spir",
-        .spir64 => "spir64",
-        .kalimba => "kalimba",
-        .shave => "shave",
-        .lanai => "lanai",
-        .wasm32 => "wasm32",
-        .wasm64 => "wasm64",
-        .renderscript32 => "renderscript32",
-        .renderscript64 => "renderscript64",
-        .ve => "ve",
-        .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII,
-    };
-    // TODO Add a sub-arch for some architectures depending on CPU features.
-
-    const llvm_os = switch (target.os.tag) {
-        .freestanding => "unknown",
-        .ananas => "ananas",
-        .cloudabi => "cloudabi",
-        .dragonfly => "dragonfly",
-        .freebsd => "freebsd",
-        .fuchsia => "fuchsia",
-        .ios => "ios",
-        .kfreebsd => "kfreebsd",
-        .linux => "linux",
-        .lv2 => "lv2",
-        .macos => "macosx",
-        .netbsd => "netbsd",
-        .openbsd => "openbsd",
-        .solaris => "solaris",
-        .windows => "windows",
-        .haiku => "haiku",
-        .minix => "minix",
-        .rtems => "rtems",
-        .nacl => "nacl",
-        .cnk => "cnk",
-        .aix => "aix",
-        .cuda => "cuda",
-        .nvcl => "nvcl",
-        .amdhsa => "amdhsa",
-        .ps4 => "ps4",
-        .elfiamcu => "elfiamcu",
-        .tvos => "tvos",
-        .watchos => "watchos",
-        .mesa3d => "mesa3d",
-        .contiki => "contiki",
-        .amdpal => "amdpal",
-        .hermit => "hermit",
-        .hurd => "hurd",
-        .wasi => "wasi",
-        .emscripten => "emscripten",
-        .uefi => "windows",
-        .other => "unknown",
-    };
-
-    const llvm_abi = switch (target.abi) {
-        .none => "unknown",
-        .gnu => "gnu",
-        .gnuabin32 => "gnuabin32",
-        .gnuabi64 => "gnuabi64",
-        .gnueabi => "gnueabi",
-        .gnueabihf => "gnueabihf",
-        .gnux32 => "gnux32",
-        .code16 => "code16",
-        .eabi => "eabi",
-        .eabihf => "eabihf",
-        .android => "android",
-        .musl => "musl",
-        .musleabi => "musleabi",
-        .musleabihf => "musleabihf",
-        .msvc => "msvc",
-        .itanium => "itanium",
-        .cygnus => "cygnus",
-        .coreclr => "coreclr",
-        .simulator => "simulator",
-        .macabi => "macabi",
-    };
-
-    return std.fmt.allocPrint(allocator, "{}-unknown-{}-{}", .{ llvm_arch, llvm_os, llvm_abi });
-}
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 6232a64e4a..479dc415e5 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -24,6 +24,7 @@ const build_options = @import("build_options");
 const target_util = @import("../target.zig");
 const glibc = @import("../glibc.zig");
 const Cache = @import("../Cache.zig");
+const llvm_backend = @import("../llvm_backend.zig");
 
 const default_entry_addr = 0x8000000;
 
@@ -33,6 +34,9 @@ base: File,
 
 ptr_width: PtrWidth,
 
+/// If this is not null, an object file is created by LLVM and linked with LLD afterwards.
+llvm_ir_module: ?*llvm_backend.LLVMIRModule = null,
+
 /// Stored in native-endian format, depending on target endianness needs to be bswapped on read/write.
 /// Same order as in the file.
 sections: std.ArrayListUnmanaged(elf.Elf64_Shdr) = std.ArrayListUnmanaged(elf.Elf64_Shdr){},
@@ -224,7 +228,13 @@ pub const SrcFn = struct {
 pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*Elf {
     assert(options.object_format == .elf);
 
-    if (options.use_llvm) return error.LLVMBackendUnimplementedForELF; // TODO
+    if (options.use_llvm) {
+        const self = try createEmpty(allocator, options);
+        errdefer self.base.destroy();
+
+        self.llvm_ir_module = try llvm_backend.LLVMIRModule.create(allocator, sub_path, options);
+        return self;
+    }
 
     const file = try options.emit.?.directory.handle.createFile(sub_path, .{
         .truncate = false,
@@ -288,6 +298,7 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Elf {
 }
 
 pub fn deinit(self: *Elf) void {
+    if (self.llvm_ir_module) |ir_module| ir_module.deinit(self.base.allocator);
     self.sections.deinit(self.base.allocator);
     self.program_headers.deinit(self.base.allocator);
     self.shstrtab.deinit(self.base.allocator);
@@ -423,6 +434,8 @@ fn updateString(self: *Elf, old_str_off: u32, new_name: []const u8) !u32 {
 }
 
 pub fn populateMissingMetadata(self: *Elf) !void {
+    if (self.llvm_ir_module) |_| return;
+
     const small_ptr = switch (self.ptr_width) {
         .p32 => true,
         .p64 => false,
@@ -727,6 +740,11 @@ pub fn flushModule(self: *Elf, comp: *Compilation) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
+    if (self.llvm_ir_module) |llvm_ir_module| {
+        try llvm_ir_module.flushModule(comp);
+        return;
+    }
+
     // TODO This linker code currently assumes there is only 1 compilation unit and it corresponds to the
     // Zig source code.
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
@@ -1261,6 +1279,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     const stack_size = self.base.options.stack_size_override orelse 16777216;
     const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;
     const compiler_rt_path: ?[]const u8 = if (self.base.options.include_compiler_rt) blk: {
+        // TODO: remove when stage2 can build compiler_rt.zig
+        if (!build_options.is_stage1) break :blk null;
+
         if (is_exe_or_dyn_lib) {
             break :blk comp.compiler_rt_static_lib.?.full_object_path;
         } else {
@@ -1552,7 +1573,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     }
 
     // libc
-    if (is_exe_or_dyn_lib and !self.base.options.skip_linker_dependencies and !self.base.options.link_libc) {
+    // TODO: enable when stage2 can build c.zig
+    if (is_exe_or_dyn_lib and
+        !self.base.options.skip_linker_dependencies and
+        !self.base.options.link_libc and
+        build_options.is_stage1)
+    {
         try argv.append(comp.libc_static_lib.?.full_object_path);
     }
 
@@ -2046,6 +2072,8 @@ fn allocateTextBlock(self: *Elf, text_block: *TextBlock, new_block_size: u64, al
 }
 
 pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void {
+    if (self.llvm_ir_module) |_| return;
+
     if (decl.link.elf.local_sym_index != 0) return;
 
     try self.local_symbols.ensureCapacity(self.base.allocator, self.local_symbols.items.len + 1);
@@ -2082,6 +2110,8 @@ pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void {
 }
 
 pub fn freeDecl(self: *Elf, decl: *Module.Decl) void {
+    if (self.llvm_ir_module) |_| return;
+
     // Appending to free lists is allowed to fail because the free lists are heuristics based anyway.
     self.freeTextBlock(&decl.link.elf);
     if (decl.link.elf.local_sym_index != 0) {
@@ -2119,6 +2149,11 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
+    if (self.llvm_ir_module) |llvm_ir_module| {
+        try llvm_ir_module.updateDecl(module, decl);
+        return;
+    }
+
     var code_buffer = std.ArrayList(u8).init(self.base.allocator);
     defer code_buffer.deinit();
 
@@ -2594,6 +2629,8 @@ pub fn updateDeclExports(
     decl: *const Module.Decl,
     exports: []const *Module.Export,
 ) !void {
+    if (self.llvm_ir_module) |_| return;
+
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -2667,6 +2704,8 @@ pub fn updateDeclLineNumber(self: *Elf, module: *Module, decl: *const Module.Dec
     const tracy = trace(@src());
     defer tracy.end();
 
+    if (self.llvm_ir_module) |_| return;
+
     const container_scope = decl.scope.cast(Module.Scope.Container).?;
     const tree = container_scope.file_scope.contents.tree;
     const file_ast_decls = tree.root_node.decls();
@@ -2685,6 +2724,8 @@ pub fn updateDeclLineNumber(self: *Elf, module: *Module, decl: *const Module.Dec
 }
 
 pub fn deleteExport(self: *Elf, exp: Export) void {
+    if (self.llvm_ir_module) |_| return;
+
     const sym_index = exp.sym_index orelse return;
     self.global_symbol_free_list.append(self.base.allocator, sym_index) catch {};
     self.global_symbols.items[sym_index].st_info = 0;
diff --git a/src/llvm_backend.zig b/src/llvm_backend.zig
new file mode 100644
index 0000000000..0745a1fe8b
--- /dev/null
+++ b/src/llvm_backend.zig
@@ -0,0 +1,410 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Compilation = @import("Compilation.zig");
+const llvm = @import("llvm_bindings.zig");
+const link = @import("link.zig");
+
+const Module = @import("Module.zig");
+const TypedValue = @import("TypedValue.zig");
+const ir = @import("ir.zig");
+const Inst = ir.Inst;
+
+const Value = @import("value.zig").Value;
+const Type = @import("type.zig").Type;
+
+pub fn targetTriple(allocator: *Allocator, target: std.Target) ![:0]u8 {
+    const llvm_arch = switch (target.cpu.arch) {
+        .arm => "arm",
+        .armeb => "armeb",
+        .aarch64 => "aarch64",
+        .aarch64_be => "aarch64_be",
+        .aarch64_32 => "aarch64_32",
+        .arc => "arc",
+        .avr => "avr",
+        .bpfel => "bpfel",
+        .bpfeb => "bpfeb",
+        .hexagon => "hexagon",
+        .mips => "mips",
+        .mipsel => "mipsel",
+        .mips64 => "mips64",
+        .mips64el => "mips64el",
+        .msp430 => "msp430",
+        .powerpc => "powerpc",
+        .powerpc64 => "powerpc64",
+        .powerpc64le => "powerpc64le",
+        .r600 => "r600",
+        .amdgcn => "amdgcn",
+        .riscv32 => "riscv32",
+        .riscv64 => "riscv64",
+        .sparc => "sparc",
+        .sparcv9 => "sparcv9",
+        .sparcel => "sparcel",
+        .s390x => "s390x",
+        .tce => "tce",
+        .tcele => "tcele",
+        .thumb => "thumb",
+        .thumbeb => "thumbeb",
+        .i386 => "i386",
+        .x86_64 => "x86_64",
+        .xcore => "xcore",
+        .nvptx => "nvptx",
+        .nvptx64 => "nvptx64",
+        .le32 => "le32",
+        .le64 => "le64",
+        .amdil => "amdil",
+        .amdil64 => "amdil64",
+        .hsail => "hsail",
+        .hsail64 => "hsail64",
+        .spir => "spir",
+        .spir64 => "spir64",
+        .kalimba => "kalimba",
+        .shave => "shave",
+        .lanai => "lanai",
+        .wasm32 => "wasm32",
+        .wasm64 => "wasm64",
+        .renderscript32 => "renderscript32",
+        .renderscript64 => "renderscript64",
+        .ve => "ve",
+        .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII,
+    };
+    // TODO Add a sub-arch for some architectures depending on CPU features.
+
+    const llvm_os = switch (target.os.tag) {
+        .freestanding => "unknown",
+        .ananas => "ananas",
+        .cloudabi => "cloudabi",
+        .dragonfly => "dragonfly",
+        .freebsd => "freebsd",
+        .fuchsia => "fuchsia",
+        .ios => "ios",
+        .kfreebsd => "kfreebsd",
+        .linux => "linux",
+        .lv2 => "lv2",
+        .macos => "macosx",
+        .netbsd => "netbsd",
+        .openbsd => "openbsd",
+        .solaris => "solaris",
+        .windows => "windows",
+        .haiku => "haiku",
+        .minix => "minix",
+        .rtems => "rtems",
+        .nacl => "nacl",
+        .cnk => "cnk",
+        .aix => "aix",
+        .cuda => "cuda",
+        .nvcl => "nvcl",
+        .amdhsa => "amdhsa",
+        .ps4 => "ps4",
+        .elfiamcu => "elfiamcu",
+        .tvos => "tvos",
+        .watchos => "watchos",
+        .mesa3d => "mesa3d",
+        .contiki => "contiki",
+        .amdpal => "amdpal",
+        .hermit => "hermit",
+        .hurd => "hurd",
+        .wasi => "wasi",
+        .emscripten => "emscripten",
+        .uefi => "windows",
+        .other => "unknown",
+    };
+
+    const llvm_abi = switch (target.abi) {
+        .none => "unknown",
+        .gnu => "gnu",
+        .gnuabin32 => "gnuabin32",
+        .gnuabi64 => "gnuabi64",
+        .gnueabi => "gnueabi",
+        .gnueabihf => "gnueabihf",
+        .gnux32 => "gnux32",
+        .code16 => "code16",
+        .eabi => "eabi",
+        .eabihf => "eabihf",
+        .android => "android",
+        .musl => "musl",
+        .musleabi => "musleabi",
+        .musleabihf => "musleabihf",
+        .msvc => "msvc",
+        .itanium => "itanium",
+        .cygnus => "cygnus",
+        .coreclr => "coreclr",
+        .simulator => "simulator",
+        .macabi => "macabi",
+    };
+
+    return std.fmt.allocPrintZ(allocator, "{}-unknown-{}-{}", .{ llvm_arch, llvm_os, llvm_abi });
+}
+
+pub const LLVMIRModule = struct {
+    llvm_module: *const llvm.ModuleRef,
+    target_machine: *const llvm.TargetMachineRef,
+    output_path: []const u8,
+
+    gpa: *Allocator,
+    err_msg: ?*Compilation.ErrorMsg = null,
+
+    pub fn create(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*LLVMIRModule {
+        const self = try allocator.create(LLVMIRModule);
+        errdefer allocator.destroy(self);
+
+        const gpa = options.module.?.gpa;
+
+        initializeLLVMTargets();
+
+        const root_nameZ = try gpa.dupeZ(u8, options.root_name);
+        defer gpa.free(root_nameZ);
+        const llvm_module = llvm.ModuleRef.createWithName(root_nameZ.ptr);
+        errdefer llvm_module.disposeModule();
+
+        const llvm_target_triple = try targetTriple(gpa, options.target);
+        defer gpa.free(llvm_target_triple);
+
+        var error_message: [*:0]const u8 = undefined;
+        var target_ref: *const llvm.TargetRef = undefined;
+        if (llvm.TargetRef.getTargetFromTriple(llvm_target_triple.ptr, &target_ref, &error_message)) {
+            defer llvm.disposeMessage(error_message);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.print(
+                \\Zig is expecting LLVM to understand this target: '{s}'
+                \\However LLVM responded with: "{s}"
+                \\Zig is unable to continue. This is a bug in Zig:
+                \\https://github.com/ziglang/zig/issues/438
+                \\
+            ,
+                .{
+                    llvm_target_triple,
+                    error_message,
+                },
+            );
+            return error.InvalidLLVMTriple;
+        }
+
+        const opt_level: llvm.CodeGenOptLevel = if (options.optimize_mode == .Debug) .None else .Aggressive;
+        const target_machine = llvm.TargetMachineRef.createTargetMachine(
+            target_ref,
+            llvm_target_triple.ptr,
+            "",
+            "",
+            opt_level,
+            .Static,
+            .Default,
+        );
+        errdefer target_machine.disposeTargetMachine();
+
+        self.* = .{
+            .llvm_module = llvm_module,
+            .target_machine = target_machine,
+            .output_path = sub_path,
+            .gpa = gpa,
+        };
+        return self;
+    }
+
+    pub fn deinit(self: *LLVMIRModule, allocator: *Allocator) void {
+        self.llvm_module.disposeModule();
+        self.target_machine.disposeTargetMachine();
+        allocator.destroy(self);
+    }
+
+    fn initializeLLVMTargets() void {
+        llvm.initializeAllTargets();
+        llvm.initializeAllTargetInfos();
+        llvm.initializeAllTargetMCs();
+        llvm.initializeAllAsmPrinters();
+        llvm.initializeAllAsmParsers();
+    }
+
+    pub fn flushModule(self: *LLVMIRModule, comp: *Compilation) !void {
+        {
+            var error_message: [*:0]const u8 = undefined;
+            // verifyModule always allocs the error_message even if there is no error
+            defer llvm.disposeMessage(error_message);
+
+            if (self.llvm_module.verifyModule(.ReturnStatus, &error_message)) {
+                const stderr = std.io.getStdErr().outStream();
+                try stderr.print("broken LLVM module found: {s}\nThis is a bug in the Zig compiler.", .{error_message});
+                return error.BrokenLLVMModule;
+            }
+        }
+
+        if (comp.verbose_llvm_ir) {
+            const dump = self.llvm_module.printToString();
+            defer llvm.disposeMessage(dump);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.writeAll(std.mem.spanZ(dump));
+        }
+
+        const output_pathZ = try self.gpa.dupeZ(u8, self.output_path);
+        defer self.gpa.free(output_pathZ);
+
+        var error_message: [*:0]const u8 = undefined;
+        // TODO: where to put the output object, zig-cache something?
+        // TODO: caching?
+        if (self.target_machine.emitToFile(
+            self.llvm_module,
+            output_pathZ.ptr,
+            .ObjectFile,
+            &error_message,
+        )) {
+            defer llvm.disposeMessage(error_message);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.print("LLVM failed to emit file: {s}\n", .{error_message});
+            return error.FailedToEmit;
+        }
+    }
+
+    pub fn updateDecl(self: *LLVMIRModule, module: *Module, decl: *Module.Decl) !void {
+        const typed_value = decl.typed_value.most_recent.typed_value;
+        self.generate(module, typed_value, decl.src()) catch |err| switch (err) {
+            error.CodegenFail => {
+                decl.analysis = .codegen_failure;
+                try module.failed_decls.put(module.gpa, decl, self.err_msg.?);
+                return;
+            },
+            else => |e| return e,
+        };
+    }
+
+    fn generate(self: *LLVMIRModule, module: *Module, typed_value: TypedValue, src: usize) !void {
+        switch (typed_value.ty.zigTypeTag()) {
+            .Fn => {
+                const func = typed_value.val.cast(Value.Payload.Function).?.func;
+
+                var codegen = CodeGen{
+                    .module = module,
+                    .llvm_module = self.llvm_module,
+                    .builder = llvm.BuilderRef.createBuilder(),
+                };
+                defer codegen.builder.disposeBuilder();
+
+                const llvm_func = try codegen.resolveLLVMFunction(func);
+
+                // We remove all the basic blocks of a function to support incremental
+                // compilation!
+                // TODO: remove all basic blocks if functions can have more than one
+                if (llvm_func.getFirstBasicBlock()) |bb| {
+                    bb.deleteBasicBlock();
+                }
+
+                const entry_block = llvm_func.appendBasicBlock("Entry");
+                codegen.builder.positionBuilderAtEnd(entry_block);
+
+                const instructions = func.analysis.success.instructions;
+                for (instructions) |inst| {
+                    switch (inst.tag) {
+                        .breakpoint => try codegen.generateBreakpoint(inst.castTag(.breakpoint).?),
+                        .call => try codegen.generateCall(inst.castTag(.call).?),
+                        .unreach => codegen.generateUnreach(inst.castTag(.unreach).?),
+                        .retvoid => codegen.generateRetVoid(inst.castTag(.retvoid).?),
+                        .dbg_stmt => {
+                            // TODO: implement debug info
+                        },
+                        else => |tag| return self.fail(src, "TODO implement LLVM codegen for Zir instruction: {}", .{tag}),
+                    }
+                }
+            },
+            else => |ty| return self.fail(src, "TODO implement LLVM codegen for top-level decl type: {}", .{ty}),
+        }
+    }
+
+    pub fn fail(self: *LLVMIRModule, src: usize, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
+        @setCold(true);
+        std.debug.assert(self.err_msg == null);
+        self.err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
+        return error.CodegenFail;
+    }
+};
+
+const CodeGen = struct {
+    module: *Module,
+    llvm_module: *const llvm.ModuleRef,
+    builder: *const llvm.BuilderRef,
+
+    fn generateCall(codegen: *CodeGen, inst: *Inst.Call) !void {
+        if (inst.func.cast(Inst.Constant)) |func_inst| {
+            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
+                const func = func_val.func;
+                const zig_fn_type = func.owner_decl.typed_value.most_recent.typed_value.ty;
+                const llvm_fn = try codegen.resolveLLVMFunction(func);
+
+                // TODO: handle more arguments, inst.args
+
+                // TODO: LLVMBuildCall2 handles opaque function pointers, according to llvm docs
+                //       Do we need that?
+                const call = codegen.builder.buildCall(llvm_fn, null, 0, "");
+
+                if (zig_fn_type.fnReturnType().zigTypeTag() == .NoReturn) {
+                    _ = codegen.builder.buildUnreachable();
+                }
+            }
+        }
+    }
+
+    fn generateRetVoid(codegen: *CodeGen, inst: *Inst.NoOp) void {
+        _ = codegen.builder.buildRetVoid();
+    }
+
+    fn generateUnreach(codegen: *CodeGen, inst: *Inst.NoOp) void {
+        _ = codegen.builder.buildUnreachable();
+    }
+
+    fn generateBreakpoint(codegen: *CodeGen, inst: *Inst.NoOp) !void {
+        // TODO: Store this function somewhere such that we dont have to add it again
+        const fn_type = llvm.TypeRef.functionType(llvm.voidType(), null, 0, false);
+        const func = codegen.llvm_module.addFunction("llvm.debugtrap", fn_type);
+        // TODO: add assertion: LLVMGetIntrinsicID
+        _ = codegen.builder.buildCall(func, null, 0, "");
+    }
+
+    /// If the llvm function does not exist, create it
+    fn resolveLLVMFunction(codegen: *CodeGen, func: *Module.Fn) !*const llvm.ValueRef {
+        // TODO: do we want to store this in our own datastructure?
+        if (codegen.llvm_module.getNamedFunction(func.owner_decl.name)) |llvm_fn| return llvm_fn;
+
+        const zig_fn_type = func.owner_decl.typed_value.most_recent.typed_value.ty;
+        const return_type = zig_fn_type.fnReturnType();
+
+        const fn_param_len = zig_fn_type.fnParamLen();
+
+        const fn_param_types = try codegen.module.gpa.alloc(Type, fn_param_len);
+        defer codegen.module.gpa.free(fn_param_types);
+        zig_fn_type.fnParamTypes(fn_param_types);
+
+        const llvm_param = try codegen.module.gpa.alloc(*const llvm.TypeRef, fn_param_len);
+        defer codegen.module.gpa.free(llvm_param);
+
+        for (fn_param_types) |fn_param, i| {
+            llvm_param[i] = codegen.getLLVMType(fn_param);
+        }
+
+        const fn_type = llvm.TypeRef.functionType(
+            codegen.getLLVMType(return_type),
+            if (fn_param_len == 0) null else llvm_param.ptr,
+            @intCast(c_uint, fn_param_len),
+            false,
+        );
+        const llvm_fn = codegen.llvm_module.addFunction(func.owner_decl.name, fn_type);
+
+        if (return_type.zigTypeTag() == .NoReturn) {
+            llvm_fn.addFnAttr("noreturn");
+        }
+
+        return llvm_fn;
+    }
+
+    fn getLLVMType(codegen: *CodeGen, t: Type) *const llvm.TypeRef {
+        switch (t.zigTypeTag()) {
+            .Void => return llvm.voidType(),
+            .NoReturn => return llvm.voidType(),
+            .Int => {
+                const info = t.intInfo(codegen.module.getTarget());
+                return llvm.intType(info.bits);
+            },
+            .Bool => return llvm.intType(1),
+            else => unreachable,
+        }
+    }
+};
diff --git a/src/llvm_bindings.zig b/src/llvm_bindings.zig
index 865e989c89..e36d71b312 100644
--- a/src/llvm_bindings.zig
+++ b/src/llvm_bindings.zig
@@ -1,6 +1,364 @@
 //! We do this instead of @cImport because the self-hosted compiler is easier
 //! to bootstrap if it does not depend on translate-c.
 
+const std = @import("std");
+const assert = std.debug.assert;
+
+const LLVMBool = bool;
+pub const LLVMAttributeIndex = c_uint;
+
+pub const ValueRef = opaque {
+    pub const addAttributeAtIndex = LLVMAddAttributeAtIndex;
+    extern fn LLVMAddAttributeAtIndex(*const ValueRef, Idx: LLVMAttributeIndex, A: *const AttributeRef) void;
+
+    pub const appendBasicBlock = LLVMAppendBasicBlock;
+    extern fn LLVMAppendBasicBlock(Fn: *const ValueRef, Name: [*:0]const u8) *const BasicBlockRef;
+
+    pub const getFirstBasicBlock = LLVMGetFirstBasicBlock;
+    extern fn LLVMGetFirstBasicBlock(Fn: *const ValueRef) ?*const BasicBlockRef;
+
+    // Helper functions
+    // TODO: Do we want to put these functions here? It allows for convienient function calls
+    //       on ValueRef: llvm_fn.addFnAttr("noreturn")
+    fn addAttr(val: *const ValueRef, index: LLVMAttributeIndex, name: []const u8) void {
+        const kind_id = getEnumAttributeKindForName(name.ptr, name.len);
+        assert(kind_id != 0);
+        const llvm_attr = ContextRef.getGlobal().createEnumAttribute(kind_id, 0);
+        val.addAttributeAtIndex(index, llvm_attr);
+    }
+
+    pub fn addFnAttr(val: *const ValueRef, attr_name: []const u8) void {
+        // TODO: improve this API, `addAttr(-1, attr_name)`
+        val.addAttr(std.math.maxInt(LLVMAttributeIndex), attr_name);
+    }
+};
+
+pub const TypeRef = opaque {
+    pub const functionType = LLVMFunctionType;
+    extern fn LLVMFunctionType(ReturnType: *const TypeRef, ParamTypes: ?[*]*const TypeRef, ParamCount: c_uint, IsVarArg: LLVMBool) *const TypeRef;
+};
+
+pub const ModuleRef = opaque {
+    pub const createWithName = LLVMModuleCreateWithName;
+    extern fn LLVMModuleCreateWithName(ModuleID: [*:0]const u8) *const ModuleRef;
+
+    pub const disposeModule = LLVMDisposeModule;
+    extern fn LLVMDisposeModule(*const ModuleRef) void;
+
+    pub const verifyModule = LLVMVerifyModule;
+    extern fn LLVMVerifyModule(*const ModuleRef, Action: VerifierFailureAction, OutMessage: *[*:0]const u8) LLVMBool;
+
+    pub const addFunction = LLVMAddFunction;
+    extern fn LLVMAddFunction(*const ModuleRef, Name: [*:0]const u8, FunctionTy: *const TypeRef) *const ValueRef;
+
+    pub const getNamedFunction = LLVMGetNamedFunction;
+    extern fn LLVMGetNamedFunction(*const ModuleRef, Name: [*:0]const u8) ?*const ValueRef;
+
+    pub const printToString = LLVMPrintModuleToString;
+    extern fn LLVMPrintModuleToString(*const ModuleRef) [*:0]const u8;
+};
+
+pub const disposeMessage = LLVMDisposeMessage;
+extern fn LLVMDisposeMessage(Message: [*:0]const u8) void;
+
+pub const VerifierFailureAction = extern enum {
+    AbortProcess,
+    PrintMessage,
+    ReturnStatus,
+};
+
+pub const voidType = LLVMVoidType;
+extern fn LLVMVoidType() *const TypeRef;
+
+pub const getEnumAttributeKindForName = LLVMGetEnumAttributeKindForName;
+extern fn LLVMGetEnumAttributeKindForName(Name: [*]const u8, SLen: usize) c_uint;
+
+pub const AttributeRef = opaque {};
+
+pub const ContextRef = opaque {
+    pub const createEnumAttribute = LLVMCreateEnumAttribute;
+    extern fn LLVMCreateEnumAttribute(*const ContextRef, KindID: c_uint, Val: u64) *const AttributeRef;
+
+    pub const getGlobal = LLVMGetGlobalContext;
+    extern fn LLVMGetGlobalContext() *const ContextRef;
+};
+
+pub const intType = LLVMIntType;
+extern fn LLVMIntType(NumBits: c_uint) *const TypeRef;
+
+pub const BuilderRef = opaque {
+    pub const createBuilder = LLVMCreateBuilder;
+    extern fn LLVMCreateBuilder() *const BuilderRef;
+
+    pub const disposeBuilder = LLVMDisposeBuilder;
+    extern fn LLVMDisposeBuilder(Builder: *const BuilderRef) void;
+
+    pub const positionBuilderAtEnd = LLVMPositionBuilderAtEnd;
+    extern fn LLVMPositionBuilderAtEnd(Builder: *const BuilderRef, Block: *const BasicBlockRef) void;
+
+    pub const getInsertBlock = LLVMGetInsertBlock;
+    extern fn LLVMGetInsertBlock(Builder: *const BuilderRef) *const BasicBlockRef;
+
+    pub const buildCall = LLVMBuildCall;
+    extern fn LLVMBuildCall(*const BuilderRef, Fn: *const ValueRef, Args: ?[*]*const ValueRef, NumArgs: c_uint, Name: [*:0]const u8) *const ValueRef;
+
+    pub const buildCall2 = LLVMBuildCall2;
+    extern fn LLVMBuildCall2(*const BuilderRef, *const TypeRef, Fn: *const ValueRef, Args: [*]*const ValueRef, NumArgs: c_uint, Name: [*:0]const u8) *const ValueRef;
+
+    pub const buildRetVoid = LLVMBuildRetVoid;
+    extern fn LLVMBuildRetVoid(*const BuilderRef) *const ValueRef;
+
+    pub const buildUnreachable = LLVMBuildUnreachable;
+    extern fn LLVMBuildUnreachable(*const BuilderRef) *const ValueRef;
+
+    pub const buildAlloca = LLVMBuildAlloca;
+    extern fn LLVMBuildAlloca(*const BuilderRef, Ty: *const TypeRef, Name: [*:0]const u8) *const ValueRef;
+};
+
+pub const BasicBlockRef = opaque {
+    pub const deleteBasicBlock = LLVMDeleteBasicBlock;
+    extern fn LLVMDeleteBasicBlock(BB: *const BasicBlockRef) void;
+};
+
+pub const TargetMachineRef = opaque {
+    pub const createTargetMachine = LLVMCreateTargetMachine;
+    extern fn LLVMCreateTargetMachine(
+        T: *const TargetRef,
+        Triple: [*:0]const u8,
+        CPU: [*:0]const u8,
+        Features: [*:0]const u8,
+        Level: CodeGenOptLevel,
+        Reloc: RelocMode,
+        CodeModel: CodeMode,
+    ) *const TargetMachineRef;
+
+    pub const disposeTargetMachine = LLVMDisposeTargetMachine;
+    extern fn LLVMDisposeTargetMachine(T: *const TargetMachineRef) void;
+
+    pub const emitToFile = LLVMTargetMachineEmitToFile;
+    extern fn LLVMTargetMachineEmitToFile(*const TargetMachineRef, M: *const ModuleRef, Filename: [*:0]const u8, codegen: CodeGenFileType, ErrorMessage: *[*:0]const u8) LLVMBool;
+};
+
+pub const CodeMode = extern enum {
+    Default,
+    JITDefault,
+    Tiny,
+    Small,
+    Kernel,
+    Medium,
+    Large,
+};
+
+pub const CodeGenOptLevel = extern enum {
+    None,
+    Less,
+    Default,
+    Aggressive,
+};
+
+pub const RelocMode = extern enum {
+    Default,
+    Static,
+    PIC,
+    DynamicNoPic,
+    ROPI,
+    RWPI,
+    ROPI_RWPI,
+};
+
+pub const CodeGenFileType = extern enum {
+    AssemblyFile,
+    ObjectFile,
+};
+
+pub const TargetRef = opaque {
+    pub const getTargetFromTriple = LLVMGetTargetFromTriple;
+    extern fn LLVMGetTargetFromTriple(Triple: [*:0]const u8, T: **const TargetRef, ErrorMessage: *[*:0]const u8) LLVMBool;
+};
+
+extern fn LLVMInitializeAArch64TargetInfo() void;
+extern fn LLVMInitializeAMDGPUTargetInfo() void;
+extern fn LLVMInitializeARMTargetInfo() void;
+extern fn LLVMInitializeAVRTargetInfo() void;
+extern fn LLVMInitializeBPFTargetInfo() void;
+extern fn LLVMInitializeHexagonTargetInfo() void;
+extern fn LLVMInitializeLanaiTargetInfo() void;
+extern fn LLVMInitializeMipsTargetInfo() void;
+extern fn LLVMInitializeMSP430TargetInfo() void;
+extern fn LLVMInitializeNVPTXTargetInfo() void;
+extern fn LLVMInitializePowerPCTargetInfo() void;
+extern fn LLVMInitializeRISCVTargetInfo() void;
+extern fn LLVMInitializeSparcTargetInfo() void;
+extern fn LLVMInitializeSystemZTargetInfo() void;
+extern fn LLVMInitializeWebAssemblyTargetInfo() void;
+extern fn LLVMInitializeX86TargetInfo() void;
+extern fn LLVMInitializeXCoreTargetInfo() void;
+extern fn LLVMInitializeAArch64Target() void;
+extern fn LLVMInitializeAMDGPUTarget() void;
+extern fn LLVMInitializeARMTarget() void;
+extern fn LLVMInitializeAVRTarget() void;
+extern fn LLVMInitializeBPFTarget() void;
+extern fn LLVMInitializeHexagonTarget() void;
+extern fn LLVMInitializeLanaiTarget() void;
+extern fn LLVMInitializeMipsTarget() void;
+extern fn LLVMInitializeMSP430Target() void;
+extern fn LLVMInitializeNVPTXTarget() void;
+extern fn LLVMInitializePowerPCTarget() void;
+extern fn LLVMInitializeRISCVTarget() void;
+extern fn LLVMInitializeSparcTarget() void;
+extern fn LLVMInitializeSystemZTarget() void;
+extern fn LLVMInitializeWebAssemblyTarget() void;
+extern fn LLVMInitializeX86Target() void;
+extern fn LLVMInitializeXCoreTarget() void;
+extern fn LLVMInitializeAArch64TargetMC() void;
+extern fn LLVMInitializeAMDGPUTargetMC() void;
+extern fn LLVMInitializeARMTargetMC() void;
+extern fn LLVMInitializeAVRTargetMC() void;
+extern fn LLVMInitializeBPFTargetMC() void;
+extern fn LLVMInitializeHexagonTargetMC() void;
+extern fn LLVMInitializeLanaiTargetMC() void;
+extern fn LLVMInitializeMipsTargetMC() void;
+extern fn LLVMInitializeMSP430TargetMC() void;
+extern fn LLVMInitializeNVPTXTargetMC() void;
+extern fn LLVMInitializePowerPCTargetMC() void;
+extern fn LLVMInitializeRISCVTargetMC() void;
+extern fn LLVMInitializeSparcTargetMC() void;
+extern fn LLVMInitializeSystemZTargetMC() void;
+extern fn LLVMInitializeWebAssemblyTargetMC() void;
+extern fn LLVMInitializeX86TargetMC() void;
+extern fn LLVMInitializeXCoreTargetMC() void;
+extern fn LLVMInitializeAArch64AsmPrinter() void;
+extern fn LLVMInitializeAMDGPUAsmPrinter() void;
+extern fn LLVMInitializeARMAsmPrinter() void;
+extern fn LLVMInitializeAVRAsmPrinter() void;
+extern fn LLVMInitializeBPFAsmPrinter() void;
+extern fn LLVMInitializeHexagonAsmPrinter() void;
+extern fn LLVMInitializeLanaiAsmPrinter() void;
+extern fn LLVMInitializeMipsAsmPrinter() void;
+extern fn LLVMInitializeMSP430AsmPrinter() void;
+extern fn LLVMInitializeNVPTXAsmPrinter() void;
+extern fn LLVMInitializePowerPCAsmPrinter() void;
+extern fn LLVMInitializeRISCVAsmPrinter() void;
+extern fn LLVMInitializeSparcAsmPrinter() void;
+extern fn LLVMInitializeSystemZAsmPrinter() void;
+extern fn LLVMInitializeWebAssemblyAsmPrinter() void;
+extern fn LLVMInitializeX86AsmPrinter() void;
+extern fn LLVMInitializeXCoreAsmPrinter() void;
+extern fn LLVMInitializeAArch64AsmParser() void;
+extern fn LLVMInitializeAMDGPUAsmParser() void;
+extern fn LLVMInitializeARMAsmParser() void;
+extern fn LLVMInitializeAVRAsmParser() void;
+extern fn LLVMInitializeBPFAsmParser() void;
+extern fn LLVMInitializeHexagonAsmParser() void;
+extern fn LLVMInitializeLanaiAsmParser() void;
+extern fn LLVMInitializeMipsAsmParser() void;
+extern fn LLVMInitializeMSP430AsmParser() void;
+extern fn LLVMInitializePowerPCAsmParser() void;
+extern fn LLVMInitializeRISCVAsmParser() void;
+extern fn LLVMInitializeSparcAsmParser() void;
+extern fn LLVMInitializeSystemZAsmParser() void;
+extern fn LLVMInitializeWebAssemblyAsmParser() void;
+extern fn LLVMInitializeX86AsmParser() void;
+
+pub const initializeAllTargetInfos = LLVMInitializeAllTargetInfos;
+fn LLVMInitializeAllTargetInfos() callconv(.C) void {
+    LLVMInitializeAArch64TargetInfo();
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeARMTargetInfo();
+    LLVMInitializeAVRTargetInfo();
+    LLVMInitializeBPFTargetInfo();
+    LLVMInitializeHexagonTargetInfo();
+    LLVMInitializeLanaiTargetInfo();
+    LLVMInitializeMipsTargetInfo();
+    LLVMInitializeMSP430TargetInfo();
+    LLVMInitializeNVPTXTargetInfo();
+    LLVMInitializePowerPCTargetInfo();
+    LLVMInitializeRISCVTargetInfo();
+    LLVMInitializeSparcTargetInfo();
+    LLVMInitializeSystemZTargetInfo();
+    LLVMInitializeWebAssemblyTargetInfo();
+    LLVMInitializeX86TargetInfo();
+    LLVMInitializeXCoreTargetInfo();
+}
+pub const initializeAllTargets = LLVMInitializeAllTargets;
+fn LLVMInitializeAllTargets() callconv(.C) void {
+    LLVMInitializeAArch64Target();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeARMTarget();
+    LLVMInitializeAVRTarget();
+    LLVMInitializeBPFTarget();
+    LLVMInitializeHexagonTarget();
+    LLVMInitializeLanaiTarget();
+    LLVMInitializeMipsTarget();
+    LLVMInitializeMSP430Target();
+    LLVMInitializeNVPTXTarget();
+    LLVMInitializePowerPCTarget();
+    LLVMInitializeRISCVTarget();
+    LLVMInitializeSparcTarget();
+    LLVMInitializeSystemZTarget();
+    LLVMInitializeWebAssemblyTarget();
+    LLVMInitializeX86Target();
+    LLVMInitializeXCoreTarget();
+}
+pub const initializeAllTargetMCs = LLVMInitializeAllTargetMCs;
+fn LLVMInitializeAllTargetMCs() callconv(.C) void {
+    LLVMInitializeAArch64TargetMC();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeARMTargetMC();
+    LLVMInitializeAVRTargetMC();
+    LLVMInitializeBPFTargetMC();
+    LLVMInitializeHexagonTargetMC();
+    LLVMInitializeLanaiTargetMC();
+    LLVMInitializeMipsTargetMC();
+    LLVMInitializeMSP430TargetMC();
+    LLVMInitializeNVPTXTargetMC();
+    LLVMInitializePowerPCTargetMC();
+    LLVMInitializeRISCVTargetMC();
+    LLVMInitializeSparcTargetMC();
+    LLVMInitializeSystemZTargetMC();
+    LLVMInitializeWebAssemblyTargetMC();
+    LLVMInitializeX86TargetMC();
+    LLVMInitializeXCoreTargetMC();
+}
+pub const initializeAllAsmPrinters = LLVMInitializeAllAsmPrinters;
+fn LLVMInitializeAllAsmPrinters() callconv(.C) void {
+    LLVMInitializeAArch64AsmPrinter();
+    LLVMInitializeAMDGPUAsmPrinter();
+    LLVMInitializeARMAsmPrinter();
+    LLVMInitializeAVRAsmPrinter();
+    LLVMInitializeBPFAsmPrinter();
+    LLVMInitializeHexagonAsmPrinter();
+    LLVMInitializeLanaiAsmPrinter();
+    LLVMInitializeMipsAsmPrinter();
+    LLVMInitializeMSP430AsmPrinter();
+    LLVMInitializeNVPTXAsmPrinter();
+    LLVMInitializePowerPCAsmPrinter();
+    LLVMInitializeRISCVAsmPrinter();
+    LLVMInitializeSparcAsmPrinter();
+    LLVMInitializeSystemZAsmPrinter();
+    LLVMInitializeWebAssemblyAsmPrinter();
+    LLVMInitializeX86AsmPrinter();
+    LLVMInitializeXCoreAsmPrinter();
+}
+pub const initializeAllAsmParsers = LLVMInitializeAllAsmParsers;
+fn LLVMInitializeAllAsmParsers() callconv(.C) void {
+    LLVMInitializeAArch64AsmParser();
+    LLVMInitializeAMDGPUAsmParser();
+    LLVMInitializeARMAsmParser();
+    LLVMInitializeAVRAsmParser();
+    LLVMInitializeBPFAsmParser();
+    LLVMInitializeHexagonAsmParser();
+    LLVMInitializeLanaiAsmParser();
+    LLVMInitializeMipsAsmParser();
+    LLVMInitializeMSP430AsmParser();
+    LLVMInitializePowerPCAsmParser();
+    LLVMInitializeRISCVAsmParser();
+    LLVMInitializeSparcAsmParser();
+    LLVMInitializeSystemZAsmParser();
+    LLVMInitializeWebAssemblyAsmParser();
+    LLVMInitializeX86AsmParser();
+}
+
 extern fn ZigLLDLinkCOFF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
 extern fn ZigLLDLinkELF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
 extern fn ZigLLDLinkMachO(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
-- 
cgit v1.2.3


From 87c6341b61aa54301aa98fea1a449fff40ba25af Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 28 Dec 2020 17:15:29 -0700
Subject: stage2: add extern functions

and improve the C backend enough to support Hello World (almost)
---
 src/Compilation.zig |   5 +-
 src/Module.zig      |  97 ++++++++++++--
 src/codegen/c.zig   | 377 ++++++++++++++++++++++++++++++++++------------------
 src/link/C.zig      |  16 +--
 src/link/cbe.h      |   4 +
 src/type.zig        |  92 ++++++++++++-
 src/value.zig       |  24 ++++
 7 files changed, 452 insertions(+), 163 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 26beeccd9e..39dd97c3a2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1431,9 +1431,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
     var c_comp_progress_node = main_progress_node.start("Compile C Objects", self.c_source_files.len);
     defer c_comp_progress_node.end();
 
-    var arena = std.heap.ArenaAllocator.init(self.gpa);
-    defer arena.deinit();
-
     self.work_queue_wait_group.reset();
     defer self.work_queue_wait_group.wait();
 
@@ -1502,7 +1499,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 };
 
                 if (self.c_header) |*header| {
-                    c_codegen.generateHeader(&arena, module, &header.*, decl) catch |err| switch (err) {
+                    c_codegen.generateHeader(self, module, header, decl) catch |err| switch (err) {
                         error.OutOfMemory => return error.OutOfMemory,
                         error.AnalysisFail => {
                             decl.analysis = .dependency_failure;
diff --git a/src/Module.zig b/src/Module.zig
index c0b7011f43..dbb845116a 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -277,6 +277,8 @@ pub const Decl = struct {
 };
 
 /// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator.
+/// Extern functions do not have this data structure; they are represented by
+/// the `Decl` only, with a `Value` tag of `extern_fn`.
 pub const Fn = struct {
     /// This memory owned by the Decl's TypedValue.Managed arena allocator.
     analysis: union(enum) {
@@ -1010,8 +1012,6 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             defer fn_type_scope.instructions.deinit(self.gpa);
 
             decl.is_pub = fn_proto.getVisibToken() != null;
-            const body_node = fn_proto.getBodyNode() orelse
-                return self.failTok(&fn_type_scope.base, fn_proto.fn_token, "TODO implement extern functions", .{});
 
             const param_decls = fn_proto.params();
             const param_types = try fn_type_scope.arena.alloc(*zir.Inst, param_decls.len);
@@ -1083,6 +1083,36 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             const fn_type = try zir_sema.analyzeBodyValueAsType(self, &block_scope, fn_type_inst, .{
                 .instructions = fn_type_scope.instructions.items,
             });
+            const body_node = fn_proto.getBodyNode() orelse {
+                // Extern function.
+                var type_changed = true;
+                if (decl.typedValueManaged()) |tvm| {
+                    type_changed = !tvm.typed_value.ty.eql(fn_type);
+
+                    tvm.deinit(self.gpa);
+                }
+                const value_payload = try decl_arena.allocator.create(Value.Payload.ExternFn);
+                value_payload.* = .{ .decl = decl };
+
+                decl_arena_state.* = decl_arena.state;
+                decl.typed_value = .{
+                    .most_recent = .{
+                        .typed_value = .{
+                            .ty = fn_type,
+                            .val = Value.initPayload(&value_payload.base),
+                        },
+                        .arena = decl_arena_state,
+                    },
+                };
+                decl.analysis = .complete;
+                decl.generation = self.generation;
+
+                try self.comp.bin_file.allocateDeclIndexes(decl);
+                try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
+
+                return type_changed;
+            };
+
             const new_func = try decl_arena.allocator.create(Fn);
             const fn_payload = try decl_arena.allocator.create(Value.Payload.Function);
 
@@ -1899,7 +1929,13 @@ pub fn resolveDefinedValue(self: *Module, scope: *Scope, base: *Inst) !?Value {
     return null;
 }
 
-pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, borrowed_symbol_name: []const u8, exported_decl: *Decl) !void {
+pub fn analyzeExport(
+    self: *Module,
+    scope: *Scope,
+    src: usize,
+    borrowed_symbol_name: []const u8,
+    exported_decl: *Decl,
+) !void {
     try self.ensureDeclAnalyzed(exported_decl);
     const typed_value = exported_decl.typed_value.most_recent.typed_value;
     switch (typed_value.ty.zigTypeTag()) {
@@ -2801,16 +2837,47 @@ pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst
         }
     }
 
-    // *[N]T to []T
-    if (inst.ty.isSinglePointer() and dest_type.isSlice() and
-        (!inst.ty.isConstPtr() or dest_type.isConstPtr()))
-    {
+    // Coercions where the source is a single pointer to an array.
+    src_array_ptr: {
+        if (!inst.ty.isSinglePointer()) break :src_array_ptr;
         const array_type = inst.ty.elemType();
+        if (array_type.zigTypeTag() != .Array) break :src_array_ptr;
+        const array_elem_type = array_type.elemType();
+        if (inst.ty.isConstPtr() and !dest_type.isConstPtr()) break :src_array_ptr;
+        if (inst.ty.isVolatilePtr() and !dest_type.isVolatilePtr()) break :src_array_ptr;
+
         const dst_elem_type = dest_type.elemType();
-        if (array_type.zigTypeTag() == .Array and
-            coerceInMemoryAllowed(dst_elem_type, array_type.elemType()) == .ok)
-        {
-            return self.coerceArrayPtrToSlice(scope, dest_type, inst);
+        switch (coerceInMemoryAllowed(dst_elem_type, array_elem_type)) {
+            .ok => {},
+            .no_match => break :src_array_ptr,
+        }
+
+        switch (dest_type.ptrSize()) {
+            .Slice => {
+                // *[N]T to []T
+                return self.coerceArrayPtrToSlice(scope, dest_type, inst);
+            },
+            .C => {
+                // *[N]T to [*c]T
+                return self.coerceArrayPtrToMany(scope, dest_type, inst);
+            },
+            .Many => {
+                // *[N]T to [*]T
+                // *[N:s]T to [*:s]T
+                const src_sentinel = array_type.sentinel();
+                const dst_sentinel = dest_type.sentinel();
+                if (src_sentinel == null and dst_sentinel == null)
+                    return self.coerceArrayPtrToMany(scope, dest_type, inst);
+
+                if (src_sentinel) |src_s| {
+                    if (dst_sentinel) |dst_s| {
+                        if (src_s.eql(dst_s)) {
+                            return self.coerceArrayPtrToMany(scope, dest_type, inst);
+                        }
+                    }
+                }
+            },
+            .One => {},
         }
     }
 
@@ -2918,6 +2985,14 @@ fn coerceArrayPtrToSlice(self: *Module, scope: *Scope, dest_type: Type, inst: *I
     return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToSlice runtime instruction", .{});
 }
 
+fn coerceArrayPtrToMany(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst {
+    if (inst.value()) |val| {
+        // The comptime Value representation is compatible with both types.
+        return self.constInst(scope, inst.src, .{ .ty = dest_type, .val = val });
+    }
+    return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToMany runtime instruction", .{});
+}
+
 pub fn fail(self: *Module, scope: *Scope, src: usize, comptime format: []const u8, args: anytype) InnerError {
     @setCold(true);
     const err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 589e2f17e0..8d706f8735 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -12,7 +12,7 @@ const C = link.File.C;
 const Decl = Module.Decl;
 const mem = std.mem;
 
-const indentation = "    ";
+const Writer = std.ArrayList(u8).Writer;
 
 /// Maps a name from Zig source to C. Currently, this will always give the same
 /// output for any given input, sometimes resulting in broken identifiers.
@@ -20,43 +20,145 @@ fn map(allocator: *std.mem.Allocator, name: []const u8) ![]const u8 {
     return allocator.dupe(u8, name);
 }
 
-fn renderType(ctx: *Context, header: *C.Header, writer: std.ArrayList(u8).Writer, T: Type) !void {
-    switch (T.zigTypeTag()) {
+fn renderType(
+    ctx: *Context,
+    header: *C.Header,
+    writer: Writer,
+    t: Type,
+) error{ OutOfMemory, AnalysisFail }!void {
+    switch (t.zigTypeTag()) {
         .NoReturn => {
             try writer.writeAll("zig_noreturn void");
         },
         .Void => try writer.writeAll("void"),
         .Bool => try writer.writeAll("bool"),
         .Int => {
-            if (T.tag() == .u8) {
-                header.need_stdint = true;
-                try writer.writeAll("uint8_t");
-            } else if (T.tag() == .u32) {
-                header.need_stdint = true;
-                try writer.writeAll("uint32_t");
-            } else if (T.tag() == .usize) {
-                header.need_stddef = true;
-                try writer.writeAll("size_t");
+            switch (t.tag()) {
+                .u8 => try writer.writeAll("uint8_t"),
+                .i8 => try writer.writeAll("int8_t"),
+                .u16 => try writer.writeAll("uint16_t"),
+                .i16 => try writer.writeAll("int16_t"),
+                .u32 => try writer.writeAll("uint32_t"),
+                .i32 => try writer.writeAll("int32_t"),
+                .u64 => try writer.writeAll("uint64_t"),
+                .i64 => try writer.writeAll("int64_t"),
+                .usize => try writer.writeAll("uintptr_t"),
+                .isize => try writer.writeAll("intptr_t"),
+                .c_short => try writer.writeAll("short"),
+                .c_ushort => try writer.writeAll("unsigned short"),
+                .c_int => try writer.writeAll("int"),
+                .c_uint => try writer.writeAll("unsigned int"),
+                .c_long => try writer.writeAll("long"),
+                .c_ulong => try writer.writeAll("unsigned long"),
+                .c_longlong => try writer.writeAll("long long"),
+                .c_ulonglong => try writer.writeAll("unsigned long long"),
+                .int_signed, .int_unsigned => {
+                    const info = t.intInfo(ctx.target);
+                    const sign_prefix = switch (info.signedness) {
+                        .signed => "i",
+                        .unsigned => "",
+                    };
+                    inline for (.{ 8, 16, 32, 64, 128 }) |nbits| {
+                        if (info.bits <= nbits) {
+                            try writer.print("{s}int{d}_t", .{ sign_prefix, nbits });
+                            break;
+                        }
+                    } else {
+                        return ctx.fail(ctx.decl.src(), "TODO: C backend: implement integer types larger than 128 bits", .{});
+                    }
+                },
+                else => unreachable,
+            }
+        },
+        .Pointer => {
+            if (t.isSlice()) {
+                return ctx.fail(ctx.decl.src(), "TODO: C backend: implement slices", .{});
             } else {
-                return ctx.fail(ctx.decl.src(), "TODO implement int type {}", .{T});
+                if (t.isConstPtr()) {
+                    try writer.writeAll("const ");
+                }
+                if (t.isVolatilePtr()) {
+                    try writer.writeAll("volatile ");
+                }
+                try renderType(ctx, header, writer, t.elemType());
+                try writer.writeAll(" *");
             }
         },
-        else => |e| return ctx.fail(ctx.decl.src(), "TODO implement type {}", .{e}),
+        .Array => {
+            try renderType(ctx, header, writer, t.elemType());
+            const sentinel_bit = @boolToInt(t.sentinel() != null);
+            const c_len = t.arrayLen() + sentinel_bit;
+            try writer.print("[{d}]", .{c_len});
+        },
+        else => |e| return ctx.fail(ctx.decl.src(), "TODO: C backend: implement type {s}", .{
+            @tagName(e),
+        }),
     }
 }
 
-fn renderValue(ctx: *Context, writer: std.ArrayList(u8).Writer, T: Type, val: Value) !void {
-    switch (T.zigTypeTag()) {
+fn renderValue(
+    ctx: *Context,
+    writer: Writer,
+    t: Type,
+    val: Value,
+) error{ OutOfMemory, AnalysisFail }!void {
+    switch (t.zigTypeTag()) {
         .Int => {
-            if (T.isSignedInt())
-                return writer.print("{}", .{val.toSignedInt()});
-            return writer.print("{}", .{val.toUnsignedInt()});
+            if (t.isSignedInt())
+                return writer.print("{d}", .{val.toSignedInt()});
+            return writer.print("{d}", .{val.toUnsignedInt()});
+        },
+        .Pointer => switch (val.tag()) {
+            .undef, .zero => try writer.writeAll("0"),
+            .one => try writer.writeAll("1"),
+            .decl_ref => {
+                const decl_ref_payload = val.cast(Value.Payload.DeclRef).?;
+                try writer.print("&{s}", .{decl_ref_payload.decl.name});
+            },
+            .function => {
+                const payload = val.cast(Value.Payload.Function).?;
+                try writer.print("{s}", .{payload.func.owner_decl.name});
+            },
+            .extern_fn => {
+                const payload = val.cast(Value.Payload.ExternFn).?;
+                try writer.print("{s}", .{payload.decl.name});
+            },
+            else => |e| return ctx.fail(
+                ctx.decl.src(),
+                "TODO: C backend: implement Pointer value {s}",
+                .{@tagName(e)},
+            ),
+        },
+        .Array => {
+            // TODO first try specific tag representations for more efficiency
+            // Fall back to inefficient generic implementation.
+            try writer.writeAll("{");
+            var index: usize = 0;
+            const len = t.arrayLen();
+            const elem_ty = t.elemType();
+            while (index < len) : (index += 1) {
+                if (index != 0) try writer.writeAll(",");
+                const elem_val = try val.elemValue(&ctx.arena.allocator, index);
+                try renderValue(ctx, writer, elem_ty, elem_val);
+            }
+            if (t.sentinel()) |sentinel_val| {
+                if (index != 0) try writer.writeAll(",");
+                try renderValue(ctx, writer, elem_ty, sentinel_val);
+            }
+            try writer.writeAll("}");
         },
-        else => |e| return ctx.fail(ctx.decl.src(), "TODO implement value {}", .{e}),
+        else => |e| return ctx.fail(ctx.decl.src(), "TODO: C backend: implement value {s}", .{
+            @tagName(e),
+        }),
     }
 }
 
-fn renderFunctionSignature(ctx: *Context, header: *C.Header, writer: std.ArrayList(u8).Writer, decl: *Decl) !void {
+fn renderFunctionSignature(
+    ctx: *Context,
+    header: *C.Header,
+    writer: Writer,
+    decl: *Decl,
+) !void {
     const tv = decl.typed_value.most_recent.typed_value;
     try renderType(ctx, header, writer, tv.ty.fnReturnType());
     // Use the child allocator directly, as we know the name can be freed before
@@ -81,27 +183,92 @@ fn renderFunctionSignature(ctx: *Context, header: *C.Header, writer: std.ArrayLi
 }
 
 pub fn generate(file: *C, decl: *Decl) !void {
-    switch (decl.typed_value.most_recent.typed_value.ty.zigTypeTag()) {
-        .Fn => try genFn(file, decl),
-        .Array => try genArray(file, decl),
-        else => |e| return file.fail(decl.src(), "TODO {}", .{e}),
+    const tv = decl.typed_value.most_recent.typed_value;
+
+    var arena = std.heap.ArenaAllocator.init(file.base.allocator);
+    defer arena.deinit();
+    var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
+    defer inst_map.deinit();
+    var ctx = Context{
+        .decl = decl,
+        .arena = &arena,
+        .inst_map = &inst_map,
+        .target = file.base.options.target,
+    };
+    defer {
+        file.error_msg = ctx.error_msg;
+        ctx.deinit();
+    }
+
+    if (tv.val.cast(Value.Payload.Function)) |func_payload| {
+        const writer = file.main.writer();
+        try renderFunctionSignature(&ctx, &file.header, writer, decl);
+
+        try writer.writeAll(" {");
+
+        const func: *Module.Fn = func_payload.func;
+        const instructions = func.analysis.success.instructions;
+        if (instructions.len > 0) {
+            try writer.writeAll("\n");
+            for (instructions) |inst| {
+                const indent_size = 4;
+                const indent_level = 1;
+                try writer.writeByteNTimes(' ', indent_size * indent_level);
+                if (switch (inst.tag) {
+                    .assembly => try genAsm(&ctx, file, inst.castTag(.assembly).?),
+                    .call => try genCall(&ctx, file, inst.castTag(.call).?),
+                    .add => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "+"),
+                    .sub => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "-"),
+                    .ret => try genRet(&ctx, file, inst.castTag(.ret).?),
+                    .retvoid => try genRetVoid(file),
+                    .arg => try genArg(&ctx),
+                    .dbg_stmt => try genDbgStmt(&ctx, inst.castTag(.dbg_stmt).?),
+                    .breakpoint => try genBreak(&ctx, inst.castTag(.breakpoint).?),
+                    .unreach => try genUnreach(file, inst.castTag(.unreach).?),
+                    .intcast => try genIntCast(&ctx, file, inst.castTag(.intcast).?),
+                    else => |e| return ctx.fail(decl.src(), "TODO implement C codegen for {}", .{e}),
+                }) |name| {
+                    try ctx.inst_map.putNoClobber(inst, name);
+                }
+            }
+        }
+
+        try writer.writeAll("}\n\n");
+    } else if (tv.val.tag() == .extern_fn) {
+        return; // handled when referenced
+    } else {
+        const writer = file.constants.writer();
+        try writer.writeAll("static ");
+
+        // TODO ask the Decl if it is const
+        // https://github.com/ziglang/zig/issues/7582
+
+        try renderType(&ctx, &file.header, writer, tv.ty);
+        try writer.print(" {s} = ", .{decl.name});
+        try renderValue(&ctx, writer, tv.ty, tv.val);
+        try writer.writeAll(";\n");
     }
 }
 
 pub fn generateHeader(
-    arena: *std.heap.ArenaAllocator,
+    comp: *Compilation,
     module: *Module,
     header: *C.Header,
     decl: *Decl,
 ) error{ AnalysisFail, OutOfMemory }!void {
     switch (decl.typed_value.most_recent.typed_value.ty.zigTypeTag()) {
         .Fn => {
-            var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
+            var inst_map = std.AutoHashMap(*Inst, []u8).init(comp.gpa);
             defer inst_map.deinit();
+
+            var arena = std.heap.ArenaAllocator.init(comp.gpa);
+            defer arena.deinit();
+
             var ctx = Context{
                 .decl = decl,
-                .arena = arena,
+                .arena = &arena,
                 .inst_map = &inst_map,
+                .target = comp.getTarget(),
             };
             const writer = header.buf.writer();
             renderFunctionSignature(&ctx, header, writer, decl) catch |err| {
@@ -116,24 +283,6 @@ pub fn generateHeader(
     }
 }
 
-fn genArray(file: *C, decl: *Decl) !void {
-    const tv = decl.typed_value.most_recent.typed_value;
-    // TODO: prevent inline asm constants from being emitted
-    const name = try map(file.base.allocator, mem.span(decl.name));
-    defer file.base.allocator.free(name);
-    if (tv.val.cast(Value.Payload.Bytes)) |payload|
-        if (tv.ty.sentinel()) |sentinel|
-            if (sentinel.toUnsignedInt() == 0)
-                // TODO: static by default
-                try file.constants.writer().print("const char *const {} = \"{}\";\n", .{ name, payload.data })
-            else
-                return file.fail(decl.src(), "TODO byte arrays with non-zero sentinels", .{})
-        else
-            return file.fail(decl.src(), "TODO byte arrays without sentinels", .{})
-    else
-        return file.fail(decl.src(), "TODO non-byte arrays", .{});
-}
-
 const Context = struct {
     decl: *Decl,
     inst_map: *std.AutoHashMap(*Inst, []u8),
@@ -141,6 +290,7 @@ const Context = struct {
     argdex: usize = 0,
     unnamed_index: usize = 0,
     error_msg: *Compilation.ErrorMsg = undefined,
+    target: std.Target,
 
     fn resolveInst(self: *Context, inst: *Inst) ![]u8 {
         if (inst.cast(Inst.Constant)) |const_inst| {
@@ -170,55 +320,6 @@ const Context = struct {
     }
 };
 
-fn genFn(file: *C, decl: *Decl) !void {
-    const writer = file.main.writer();
-    const tv = decl.typed_value.most_recent.typed_value;
-
-    var arena = std.heap.ArenaAllocator.init(file.base.allocator);
-    defer arena.deinit();
-    var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
-    defer inst_map.deinit();
-    var ctx = Context{
-        .decl = decl,
-        .arena = &arena,
-        .inst_map = &inst_map,
-    };
-    defer {
-        file.error_msg = ctx.error_msg;
-        ctx.deinit();
-    }
-
-    try renderFunctionSignature(&ctx, &file.header, writer, decl);
-
-    try writer.writeAll(" {");
-
-    const func: *Module.Fn = tv.val.cast(Value.Payload.Function).?.func;
-    const instructions = func.analysis.success.instructions;
-    if (instructions.len > 0) {
-        try writer.writeAll("\n");
-        for (instructions) |inst| {
-            if (switch (inst.tag) {
-                .assembly => try genAsm(&ctx, file, inst.castTag(.assembly).?),
-                .call => try genCall(&ctx, file, inst.castTag(.call).?),
-                .add => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "+"),
-                .sub => try genBinOp(&ctx, file, inst.cast(Inst.BinOp).?, "-"),
-                .ret => try genRet(&ctx, inst.castTag(.ret).?),
-                .retvoid => try genRetVoid(file),
-                .arg => try genArg(&ctx),
-                .dbg_stmt => try genDbgStmt(&ctx, inst.castTag(.dbg_stmt).?),
-                .breakpoint => try genBreak(&ctx, inst.castTag(.breakpoint).?),
-                .unreach => try genUnreach(file, inst.castTag(.unreach).?),
-                .intcast => try genIntCast(&ctx, file, inst.castTag(.intcast).?),
-                else => |e| return ctx.fail(decl.src(), "TODO implement C codegen for {}", .{e}),
-            }) |name| {
-                try ctx.inst_map.putNoClobber(inst, name);
-            }
-        }
-    }
-
-    try writer.writeAll("}\n\n");
-}
-
 fn genArg(ctx: *Context) !?[]u8 {
     const name = try std.fmt.allocPrint(&ctx.arena.allocator, "arg{}", .{ctx.argdex});
     ctx.argdex += 1;
@@ -226,12 +327,24 @@ fn genArg(ctx: *Context) !?[]u8 {
 }
 
 fn genRetVoid(file: *C) !?[]u8 {
-    try file.main.writer().print(indentation ++ "return;\n", .{});
+    try file.main.writer().print("return;\n", .{});
     return null;
 }
 
-fn genRet(ctx: *Context, inst: *Inst.UnOp) !?[]u8 {
-    return ctx.fail(ctx.decl.src(), "TODO return", .{});
+fn genRet(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
+    const writer = file.main.writer();
+    try writer.writeAll("return ");
+    try genValue(ctx, writer, inst.operand);
+    try writer.writeAll(";\n");
+    return null;
+}
+
+fn genValue(ctx: *Context, writer: Writer, inst: *Inst) !void {
+    if (inst.value()) |val| {
+        try renderValue(ctx, writer, inst.ty, val);
+        return;
+    }
+    return ctx.fail(ctx.decl.src(), "TODO: C backend: genValue for non-constant value", .{});
 }
 
 fn genIntCast(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
@@ -241,7 +354,7 @@ fn genIntCast(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
     const writer = file.main.writer();
     const name = try ctx.name();
     const from = try ctx.resolveInst(inst.operand);
-    try writer.writeAll(indentation ++ "const ");
+    try writer.writeAll("const ");
     try renderType(ctx, &file.header, writer, inst.base.ty);
     try writer.print(" {} = (", .{name});
     try renderType(ctx, &file.header, writer, inst.base.ty);
@@ -256,7 +369,7 @@ fn genBinOp(ctx: *Context, file: *C, inst: *Inst.BinOp, comptime operator: []con
     const rhs = ctx.resolveInst(inst.rhs);
     const writer = file.main.writer();
     const name = try ctx.name();
-    try writer.writeAll(indentation ++ "const ");
+    try writer.writeAll("const ");
     try renderType(ctx, &file.header, writer, inst.base.ty);
     try writer.print(" {} = {} " ++ operator ++ " {};\n", .{ name, lhs, rhs });
     return name;
@@ -265,41 +378,42 @@ fn genBinOp(ctx: *Context, file: *C, inst: *Inst.BinOp, comptime operator: []con
 fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
     const writer = file.main.writer();
     const header = file.header.buf.writer();
-    try writer.writeAll(indentation);
     if (inst.func.castTag(.constant)) |func_inst| {
-        if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-            const target = func_val.func.owner_decl;
-            const target_ty = target.typed_value.most_recent.typed_value.ty;
-            const ret_ty = target_ty.fnReturnType().tag();
-            if (target_ty.fnReturnType().hasCodeGenBits() and inst.base.isUnused()) {
-                try writer.print("(void)", .{});
-            }
-            const tname = mem.spanZ(target.name);
-            if (file.called.get(tname) == null) {
-                try file.called.put(tname, void{});
-                try renderFunctionSignature(ctx, &file.header, header, target);
-                try header.writeAll(";\n");
-            }
-            try writer.print("{}(", .{tname});
-            if (inst.args.len != 0) {
-                for (inst.args) |arg, i| {
-                    if (i > 0) {
-                        try writer.writeAll(", ");
-                    }
-                    if (arg.cast(Inst.Constant)) |con| {
-                        try renderValue(ctx, writer, arg.ty, con.val);
-                    } else {
-                        const val = try ctx.resolveInst(arg);
-                        try writer.print("{}", .{val});
-                    }
+        const fn_decl = if (func_inst.val.cast(Value.Payload.ExternFn)) |extern_fn|
+            extern_fn.decl
+        else if (func_inst.val.cast(Value.Payload.Function)) |func_val|
+            func_val.func.owner_decl
+        else
+            unreachable;
+
+        const fn_ty = fn_decl.typed_value.most_recent.typed_value.ty;
+        const ret_ty = fn_ty.fnReturnType().tag();
+        if (fn_ty.fnReturnType().hasCodeGenBits() and inst.base.isUnused()) {
+            try writer.print("(void)", .{});
+        }
+        const fn_name = mem.spanZ(fn_decl.name);
+        if (file.called.get(fn_name) == null) {
+            try file.called.put(fn_name, void{});
+            try renderFunctionSignature(ctx, &file.header, header, fn_decl);
+            try header.writeAll(";\n");
+        }
+        try writer.print("{s}(", .{fn_name});
+        if (inst.args.len != 0) {
+            for (inst.args) |arg, i| {
+                if (i > 0) {
+                    try writer.writeAll(", ");
+                }
+                if (arg.cast(Inst.Constant)) |con| {
+                    try renderValue(ctx, writer, arg.ty, con.val);
+                } else {
+                    const val = try ctx.resolveInst(arg);
+                    try writer.print("{}", .{val});
                 }
             }
-            try writer.writeAll(");\n");
-        } else {
-            return ctx.fail(ctx.decl.src(), "TODO non-function call target?", .{});
         }
+        try writer.writeAll(");\n");
     } else {
-        return ctx.fail(ctx.decl.src(), "TODO non-constant call inst?", .{});
+        return ctx.fail(ctx.decl.src(), "TODO: C backend: implement function pointers", .{});
     }
     return null;
 }
@@ -315,13 +429,12 @@ fn genBreak(ctx: *Context, inst: *Inst.NoOp) !?[]u8 {
 }
 
 fn genUnreach(file: *C, inst: *Inst.NoOp) !?[]u8 {
-    try file.main.writer().writeAll(indentation ++ "zig_unreachable();\n");
+    try file.main.writer().writeAll("zig_unreachable();\n");
     return null;
 }
 
 fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
     const writer = file.main.writer();
-    try writer.writeAll(indentation);
     for (as.inputs) |i, index| {
         if (i[0] == '{' and i[i.len - 1] == '}') {
             const reg = i[1 .. i.len - 1];
diff --git a/src/link/C.zig b/src/link/C.zig
index 325f2ee048..62a350826e 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -15,8 +15,6 @@ pub const base_tag: File.Tag = .c;
 
 pub const Header = struct {
     buf: std.ArrayList(u8),
-    need_stddef: bool = false,
-    need_stdint: bool = false,
     emit_loc: ?Compilation.EmitLoc,
 
     pub fn init(allocator: *Allocator, emit_loc: ?Compilation.EmitLoc) Header {
@@ -31,20 +29,8 @@ pub const Header = struct {
         defer tracy.end();
 
         try writer.writeAll(@embedFile("cbe.h"));
-        var includes = false;
-        if (self.need_stddef) {
-            try writer.writeAll("#include <stddef.h>\n");
-            includes = true;
-        }
-        if (self.need_stdint) {
-            try writer.writeAll("#include <stdint.h>\n");
-            includes = true;
-        }
-        if (includes) {
-            try writer.writeByte('\n');
-        }
         if (self.buf.items.len > 0) {
-            try writer.print("{}", .{self.buf.items});
+            try writer.print("{s}", .{self.buf.items});
         }
     }
 
diff --git a/src/link/cbe.h b/src/link/cbe.h
index c3f889c6f1..cd37ba2f2e 100644
--- a/src/link/cbe.h
+++ b/src/link/cbe.h
@@ -23,3 +23,7 @@
 #define zig_unreachable()
 #endif
 
+#include <stdint.h>
+#define int128_t __int128
+#define uint128_t unsigned __int128
+
diff --git a/src/type.zig b/src/type.zig
index f3ca23014d..f08408738a 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -172,7 +172,15 @@ pub const Type = extern union {
                 const is_slice_b = isSlice(b);
                 if (is_slice_a != is_slice_b)
                     return false;
-                @panic("TODO implement more pointer Type equality comparison");
+
+                const ptr_size_a = ptrSize(a);
+                const ptr_size_b = ptrSize(b);
+                if (ptr_size_a != ptr_size_b)
+                    return false;
+
+                std.debug.panic("TODO implement more pointer Type equality comparison: {} and {}", .{
+                    a, b,
+                });
             },
             .Int => {
                 // Detect that e.g. u64 != usize, even if the bits match on a particular target.
@@ -1128,6 +1136,88 @@ pub const Type = extern union {
         };
     }
 
+    /// Asserts the `Type` is a pointer.
+    pub fn ptrSize(self: Type) std.builtin.TypeInfo.Pointer.Size {
+        return switch (self.tag()) {
+            .u8,
+            .i8,
+            .u16,
+            .i16,
+            .u32,
+            .i32,
+            .u64,
+            .i64,
+            .usize,
+            .isize,
+            .c_short,
+            .c_ushort,
+            .c_int,
+            .c_uint,
+            .c_long,
+            .c_ulong,
+            .c_longlong,
+            .c_ulonglong,
+            .c_longdouble,
+            .f16,
+            .f32,
+            .f64,
+            .f128,
+            .c_void,
+            .bool,
+            .void,
+            .type,
+            .anyerror,
+            .comptime_int,
+            .comptime_float,
+            .noreturn,
+            .@"null",
+            .@"undefined",
+            .array,
+            .array_sentinel,
+            .array_u8,
+            .array_u8_sentinel_0,
+            .fn_noreturn_no_args,
+            .fn_void_no_args,
+            .fn_naked_noreturn_no_args,
+            .fn_ccc_void_no_args,
+            .function,
+            .int_unsigned,
+            .int_signed,
+            .optional,
+            .optional_single_mut_pointer,
+            .optional_single_const_pointer,
+            .enum_literal,
+            .error_union,
+            .@"anyframe",
+            .anyframe_T,
+            .anyerror_void_error_union,
+            .error_set,
+            .error_set_single,
+            .empty_struct,
+            => unreachable,
+
+            .const_slice,
+            .mut_slice,
+            .const_slice_u8,
+            => .Slice,
+
+            .many_const_pointer,
+            .many_mut_pointer,
+            => .Many,
+
+            .c_const_pointer,
+            .c_mut_pointer,
+            => .C,
+
+            .single_const_pointer,
+            .single_mut_pointer,
+            .single_const_pointer_to_comptime_int,
+            => .One,
+
+            .pointer => self.cast(Payload.Pointer).?.size,
+        };
+    }
+
     pub fn isSlice(self: Type) bool {
         return switch (self.tag()) {
             .u8,
diff --git a/src/value.zig b/src/value.zig
index 4271ae66f4..53c4b2d540 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -82,6 +82,7 @@ pub const Value = extern union {
         int_big_positive,
         int_big_negative,
         function,
+        extern_fn,
         variable,
         ref_val,
         decl_ref,
@@ -205,6 +206,7 @@ pub const Value = extern union {
                 @panic("TODO implement copying of big ints");
             },
             .function => return self.copyPayloadShallow(allocator, Payload.Function),
+            .extern_fn => return self.copyPayloadShallow(allocator, Payload.ExternFn),
             .variable => return self.copyPayloadShallow(allocator, Payload.Variable),
             .ref_val => {
                 const payload = @fieldParentPtr(Payload.RefVal, "base", self.ptr_otherwise);
@@ -337,6 +339,7 @@ pub const Value = extern union {
             .int_big_positive => return out_stream.print("{}", .{val.cast(Payload.IntBigPositive).?.asBigInt()}),
             .int_big_negative => return out_stream.print("{}", .{val.cast(Payload.IntBigNegative).?.asBigInt()}),
             .function => return out_stream.writeAll("(function)"),
+            .extern_fn => return out_stream.writeAll("(extern function)"),
             .variable => return out_stream.writeAll("(variable)"),
             .ref_val => {
                 const ref_val = val.cast(Payload.RefVal).?;
@@ -468,6 +471,7 @@ pub const Value = extern union {
             .int_big_positive,
             .int_big_negative,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -533,6 +537,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -617,6 +622,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -701,6 +707,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -812,6 +819,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -901,6 +909,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -1071,6 +1080,7 @@ pub const Value = extern union {
             .bool_false,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -1150,6 +1160,7 @@ pub const Value = extern union {
             .anyframe_type,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .ref_val,
             .decl_ref,
@@ -1383,6 +1394,10 @@ pub const Value = extern union {
                 const payload = @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise);
                 std.hash.autoHash(&hasher, payload.func);
             },
+            .extern_fn => {
+                const payload = @fieldParentPtr(Payload.ExternFn, "base", self.ptr_otherwise);
+                std.hash.autoHash(&hasher, payload.decl);
+            },
             .variable => {
                 const payload = @fieldParentPtr(Payload.Variable, "base", self.ptr_otherwise);
                 std.hash.autoHash(&hasher, payload.variable);
@@ -1449,6 +1464,7 @@ pub const Value = extern union {
             .bool_false,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .int_u64,
             .int_i64,
@@ -1533,6 +1549,7 @@ pub const Value = extern union {
             .bool_false,
             .null_value,
             .function,
+            .extern_fn,
             .variable,
             .int_u64,
             .int_i64,
@@ -1634,6 +1651,7 @@ pub const Value = extern union {
             .bool_true,
             .bool_false,
             .function,
+            .extern_fn,
             .variable,
             .int_u64,
             .int_i64,
@@ -1730,6 +1748,7 @@ pub const Value = extern union {
             .bool_true,
             .bool_false,
             .function,
+            .extern_fn,
             .variable,
             .int_u64,
             .int_i64,
@@ -1793,6 +1812,11 @@ pub const Value = extern union {
             func: *Module.Fn,
         };
 
+        pub const ExternFn = struct {
+            base: Payload = Payload{ .tag = .extern_fn },
+            decl: *Module.Decl,
+        };
+
         pub const Variable = struct {
             base: Payload = Payload{ .tag = .variable },
             variable: *Module.Var,
-- 
cgit v1.2.3


From e0a78d10ccb3c9b5d6ebb17f32ef3b79363cd4d0 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 28 Dec 2020 21:48:56 -0700
Subject: stage2: better error message for root zig source file not found

closes #6777
closes #6893
---
 src/Cache.zig       | 20 +++++++++++++++++---
 src/Compilation.zig |  9 +++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Cache.zig b/src/Cache.zig
index 7d77782a6f..3d33226f27 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -194,6 +194,9 @@ pub const Manifest = struct {
     files: std.ArrayListUnmanaged(File) = .{},
     hex_digest: [hex_digest_len]u8,
     debug_bin_digest: DebugBinDigest = null_debug_bin_digest,
+    /// Populated when hit() returns an error because of one
+    /// of the files listed in the manifest.
+    failed_file_index: ?usize = null,
 
     /// Add a file as a dependency of process being cached. When `hit` is
     /// called, the file's contents will be checked to ensure that it matches
@@ -255,6 +258,8 @@ pub const Manifest = struct {
     pub fn hit(self: *Manifest) !bool {
         assert(self.manifest_file == null);
 
+        self.failed_file_index = null;
+
         const ext = ".txt";
         var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
 
@@ -366,7 +371,10 @@ pub const Manifest = struct {
             };
             defer this_file.close();
 
-            const actual_stat = try this_file.stat();
+            const actual_stat = this_file.stat() catch |err| {
+                self.failed_file_index = idx;
+                return err;
+            };
             const size_match = actual_stat.size == cache_hash_file.stat.size;
             const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
             const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
@@ -382,7 +390,10 @@ pub const Manifest = struct {
                 }
 
                 var actual_digest: BinDigest = undefined;
-                try hashFile(this_file, &actual_digest);
+                hashFile(this_file, &actual_digest) catch |err| {
+                    self.failed_file_index = idx;
+                    return err;
+                };
 
                 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
                     cache_hash_file.bin_digest = actual_digest;
@@ -407,7 +418,10 @@ pub const Manifest = struct {
             self.manifest_dirty = true;
             while (idx < input_file_count) : (idx += 1) {
                 const ch_file = &self.files.items[idx];
-                try self.populateFileHash(ch_file);
+                self.populateFileHash(ch_file) catch |err| {
+                    self.failed_file_index = idx;
+                    return err;
+                };
             }
             return false;
         }
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 39dd97c3a2..8a0a6ee58d 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1629,7 +1629,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 unreachable;
 
             self.updateStage1Module(main_progress_node) catch |err| {
-                fatal("unable to build stage1 zig object: {}", .{@errorName(err)});
+                fatal("unable to build stage1 zig object: {s}", .{@errorName(err)});
             };
         },
     };
@@ -3001,7 +3001,12 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     const prev_hash_state = man.hash.peekBin();
     const input_file_count = man.files.items.len;
 
-    if (try man.hit()) {
+    const hit = man.hit() catch |err| {
+        const i = man.failed_file_index orelse return err;
+        const file_path = man.files.items[i].path orelse return err;
+        fatal("unable to build stage1 zig object: {s}: {s}", .{ @errorName(err), file_path });
+    };
+    if (hit) {
         const digest = man.final();
 
         // We use an extra hex-encoded byte here to store some flags.
-- 
cgit v1.2.3


From 133da8692e80532797dd91b32539cf2175280a95 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 30 Dec 2020 19:57:11 -0700
Subject: stage2: rework Type Payload layout

Add `Type.castTag` and note that it is preferable to call than
`Type.cast`. This matches other abstractions in the codebase.

Added a convenience function `Type.Tag.create` which really cleans up
the callsites of creating `Type` objects.

`Type` payloads can now share types. This is in preparation for another
improvement that I want to do.
---
 src/Compilation.zig |   8 +-
 src/Module.zig      | 127 +++++------
 src/astgen.zig      |   2 +-
 src/codegen.zig     |   2 +-
 src/type.zig        | 613 +++++++++++++++++++++++++++++-----------------------
 src/value.zig       |  35 ++-
 src/zir.zig         |   2 +-
 src/zir_sema.zig    |  30 +--
 8 files changed, 424 insertions(+), 395 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 8a0a6ee58d..11c8303fac 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -825,9 +825,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
 
             const root_scope = rs: {
                 if (mem.endsWith(u8, root_pkg.root_src_path, ".zig")) {
-                    const struct_payload = try gpa.create(Type.Payload.EmptyStruct);
                     const root_scope = try gpa.create(Module.Scope.File);
-                    struct_payload.* = .{ .scope = &root_scope.root_container };
+                    const struct_ty = try Type.Tag.empty_struct.create(
+                        gpa,
+                        &root_scope.root_container,
+                    );
                     root_scope.* = .{
                         // TODO this is duped so it can be freed in Container.deinit
                         .sub_file_path = try gpa.dupe(u8, root_pkg.root_src_path),
@@ -838,7 +840,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                         .root_container = .{
                             .file_scope = root_scope,
                             .decls = .{},
-                            .ty = Type.initPayload(&struct_payload.base),
+                            .ty = struct_ty,
                         },
                     };
                     break :rs &root_scope.base;
diff --git a/src/Module.zig b/src/Module.zig
index 089bc51893..3e937fe49b 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -562,7 +562,7 @@ pub const Scope = struct {
         pub fn deinit(self: *Container, gpa: *Allocator) void {
             self.decls.deinit(gpa);
             // TODO either Container of File should have an arena for sub_file_path and ty
-            gpa.destroy(self.ty.cast(Type.Payload.EmptyStruct).?);
+            gpa.destroy(self.ty.castTag(.empty_struct).?);
             gpa.free(self.file_scope.sub_file_path);
             self.* = undefined;
         }
@@ -2528,12 +2528,11 @@ pub fn analyzeImport(self: *Module, scope: *Scope, src: usize, target_string: []
     }
 
     // TODO Scope.Container arena for ty and sub_file_path
-    const struct_payload = try self.gpa.create(Type.Payload.EmptyStruct);
-    errdefer self.gpa.destroy(struct_payload);
     const file_scope = try self.gpa.create(Scope.File);
     errdefer self.gpa.destroy(file_scope);
+    const struct_ty = try Type.Tag.empty_struct.create(self.gpa, &file_scope.root_container);
+    errdefer self.gpa.destroy(struct_ty.castTag(.empty_struct).?);
 
-    struct_payload.* = .{ .scope = &file_scope.root_container };
     file_scope.* = .{
         .sub_file_path = resolved_path,
         .source = .{ .unloaded = {} },
@@ -2543,7 +2542,7 @@ pub fn analyzeImport(self: *Module, scope: *Scope, src: usize, target_string: []
         .root_container = .{
             .file_scope = file_scope,
             .decls = .{},
-            .ty = Type.initPayload(&struct_payload.base),
+            .ty = struct_ty,
         },
     };
     self.analyzeContainer(&file_scope.root_container) catch |err| switch (err) {
@@ -2564,7 +2563,7 @@ pub fn cmpNumeric(
     lhs: *Inst,
     rhs: *Inst,
     op: std.math.CompareOperator,
-) !*Inst {
+) InnerError!*Inst {
     assert(lhs.ty.isNumeric());
     assert(rhs.ty.isNumeric());
 
@@ -2738,15 +2737,14 @@ fn wrapOptional(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*In
 }
 
 fn makeIntType(self: *Module, scope: *Scope, signed: bool, bits: u16) !Type {
-    if (signed) {
-        const int_payload = try scope.arena().create(Type.Payload.IntSigned);
-        int_payload.* = .{ .bits = bits };
-        return Type.initPayload(&int_payload.base);
-    } else {
-        const int_payload = try scope.arena().create(Type.Payload.IntUnsigned);
-        int_payload.* = .{ .bits = bits };
-        return Type.initPayload(&int_payload.base);
-    }
+    const int_payload = try scope.arena().create(Type.Payload.Bits);
+    int_payload.* = .{
+        .base = .{
+            .tag = if (signed) .int_signed else .int_unsigned,
+        },
+        .data = bits,
+    };
+    return Type.initPayload(&int_payload.base);
 }
 
 pub fn resolvePeerTypes(self: *Module, scope: *Scope, instructions: []*Inst) !Type {
@@ -2829,7 +2827,7 @@ pub fn coerce(self: *Module, scope: *Scope, dest_type: Type, inst: *Inst) !*Inst
 
     // T to ?T
     if (dest_type.zigTypeTag() == .Optional) {
-        var buf: Type.Payload.PointerSimple = undefined;
+        var buf: Type.Payload.ElemType = undefined;
         const child_type = dest_type.optionalChild(&buf);
         if (child_type.eql(inst.ty)) {
             return self.wrapOptional(scope, dest_type, inst);
@@ -3225,7 +3223,7 @@ pub fn simplePtrType(self: *Module, scope: *Scope, src: usize, elem_ty: Type, mu
     // TODO stage1 type inference bug
     const T = Type.Tag;
 
-    const type_payload = try scope.arena().create(Type.Payload.PointerSimple);
+    const type_payload = try scope.arena().create(Type.Payload.ElemType);
     type_payload.* = .{
         .base = .{
             .tag = switch (size) {
@@ -3235,7 +3233,7 @@ pub fn simplePtrType(self: *Module, scope: *Scope, src: usize, elem_ty: Type, mu
                 .Slice => if (mutable) T.mut_slice else T.const_slice,
             },
         },
-        .pointee_type = elem_ty,
+        .data = elem_ty,
     };
     return Type.initPayload(&type_payload.base);
 }
@@ -3257,8 +3255,7 @@ pub fn ptrType(
     assert(host_size == 0 or bit_offset < host_size * 8);
 
     // TODO check if type can be represented by simplePtrType
-    const type_payload = try scope.arena().create(Type.Payload.Pointer);
-    type_payload.* = .{
+    return Type.Tag.pointer.create(scope.arena(), .{
         .pointee_type = elem_ty,
         .sentinel = sentinel,
         .@"align" = @"align",
@@ -3268,95 +3265,73 @@ pub fn ptrType(
         .mutable = mutable,
         .@"volatile" = @"volatile",
         .size = size,
-    };
-    return Type.initPayload(&type_payload.base);
+    });
 }
 
 pub fn optionalType(self: *Module, scope: *Scope, child_type: Type) Allocator.Error!Type {
-    return Type.initPayload(switch (child_type.tag()) {
-        .single_const_pointer => blk: {
-            const payload = try scope.arena().create(Type.Payload.PointerSimple);
-            payload.* = .{
-                .base = .{ .tag = .optional_single_const_pointer },
-                .pointee_type = child_type.elemType(),
-            };
-            break :blk &payload.base;
-        },
-        .single_mut_pointer => blk: {
-            const payload = try scope.arena().create(Type.Payload.PointerSimple);
-            payload.* = .{
-                .base = .{ .tag = .optional_single_mut_pointer },
-                .pointee_type = child_type.elemType(),
-            };
-            break :blk &payload.base;
-        },
-        else => blk: {
-            const payload = try scope.arena().create(Type.Payload.Optional);
-            payload.* = .{
-                .child_type = child_type,
-            };
-            break :blk &payload.base;
-        },
-    });
+    switch (child_type.tag()) {
+        .single_const_pointer => return Type.Tag.optional_single_const_pointer.create(
+            scope.arena(),
+            child_type.elemType(),
+        ),
+        .single_mut_pointer => return Type.Tag.optional_single_mut_pointer.create(
+            scope.arena(),
+            child_type.elemType(),
+        ),
+        else => return Type.Tag.optional.create(scope.arena(), child_type),
+    }
 }
 
-pub fn arrayType(self: *Module, scope: *Scope, len: u64, sentinel: ?Value, elem_type: Type) Allocator.Error!Type {
+pub fn arrayType(
+    self: *Module,
+    scope: *Scope,
+    len: u64,
+    sentinel: ?Value,
+    elem_type: Type,
+) Allocator.Error!Type {
     if (elem_type.eql(Type.initTag(.u8))) {
         if (sentinel) |some| {
             if (some.eql(Value.initTag(.zero))) {
-                const payload = try scope.arena().create(Type.Payload.Array_u8_Sentinel0);
-                payload.* = .{
-                    .len = len,
-                };
-                return Type.initPayload(&payload.base);
+                return Type.Tag.array_u8_sentinel_0.create(scope.arena(), len);
             }
         } else {
-            const payload = try scope.arena().create(Type.Payload.Array_u8);
-            payload.* = .{
-                .len = len,
-            };
-            return Type.initPayload(&payload.base);
+            return Type.Tag.array_u8.create(scope.arena(), len);
         }
     }
 
     if (sentinel) |some| {
-        const payload = try scope.arena().create(Type.Payload.ArraySentinel);
-        payload.* = .{
+        return Type.Tag.array_sentinel.create(scope.arena(), .{
             .len = len,
             .sentinel = some,
             .elem_type = elem_type,
-        };
-        return Type.initPayload(&payload.base);
+        });
     }
 
-    const payload = try scope.arena().create(Type.Payload.Array);
-    payload.* = .{
+    return Type.Tag.array.create(scope.arena(), .{
         .len = len,
         .elem_type = elem_type,
-    };
-    return Type.initPayload(&payload.base);
+    });
 }
 
-pub fn errorUnionType(self: *Module, scope: *Scope, error_set: Type, payload: Type) Allocator.Error!Type {
+pub fn errorUnionType(
+    self: *Module,
+    scope: *Scope,
+    error_set: Type,
+    payload: Type,
+) Allocator.Error!Type {
     assert(error_set.zigTypeTag() == .ErrorSet);
     if (error_set.eql(Type.initTag(.anyerror)) and payload.eql(Type.initTag(.void))) {
         return Type.initTag(.anyerror_void_error_union);
     }
 
-    const result = try scope.arena().create(Type.Payload.ErrorUnion);
-    result.* = .{
+    return Type.Tag.error_union.create(scope.arena(), .{
         .error_set = error_set,
         .payload = payload,
-    };
-    return Type.initPayload(&result.base);
+    });
 }
 
 pub fn anyframeType(self: *Module, scope: *Scope, return_type: Type) Allocator.Error!Type {
-    const result = try scope.arena().create(Type.Payload.AnyFrame);
-    result.* = .{
-        .return_type = return_type,
-    };
-    return Type.initPayload(&result.base);
+    return Type.Tag.anyframe_T.create(scope.arena(), return_type);
 }
 
 pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void {
diff --git a/src/astgen.zig b/src/astgen.zig
index 672fe343e2..1fc8a0d19e 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -2723,7 +2723,7 @@ fn rlWrap(mod: *Module, scope: *Scope, rl: ResultLoc, result: *zir.Inst) InnerEr
             return mod.fail(scope, result.src, "TODO implement rlWrap .bitcasted_ptr", .{});
         },
         .inferred_ptr => |alloc| {
-            return mod.fail(scope, result.src, "TODO implement rlWrap .inferred_ptr", .{});
+            return addZIRBinOp(mod, scope, result.src, .store, &alloc.base, result);
         },
         .block_ptr => |block_ptr| {
             return mod.fail(scope, result.src, "TODO implement rlWrap .block_ptr", .{});
diff --git a/src/codegen.zig b/src/codegen.zig
index 01deb13c0e..d98a87a440 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -3262,7 +3262,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         if (typed_value.val.isNull())
                             return MCValue{ .immediate = 0 };
 
-                        var buf: Type.Payload.PointerSimple = undefined;
+                        var buf: Type.Payload.ElemType = undefined;
                         return self.genTypedValue(src, .{
                             .ty = typed_value.ty.optionalChild(&buf),
                             .val = typed_value.val,
diff --git a/src/type.zig b/src/type.zig
index f08408738a..ce237f89c7 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -112,18 +112,39 @@ pub const Type = extern union {
         }
     }
 
+    /// Prefer `castTag` to this.
     pub fn cast(self: Type, comptime T: type) ?*T {
-        if (self.tag_if_small_enough < Tag.no_payload_count)
+        if (@hasField(T, "base_tag")) {
+            return base.castTag(T.base_tag);
+        }
+        if (self.tag_if_small_enough < Tag.no_payload_count) {
             return null;
+        }
+        inline for (@typeInfo(Tag).Enum.fields) |field| {
+            if (field.value < Tag.no_payload_count)
+                continue;
+            const t = @intToEnum(Tag, field.value);
+            if (self.ptr_otherwise.tag == t) {
+                if (T == t.Type()) {
+                    return @fieldParentPtr(T, "base", self.ptr_otherwise);
+                }
+                return null;
+            }
+        }
+        unreachable;
+    }
 
-        const expected_tag = std.meta.fieldInfo(T, "base").default_value.?.tag;
-        if (self.ptr_otherwise.tag != expected_tag)
+    pub fn castTag(self: Type, comptime t: Tag) ?*t.Type() {
+        if (self.tag_if_small_enough < Tag.no_payload_count)
             return null;
 
-        return @fieldParentPtr(T, "base", self.ptr_otherwise);
+        if (self.ptr_otherwise.tag == t)
+            return @fieldParentPtr(t.Type(), "base", self.ptr_otherwise);
+
+        return null;
     }
 
-    pub fn castPointer(self: Type) ?*Payload.PointerSimple {
+    pub fn castPointer(self: Type) ?*Payload.ElemType {
         return switch (self.tag()) {
             .single_const_pointer,
             .single_mut_pointer,
@@ -135,7 +156,8 @@ pub const Type = extern union {
             .mut_slice,
             .optional_single_const_pointer,
             .optional_single_mut_pointer,
-            => @fieldParentPtr(Payload.PointerSimple, "base", self.ptr_otherwise),
+            => self.cast(Payload.ElemType),
+
             else => null,
         };
     }
@@ -165,7 +187,7 @@ pub const Type = extern union {
                 // Hot path for common case:
                 if (a.castPointer()) |a_payload| {
                     if (b.castPointer()) |b_payload| {
-                        return a.tag() == b.tag() and eql(a_payload.pointee_type, b_payload.pointee_type);
+                        return a.tag() == b.tag() and eql(a_payload.data, b_payload.data);
                     }
                 }
                 const is_slice_a = isSlice(a);
@@ -230,8 +252,8 @@ pub const Type = extern union {
                 return true;
             },
             .Optional => {
-                var buf_a: Payload.PointerSimple = undefined;
-                var buf_b: Payload.PointerSimple = undefined;
+                var buf_a: Payload.ElemType = undefined;
+                var buf_b: Payload.ElemType = undefined;
                 return a.optionalChild(&buf_a).eql(b.optionalChild(&buf_b));
             },
             .Float,
@@ -294,7 +316,7 @@ pub const Type = extern union {
                 }
             },
             .Optional => {
-                var buf: Payload.PointerSimple = undefined;
+                var buf: Payload.ElemType = undefined;
                 std.hash.autoHash(&hasher, self.optionalChild(&buf).hash());
             },
             .Float,
@@ -364,68 +386,64 @@ pub const Type = extern union {
             .@"anyframe",
             => unreachable,
 
-            .array_u8_sentinel_0 => return self.copyPayloadShallow(allocator, Payload.Array_u8_Sentinel0),
-            .array_u8 => return self.copyPayloadShallow(allocator, Payload.Array_u8),
+            .array_u8,
+            .array_u8_sentinel_0,
+            => return self.copyPayloadShallow(allocator, Payload.Len),
+
+            .single_const_pointer,
+            .single_mut_pointer,
+            .many_const_pointer,
+            .many_mut_pointer,
+            .c_const_pointer,
+            .c_mut_pointer,
+            .const_slice,
+            .mut_slice,
+            .optional,
+            .optional_single_mut_pointer,
+            .optional_single_const_pointer,
+            .anyframe_T,
+            => return self.copyPayloadShallow(allocator, Payload.ElemType),
+
+            .int_signed,
+            .int_unsigned,
+            => return self.copyPayloadShallow(allocator, Payload.Bits),
+
             .array => {
-                const payload = @fieldParentPtr(Payload.Array, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.Array);
-                new_payload.* = .{
-                    .base = payload.base,
+                const payload = self.castTag(.array).?.data;
+                return Tag.array.create(allocator, .{
                     .len = payload.len,
                     .elem_type = try payload.elem_type.copy(allocator),
-                };
-                return Type{ .ptr_otherwise = &new_payload.base };
+                });
             },
             .array_sentinel => {
-                const payload = @fieldParentPtr(Payload.ArraySentinel, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.ArraySentinel);
-                new_payload.* = .{
-                    .base = payload.base,
+                const payload = self.castTag(.array_sentinel).?.data;
+                return Tag.array_sentinel.create(allocator, .{
                     .len = payload.len,
                     .sentinel = try payload.sentinel.copy(allocator),
                     .elem_type = try payload.elem_type.copy(allocator),
-                };
-                return Type{ .ptr_otherwise = &new_payload.base };
+                });
             },
-            .int_signed => return self.copyPayloadShallow(allocator, Payload.IntSigned),
-            .int_unsigned => return self.copyPayloadShallow(allocator, Payload.IntUnsigned),
             .function => {
-                const payload = @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.Function);
+                const payload = self.castTag(.function).?.data;
                 const param_types = try allocator.alloc(Type, payload.param_types.len);
                 for (payload.param_types) |param_type, i| {
                     param_types[i] = try param_type.copy(allocator);
                 }
-                new_payload.* = .{
-                    .base = payload.base,
+                return Tag.function.create(allocator, .{
                     .return_type = try payload.return_type.copy(allocator),
                     .param_types = param_types,
                     .cc = payload.cc,
-                };
-                return Type{ .ptr_otherwise = &new_payload.base };
+                });
             },
-            .optional => return self.copyPayloadSingleField(allocator, Payload.Optional, "child_type"),
-            .single_const_pointer,
-            .single_mut_pointer,
-            .many_const_pointer,
-            .many_mut_pointer,
-            .c_const_pointer,
-            .c_mut_pointer,
-            .const_slice,
-            .mut_slice,
-            .optional_single_mut_pointer,
-            .optional_single_const_pointer,
-            => return self.copyPayloadSingleField(allocator, Payload.PointerSimple, "pointee_type"),
-            .anyframe_T => return self.copyPayloadSingleField(allocator, Payload.AnyFrame, "return_type"),
-
             .pointer => {
-                const payload = @fieldParentPtr(Payload.Pointer, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.Pointer);
-                new_payload.* = .{
-                    .base = payload.base,
-
+                const payload = self.castTag(.pointer).?.data;
+                const sent: ?Value = if (payload.sentinel) |some|
+                    try some.copy(allocator)
+                else
+                    null;
+                return Tag.pointer.create(allocator, .{
                     .pointee_type = try payload.pointee_type.copy(allocator),
-                    .sentinel = if (payload.sentinel) |some| try some.copy(allocator) else null,
+                    .sentinel = sent,
                     .@"align" = payload.@"align",
                     .bit_offset = payload.bit_offset,
                     .host_size = payload.host_size,
@@ -433,41 +451,28 @@ pub const Type = extern union {
                     .mutable = payload.mutable,
                     .@"volatile" = payload.@"volatile",
                     .size = payload.size,
-                };
-                return Type{ .ptr_otherwise = &new_payload.base };
+                });
             },
             .error_union => {
-                const payload = @fieldParentPtr(Payload.ErrorUnion, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.ErrorUnion);
-                new_payload.* = .{
-                    .base = payload.base,
-
+                const payload = self.castTag(.error_union).?.data;
+                return Tag.error_union.create(allocator, .{
                     .error_set = try payload.error_set.copy(allocator),
                     .payload = try payload.payload.copy(allocator),
-                };
-                return Type{ .ptr_otherwise = &new_payload.base };
+                });
             },
-            .error_set => return self.copyPayloadShallow(allocator, Payload.ErrorSet),
-            .error_set_single => return self.copyPayloadShallow(allocator, Payload.ErrorSetSingle),
-            .empty_struct => return self.copyPayloadShallow(allocator, Payload.EmptyStruct),
+            .error_set => return self.copyPayloadShallow(allocator, Payload.Decl),
+            .error_set_single => return self.copyPayloadShallow(allocator, Payload.Name),
+            .empty_struct => return self.copyPayloadShallow(allocator, Payload.ContainerScope),
         }
     }
 
     fn copyPayloadShallow(self: Type, allocator: *Allocator, comptime T: type) error{OutOfMemory}!Type {
-        const payload = @fieldParentPtr(T, "base", self.ptr_otherwise);
+        const payload = self.cast(T).?;
         const new_payload = try allocator.create(T);
         new_payload.* = payload.*;
         return Type{ .ptr_otherwise = &new_payload.base };
     }
 
-    fn copyPayloadSingleField(self: Type, allocator: *Allocator, comptime T: type, comptime field_name: []const u8) error{OutOfMemory}!Type {
-        const payload = @fieldParentPtr(T, "base", self.ptr_otherwise);
-        const new_payload = try allocator.create(T);
-        new_payload.base = payload.base;
-        @field(new_payload, field_name) = try @field(payload, field_name).copy(allocator);
-        return Type{ .ptr_otherwise = &new_payload.base };
-    }
-
     pub fn format(
         self: Type,
         comptime fmt: []const u8,
@@ -527,7 +532,7 @@ pub const Type = extern union {
                 .fn_ccc_void_no_args => return out_stream.writeAll("fn() callconv(.C) void"),
                 .single_const_pointer_to_comptime_int => return out_stream.writeAll("*const comptime_int"),
                 .function => {
-                    const payload = @fieldParentPtr(Payload.Function, "base", ty.ptr_otherwise);
+                    const payload = ty.castTag(.function).?.data;
                     try out_stream.writeAll("fn(");
                     for (payload.param_types) |param_type, i| {
                         if (i != 0) try out_stream.writeAll(", ");
@@ -539,108 +544,108 @@ pub const Type = extern union {
                 },
 
                 .anyframe_T => {
-                    const payload = @fieldParentPtr(Payload.AnyFrame, "base", ty.ptr_otherwise);
+                    const return_type = ty.castTag(.anyframe_T).?.data;
                     try out_stream.print("anyframe->", .{});
-                    ty = payload.return_type;
+                    ty = return_type;
                     continue;
                 },
                 .array_u8 => {
-                    const payload = @fieldParentPtr(Payload.Array_u8, "base", ty.ptr_otherwise);
-                    return out_stream.print("[{}]u8", .{payload.len});
+                    const len = ty.castTag(.array_u8).?.data;
+                    return out_stream.print("[{}]u8", .{len});
                 },
                 .array_u8_sentinel_0 => {
-                    const payload = @fieldParentPtr(Payload.Array_u8_Sentinel0, "base", ty.ptr_otherwise);
-                    return out_stream.print("[{}:0]u8", .{payload.len});
+                    const len = ty.castTag(.array_u8_sentinel_0).?.data;
+                    return out_stream.print("[{}:0]u8", .{len});
                 },
                 .array => {
-                    const payload = @fieldParentPtr(Payload.Array, "base", ty.ptr_otherwise);
+                    const payload = ty.castTag(.array).?.data;
                     try out_stream.print("[{}]", .{payload.len});
                     ty = payload.elem_type;
                     continue;
                 },
                 .array_sentinel => {
-                    const payload = @fieldParentPtr(Payload.ArraySentinel, "base", ty.ptr_otherwise);
+                    const payload = ty.castTag(.array_sentinel).?.data;
                     try out_stream.print("[{}:{}]", .{ payload.len, payload.sentinel });
                     ty = payload.elem_type;
                     continue;
                 },
                 .single_const_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.single_const_pointer).?.data;
                     try out_stream.writeAll("*const ");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .single_mut_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.single_mut_pointer).?.data;
                     try out_stream.writeAll("*");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .many_const_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.many_const_pointer).?.data;
                     try out_stream.writeAll("[*]const ");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .many_mut_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.many_mut_pointer).?.data;
                     try out_stream.writeAll("[*]");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .c_const_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.c_const_pointer).?.data;
                     try out_stream.writeAll("[*c]const ");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .c_mut_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.c_mut_pointer).?.data;
                     try out_stream.writeAll("[*c]");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .const_slice => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.const_slice).?.data;
                     try out_stream.writeAll("[]const ");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .mut_slice => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.mut_slice).?.data;
                     try out_stream.writeAll("[]");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .int_signed => {
-                    const payload = @fieldParentPtr(Payload.IntSigned, "base", ty.ptr_otherwise);
-                    return out_stream.print("i{}", .{payload.bits});
+                    const bits = ty.castTag(.int_signed).?.data;
+                    return out_stream.print("i{d}", .{bits});
                 },
                 .int_unsigned => {
-                    const payload = @fieldParentPtr(Payload.IntUnsigned, "base", ty.ptr_otherwise);
-                    return out_stream.print("u{}", .{payload.bits});
+                    const bits = ty.castTag(.int_unsigned).?.data;
+                    return out_stream.print("u{d}", .{bits});
                 },
                 .optional => {
-                    const payload = @fieldParentPtr(Payload.Optional, "base", ty.ptr_otherwise);
+                    const child_type = ty.castTag(.optional).?.data;
                     try out_stream.writeByte('?');
-                    ty = payload.child_type;
+                    ty = child_type;
                     continue;
                 },
                 .optional_single_const_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.optional_single_const_pointer).?.data;
                     try out_stream.writeAll("?*const ");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
                 .optional_single_mut_pointer => {
-                    const payload = @fieldParentPtr(Payload.PointerSimple, "base", ty.ptr_otherwise);
+                    const pointee_type = ty.castTag(.optional_single_mut_pointer).?.data;
                     try out_stream.writeAll("?*");
-                    ty = payload.pointee_type;
+                    ty = pointee_type;
                     continue;
                 },
 
                 .pointer => {
-                    const payload = @fieldParentPtr(Payload.Pointer, "base", ty.ptr_otherwise);
+                    const payload = ty.castTag(.pointer).?.data;
                     if (payload.sentinel) |some| switch (payload.size) {
                         .One, .C => unreachable,
                         .Many => try out_stream.print("[*:{}]", .{some}),
@@ -652,10 +657,10 @@ pub const Type = extern union {
                         .Slice => try out_stream.writeAll("[]"),
                     }
                     if (payload.@"align" != 0) {
-                        try out_stream.print("align({}", .{payload.@"align"});
+                        try out_stream.print("align({d}", .{payload.@"align"});
 
                         if (payload.bit_offset != 0) {
-                            try out_stream.print(":{}:{}", .{ payload.bit_offset, payload.host_size });
+                            try out_stream.print(":{d}:{d}", .{ payload.bit_offset, payload.host_size });
                         }
                         try out_stream.writeAll(") ");
                     }
@@ -667,19 +672,19 @@ pub const Type = extern union {
                     continue;
                 },
                 .error_union => {
-                    const payload = @fieldParentPtr(Payload.ErrorUnion, "base", ty.ptr_otherwise);
+                    const payload = ty.castTag(.error_union).?.data;
                     try payload.error_set.format("", .{}, out_stream);
                     try out_stream.writeAll("!");
                     ty = payload.payload;
                     continue;
                 },
                 .error_set => {
-                    const payload = @fieldParentPtr(Payload.ErrorSet, "base", ty.ptr_otherwise);
-                    return out_stream.writeAll(std.mem.spanZ(payload.decl.name));
+                    const decl = ty.castTag(.error_set).?.data;
+                    return out_stream.writeAll(std.mem.spanZ(decl.name));
                 },
                 .error_set_single => {
-                    const payload = @fieldParentPtr(Payload.ErrorSetSingle, "base", ty.ptr_otherwise);
-                    return out_stream.print("error{{{}}}", .{payload.name});
+                    const name = ty.castTag(.error_set_single).?.data;
+                    return out_stream.print("error{{{s}}}", .{name});
                 },
             }
             unreachable;
@@ -784,11 +789,10 @@ pub const Type = extern union {
             .array => self.elemType().hasCodeGenBits() and self.arrayLen() != 0,
             .array_u8 => self.arrayLen() != 0,
             .array_sentinel, .single_const_pointer, .single_mut_pointer, .many_const_pointer, .many_mut_pointer, .c_const_pointer, .c_mut_pointer, .const_slice, .mut_slice, .pointer => self.elemType().hasCodeGenBits(),
-            .int_signed => self.cast(Payload.IntSigned).?.bits != 0,
-            .int_unsigned => self.cast(Payload.IntUnsigned).?.bits != 0,
+            .int_signed, .int_unsigned => self.cast(Payload.Bits).?.data != 0,
 
             .error_union => {
-                const payload = self.cast(Payload.ErrorUnion).?;
+                const payload = self.castTag(.error_union).?.data;
                 return payload.error_set.hasCodeGenBits() or payload.payload.hasCodeGenBits();
             },
 
@@ -855,7 +859,7 @@ pub const Type = extern union {
             => return @divExact(target.cpu.arch.ptrBitWidth(), 8),
 
             .pointer => {
-                const payload = @fieldParentPtr(Payload.Pointer, "base", self.ptr_otherwise);
+                const payload = self.castTag(.pointer).?.data;
 
                 if (payload.@"align" != 0) return payload.@"align";
                 return @divExact(target.cpu.arch.ptrBitWidth(), 8);
@@ -885,18 +889,12 @@ pub const Type = extern union {
             .array, .array_sentinel => return self.elemType().abiAlignment(target),
 
             .int_signed, .int_unsigned => {
-                const bits: u16 = if (self.cast(Payload.IntSigned)) |pl|
-                    pl.bits
-                else if (self.cast(Payload.IntUnsigned)) |pl|
-                    pl.bits
-                else
-                    unreachable;
-
+                const bits: u16 = self.cast(Payload.Bits).?.data;
                 return std.math.ceilPowerOfTwoPromote(u16, (bits + 7) / 8);
             },
 
             .optional => {
-                var buf: Payload.PointerSimple = undefined;
+                var buf: Payload.ElemType = undefined;
                 const child_type = self.optionalChild(&buf);
                 if (!child_type.hasCodeGenBits()) return 1;
 
@@ -907,7 +905,7 @@ pub const Type = extern union {
             },
 
             .error_union => {
-                const payload = self.cast(Payload.ErrorUnion).?;
+                const payload = self.castTag(.error_union).?.data;
                 if (!payload.error_set.hasCodeGenBits()) {
                     return payload.payload.abiAlignment(target);
                 } else if (!payload.payload.hasCodeGenBits()) {
@@ -955,16 +953,19 @@ pub const Type = extern union {
             .bool,
             => return 1,
 
-            .array_u8 => @fieldParentPtr(Payload.Array_u8_Sentinel0, "base", self.ptr_otherwise).len,
-            .array_u8_sentinel_0 => @fieldParentPtr(Payload.Array_u8_Sentinel0, "base", self.ptr_otherwise).len + 1,
+            .array_u8 => self.castTag(.array_u8).?.data,
+            .array_u8_sentinel_0 => self.castTag(.array_u8_sentinel_0).?.data + 1,
             .array => {
-                const payload = @fieldParentPtr(Payload.Array, "base", self.ptr_otherwise);
+                const payload = self.castTag(.array).?.data;
                 const elem_size = std.math.max(payload.elem_type.abiAlignment(target), payload.elem_type.abiSize(target));
                 return payload.len * elem_size;
             },
             .array_sentinel => {
-                const payload = @fieldParentPtr(Payload.ArraySentinel, "base", self.ptr_otherwise);
-                const elem_size = std.math.max(payload.elem_type.abiAlignment(target), payload.elem_type.abiSize(target));
+                const payload = self.castTag(.array_sentinel).?.data;
+                const elem_size = std.math.max(
+                    payload.elem_type.abiAlignment(target),
+                    payload.elem_type.abiSize(target),
+                );
                 return (payload.len + 1) * elem_size;
             },
             .i16, .u16 => return 2,
@@ -1022,18 +1023,12 @@ pub const Type = extern union {
             => return 2, // TODO revisit this when we have the concept of the error tag type
 
             .int_signed, .int_unsigned => {
-                const bits: u16 = if (self.cast(Payload.IntSigned)) |pl|
-                    pl.bits
-                else if (self.cast(Payload.IntUnsigned)) |pl|
-                    pl.bits
-                else
-                    unreachable;
-
+                const bits: u16 = self.cast(Payload.Bits).?.data;
                 return std.math.ceilPowerOfTwoPromote(u16, (bits + 7) / 8);
             },
 
             .optional => {
-                var buf: Payload.PointerSimple = undefined;
+                var buf: Payload.ElemType = undefined;
                 const child_type = self.optionalChild(&buf);
                 if (!child_type.hasCodeGenBits()) return 1;
 
@@ -1048,7 +1043,7 @@ pub const Type = extern union {
             },
 
             .error_union => {
-                const payload = self.cast(Payload.ErrorUnion).?;
+                const payload = self.castTag(.error_union).?.data;
                 if (!payload.error_set.hasCodeGenBits() and !payload.payload.hasCodeGenBits()) {
                     return 0;
                 } else if (!payload.error_set.hasCodeGenBits()) {
@@ -1132,7 +1127,7 @@ pub const Type = extern union {
             .single_const_pointer_to_comptime_int,
             => true,
 
-            .pointer => self.cast(Payload.Pointer).?.size == .One,
+            .pointer => self.castTag(.pointer).?.data.size == .One,
         };
     }
 
@@ -1214,7 +1209,7 @@ pub const Type = extern union {
             .single_const_pointer_to_comptime_int,
             => .One,
 
-            .pointer => self.cast(Payload.Pointer).?.size,
+            .pointer => self.castTag(.pointer).?.data.size,
         };
     }
 
@@ -1289,7 +1284,7 @@ pub const Type = extern union {
             .const_slice_u8,
             => true,
 
-            .pointer => self.cast(Payload.Pointer).?.size == .Slice,
+            .pointer => self.castTag(.pointer).?.data.size == .Slice,
         };
     }
 
@@ -1364,7 +1359,7 @@ pub const Type = extern union {
             .const_slice,
             => true,
 
-            .pointer => !self.cast(Payload.Pointer).?.mutable,
+            .pointer => !self.castTag(.pointer).?.data.mutable,
         };
     }
 
@@ -1438,7 +1433,7 @@ pub const Type = extern union {
             => false,
 
             .pointer => {
-                const payload = @fieldParentPtr(Payload.Pointer, "base", self.ptr_otherwise);
+                const payload = self.castTag(.pointer).?.data;
                 return payload.@"volatile";
             },
         };
@@ -1514,7 +1509,7 @@ pub const Type = extern union {
             => false,
 
             .pointer => {
-                const payload = @fieldParentPtr(Payload.Pointer, "base", self.ptr_otherwise);
+                const payload = self.castTag(.pointer).?.data;
                 return payload.@"allowzero";
             },
         };
@@ -1525,7 +1520,7 @@ pub const Type = extern union {
         switch (self.tag()) {
             .optional_single_const_pointer, .optional_single_mut_pointer => return true,
             .optional => {
-                var buf: Payload.PointerSimple = undefined;
+                var buf: Payload.ElemType = undefined;
                 const child_type = self.optionalChild(&buf);
                 // optionals of zero sized pointers behave like bools
                 if (!child_type.hasCodeGenBits()) return false;
@@ -1563,7 +1558,7 @@ pub const Type = extern union {
             => return false,
 
             .Optional => {
-                var buf: Payload.PointerSimple = undefined;
+                var buf: Payload.ElemType = undefined;
                 return ty.optionalChild(&buf).isValidVarType(is_extern);
             },
             .Pointer, .Array => ty = ty.elemType(),
@@ -1631,8 +1626,8 @@ pub const Type = extern union {
             .empty_struct,
             => unreachable,
 
-            .array => self.cast(Payload.Array).?.elem_type,
-            .array_sentinel => self.cast(Payload.ArraySentinel).?.elem_type,
+            .array => self.castTag(.array).?.data.elem_type,
+            .array_sentinel => self.castTag(.array_sentinel).?.data.elem_type,
             .single_const_pointer,
             .single_mut_pointer,
             .many_const_pointer,
@@ -1641,28 +1636,29 @@ pub const Type = extern union {
             .c_mut_pointer,
             .const_slice,
             .mut_slice,
-            => self.castPointer().?.pointee_type,
+            => self.castPointer().?.data,
             .array_u8, .array_u8_sentinel_0, .const_slice_u8 => Type.initTag(.u8),
             .single_const_pointer_to_comptime_int => Type.initTag(.comptime_int),
-            .pointer => self.cast(Payload.Pointer).?.pointee_type,
+            .pointer => self.castTag(.pointer).?.data.pointee_type,
         };
     }
 
     /// Asserts that the type is an optional.
-    pub fn optionalChild(self: Type, buf: *Payload.PointerSimple) Type {
+    /// Resulting `Type` will have inner memory referencing `buf`.
+    pub fn optionalChild(self: Type, buf: *Payload.ElemType) Type {
         return switch (self.tag()) {
-            .optional => self.cast(Payload.Optional).?.child_type,
+            .optional => self.castTag(.optional).?.data,
             .optional_single_mut_pointer => {
                 buf.* = .{
                     .base = .{ .tag = .single_mut_pointer },
-                    .pointee_type = self.castPointer().?.pointee_type,
+                    .data = self.castPointer().?.data,
                 };
                 return Type.initPayload(&buf.base);
             },
             .optional_single_const_pointer => {
                 buf.* = .{
                     .base = .{ .tag = .single_const_pointer },
-                    .pointee_type = self.castPointer().?.pointee_type,
+                    .data = self.castPointer().?.data,
                 };
                 return Type.initPayload(&buf.base);
             },
@@ -1673,23 +1669,16 @@ pub const Type = extern union {
     /// Asserts that the type is an optional.
     /// Same as `optionalChild` but allocates the buffer if needed.
     pub fn optionalChildAlloc(self: Type, allocator: *Allocator) !Type {
-        return switch (self.tag()) {
-            .optional => self.cast(Payload.Optional).?.child_type,
-            .optional_single_mut_pointer, .optional_single_const_pointer => {
-                const payload = try allocator.create(Payload.PointerSimple);
-                payload.* = .{
-                    .base = .{
-                        .tag = if (self.tag() == .optional_single_const_pointer)
-                            .single_const_pointer
-                        else
-                            .single_mut_pointer,
-                    },
-                    .pointee_type = self.castPointer().?.pointee_type,
-                };
-                return Type.initPayload(&payload.base);
+        switch (self.tag()) {
+            .optional => return self.castTag(.optional).?.data,
+            .optional_single_mut_pointer => {
+                return Tag.single_mut_pointer.create(allocator, self.castPointer().?.data);
+            },
+            .optional_single_const_pointer => {
+                return Tag.single_const_pointer.create(allocator, self.castPointer().?.data);
             },
             else => unreachable,
-        };
+        }
     }
 
     /// Asserts the type is an array or vector.
@@ -1759,10 +1748,10 @@ pub const Type = extern union {
             .empty_struct,
             => unreachable,
 
-            .array => self.cast(Payload.Array).?.len,
-            .array_sentinel => self.cast(Payload.ArraySentinel).?.len,
-            .array_u8 => self.cast(Payload.Array_u8).?.len,
-            .array_u8_sentinel_0 => self.cast(Payload.Array_u8_Sentinel0).?.len,
+            .array => self.castTag(.array).?.data.len,
+            .array_sentinel => self.castTag(.array_sentinel).?.data.len,
+            .array_u8 => self.castTag(.array_u8).?.data,
+            .array_u8_sentinel_0 => self.castTag(.array_u8_sentinel_0).?.data,
         };
     }
 
@@ -1836,8 +1825,8 @@ pub const Type = extern union {
             .array_u8,
             => return null,
 
-            .pointer => return self.cast(Payload.Pointer).?.sentinel,
-            .array_sentinel => return self.cast(Payload.ArraySentinel).?.sentinel,
+            .pointer => return self.castTag(.pointer).?.data.sentinel,
+            .array_sentinel => return self.castTag(.array_sentinel).?.data.sentinel,
             .array_u8_sentinel_0 => return Value.initTag(.zero),
         };
     }
@@ -2048,8 +2037,14 @@ pub const Type = extern union {
             .empty_struct,
             => unreachable,
 
-            .int_unsigned => .{ .signedness = .unsigned, .bits = self.cast(Payload.IntUnsigned).?.bits },
-            .int_signed => .{ .signedness = .signed, .bits = self.cast(Payload.IntSigned).?.bits },
+            .int_unsigned => .{
+                .signedness = .unsigned,
+                .bits = self.castTag(.int_unsigned).?.data,
+            },
+            .int_signed => .{
+                .signedness = .signed,
+                .bits = self.castTag(.int_signed).?.data,
+            },
             .u8 => .{ .signedness = .unsigned, .bits = 8 },
             .i8 => .{ .signedness = .signed, .bits = 8 },
             .u16 => .{ .signedness = .unsigned, .bits = 16 },
@@ -2178,7 +2173,7 @@ pub const Type = extern union {
             .fn_void_no_args => 0,
             .fn_naked_noreturn_no_args => 0,
             .fn_ccc_void_no_args => 0,
-            .function => @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise).param_types.len,
+            .function => self.castTag(.function).?.data.param_types.len,
 
             .f16,
             .f32,
@@ -2254,7 +2249,7 @@ pub const Type = extern union {
             .fn_naked_noreturn_no_args => return,
             .fn_ccc_void_no_args => return,
             .function => {
-                const payload = @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise);
+                const payload = self.castTag(.function).?.data;
                 std.mem.copy(Type, types, payload.param_types);
             },
 
@@ -2327,7 +2322,7 @@ pub const Type = extern union {
     pub fn fnParamType(self: Type, index: usize) Type {
         switch (self.tag()) {
             .function => {
-                const payload = @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise);
+                const payload = self.castTag(.function).?.data;
                 return payload.param_types[index];
             },
 
@@ -2410,7 +2405,7 @@ pub const Type = extern union {
             .fn_ccc_void_no_args,
             => Type.initTag(.void),
 
-            .function => @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise).return_type,
+            .function => self.castTag(.function).?.data.return_type,
 
             .f16,
             .f32,
@@ -2484,7 +2479,7 @@ pub const Type = extern union {
             .fn_void_no_args => .Unspecified,
             .fn_naked_noreturn_no_args => .Naked,
             .fn_ccc_void_no_args => .C,
-            .function => @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise).cc,
+            .function => self.castTag(.function).?.data.cc,
 
             .f16,
             .f32,
@@ -2760,15 +2755,8 @@ pub const Type = extern union {
             .@"null" => return Value.initTag(.null_value),
             .@"undefined" => return Value.initTag(.undef),
 
-            .int_unsigned => {
-                if (ty.cast(Payload.IntUnsigned).?.bits == 0) {
-                    return Value.initTag(.zero);
-                } else {
-                    return null;
-                }
-            },
-            .int_signed => {
-                if (ty.cast(Payload.IntSigned).?.bits == 0) {
+            .int_unsigned, .int_signed => {
+                if (ty.cast(Payload.Bits).?.data == 0) {
                     return Value.initTag(.zero);
                 } else {
                     return null;
@@ -2787,12 +2775,11 @@ pub const Type = extern union {
             .single_const_pointer,
             .single_mut_pointer,
             => {
-                const ptr = ty.castPointer().?;
-                ty = ptr.pointee_type;
+                ty = ty.castPointer().?.data;
                 continue;
             },
             .pointer => {
-                ty = ty.cast(Payload.Pointer).?.pointee_type;
+                ty = ty.castTag(.pointer).?.data.pointee_type;
                 continue;
             },
         };
@@ -2869,7 +2856,7 @@ pub const Type = extern union {
             .c_mut_pointer,
             => return true,
 
-            .pointer => self.cast(Payload.Pointer).?.size == .C,
+            .pointer => self.castTag(.pointer).?.data.size == .C,
         };
     }
 
@@ -2950,7 +2937,7 @@ pub const Type = extern union {
             .pointer,
             => unreachable,
 
-            .empty_struct => self.cast(Type.Payload.EmptyStruct).?.scope,
+            .empty_struct => self.castTag(.empty_struct).?.data,
         };
     }
 
@@ -3105,117 +3092,195 @@ pub const Type = extern union {
 
         pub const last_no_payload_tag = Tag.const_slice_u8;
         pub const no_payload_count = @enumToInt(last_no_payload_tag) + 1;
-    };
 
-    pub const Payload = struct {
-        tag: Tag,
+        pub fn Type(comptime t: Tag) type {
+            return switch (t) {
+                .u8,
+                .i8,
+                .u16,
+                .i16,
+                .u32,
+                .i32,
+                .u64,
+                .i64,
+                .usize,
+                .isize,
+                .c_short,
+                .c_ushort,
+                .c_int,
+                .c_uint,
+                .c_long,
+                .c_ulong,
+                .c_longlong,
+                .c_ulonglong,
+                .c_longdouble,
+                .f16,
+                .f32,
+                .f64,
+                .f128,
+                .c_void,
+                .bool,
+                .void,
+                .type,
+                .anyerror,
+                .comptime_int,
+                .comptime_float,
+                .noreturn,
+                .enum_literal,
+                .@"null",
+                .@"undefined",
+                .fn_noreturn_no_args,
+                .fn_void_no_args,
+                .fn_naked_noreturn_no_args,
+                .fn_ccc_void_no_args,
+                .single_const_pointer_to_comptime_int,
+                .anyerror_void_error_union,
+                .@"anyframe",
+                .const_slice_u8,
+                => @compileError("Type Tag " ++ @tagName(t) ++ " has no payload"),
+
+                .array_u8,
+                .array_u8_sentinel_0,
+                => Payload.Len,
+
+                .single_const_pointer,
+                .single_mut_pointer,
+                .many_const_pointer,
+                .many_mut_pointer,
+                .c_const_pointer,
+                .c_mut_pointer,
+                .const_slice,
+                .mut_slice,
+                .optional,
+                .optional_single_mut_pointer,
+                .optional_single_const_pointer,
+                .anyframe_T,
+                => Payload.ElemType,
+
+                .int_signed,
+                .int_unsigned,
+                => Payload.Bits,
+
+                .array => Payload.Array,
+                .array_sentinel => Payload.ArraySentinel,
+                .pointer => Payload.Pointer,
+                .function => Payload.Function,
+                .error_union => Payload.ErrorUnion,
+                .error_set => Payload.Decl,
+                .error_set_single => Payload.Name,
+                .empty_struct => Payload.ContainerScope,
+            };
+        }
 
-        pub const Array_u8_Sentinel0 = struct {
-            base: Payload = Payload{ .tag = .array_u8_sentinel_0 },
+        pub fn create(comptime t: Tag, ally: *Allocator, data: Data(t)) error{OutOfMemory}!Type {
+            const ptr = try ally.create(t.Type());
+            ptr.* = .{
+                .base = .{ .tag = t },
+                .data = data,
+            };
+            return Type{ .ptr_otherwise = &ptr.base };
+        }
 
-            len: u64,
-        };
+        pub fn Data(comptime t: Tag) type {
+            return std.meta.fieldInfo(t.Type(), "data").field_type;
+        }
+    };
 
-        pub const Array_u8 = struct {
-            base: Payload = Payload{ .tag = .array_u8 },
+    /// The sub-types are named after what fields they contain.
+    pub const Payload = struct {
+        tag: Tag,
 
-            len: u64,
+        pub const Len = struct {
+            base: Payload,
+            data: u64,
         };
 
         pub const Array = struct {
-            base: Payload = Payload{ .tag = .array },
+            pub const base_tag = Tag.array;
 
-            len: u64,
-            elem_type: Type,
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                len: u64,
+                elem_type: Type,
+            },
         };
 
         pub const ArraySentinel = struct {
-            base: Payload = Payload{ .tag = .array_sentinel },
+            pub const base_tag = Tag.array_sentinel;
 
-            len: u64,
-            sentinel: Value,
-            elem_type: Type,
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                len: u64,
+                sentinel: Value,
+                elem_type: Type,
+            },
         };
 
-        pub const PointerSimple = struct {
+        pub const ElemType = struct {
             base: Payload,
-
-            pointee_type: Type,
+            data: Type,
         };
 
-        pub const IntSigned = struct {
-            base: Payload = Payload{ .tag = .int_signed },
-
-            bits: u16,
-        };
-
-        pub const IntUnsigned = struct {
-            base: Payload = Payload{ .tag = .int_unsigned },
-
-            bits: u16,
+        pub const Bits = struct {
+            base: Payload,
+            data: u16,
         };
 
         pub const Function = struct {
-            base: Payload = Payload{ .tag = .function },
-
-            param_types: []Type,
-            return_type: Type,
-            cc: std.builtin.CallingConvention,
-        };
-
-        pub const Optional = struct {
-            base: Payload = Payload{ .tag = .optional },
+            pub const base_tag = Tag.function;
 
-            child_type: Type,
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                param_types: []Type,
+                return_type: Type,
+                cc: std.builtin.CallingConvention,
+            },
         };
 
         pub const Pointer = struct {
-            base: Payload = .{ .tag = .pointer },
-
-            pointee_type: Type,
-            sentinel: ?Value,
-            /// If zero use pointee_type.AbiAlign()
-            @"align": u32,
-            bit_offset: u16,
-            host_size: u16,
-            @"allowzero": bool,
-            mutable: bool,
-            @"volatile": bool,
-            size: std.builtin.TypeInfo.Pointer.Size,
+            pub const base_tag = Tag.pointer;
+
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                pointee_type: Type,
+                sentinel: ?Value,
+                /// If zero use pointee_type.AbiAlign()
+                @"align": u32,
+                bit_offset: u16,
+                host_size: u16,
+                @"allowzero": bool,
+                mutable: bool,
+                @"volatile": bool,
+                size: std.builtin.TypeInfo.Pointer.Size,
+            },
         };
 
         pub const ErrorUnion = struct {
-            base: Payload = .{ .tag = .error_union },
+            pub const base_tag = Tag.error_union;
 
-            error_set: Type,
-            payload: Type,
-        };
-
-        pub const AnyFrame = struct {
-            base: Payload = .{ .tag = .anyframe_T },
-
-            return_type: Type,
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                error_set: Type,
+                payload: Type,
+            },
         };
 
-        pub const ErrorSet = struct {
-            base: Payload = .{ .tag = .error_set },
-
-            decl: *Module.Decl,
+        pub const Decl = struct {
+            base: Payload,
+            data: *Module.Decl,
         };
 
-        pub const ErrorSetSingle = struct {
-            base: Payload = .{ .tag = .error_set_single },
-
+        pub const Name = struct {
+            base: Payload,
             /// memory is owned by `Module`
-            name: []const u8,
+            data: []const u8,
         };
 
         /// Mostly used for namespace like structs with zero fields.
         /// Most commonly used for files.
-        pub const EmptyStruct = struct {
-            base: Payload = .{ .tag = .empty_struct },
-
-            scope: *Module.Scope.Container,
+        pub const ContainerScope = struct {
+            base: Payload,
+            data: *Module.Scope.Container,
         };
     };
 };
diff --git a/src/value.zig b/src/value.zig
index 53c4b2d540..f26c8d8772 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -440,21 +440,18 @@ pub const Value = extern union {
 
             .int_type => {
                 const payload = self.cast(Payload.IntType).?;
-                if (payload.signed) {
-                    const new = try allocator.create(Type.Payload.IntSigned);
-                    new.* = .{ .bits = payload.bits };
-                    return Type.initPayload(&new.base);
-                } else {
-                    const new = try allocator.create(Type.Payload.IntUnsigned);
-                    new.* = .{ .bits = payload.bits };
-                    return Type.initPayload(&new.base);
-                }
+                const new = try allocator.create(Type.Payload.Bits);
+                new.* = .{
+                    .base = .{
+                        .tag = if (payload.signed) .int_signed else .int_unsigned,
+                    },
+                    .data = payload.bits,
+                };
+                return Type.initPayload(&new.base);
             },
             .error_set => {
                 const payload = self.cast(Payload.ErrorSet).?;
-                const new = try allocator.create(Type.Payload.ErrorSet);
-                new.* = .{ .decl = payload.decl };
-                return Type.initPayload(&new.base);
+                return Type.Tag.error_set.create(allocator, payload.decl);
             },
 
             .undef,
@@ -1321,13 +1318,13 @@ pub const Value = extern union {
             },
             .int_type => {
                 const payload = self.cast(Payload.IntType).?;
-                if (payload.signed) {
-                    var new = Type.Payload.IntSigned{ .bits = payload.bits };
-                    return Type.initPayload(&new.base).hash();
-                } else {
-                    var new = Type.Payload.IntUnsigned{ .bits = payload.bits };
-                    return Type.initPayload(&new.base).hash();
-                }
+                var int_payload = Type.Payload.Bits{
+                    .base = .{
+                        .tag = if (payload.signed) .int_signed else .int_unsigned,
+                    },
+                    .data = payload.bits,
+                };
+                return Type.initPayload(&int_payload.base).hash();
             },
 
             .empty_struct_value,
diff --git a/src/zir.zig b/src/zir.zig
index 47e1abc24b..bd9ab2c538 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -2785,7 +2785,7 @@ const EmitZIR = struct {
                     }
                 },
                 .Optional => {
-                    var buf: Type.Payload.PointerSimple = undefined;
+                    var buf: Type.Payload.ElemType = undefined;
                     const inst = try self.arena.allocator.create(Inst.UnOp);
                     inst.* = .{
                         .base = .{
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 3a9f511373..8960f3ba4d 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -480,14 +480,11 @@ fn analyzeInstStr(mod: *Module, scope: *Scope, str_inst: *zir.Inst.Str) InnerErr
     errdefer new_decl_arena.deinit();
     const arena_bytes = try new_decl_arena.allocator.dupe(u8, str_inst.positionals.bytes);
 
-    const ty_payload = try scope.arena().create(Type.Payload.Array_u8_Sentinel0);
-    ty_payload.* = .{ .len = arena_bytes.len };
-
     const bytes_payload = try scope.arena().create(Value.Payload.Bytes);
     bytes_payload.* = .{ .data = arena_bytes };
 
     const new_decl = try mod.createAnonymousDecl(scope, &new_decl_arena, .{
-        .ty = Type.initPayload(&ty_payload.base),
+        .ty = try Type.Tag.array_u8_sentinel_0.create(scope.arena(), arena_bytes.len),
         .val = Value.initPayload(&bytes_payload.base),
     });
     return mod.analyzeDeclRef(scope, str_inst.base.src, new_decl);
@@ -952,13 +949,12 @@ fn analyzeInstFnType(mod: *Module, scope: *Scope, fntype: *zir.Inst.FnType) Inne
         param_types[i] = resolved;
     }
 
-    const payload = try arena.create(Type.Payload.Function);
-    payload.* = .{
+    const fn_ty = try Type.Tag.function.create(arena, .{
         .cc = fntype.kw_args.cc,
         .return_type = return_type,
         .param_types = param_types,
-    };
-    return mod.constType(scope, fntype.base.src, Type.initPayload(&payload.base));
+    });
+    return mod.constType(scope, fntype.base.src, fn_ty);
 }
 
 fn analyzeInstPrimitive(mod: *Module, scope: *Scope, primitive: *zir.Inst.Primitive) InnerError!*Inst {
@@ -1062,11 +1058,10 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                     const ref_payload = try scope.arena().create(Value.Payload.RefVal);
                     ref_payload.* = .{ .val = Value.initPayload(&error_payload.base) };
 
-                    const result_type = if (child_type.tag() == .anyerror) blk: {
-                        const result_payload = try scope.arena().create(Type.Payload.ErrorSetSingle);
-                        result_payload.* = .{ .name = entry.key };
-                        break :blk Type.initPayload(&result_payload.base);
-                    } else child_type;
+                    const result_type = if (child_type.tag() == .anyerror)
+                        try Type.Tag.error_set_single.create(scope.arena(), entry.key)
+                    else
+                        child_type;
 
                     return mod.constInst(scope, fieldptr.base.src, .{
                         .ty = try mod.simplePtrType(scope, fieldptr.base.src, result_type, false, .One),
@@ -1195,15 +1190,10 @@ fn analyzeInstElemPtr(mod: *Module, scope: *Scope, inst: *zir.Inst.ElemPtr) Inne
                 // @intCast here because it would have been impossible to construct a value that
                 // required a larger index.
                 const elem_ptr = try array_ptr_val.elemPtr(scope.arena(), @intCast(usize, index_u64));
-
-                const type_payload = try scope.arena().create(Type.Payload.PointerSimple);
-                type_payload.* = .{
-                    .base = .{ .tag = .single_const_pointer },
-                    .pointee_type = elem_ty.elemType().elemType(),
-                };
+                const pointee_type = elem_ty.elemType().elemType();
 
                 return mod.constInst(scope, inst.base.src, .{
-                    .ty = Type.initPayload(&type_payload.base),
+                    .ty = try Type.Tag.single_const_pointer.create(scope.arena(), pointee_type),
                     .val = elem_ptr,
                 });
             }
-- 
cgit v1.2.3


From 3f7d9b5fc19e4081236b3b63aebbc80e1b17f5b5 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 30 Dec 2020 22:31:56 -0700
Subject: stage2: rework Value Payload layout

This is the same as the previous commit but for Value instead of Type.

Add `Value.castTag` and note that it is preferable to call than
`Value.cast`. This matches other abstractions in the codebase.

Added a convenience function `Value.Tag.create` which really cleans up
the callsites of creating `Value` objects.

`Value` tags can now share payload types. This is in preparation for
another improvement that I want to do.
---
 src/Compilation.zig  |  11 +-
 src/Module.zig       | 193 +++++++----------
 src/astgen.zig       |  26 +--
 src/codegen.zig      |  52 ++---
 src/codegen/c.zig    |  36 ++--
 src/codegen/wasm.zig |   6 +-
 src/link/Elf.zig     |   2 +-
 src/llvm_backend.zig |   8 +-
 src/type.zig         |  43 +---
 src/value.zig        | 599 +++++++++++++++++++++++++++++++--------------------
 src/zir.zig          |  34 +--
 src/zir_sema.zig     |  78 +++----
 12 files changed, 573 insertions(+), 515 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 11c8303fac..cd3db84ec2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1457,11 +1457,12 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
 
             .complete, .codegen_failure_retryable => {
                 const module = self.bin_file.options.module.?;
-                if (decl.typed_value.most_recent.typed_value.val.cast(Value.Payload.Function)) |payload| {
-                    switch (payload.func.analysis) {
-                        .queued => module.analyzeFnBody(decl, payload.func) catch |err| switch (err) {
+                if (decl.typed_value.most_recent.typed_value.val.castTag(.function)) |payload| {
+                    const func = payload.data;
+                    switch (func.analysis) {
+                        .queued => module.analyzeFnBody(decl, func) catch |err| switch (err) {
                             error.AnalysisFail => {
-                                assert(payload.func.analysis != .in_progress);
+                                assert(func.analysis != .in_progress);
                                 continue;
                             },
                             error.OutOfMemory => return error.OutOfMemory,
@@ -1475,7 +1476,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa);
                     defer decl.typed_value.most_recent.arena.?.* = decl_arena.state;
                     log.debug("analyze liveness of {}\n", .{decl.name});
-                    try liveness.analyze(module.gpa, &decl_arena.allocator, payload.func.analysis.success);
+                    try liveness.analyze(module.gpa, &decl_arena.allocator, func.analysis.success);
                 }
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
diff --git a/src/Module.zig b/src/Module.zig
index 3e937fe49b..ca0718c3d5 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -1092,16 +1092,12 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
 
                     tvm.deinit(self.gpa);
                 }
-                const value_payload = try decl_arena.allocator.create(Value.Payload.ExternFn);
-                value_payload.* = .{ .decl = decl };
+                const fn_val = try Value.Tag.extern_fn.create(&decl_arena.allocator, decl);
 
                 decl_arena_state.* = decl_arena.state;
                 decl.typed_value = .{
                     .most_recent = .{
-                        .typed_value = .{
-                            .ty = fn_type,
-                            .val = Value.initPayload(&value_payload.base),
-                        },
+                        .typed_value = .{ .ty = fn_type, .val = fn_val },
                         .arena = decl_arena_state,
                     },
                 };
@@ -1187,7 +1183,10 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .analysis = .{ .queued = fn_zir },
                 .owner_decl = decl,
             };
-            fn_payload.* = .{ .func = new_func };
+            fn_payload.* = .{
+                .base = .{ .tag = .function },
+                .data = new_func,
+            };
 
             var prev_type_has_bits = false;
             var type_changed = true;
@@ -1375,7 +1374,6 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             }
 
             const new_variable = try decl_arena.allocator.create(Var);
-            const var_payload = try decl_arena.allocator.create(Value.Payload.Variable);
             new_variable.* = .{
                 .owner_decl = decl,
                 .init = var_info.val orelse undefined,
@@ -1383,14 +1381,14 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .is_mutable = is_mutable,
                 .is_threadlocal = is_threadlocal,
             };
-            var_payload.* = .{ .variable = new_variable };
+            const var_val = try Value.Tag.variable.create(&decl_arena.allocator, new_variable);
 
             decl_arena_state.* = decl_arena.state;
             decl.typed_value = .{
                 .most_recent = .{
                     .typed_value = .{
                         .ty = var_info.ty,
-                        .val = Value.initPayload(&var_payload.base),
+                        .val = var_val,
                     },
                     .arena = decl_arena_state,
                 },
@@ -2232,52 +2230,43 @@ pub fn constBool(self: *Module, scope: *Scope, src: usize, v: bool) !*Inst {
 }
 
 pub fn constIntUnsigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: u64) !*Inst {
-    const int_payload = try scope.arena().create(Value.Payload.Int_u64);
-    int_payload.* = .{ .int = int };
-
     return self.constInst(scope, src, .{
         .ty = ty,
-        .val = Value.initPayload(&int_payload.base),
+        .val = try Value.Tag.int_u64.create(scope.arena(), int),
     });
 }
 
 pub fn constIntSigned(self: *Module, scope: *Scope, src: usize, ty: Type, int: i64) !*Inst {
-    const int_payload = try scope.arena().create(Value.Payload.Int_i64);
-    int_payload.* = .{ .int = int };
-
     return self.constInst(scope, src, .{
         .ty = ty,
-        .val = Value.initPayload(&int_payload.base),
+        .val = try Value.Tag.int_i64.create(scope.arena(), int),
     });
 }
 
 pub fn constIntBig(self: *Module, scope: *Scope, src: usize, ty: Type, big_int: BigIntConst) !*Inst {
-    const val_payload = if (big_int.positive) blk: {
+    if (big_int.positive) {
         if (big_int.to(u64)) |x| {
             return self.constIntUnsigned(scope, src, ty, x);
         } else |err| switch (err) {
             error.NegativeIntoUnsigned => unreachable,
             error.TargetTooSmall => {}, // handled below
         }
-        const big_int_payload = try scope.arena().create(Value.Payload.IntBigPositive);
-        big_int_payload.* = .{ .limbs = big_int.limbs };
-        break :blk &big_int_payload.base;
-    } else blk: {
+        return self.constInst(scope, src, .{
+            .ty = ty,
+            .val = try Value.Tag.int_big_positive.create(scope.arena(), big_int.limbs),
+        });
+    } else {
         if (big_int.to(i64)) |x| {
             return self.constIntSigned(scope, src, ty, x);
         } else |err| switch (err) {
             error.NegativeIntoUnsigned => unreachable,
             error.TargetTooSmall => {}, // handled below
         }
-        const big_int_payload = try scope.arena().create(Value.Payload.IntBigNegative);
-        big_int_payload.* = .{ .limbs = big_int.limbs };
-        break :blk &big_int_payload.base;
-    };
-
-    return self.constInst(scope, src, .{
-        .ty = ty,
-        .val = Value.initPayload(val_payload),
-    });
+        return self.constInst(scope, src, .{
+            .ty = ty,
+            .val = try Value.Tag.int_big_negative.create(scope.arena(), big_int.limbs),
+        });
+    }
 }
 
 pub fn createAnonymousDecl(
@@ -2346,26 +2335,20 @@ pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) Inn
     if (decl_tv.val.tag() == .variable) {
         return self.analyzeVarRef(scope, src, decl_tv);
     }
-    const ty = try self.simplePtrType(scope, src, decl_tv.ty, false, .One);
-    const val_payload = try scope.arena().create(Value.Payload.DeclRef);
-    val_payload.* = .{ .decl = decl };
-
     return self.constInst(scope, src, .{
-        .ty = ty,
-        .val = Value.initPayload(&val_payload.base),
+        .ty = try self.simplePtrType(scope, src, decl_tv.ty, false, .One),
+        .val = try Value.Tag.decl_ref.create(scope.arena(), decl),
     });
 }
 
 fn analyzeVarRef(self: *Module, scope: *Scope, src: usize, tv: TypedValue) InnerError!*Inst {
-    const variable = tv.val.cast(Value.Payload.Variable).?.variable;
+    const variable = tv.val.castTag(.variable).?.data;
 
     const ty = try self.simplePtrType(scope, src, tv.ty, variable.is_mutable, .One);
     if (!variable.is_mutable and !variable.is_extern) {
-        const val_payload = try scope.arena().create(Value.Payload.RefVal);
-        val_payload.* = .{ .val = variable.init };
         return self.constInst(scope, src, .{
             .ty = ty,
-            .val = Value.initPayload(&val_payload.base),
+            .val = try Value.Tag.ref_val.create(scope.arena(), variable.init),
         });
     }
 
@@ -3107,17 +3090,11 @@ pub fn intAdd(allocator: *Allocator, lhs: Value, rhs: Value) !Value {
     result_bigint.add(lhs_bigint, rhs_bigint);
     const result_limbs = result_bigint.limbs[0..result_bigint.len];
 
-    const val_payload = if (result_bigint.positive) blk: {
-        const val_payload = try allocator.create(Value.Payload.IntBigPositive);
-        val_payload.* = .{ .limbs = result_limbs };
-        break :blk &val_payload.base;
-    } else blk: {
-        const val_payload = try allocator.create(Value.Payload.IntBigNegative);
-        val_payload.* = .{ .limbs = result_limbs };
-        break :blk &val_payload.base;
-    };
-
-    return Value.initPayload(val_payload);
+    if (result_bigint.positive) {
+        return Value.Tag.int_big_positive.create(allocator, result_limbs);
+    } else {
+        return Value.Tag.int_big_negative.create(allocator, result_limbs);
+    }
 }
 
 pub fn intSub(allocator: *Allocator, lhs: Value, rhs: Value) !Value {
@@ -3135,85 +3112,81 @@ pub fn intSub(allocator: *Allocator, lhs: Value, rhs: Value) !Value {
     result_bigint.sub(lhs_bigint, rhs_bigint);
     const result_limbs = result_bigint.limbs[0..result_bigint.len];
 
-    const val_payload = if (result_bigint.positive) blk: {
-        const val_payload = try allocator.create(Value.Payload.IntBigPositive);
-        val_payload.* = .{ .limbs = result_limbs };
-        break :blk &val_payload.base;
-    } else blk: {
-        const val_payload = try allocator.create(Value.Payload.IntBigNegative);
-        val_payload.* = .{ .limbs = result_limbs };
-        break :blk &val_payload.base;
-    };
-
-    return Value.initPayload(val_payload);
+    if (result_bigint.positive) {
+        return Value.Tag.int_big_positive.create(allocator, result_limbs);
+    } else {
+        return Value.Tag.int_big_negative.create(allocator, result_limbs);
+    }
 }
 
-pub fn floatAdd(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value {
-    var bit_count = switch (float_type.tag()) {
-        .comptime_float => 128,
-        else => float_type.floatBits(self.getTarget()),
-    };
-
-    const allocator = scope.arena();
-    const val_payload = switch (bit_count) {
-        16 => {
-            return self.fail(scope, src, "TODO Implement addition for soft floats", .{});
+pub fn floatAdd(
+    self: *Module,
+    scope: *Scope,
+    float_type: Type,
+    src: usize,
+    lhs: Value,
+    rhs: Value,
+) !Value {
+    const arena = scope.arena();
+    switch (float_type.tag()) {
+        .f16 => {
+            @panic("TODO add __trunctfhf2 to compiler-rt");
+            //const lhs_val = lhs.toFloat(f16);
+            //const rhs_val = rhs.toFloat(f16);
+            //return Value.Tag.float_16.create(arena, lhs_val + rhs_val);
         },
-        32 => blk: {
+        .f32 => {
             const lhs_val = lhs.toFloat(f32);
             const rhs_val = rhs.toFloat(f32);
-            const val_payload = try allocator.create(Value.Payload.Float_32);
-            val_payload.* = .{ .val = lhs_val + rhs_val };
-            break :blk &val_payload.base;
+            return Value.Tag.float_32.create(arena, lhs_val + rhs_val);
         },
-        64 => blk: {
+        .f64 => {
             const lhs_val = lhs.toFloat(f64);
             const rhs_val = rhs.toFloat(f64);
-            const val_payload = try allocator.create(Value.Payload.Float_64);
-            val_payload.* = .{ .val = lhs_val + rhs_val };
-            break :blk &val_payload.base;
+            return Value.Tag.float_64.create(arena, lhs_val + rhs_val);
         },
-        128 => {
-            return self.fail(scope, src, "TODO Implement addition for big floats", .{});
+        .f128, .comptime_float, .c_longdouble => {
+            const lhs_val = lhs.toFloat(f128);
+            const rhs_val = rhs.toFloat(f128);
+            return Value.Tag.float_128.create(arena, lhs_val + rhs_val);
         },
         else => unreachable,
-    };
-
-    return Value.initPayload(val_payload);
+    }
 }
 
-pub fn floatSub(self: *Module, scope: *Scope, float_type: Type, src: usize, lhs: Value, rhs: Value) !Value {
-    var bit_count = switch (float_type.tag()) {
-        .comptime_float => 128,
-        else => float_type.floatBits(self.getTarget()),
-    };
-
-    const allocator = scope.arena();
-    const val_payload = switch (bit_count) {
-        16 => {
-            return self.fail(scope, src, "TODO Implement substraction for soft floats", .{});
+pub fn floatSub(
+    self: *Module,
+    scope: *Scope,
+    float_type: Type,
+    src: usize,
+    lhs: Value,
+    rhs: Value,
+) !Value {
+    const arena = scope.arena();
+    switch (float_type.tag()) {
+        .f16 => {
+            @panic("TODO add __trunctfhf2 to compiler-rt");
+            //const lhs_val = lhs.toFloat(f16);
+            //const rhs_val = rhs.toFloat(f16);
+            //return Value.Tag.float_16.create(arena, lhs_val - rhs_val);
         },
-        32 => blk: {
+        .f32 => {
             const lhs_val = lhs.toFloat(f32);
             const rhs_val = rhs.toFloat(f32);
-            const val_payload = try allocator.create(Value.Payload.Float_32);
-            val_payload.* = .{ .val = lhs_val - rhs_val };
-            break :blk &val_payload.base;
+            return Value.Tag.float_32.create(arena, lhs_val - rhs_val);
         },
-        64 => blk: {
+        .f64 => {
             const lhs_val = lhs.toFloat(f64);
             const rhs_val = rhs.toFloat(f64);
-            const val_payload = try allocator.create(Value.Payload.Float_64);
-            val_payload.* = .{ .val = lhs_val - rhs_val };
-            break :blk &val_payload.base;
+            return Value.Tag.float_64.create(arena, lhs_val - rhs_val);
         },
-        128 => {
-            return self.fail(scope, src, "TODO Implement substraction for big floats", .{});
+        .f128, .comptime_float, .c_longdouble => {
+            const lhs_val = lhs.toFloat(f128);
+            const rhs_val = rhs.toFloat(f128);
+            return Value.Tag.float_128.create(arena, lhs_val - rhs_val);
         },
         else => unreachable,
-    };
-
-    return Value.initPayload(val_payload);
+    }
 }
 
 pub fn simplePtrType(self: *Module, scope: *Scope, src: usize, elem_ty: Type, mutable: bool, size: std.builtin.TypeInfo.Pointer.Size) Allocator.Error!Type {
diff --git a/src/astgen.zig b/src/astgen.zig
index 1fc8a0d19e..c5261c4073 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -1956,13 +1956,13 @@ fn identifier(mod: *Module, scope: *Scope, rl: ResultLoc, ident: *ast.Node.OneTo
                 32 => if (is_signed) Value.initTag(.i32_type) else Value.initTag(.u32_type),
                 64 => if (is_signed) Value.initTag(.i64_type) else Value.initTag(.u64_type),
                 else => {
-                    const int_type_payload = try scope.arena().create(Value.Payload.IntType);
-                    int_type_payload.* = .{ .signed = is_signed, .bits = bit_count };
-                    const result = try addZIRInstConst(mod, scope, src, .{
+                    return rlWrap(mod, scope, rl, try addZIRInstConst(mod, scope, src, .{
                         .ty = Type.initTag(.type),
-                        .val = Value.initPayload(&int_type_payload.base),
-                    });
-                    return rlWrap(mod, scope, rl, result);
+                        .val = try Value.Tag.int_type.create(scope.arena(), .{
+                            .signed = is_signed,
+                            .bits = bit_count,
+                        }),
+                    }));
                 },
             };
             const result = try addZIRInstConst(mod, scope, src, .{
@@ -2062,11 +2062,9 @@ fn charLiteral(mod: *Module, scope: *Scope, node: *ast.Node.OneToken) !*zir.Inst
         },
     };
 
-    const int_payload = try scope.arena().create(Value.Payload.Int_u64);
-    int_payload.* = .{ .int = value };
     return addZIRInstConst(mod, scope, src, .{
         .ty = Type.initTag(.comptime_int),
-        .val = Value.initPayload(&int_payload.base),
+        .val = try Value.Tag.int_u64.create(scope.arena(), value),
     });
 }
 
@@ -2089,12 +2087,10 @@ fn integerLiteral(mod: *Module, scope: *Scope, int_lit: *ast.Node.OneToken) Inne
         prefixed_bytes[2..];
 
     if (std.fmt.parseInt(u64, bytes, base)) |small_int| {
-        const int_payload = try arena.create(Value.Payload.Int_u64);
-        int_payload.* = .{ .int = small_int };
         const src = tree.token_locs[int_lit.token].start;
         return addZIRInstConst(mod, scope, src, .{
             .ty = Type.initTag(.comptime_int),
-            .val = Value.initPayload(&int_payload.base),
+            .val = try Value.Tag.int_u64.create(arena, small_int),
         });
     } else |err| {
         return mod.failTok(scope, int_lit.token, "TODO implement int literals that don't fit in a u64", .{});
@@ -2109,15 +2105,13 @@ fn floatLiteral(mod: *Module, scope: *Scope, float_lit: *ast.Node.OneToken) Inne
         return mod.failTok(scope, float_lit.token, "TODO hex floats", .{});
     }
 
-    const val = std.fmt.parseFloat(f128, bytes) catch |e| switch (e) {
+    const float_number = std.fmt.parseFloat(f128, bytes) catch |e| switch (e) {
         error.InvalidCharacter => unreachable, // validated by tokenizer
     };
-    const float_payload = try arena.create(Value.Payload.Float_128);
-    float_payload.* = .{ .val = val };
     const src = tree.token_locs[float_lit.token].start;
     return addZIRInstConst(mod, scope, src, .{
         .ty = Type.initTag(.comptime_float),
-        .val = Value.initPayload(&float_payload.base),
+        .val = try Value.Tag.float_128.create(arena, float_number),
     });
 }
 
diff --git a/src/codegen.zig b/src/codegen.zig
index d98a87a440..f978115ebc 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -137,7 +137,7 @@ pub fn generateSymbol(
         },
         .Array => {
             // TODO populate .debug_info for the array
-            if (typed_value.val.cast(Value.Payload.Bytes)) |payload| {
+            if (typed_value.val.castTag(.bytes)) |payload| {
                 if (typed_value.ty.sentinel()) |sentinel| {
                     try code.ensureCapacity(code.items.len + payload.data.len + 1);
                     code.appendSliceAssumeCapacity(payload.data);
@@ -168,8 +168,8 @@ pub fn generateSymbol(
         },
         .Pointer => {
             // TODO populate .debug_info for the pointer
-            if (typed_value.val.cast(Value.Payload.DeclRef)) |payload| {
-                const decl = payload.decl;
+            if (typed_value.val.castTag(.decl_ref)) |payload| {
+                const decl = payload.data;
                 if (decl.analysis != .complete) return error.AnalysisFail;
                 // TODO handle the dependency of this symbol on the decl's vaddr.
                 // If the decl changes vaddr, then this symbol needs to get regenerated.
@@ -432,7 +432,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 @panic("Attempted to compile for architecture that was disabled by build configuration");
             }
 
-            const module_fn = typed_value.val.cast(Value.Payload.Function).?.func;
+            const module_fn = typed_value.val.castTag(.function).?.data;
 
             const fn_type = module_fn.owner_decl.typed_value.most_recent.typed_value.ty;
 
@@ -1579,9 +1579,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                             }
                         }
 
-                        if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
-                            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                                const func = func_val.func;
+                        if (inst.func.value()) |func_value| {
+                            if (func_value.castTag(.function)) |func_payload| {
+                                const func = func_payload.data;
 
                                 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
                                 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
@@ -1607,9 +1607,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                     .riscv64 => {
                         if (info.args.len > 0) return self.fail(inst.base.src, "TODO implement fn args for {}", .{self.target.cpu.arch});
 
-                        if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
-                            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                                const func = func_val.func;
+                        if (inst.func.value()) |func_value| {
+                            if (func_value.castTag(.function)) |func_payload| {
+                                const func = func_payload.data;
 
                                 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
                                 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
@@ -1631,12 +1631,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         }
                     },
                     .spu_2 => {
-                        if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
+                        if (inst.func.value()) |func_value| {
                             if (info.args.len != 0) {
                                 return self.fail(inst.base.src, "TODO implement call with more than 0 parameters", .{});
                             }
-                            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                                const func = func_val.func;
+                            if (func_value.castTag(.function)) |func_payload| {
+                                const func = func_payload.data;
                                 const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: {
                                     const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
                                     break :blk @intCast(u16, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * 2);
@@ -1705,9 +1705,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                             }
                         }
 
-                        if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
-                            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                                const func = func_val.func;
+                        if (inst.func.value()) |func_value| {
+                            if (func_value.castTag(.function)) |func_payload| {
+                                const func = func_payload.data;
                                 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
                                 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
                                 const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: {
@@ -1766,9 +1766,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                             }
                         }
 
-                        if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
-                            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                                const func = func_val.func;
+                        if (inst.func.value()) |func_value| {
+                            if (func_value.castTag(.function)) |func_payload| {
+                                const func = func_payload.data;
                                 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
                                 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
                                 const got_addr = if (self.bin_file.cast(link.File.Elf)) |elf_file| blk: {
@@ -1825,9 +1825,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                     }
                 }
 
-                if (inst.func.cast(ir.Inst.Constant)) |func_inst| {
-                    if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                        const func = func_val.func;
+                if (inst.func.value()) |func_value| {
+                    if (func_value.castTag(.function)) |func_payload| {
+                        const func = func_payload.data;
                         const text_segment = &macho_file.load_commands.items[macho_file.text_segment_cmd_index.?].Segment;
                         const got = &text_segment.sections.items[macho_file.got_section_index.?];
                         const got_addr = got.addr + func.owner_decl.link.macho.offset_table_index * @sizeOf(u64);
@@ -3223,20 +3223,20 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
             const ptr_bytes: u64 = @divExact(ptr_bits, 8);
             switch (typed_value.ty.zigTypeTag()) {
                 .Pointer => {
-                    if (typed_value.val.cast(Value.Payload.DeclRef)) |payload| {
+                    if (typed_value.val.castTag(.decl_ref)) |payload| {
                         if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-                            const decl = payload.decl;
+                            const decl = payload.data;
                             const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
                             const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
                             return MCValue{ .memory = got_addr };
                         } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-                            const decl = payload.decl;
+                            const decl = payload.data;
                             const text_segment = &macho_file.load_commands.items[macho_file.text_segment_cmd_index.?].Segment;
                             const got = &text_segment.sections.items[macho_file.got_section_index.?];
                             const got_addr = got.addr + decl.link.macho.offset_table_index * ptr_bytes;
                             return MCValue{ .memory = got_addr };
                         } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-                            const decl = payload.decl;
+                            const decl = payload.data;
                             const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
                             return MCValue{ .memory = got_addr };
                         } else {
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 828061fac6..7cd4479bd9 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -138,25 +138,25 @@ fn renderValue(
             .undef, .zero => try writer.writeAll("0"),
             .one => try writer.writeAll("1"),
             .decl_ref => {
-                const decl_ref_payload = val.cast(Value.Payload.DeclRef).?;
+                const decl = val.castTag(.decl_ref).?.data;
 
                 // Determine if we must pointer cast.
-                const decl_tv = decl_ref_payload.decl.typed_value.most_recent.typed_value;
+                const decl_tv = decl.typed_value.most_recent.typed_value;
                 if (t.eql(decl_tv.ty)) {
-                    try writer.print("&{s}", .{decl_ref_payload.decl.name});
+                    try writer.print("&{s}", .{decl.name});
                 } else {
                     try writer.writeAll("(");
                     try renderType(ctx, writer, t);
-                    try writer.print(")&{s}", .{decl_ref_payload.decl.name});
+                    try writer.print(")&{s}", .{decl.name});
                 }
             },
             .function => {
-                const payload = val.cast(Value.Payload.Function).?;
-                try writer.print("{s}", .{payload.func.owner_decl.name});
+                const func = val.castTag(.function).?.data;
+                try writer.print("{s}", .{func.owner_decl.name});
             },
             .extern_fn => {
-                const payload = val.cast(Value.Payload.ExternFn).?;
-                try writer.print("{s}", .{payload.decl.name});
+                const decl = val.castTag(.extern_fn).?.data;
+                try writer.print("{s}", .{decl.name});
             },
             else => |e| return ctx.fail(
                 ctx.decl.src(),
@@ -169,7 +169,7 @@ fn renderValue(
             switch (val.tag()) {
                 .undef, .empty_struct_value, .empty_array => try writer.writeAll("{}"),
                 .bytes => {
-                    const bytes = val.cast(Value.Payload.Bytes).?.data;
+                    const bytes = val.castTag(.bytes).?.data;
                     // TODO: make our own C string escape instead of using {Z}
                     try writer.print("\"{Z}\"", .{bytes});
                 },
@@ -209,7 +209,7 @@ fn renderFunctionSignature(
         switch (tv.val.tag()) {
             .extern_fn => break :blk true,
             .function => {
-                const func = tv.val.cast(Value.Payload.Function).?.func;
+                const func = tv.val.castTag(.function).?.data;
                 break :blk ctx.module.decl_exports.contains(func.owner_decl);
             },
             else => unreachable,
@@ -268,13 +268,13 @@ pub fn generate(file: *C, module: *Module, decl: *Decl) !void {
         ctx.deinit();
     }
 
-    if (tv.val.cast(Value.Payload.Function)) |func_payload| {
+    if (tv.val.castTag(.function)) |func_payload| {
         const writer = file.main.writer();
         try renderFunctionSignature(&ctx, writer, decl);
 
         try writer.writeAll(" {");
 
-        const func: *Module.Fn = func_payload.func;
+        const func: *Module.Fn = func_payload.data;
         const instructions = func.analysis.success.instructions;
         if (instructions.len > 0) {
             try writer.writeAll("\n");
@@ -480,10 +480,10 @@ fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
     const writer = file.main.writer();
     const header = file.header.buf.writer();
     if (inst.func.castTag(.constant)) |func_inst| {
-        const fn_decl = if (func_inst.val.cast(Value.Payload.ExternFn)) |extern_fn|
-            extern_fn.decl
-        else if (func_inst.val.cast(Value.Payload.Function)) |func_val|
-            func_val.func.owner_decl
+        const fn_decl = if (func_inst.val.castTag(.extern_fn)) |extern_fn|
+            extern_fn.data
+        else if (func_inst.val.castTag(.function)) |func_payload|
+            func_payload.data.owner_decl
         else
             unreachable;
 
@@ -513,8 +513,8 @@ fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
                 if (i > 0) {
                     try writer.writeAll(", ");
                 }
-                if (arg.cast(Inst.Constant)) |con| {
-                    try renderValue(ctx, writer, arg.ty, con.val);
+                if (arg.value()) |val| {
+                    try renderValue(ctx, writer, arg.ty, val);
                 } else {
                     const val = try ctx.resolveInst(arg);
                     try writer.print("{}", .{val});
diff --git a/src/codegen/wasm.zig b/src/codegen/wasm.zig
index 7297ea1d54..c7ad59f5d1 100644
--- a/src/codegen/wasm.zig
+++ b/src/codegen/wasm.zig
@@ -62,7 +62,7 @@ pub fn genCode(buf: *ArrayList(u8), decl: *Decl) !void {
     // Write instructions
     // TODO: check for and handle death of instructions
     const tv = decl.typed_value.most_recent.typed_value;
-    const mod_fn = tv.val.cast(Value.Payload.Function).?.func;
+    const mod_fn = tv.val.castTag(.function).?.data;
     for (mod_fn.analysis.success.instructions) |inst| try genInst(buf, decl, inst);
 
     // Write 'end' opcode
@@ -125,8 +125,8 @@ fn genRet(buf: *ArrayList(u8), decl: *Decl, inst: *Inst.UnOp) !void {
 
 fn genCall(buf: *ArrayList(u8), decl: *Decl, inst: *Inst.Call) !void {
     const func_inst = inst.func.castTag(.constant).?;
-    const func_val = func_inst.val.cast(Value.Payload.Function).?;
-    const target = func_val.func.owner_decl;
+    const func = func_inst.val.castTag(.function).?.data;
+    const target = func.owner_decl;
     const target_ty = target.typed_value.most_recent.typed_value.ty;
 
     if (inst.args.len != 0) return error.TODOImplementMoreWasmCodegen;
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 25b883f8c6..4b2b95fc72 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2183,7 +2183,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
             for (zir_dumps) |fn_name| {
                 if (mem.eql(u8, mem.spanZ(decl.name), fn_name)) {
                     std.debug.print("\n{}\n", .{decl.name});
-                    typed_value.val.cast(Value.Payload.Function).?.func.dump(module.*);
+                    typed_value.val.castTag(.function).?.data.dump(module.*);
                 }
             }
         }
diff --git a/src/llvm_backend.zig b/src/llvm_backend.zig
index 77aa4d3bd5..294e6f5400 100644
--- a/src/llvm_backend.zig
+++ b/src/llvm_backend.zig
@@ -280,7 +280,7 @@ pub const LLVMIRModule = struct {
     fn gen(self: *LLVMIRModule, module: *Module, typed_value: TypedValue, src: usize) !void {
         switch (typed_value.ty.zigTypeTag()) {
             .Fn => {
-                const func = typed_value.val.cast(Value.Payload.Function).?.func;
+                const func = typed_value.val.castTag(.function).?.data;
 
                 const llvm_func = try self.resolveLLVMFunction(func);
 
@@ -314,9 +314,9 @@ pub const LLVMIRModule = struct {
     }
 
     fn genCall(self: *LLVMIRModule, inst: *Inst.Call) !void {
-        if (inst.func.cast(Inst.Constant)) |func_inst| {
-            if (func_inst.val.cast(Value.Payload.Function)) |func_val| {
-                const func = func_val.func;
+        if (inst.func.value()) |func_value| {
+            if (func_value.castTag(.function)) |func_payload| {
+                const func = func_payload.data;
                 const zig_fn_type = func.owner_decl.typed_value.most_recent.typed_value.ty;
                 const llvm_fn = try self.resolveLLVMFunction(func);
 
diff --git a/src/type.zig b/src/type.zig
index ce237f89c7..9d834a19f2 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -733,11 +733,7 @@ pub const Type = extern union {
             .single_const_pointer_to_comptime_int => return Value.initTag(.single_const_pointer_to_comptime_int_type),
             .const_slice_u8 => return Value.initTag(.const_slice_u8_type),
             .enum_literal => return Value.initTag(.enum_literal_type),
-            else => {
-                const ty_payload = try allocator.create(Value.Payload.Ty);
-                ty_payload.* = .{ .ty = self };
-                return Value.initPayload(&ty_payload.base);
-            },
+            else => return Value.Tag.ty.create(allocator, self),
         }
     }
 
@@ -2951,11 +2947,8 @@ pub const Type = extern union {
         }
 
         if ((info.bits - 1) <= std.math.maxInt(u6)) {
-            const payload = try arena.allocator.create(Value.Payload.Int_i64);
-            payload.* = .{
-                .int = -(@as(i64, 1) << @truncate(u6, info.bits - 1)),
-            };
-            return Value.initPayload(&payload.base);
+            const n: i64 = -(@as(i64, 1) << @truncate(u6, info.bits - 1));
+            return Value.Tag.int_i64.create(&arena.allocator, n);
         }
 
         var res = try std.math.big.int.Managed.initSet(&arena.allocator, 1);
@@ -2964,13 +2957,9 @@ pub const Type = extern union {
 
         const res_const = res.toConst();
         if (res_const.positive) {
-            const val_payload = try arena.allocator.create(Value.Payload.IntBigPositive);
-            val_payload.* = .{ .limbs = res_const.limbs };
-            return Value.initPayload(&val_payload.base);
+            return Value.Tag.int_big_positive.create(&arena.allocator, res_const.limbs);
         } else {
-            const val_payload = try arena.allocator.create(Value.Payload.IntBigNegative);
-            val_payload.* = .{ .limbs = res_const.limbs };
-            return Value.initPayload(&val_payload.base);
+            return Value.Tag.int_big_negative.create(&arena.allocator, res_const.limbs);
         }
     }
 
@@ -2980,17 +2969,11 @@ pub const Type = extern union {
         const info = self.intInfo(target);
 
         if (info.signedness == .signed and (info.bits - 1) <= std.math.maxInt(u6)) {
-            const payload = try arena.allocator.create(Value.Payload.Int_i64);
-            payload.* = .{
-                .int = (@as(i64, 1) << @truncate(u6, info.bits - 1)) - 1,
-            };
-            return Value.initPayload(&payload.base);
+            const n: i64 = (@as(i64, 1) << @truncate(u6, info.bits - 1)) - 1;
+            return Value.Tag.int_i64.create(&arena.allocator, n);
         } else if (info.signedness == .signed and info.bits <= std.math.maxInt(u6)) {
-            const payload = try arena.allocator.create(Value.Payload.Int_u64);
-            payload.* = .{
-                .int = (@as(u64, 1) << @truncate(u6, info.bits)) - 1,
-            };
-            return Value.initPayload(&payload.base);
+            const n: u64 = (@as(u64, 1) << @truncate(u6, info.bits)) - 1;
+            return Value.Tag.int_u64.create(&arena.allocator, n);
         }
 
         var res = try std.math.big.int.Managed.initSet(&arena.allocator, 1);
@@ -3003,13 +2986,9 @@ pub const Type = extern union {
 
         const res_const = res.toConst();
         if (res_const.positive) {
-            const val_payload = try arena.allocator.create(Value.Payload.IntBigPositive);
-            val_payload.* = .{ .limbs = res_const.limbs };
-            return Value.initPayload(&val_payload.base);
+            return Value.Tag.int_big_positive.create(&arena.allocator, res_const.limbs);
         } else {
-            const val_payload = try arena.allocator.create(Value.Payload.IntBigNegative);
-            val_payload.* = .{ .limbs = res_const.limbs };
-            return Value.initPayload(&val_payload.base);
+            return Value.Tag.int_big_negative.create(&arena.allocator, res_const.limbs);
         }
     }
 
diff --git a/src/value.zig b/src/value.zig
index f26c8d8772..91b21511d4 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -84,11 +84,16 @@ pub const Value = extern union {
         function,
         extern_fn,
         variable,
+        /// Represents a pointer to another immutable value.
         ref_val,
+        /// Represents a pointer to a decl, not the value of the decl.
         decl_ref,
         elem_ptr,
+        /// A slice of u8 whose memory is managed externally.
         bytes,
-        repeated, // the value is a value repeated some number of times
+        /// This value is repeated some number of times. The amount of times to repeat
+        /// is stored externally.
+        repeated,
         float_16,
         float_32,
         float_64,
@@ -99,6 +104,106 @@ pub const Value = extern union {
 
         pub const last_no_payload_tag = Tag.bool_false;
         pub const no_payload_count = @enumToInt(last_no_payload_tag) + 1;
+
+        pub fn Type(comptime t: Tag) type {
+            return switch (t) {
+                .u8_type,
+                .i8_type,
+                .u16_type,
+                .i16_type,
+                .u32_type,
+                .i32_type,
+                .u64_type,
+                .i64_type,
+                .usize_type,
+                .isize_type,
+                .c_short_type,
+                .c_ushort_type,
+                .c_int_type,
+                .c_uint_type,
+                .c_long_type,
+                .c_ulong_type,
+                .c_longlong_type,
+                .c_ulonglong_type,
+                .c_longdouble_type,
+                .f16_type,
+                .f32_type,
+                .f64_type,
+                .f128_type,
+                .c_void_type,
+                .bool_type,
+                .void_type,
+                .type_type,
+                .anyerror_type,
+                .comptime_int_type,
+                .comptime_float_type,
+                .noreturn_type,
+                .null_type,
+                .undefined_type,
+                .fn_noreturn_no_args_type,
+                .fn_void_no_args_type,
+                .fn_naked_noreturn_no_args_type,
+                .fn_ccc_void_no_args_type,
+                .single_const_pointer_to_comptime_int_type,
+                .const_slice_u8_type,
+                .enum_literal_type,
+                .anyframe_type,
+                .undef,
+                .zero,
+                .one,
+                .void_value,
+                .unreachable_value,
+                .empty_struct_value,
+                .empty_array,
+                .null_value,
+                .bool_true,
+                .bool_false,
+                => @compileError("Value Tag " ++ @tagName(t) ++ " has no payload"),
+
+                .int_big_positive,
+                .int_big_negative,
+                => Payload.BigInt,
+
+                .extern_fn,
+                .decl_ref,
+                => Payload.Decl,
+
+                .ref_val,
+                .repeated,
+                => Payload.SubValue,
+
+                .bytes,
+                .enum_literal,
+                => Payload.Bytes,
+
+                .ty => Payload.Ty,
+                .int_type => Payload.IntType,
+                .int_u64 => Payload.U64,
+                .int_i64 => Payload.I64,
+                .function => Payload.Function,
+                .variable => Payload.Variable,
+                .elem_ptr => Payload.ElemPtr,
+                .float_16 => Payload.Float_16,
+                .float_32 => Payload.Float_32,
+                .float_64 => Payload.Float_64,
+                .float_128 => Payload.Float_128,
+                .error_set => Payload.ErrorSet,
+                .@"error" => Payload.Error,
+            };
+        }
+
+        pub fn create(comptime t: Tag, ally: *Allocator, data: Data(t)) error{OutOfMemory}!Value {
+            const ptr = try ally.create(t.Type());
+            ptr.* = .{
+                .base = .{ .tag = t },
+                .data = data,
+            };
+            return Value{ .ptr_otherwise = &ptr.base };
+        }
+
+        pub fn Data(comptime t: Tag) type {
+            return std.meta.fieldInfo(t.Type(), "data").field_type;
+        }
     };
 
     pub fn initTag(small_tag: Tag) Value {
@@ -119,15 +224,36 @@ pub const Value = extern union {
         }
     }
 
+    /// Prefer `castTag` to this.
     pub fn cast(self: Value, comptime T: type) ?*T {
-        if (self.tag_if_small_enough < Tag.no_payload_count)
+        if (@hasField(T, "base_tag")) {
+            return base.castTag(T.base_tag);
+        }
+        if (self.tag_if_small_enough < Tag.no_payload_count) {
             return null;
+        }
+        inline for (@typeInfo(Tag).Enum.fields) |field| {
+            if (field.value < Tag.no_payload_count)
+                continue;
+            const t = @intToEnum(Tag, field.value);
+            if (self.ptr_otherwise.tag == t) {
+                if (T == t.Type()) {
+                    return @fieldParentPtr(T, "base", self.ptr_otherwise);
+                }
+                return null;
+            }
+        }
+        unreachable;
+    }
 
-        const expected_tag = std.meta.fieldInfo(T, "base").default_value.?.tag;
-        if (self.ptr_otherwise.tag != expected_tag)
+    pub fn castTag(self: Value, comptime t: Tag) ?*t.Type() {
+        if (self.tag_if_small_enough < Tag.no_payload_count)
             return null;
 
-        return @fieldParentPtr(T, "base", self.ptr_otherwise);
+        if (self.ptr_otherwise.tag == t)
+            return @fieldParentPtr(t.Type(), "base", self.ptr_otherwise);
+
+        return null;
     }
 
     pub fn copy(self: Value, allocator: *Allocator) error{OutOfMemory}!Value {
@@ -188,17 +314,17 @@ pub const Value = extern union {
             => unreachable,
 
             .ty => {
-                const payload = @fieldParentPtr(Payload.Ty, "base", self.ptr_otherwise);
+                const payload = self.castTag(.ty).?;
                 const new_payload = try allocator.create(Payload.Ty);
                 new_payload.* = .{
                     .base = payload.base,
-                    .ty = try payload.ty.copy(allocator),
+                    .data = try payload.data.copy(allocator),
                 };
                 return Value{ .ptr_otherwise = &new_payload.base };
             },
             .int_type => return self.copyPayloadShallow(allocator, Payload.IntType),
-            .int_u64 => return self.copyPayloadShallow(allocator, Payload.Int_u64),
-            .int_i64 => return self.copyPayloadShallow(allocator, Payload.Int_i64),
+            .int_u64 => return self.copyPayloadShallow(allocator, Payload.U64),
+            .int_i64 => return self.copyPayloadShallow(allocator, Payload.I64),
             .int_big_positive => {
                 @panic("TODO implement copying of big ints");
             },
@@ -206,35 +332,37 @@ pub const Value = extern union {
                 @panic("TODO implement copying of big ints");
             },
             .function => return self.copyPayloadShallow(allocator, Payload.Function),
-            .extern_fn => return self.copyPayloadShallow(allocator, Payload.ExternFn),
+            .extern_fn => return self.copyPayloadShallow(allocator, Payload.Decl),
             .variable => return self.copyPayloadShallow(allocator, Payload.Variable),
             .ref_val => {
-                const payload = @fieldParentPtr(Payload.RefVal, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.RefVal);
+                const payload = self.castTag(.ref_val).?;
+                const new_payload = try allocator.create(Payload.SubValue);
                 new_payload.* = .{
                     .base = payload.base,
-                    .val = try payload.val.copy(allocator),
+                    .data = try payload.data.copy(allocator),
                 };
                 return Value{ .ptr_otherwise = &new_payload.base };
             },
-            .decl_ref => return self.copyPayloadShallow(allocator, Payload.DeclRef),
+            .decl_ref => return self.copyPayloadShallow(allocator, Payload.Decl),
             .elem_ptr => {
-                const payload = @fieldParentPtr(Payload.ElemPtr, "base", self.ptr_otherwise);
+                const payload = self.castTag(.elem_ptr).?;
                 const new_payload = try allocator.create(Payload.ElemPtr);
                 new_payload.* = .{
                     .base = payload.base,
-                    .array_ptr = try payload.array_ptr.copy(allocator),
-                    .index = payload.index,
+                    .data = .{
+                        .array_ptr = try payload.data.array_ptr.copy(allocator),
+                        .index = payload.data.index,
+                    },
                 };
                 return Value{ .ptr_otherwise = &new_payload.base };
             },
             .bytes => return self.copyPayloadShallow(allocator, Payload.Bytes),
             .repeated => {
-                const payload = @fieldParentPtr(Payload.Repeated, "base", self.ptr_otherwise);
-                const new_payload = try allocator.create(Payload.Repeated);
+                const payload = self.castTag(.repeated).?;
+                const new_payload = try allocator.create(Payload.SubValue);
                 new_payload.* = .{
                     .base = payload.base,
-                    .val = try payload.val.copy(allocator),
+                    .data = try payload.data.copy(allocator),
                 };
                 return Value{ .ptr_otherwise = &new_payload.base };
             },
@@ -243,7 +371,7 @@ pub const Value = extern union {
             .float_64 => return self.copyPayloadShallow(allocator, Payload.Float_64),
             .float_128 => return self.copyPayloadShallow(allocator, Payload.Float_128),
             .enum_literal => {
-                const payload = @fieldParentPtr(Payload.Bytes, "base", self.ptr_otherwise);
+                const payload = self.castTag(.enum_literal).?;
                 const new_payload = try allocator.create(Payload.Bytes);
                 new_payload.* = .{
                     .base = payload.base,
@@ -259,7 +387,7 @@ pub const Value = extern union {
     }
 
     fn copyPayloadShallow(self: Value, allocator: *Allocator, comptime T: type) error{OutOfMemory}!Value {
-        const payload = @fieldParentPtr(T, "base", self.ptr_otherwise);
+        const payload = self.cast(T).?;
         const new_payload = try allocator.create(T);
         new_payload.* = payload.*;
         return Value{ .ptr_otherwise = &new_payload.base };
@@ -326,45 +454,45 @@ pub const Value = extern union {
             .unreachable_value => return out_stream.writeAll("unreachable"),
             .bool_true => return out_stream.writeAll("true"),
             .bool_false => return out_stream.writeAll("false"),
-            .ty => return val.cast(Payload.Ty).?.ty.format("", options, out_stream),
+            .ty => return val.castTag(.ty).?.data.format("", options, out_stream),
             .int_type => {
-                const int_type = val.cast(Payload.IntType).?;
+                const int_type = val.castTag(.int_type).?.data;
                 return out_stream.print("{}{}", .{
                     if (int_type.signed) "s" else "u",
                     int_type.bits,
                 });
             },
-            .int_u64 => return std.fmt.formatIntValue(val.cast(Payload.Int_u64).?.int, "", options, out_stream),
-            .int_i64 => return std.fmt.formatIntValue(val.cast(Payload.Int_i64).?.int, "", options, out_stream),
-            .int_big_positive => return out_stream.print("{}", .{val.cast(Payload.IntBigPositive).?.asBigInt()}),
-            .int_big_negative => return out_stream.print("{}", .{val.cast(Payload.IntBigNegative).?.asBigInt()}),
+            .int_u64 => return std.fmt.formatIntValue(val.castTag(.int_u64).?.data, "", options, out_stream),
+            .int_i64 => return std.fmt.formatIntValue(val.castTag(.int_i64).?.data, "", options, out_stream),
+            .int_big_positive => return out_stream.print("{}", .{val.castTag(.int_big_positive).?.asBigInt()}),
+            .int_big_negative => return out_stream.print("{}", .{val.castTag(.int_big_negative).?.asBigInt()}),
             .function => return out_stream.writeAll("(function)"),
             .extern_fn => return out_stream.writeAll("(extern function)"),
             .variable => return out_stream.writeAll("(variable)"),
             .ref_val => {
-                const ref_val = val.cast(Payload.RefVal).?;
+                const ref_val = val.castTag(.ref_val).?.data;
                 try out_stream.writeAll("&const ");
-                val = ref_val.val;
+                val = ref_val;
             },
             .decl_ref => return out_stream.writeAll("(decl ref)"),
             .elem_ptr => {
-                const elem_ptr = val.cast(Payload.ElemPtr).?;
+                const elem_ptr = val.castTag(.elem_ptr).?.data;
                 try out_stream.print("&[{}] ", .{elem_ptr.index});
                 val = elem_ptr.array_ptr;
             },
             .empty_array => return out_stream.writeAll(".{}"),
-            .enum_literal => return out_stream.print(".{z}", .{self.cast(Payload.Bytes).?.data}),
-            .bytes => return out_stream.print("\"{Z}\"", .{self.cast(Payload.Bytes).?.data}),
+            .enum_literal => return out_stream.print(".{z}", .{self.castTag(.enum_literal).?.data}),
+            .bytes => return out_stream.print("\"{Z}\"", .{self.castTag(.bytes).?.data}),
             .repeated => {
                 try out_stream.writeAll("(repeated) ");
-                val = val.cast(Payload.Repeated).?.val;
+                val = val.castTag(.repeated).?.data;
             },
-            .float_16 => return out_stream.print("{}", .{val.cast(Payload.Float_16).?.val}),
-            .float_32 => return out_stream.print("{}", .{val.cast(Payload.Float_32).?.val}),
-            .float_64 => return out_stream.print("{}", .{val.cast(Payload.Float_64).?.val}),
-            .float_128 => return out_stream.print("{}", .{val.cast(Payload.Float_128).?.val}),
+            .float_16 => return out_stream.print("{}", .{val.castTag(.float_16).?.data}),
+            .float_32 => return out_stream.print("{}", .{val.castTag(.float_32).?.data}),
+            .float_64 => return out_stream.print("{}", .{val.castTag(.float_64).?.data}),
+            .float_128 => return out_stream.print("{}", .{val.castTag(.float_128).?.data}),
             .error_set => {
-                const error_set = val.cast(Payload.ErrorSet).?;
+                const error_set = val.castTag(.error_set).?.data;
                 try out_stream.writeAll("error{");
                 var it = error_set.fields.iterator();
                 while (it.next()) |entry| {
@@ -372,21 +500,24 @@ pub const Value = extern union {
                 }
                 return out_stream.writeAll("}");
             },
-            .@"error" => return out_stream.print("error.{}", .{val.cast(Payload.Error).?.name}),
+            .@"error" => return out_stream.print("error.{}", .{val.castTag(.@"error").?.data.name}),
         };
     }
 
     /// Asserts that the value is representable as an array of bytes.
     /// Copies the value into a freshly allocated slice of memory, which is owned by the caller.
     pub fn toAllocatedBytes(self: Value, allocator: *Allocator) ![]u8 {
-        if (self.cast(Payload.Bytes)) |bytes| {
-            return std.mem.dupe(allocator, u8, bytes.data);
+        if (self.castTag(.bytes)) |payload| {
+            return std.mem.dupe(allocator, u8, payload.data);
         }
-        if (self.cast(Payload.Repeated)) |repeated| {
+        if (self.castTag(.enum_literal)) |payload| {
+            return std.mem.dupe(allocator, u8, payload.data);
+        }
+        if (self.castTag(.repeated)) |payload| {
             @panic("TODO implement toAllocatedBytes for this Value tag");
         }
-        if (self.cast(Payload.DeclRef)) |declref| {
-            const val = try declref.decl.value();
+        if (self.castTag(.decl_ref)) |payload| {
+            const val = try payload.data.value();
             return val.toAllocatedBytes(allocator);
         }
         unreachable;
@@ -395,7 +526,7 @@ pub const Value = extern union {
     /// Asserts that the value is representable as a type.
     pub fn toType(self: Value, allocator: *Allocator) !Type {
         return switch (self.tag()) {
-            .ty => self.cast(Payload.Ty).?.ty,
+            .ty => self.castTag(.ty).?.data,
             .u8_type => Type.initTag(.u8),
             .i8_type => Type.initTag(.i8),
             .u16_type => Type.initTag(.u16),
@@ -439,7 +570,7 @@ pub const Value = extern union {
             .anyframe_type => Type.initTag(.@"anyframe"),
 
             .int_type => {
-                const payload = self.cast(Payload.IntType).?;
+                const payload = self.castTag(.int_type).?.data;
                 const new = try allocator.create(Type.Payload.Bits);
                 new.* = .{
                     .base = .{
@@ -450,7 +581,7 @@ pub const Value = extern union {
                 return Type.initPayload(&new.base);
             },
             .error_set => {
-                const payload = self.cast(Payload.ErrorSet).?;
+                const payload = self.castTag(.error_set).?.data;
                 return Type.Tag.error_set.create(allocator, payload.decl);
             },
 
@@ -564,10 +695,10 @@ pub const Value = extern union {
             .bool_true,
             => return BigIntMutable.init(&space.limbs, 1).toConst(),
 
-            .int_u64 => return BigIntMutable.init(&space.limbs, self.cast(Payload.Int_u64).?.int).toConst(),
-            .int_i64 => return BigIntMutable.init(&space.limbs, self.cast(Payload.Int_i64).?.int).toConst(),
-            .int_big_positive => return self.cast(Payload.IntBigPositive).?.asBigInt(),
-            .int_big_negative => return self.cast(Payload.IntBigNegative).?.asBigInt(),
+            .int_u64 => return BigIntMutable.init(&space.limbs, self.castTag(.int_u64).?.data).toConst(),
+            .int_i64 => return BigIntMutable.init(&space.limbs, self.castTag(.int_i64).?.data).toConst(),
+            .int_big_positive => return self.castTag(.int_big_positive).?.asBigInt(),
+            .int_big_negative => return self.castTag(.int_big_negative).?.asBigInt(),
         }
     }
 
@@ -649,10 +780,10 @@ pub const Value = extern union {
             .bool_true,
             => return 1,
 
-            .int_u64 => return self.cast(Payload.Int_u64).?.int,
-            .int_i64 => return @intCast(u64, self.cast(Payload.Int_i64).?.int),
-            .int_big_positive => return self.cast(Payload.IntBigPositive).?.asBigInt().to(u64) catch unreachable,
-            .int_big_negative => return self.cast(Payload.IntBigNegative).?.asBigInt().to(u64) catch unreachable,
+            .int_u64 => return self.castTag(.int_u64).?.data,
+            .int_i64 => return @intCast(u64, self.castTag(.int_i64).?.data),
+            .int_big_positive => return self.castTag(.int_big_positive).?.asBigInt().to(u64) catch unreachable,
+            .int_big_negative => return self.castTag(.int_big_negative).?.asBigInt().to(u64) catch unreachable,
         }
     }
 
@@ -734,10 +865,10 @@ pub const Value = extern union {
             .bool_true,
             => return 1,
 
-            .int_u64 => return @intCast(i64, self.cast(Payload.Int_u64).?.int),
-            .int_i64 => return self.cast(Payload.Int_i64).?.int,
-            .int_big_positive => return self.cast(Payload.IntBigPositive).?.asBigInt().to(i64) catch unreachable,
-            .int_big_negative => return self.cast(Payload.IntBigNegative).?.asBigInt().to(i64) catch unreachable,
+            .int_u64 => return @intCast(i64, self.castTag(.int_u64).?.data),
+            .int_i64 => return self.castTag(.int_i64).?.data,
+            .int_big_positive => return self.castTag(.int_big_positive).?.asBigInt().to(i64) catch unreachable,
+            .int_big_negative => return self.castTag(.int_big_negative).?.asBigInt().to(i64) catch unreachable,
         }
     }
 
@@ -753,14 +884,14 @@ pub const Value = extern union {
     pub fn toFloat(self: Value, comptime T: type) T {
         return switch (self.tag()) {
             .float_16 => @panic("TODO soft float"),
-            .float_32 => @floatCast(T, self.cast(Payload.Float_32).?.val),
-            .float_64 => @floatCast(T, self.cast(Payload.Float_64).?.val),
-            .float_128 => @floatCast(T, self.cast(Payload.Float_128).?.val),
+            .float_32 => @floatCast(T, self.castTag(.float_32).?.data),
+            .float_64 => @floatCast(T, self.castTag(.float_64).?.data),
+            .float_128 => @floatCast(T, self.castTag(.float_128).?.data),
 
             .zero => 0,
             .one => 1,
-            .int_u64 => @intToFloat(T, self.cast(Payload.Int_u64).?.int),
-            .int_i64 => @intToFloat(T, self.cast(Payload.Int_i64).?.int),
+            .int_u64 => @intToFloat(T, self.castTag(.int_u64).?.data),
+            .int_i64 => @intToFloat(T, self.castTag(.int_i64).?.data),
 
             .int_big_positive, .int_big_negative => @panic("big int to f128"),
             else => unreachable,
@@ -846,15 +977,15 @@ pub const Value = extern union {
             => return 1,
 
             .int_u64 => {
-                const x = self.cast(Payload.Int_u64).?.int;
+                const x = self.castTag(.int_u64).?.data;
                 if (x == 0) return 0;
                 return @intCast(usize, std.math.log2(x) + 1);
             },
             .int_i64 => {
                 @panic("TODO implement i64 intBitCountTwosComp");
             },
-            .int_big_positive => return self.cast(Payload.IntBigPositive).?.asBigInt().bitCountTwosComp(),
-            .int_big_negative => return self.cast(Payload.IntBigNegative).?.asBigInt().bitCountTwosComp(),
+            .int_big_positive => return self.castTag(.int_big_positive).?.asBigInt().bitCountTwosComp(),
+            .int_big_negative => return self.castTag(.int_big_negative).?.asBigInt().bitCountTwosComp(),
         }
     }
 
@@ -943,7 +1074,7 @@ pub const Value = extern union {
 
             .int_u64 => switch (ty.zigTypeTag()) {
                 .Int => {
-                    const x = self.cast(Payload.Int_u64).?.int;
+                    const x = self.castTag(.int_u64).?.data;
                     if (x == 0) return true;
                     const info = ty.intInfo(target);
                     const needed_bits = std.math.log2(x) + 1 + @boolToInt(info.signedness == .signed);
@@ -954,7 +1085,7 @@ pub const Value = extern union {
             },
             .int_i64 => switch (ty.zigTypeTag()) {
                 .Int => {
-                    const x = self.cast(Payload.Int_i64).?.int;
+                    const x = self.castTag(.int_i64).?.data;
                     if (x == 0) return true;
                     const info = ty.intInfo(target);
                     if (info.signedness == .unsigned and x < 0)
@@ -967,7 +1098,7 @@ pub const Value = extern union {
             .int_big_positive => switch (ty.zigTypeTag()) {
                 .Int => {
                     const info = ty.intInfo(target);
-                    return self.cast(Payload.IntBigPositive).?.asBigInt().fitsInTwosComp(info.signedness, info.bits);
+                    return self.castTag(.int_big_positive).?.asBigInt().fitsInTwosComp(info.signedness, info.bits);
                 },
                 .ComptimeInt => return true,
                 else => unreachable,
@@ -975,7 +1106,7 @@ pub const Value = extern union {
             .int_big_negative => switch (ty.zigTypeTag()) {
                 .Int => {
                     const info = ty.intInfo(target);
-                    return self.cast(Payload.IntBigNegative).?.asBigInt().fitsInTwosComp(info.signedness, info.bits);
+                    return self.castTag(.int_big_negative).?.asBigInt().fitsInTwosComp(info.signedness, info.bits);
                 },
                 .ComptimeInt => return true,
                 else => unreachable,
@@ -986,42 +1117,28 @@ pub const Value = extern union {
     /// Converts an integer or a float to a float.
     /// Returns `error.Overflow` if the value does not fit in the new type.
     pub fn floatCast(self: Value, allocator: *Allocator, ty: Type, target: Target) !Value {
-        const dest_bit_count = switch (ty.tag()) {
-            .comptime_float => 128,
-            else => ty.floatBits(target),
-        };
-        switch (dest_bit_count) {
-            16, 32, 64, 128 => {},
-            else => std.debug.panic("TODO float cast bit count {}\n", .{dest_bit_count}),
-        }
-        if (ty.isInt()) {
-            @panic("TODO int to float");
-        }
-
-        switch (dest_bit_count) {
-            16 => {
-                @panic("TODO soft float");
-                // var res_payload = Value.Payload.Float_16{.val = self.toFloat(f16)};
-                // if (!self.eql(Value.initPayload(&res_payload.base)))
-                //     return error.Overflow;
-                // return Value.initPayload(&res_payload.base).copy(allocator);
+        switch (ty.tag()) {
+            .f16 => {
+                @panic("TODO add __trunctfhf2 to compiler-rt");
+                //const res = try Value.Tag.float_16.create(allocator, self.toFloat(f16));
+                //if (!self.eql(res))
+                //    return error.Overflow;
+                //return res;
             },
-            32 => {
-                var res_payload = Value.Payload.Float_32{ .val = self.toFloat(f32) };
-                if (!self.eql(Value.initPayload(&res_payload.base)))
+            .f32 => {
+                const res = try Value.Tag.float_32.create(allocator, self.toFloat(f32));
+                if (!self.eql(res))
                     return error.Overflow;
-                return Value.initPayload(&res_payload.base).copy(allocator);
+                return res;
             },
-            64 => {
-                var res_payload = Value.Payload.Float_64{ .val = self.toFloat(f64) };
-                if (!self.eql(Value.initPayload(&res_payload.base)))
+            .f64 => {
+                const res = try Value.Tag.float_64.create(allocator, self.toFloat(f64));
+                if (!self.eql(res))
                     return error.Overflow;
-                return Value.initPayload(&res_payload.base).copy(allocator);
+                return res;
             },
-            128 => {
-                const float_payload = try allocator.create(Value.Payload.Float_128);
-                float_payload.* = .{ .val = self.toFloat(f128) };
-                return Value.initPayload(&float_payload.base);
+            .f128, .comptime_float, .c_longdouble => {
+                return Value.Tag.float_128.create(allocator, self.toFloat(f128));
             },
             else => unreachable,
         }
@@ -1102,10 +1219,10 @@ pub const Value = extern union {
             .one,
             => false,
 
-            .float_16 => @rem(self.cast(Payload.Float_16).?.val, 1) != 0,
-            .float_32 => @rem(self.cast(Payload.Float_32).?.val, 1) != 0,
-            .float_64 => @rem(self.cast(Payload.Float_64).?.val, 1) != 0,
-            // .float_128 => @rem(self.cast(Payload.Float_128).?.val, 1) != 0,
+            .float_16 => @rem(self.castTag(.float_16).?.data, 1) != 0,
+            .float_32 => @rem(self.castTag(.float_32).?.data, 1) != 0,
+            .float_64 => @rem(self.castTag(.float_64).?.data, 1) != 0,
+            // .float_128 => @rem(self.castTag(.float_128).?.data, 1) != 0,
             .float_128 => @panic("TODO lld: error: undefined symbol: fmodl"),
         };
     }
@@ -1182,15 +1299,15 @@ pub const Value = extern union {
             .bool_true,
             => .gt,
 
-            .int_u64 => std.math.order(lhs.cast(Payload.Int_u64).?.int, 0),
-            .int_i64 => std.math.order(lhs.cast(Payload.Int_i64).?.int, 0),
-            .int_big_positive => lhs.cast(Payload.IntBigPositive).?.asBigInt().orderAgainstScalar(0),
-            .int_big_negative => lhs.cast(Payload.IntBigNegative).?.asBigInt().orderAgainstScalar(0),
+            .int_u64 => std.math.order(lhs.castTag(.int_u64).?.data, 0),
+            .int_i64 => std.math.order(lhs.castTag(.int_i64).?.data, 0),
+            .int_big_positive => lhs.castTag(.int_big_positive).?.asBigInt().orderAgainstScalar(0),
+            .int_big_negative => lhs.castTag(.int_big_negative).?.asBigInt().orderAgainstScalar(0),
 
-            .float_16 => std.math.order(lhs.cast(Payload.Float_16).?.val, 0),
-            .float_32 => std.math.order(lhs.cast(Payload.Float_32).?.val, 0),
-            .float_64 => std.math.order(lhs.cast(Payload.Float_64).?.val, 0),
-            .float_128 => std.math.order(lhs.cast(Payload.Float_128).?.val, 0),
+            .float_16 => std.math.order(lhs.castTag(.float_16).?.data, 0),
+            .float_32 => std.math.order(lhs.castTag(.float_32).?.data, 0),
+            .float_64 => std.math.order(lhs.castTag(.float_64).?.data, 0),
+            .float_128 => std.math.order(lhs.castTag(.float_128).?.data, 0),
         };
     }
 
@@ -1208,10 +1325,10 @@ pub const Value = extern union {
         if (lhs_float and rhs_float) {
             if (lhs_tag == rhs_tag) {
                 return switch (lhs.tag()) {
-                    .float_16 => return std.math.order(lhs.cast(Payload.Float_16).?.val, rhs.cast(Payload.Float_16).?.val),
-                    .float_32 => return std.math.order(lhs.cast(Payload.Float_32).?.val, rhs.cast(Payload.Float_32).?.val),
-                    .float_64 => return std.math.order(lhs.cast(Payload.Float_64).?.val, rhs.cast(Payload.Float_64).?.val),
-                    .float_128 => return std.math.order(lhs.cast(Payload.Float_128).?.val, rhs.cast(Payload.Float_128).?.val),
+                    .float_16 => return std.math.order(lhs.castTag(.float_16).?.data, rhs.castTag(.float_16).?.data),
+                    .float_32 => return std.math.order(lhs.castTag(.float_32).?.data, rhs.castTag(.float_32).?.data),
+                    .float_64 => return std.math.order(lhs.castTag(.float_64).?.data, rhs.castTag(.float_64).?.data),
+                    .float_128 => return std.math.order(lhs.castTag(.float_128).?.data, rhs.castTag(.float_128).?.data),
                     else => unreachable,
                 };
             }
@@ -1244,8 +1361,8 @@ pub const Value = extern union {
             if (a.tag() == .void_value or a.tag() == .null_value) {
                 return true;
             } else if (a.tag() == .enum_literal) {
-                const a_name = @fieldParentPtr(Payload.Bytes, "base", a.ptr_otherwise).data;
-                const b_name = @fieldParentPtr(Payload.Bytes, "base", b.ptr_otherwise).data;
+                const a_name = a.castTag(.enum_literal).?.data;
+                const b_name = b.castTag(.enum_literal).?.data;
                 return std.mem.eql(u8, a_name, b_name);
             }
         }
@@ -1313,11 +1430,11 @@ pub const Value = extern union {
             },
             .error_set => {
                 // Payload.decl should be same for all instances of the type.
-                const payload = @fieldParentPtr(Payload.ErrorSet, "base", self.ptr_otherwise);
+                const payload = self.castTag(.error_set).?.data;
                 std.hash.autoHash(&hasher, payload.decl);
             },
             .int_type => {
-                const payload = self.cast(Payload.IntType).?;
+                const payload = self.castTag(.int_type).?.data;
                 var int_payload = Type.Payload.Bits{
                     .base = .{
                         .tag = if (payload.signed) .int_signed else .int_unsigned,
@@ -1341,25 +1458,29 @@ pub const Value = extern union {
             .one, .bool_true => std.hash.autoHash(&hasher, @as(u64, 1)),
 
             .float_16, .float_32, .float_64, .float_128 => {},
-            .enum_literal, .bytes => {
-                const payload = @fieldParentPtr(Payload.Bytes, "base", self.ptr_otherwise);
+            .enum_literal => {
+                const payload = self.castTag(.enum_literal).?;
+                hasher.update(payload.data);
+            },
+            .bytes => {
+                const payload = self.castTag(.bytes).?;
                 hasher.update(payload.data);
             },
             .int_u64 => {
-                const payload = @fieldParentPtr(Payload.Int_u64, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.int);
+                const payload = self.castTag(.int_u64).?;
+                std.hash.autoHash(&hasher, payload.data);
             },
             .int_i64 => {
-                const payload = @fieldParentPtr(Payload.Int_i64, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.int);
+                const payload = self.castTag(.int_i64).?;
+                std.hash.autoHash(&hasher, payload.data);
             },
             .repeated => {
-                const payload = @fieldParentPtr(Payload.Repeated, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.val.hash());
+                const payload = self.castTag(.repeated).?;
+                std.hash.autoHash(&hasher, payload.data.hash());
             },
             .ref_val => {
-                const payload = @fieldParentPtr(Payload.RefVal, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.val.hash());
+                const payload = self.castTag(.ref_val).?;
+                std.hash.autoHash(&hasher, payload.data.hash());
             },
             .int_big_positive, .int_big_negative => {
                 var space: BigIntSpace = undefined;
@@ -1379,28 +1500,28 @@ pub const Value = extern union {
                 }
             },
             .elem_ptr => {
-                const payload = @fieldParentPtr(Payload.ElemPtr, "base", self.ptr_otherwise);
+                const payload = self.castTag(.elem_ptr).?.data;
                 std.hash.autoHash(&hasher, payload.array_ptr.hash());
                 std.hash.autoHash(&hasher, payload.index);
             },
             .decl_ref => {
-                const payload = @fieldParentPtr(Payload.DeclRef, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.decl);
+                const decl = self.castTag(.decl_ref).?.data;
+                std.hash.autoHash(&hasher, decl);
             },
             .function => {
-                const payload = @fieldParentPtr(Payload.Function, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.func);
+                const func = self.castTag(.function).?.data;
+                std.hash.autoHash(&hasher, func);
             },
             .extern_fn => {
-                const payload = @fieldParentPtr(Payload.ExternFn, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.decl);
+                const decl = self.castTag(.extern_fn).?.data;
+                std.hash.autoHash(&hasher, decl);
             },
             .variable => {
-                const payload = @fieldParentPtr(Payload.Variable, "base", self.ptr_otherwise);
-                std.hash.autoHash(&hasher, payload.variable);
+                const variable = self.castTag(.variable).?.data;
+                std.hash.autoHash(&hasher, variable);
             },
             .@"error" => {
-                const payload = @fieldParentPtr(Payload.Error, "base", self.ptr_otherwise);
+                const payload = self.castTag(.@"error").?.data;
                 hasher.update(payload.name);
                 std.hash.autoHash(&hasher, payload.value);
             },
@@ -1483,10 +1604,10 @@ pub const Value = extern union {
             .empty_struct_value,
             => unreachable,
 
-            .ref_val => self.cast(Payload.RefVal).?.val,
-            .decl_ref => self.cast(Payload.DeclRef).?.decl.value(),
+            .ref_val => self.castTag(.ref_val).?.data,
+            .decl_ref => self.castTag(.decl_ref).?.data.value(),
             .elem_ptr => {
-                const elem_ptr = self.cast(Payload.ElemPtr).?;
+                const elem_ptr = self.castTag(.elem_ptr).?.data;
                 const array_val = try elem_ptr.array_ptr.pointerDeref(allocator);
                 return array_val.elemValue(allocator, elem_ptr.index);
             },
@@ -1570,26 +1691,26 @@ pub const Value = extern union {
 
             .empty_array => unreachable, // out of bounds array index
 
-            .bytes => {
-                const int_payload = try allocator.create(Payload.Int_u64);
-                int_payload.* = .{ .int = self.cast(Payload.Bytes).?.data[index] };
-                return Value.initPayload(&int_payload.base);
-            },
+            .bytes => return Tag.int_u64.create(allocator, self.castTag(.bytes).?.data[index]),
 
             // No matter the index; all the elements are the same!
-            .repeated => return self.cast(Payload.Repeated).?.val,
+            .repeated => return self.castTag(.repeated).?.data,
         }
     }
 
     /// Returns a pointer to the element value at the index.
     pub fn elemPtr(self: Value, allocator: *Allocator, index: usize) !Value {
-        const payload = try allocator.create(Payload.ElemPtr);
-        if (self.cast(Payload.ElemPtr)) |elem_ptr| {
-            payload.* = .{ .array_ptr = elem_ptr.array_ptr, .index = elem_ptr.index + index };
-        } else {
-            payload.* = .{ .array_ptr = self, .index = index };
+        if (self.castTag(.elem_ptr)) |elem_ptr| {
+            return Tag.elem_ptr.create(allocator, .{
+                .array_ptr = elem_ptr.data.array_ptr,
+                .index = elem_ptr.data.index + index,
+            });
         }
-        return Value.initPayload(&payload.base);
+
+        return Tag.elem_ptr.create(allocator, .{
+            .array_ptr = self,
+            .index = index,
+        });
     }
 
     pub fn isUndef(self: Value) bool {
@@ -1776,131 +1897,128 @@ pub const Value = extern union {
     pub const Payload = struct {
         tag: Tag,
 
-        pub const Int_u64 = struct {
-            base: Payload = Payload{ .tag = .int_u64 },
-            int: u64,
+        pub const U64 = struct {
+            base: Payload,
+            data: u64,
         };
 
-        pub const Int_i64 = struct {
-            base: Payload = Payload{ .tag = .int_i64 },
-            int: i64,
-        };
-
-        pub const IntBigPositive = struct {
-            base: Payload = Payload{ .tag = .int_big_positive },
-            limbs: []const std.math.big.Limb,
-
-            pub fn asBigInt(self: IntBigPositive) BigIntConst {
-                return BigIntConst{ .limbs = self.limbs, .positive = true };
-            }
+        pub const I64 = struct {
+            base: Payload,
+            data: i64,
         };
 
-        pub const IntBigNegative = struct {
-            base: Payload = Payload{ .tag = .int_big_negative },
-            limbs: []const std.math.big.Limb,
+        pub const BigInt = struct {
+            base: Payload,
+            data: []const std.math.big.Limb,
 
-            pub fn asBigInt(self: IntBigNegative) BigIntConst {
-                return BigIntConst{ .limbs = self.limbs, .positive = false };
+            pub fn asBigInt(self: BigInt) BigIntConst {
+                const positive = switch (self.base.tag) {
+                    .int_big_positive => true,
+                    .int_big_negative => false,
+                    else => unreachable,
+                };
+                return BigIntConst{ .limbs = self.data, .positive = positive };
             }
         };
 
         pub const Function = struct {
-            base: Payload = Payload{ .tag = .function },
-            func: *Module.Fn,
+            base: Payload,
+            data: *Module.Fn,
         };
 
-        pub const ExternFn = struct {
-            base: Payload = Payload{ .tag = .extern_fn },
-            decl: *Module.Decl,
+        pub const Decl = struct {
+            base: Payload,
+            data: *Module.Decl,
         };
 
         pub const Variable = struct {
-            base: Payload = Payload{ .tag = .variable },
-            variable: *Module.Var,
-        };
-
-        pub const ArraySentinel0_u8_Type = struct {
-            base: Payload = Payload{ .tag = .array_sentinel_0_u8_type },
-            len: u64,
-        };
-
-        /// Represents a pointer to another immutable value.
-        pub const RefVal = struct {
-            base: Payload = Payload{ .tag = .ref_val },
-            val: Value,
+            base: Payload,
+            data: *Module.Var,
         };
 
-        /// Represents a pointer to a decl, not the value of the decl.
-        pub const DeclRef = struct {
-            base: Payload = Payload{ .tag = .decl_ref },
-            decl: *Module.Decl,
+        pub const SubValue = struct {
+            base: Payload,
+            data: Value,
         };
 
         pub const ElemPtr = struct {
-            base: Payload = Payload{ .tag = .elem_ptr },
-            array_ptr: Value,
-            index: usize,
+            pub const base_tag = Tag.elem_ptr;
+
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                array_ptr: Value,
+                index: usize,
+            },
         };
 
         pub const Bytes = struct {
-            base: Payload = Payload{ .tag = .bytes },
+            base: Payload,
             data: []const u8,
         };
 
         pub const Ty = struct {
-            base: Payload = Payload{ .tag = .ty },
-            ty: Type,
+            base: Payload,
+            data: Type,
         };
 
         pub const IntType = struct {
-            base: Payload = Payload{ .tag = .int_type },
-            bits: u16,
-            signed: bool,
-        };
+            pub const base_tag = Tag.int_type;
 
-        pub const Repeated = struct {
-            base: Payload = Payload{ .tag = .ty },
-            /// This value is repeated some number of times. The amount of times to repeat
-            /// is stored externally.
-            val: Value,
+            base: Payload = Payload{ .tag = base_tag },
+            data: struct {
+                bits: u16,
+                signed: bool,
+            },
         };
 
         pub const Float_16 = struct {
-            base: Payload = .{ .tag = .float_16 },
-            val: f16,
+            pub const base_tag = Tag.float_16;
+
+            base: Payload = .{ .tag = base_tag },
+            data: f16,
         };
 
         pub const Float_32 = struct {
-            base: Payload = .{ .tag = .float_32 },
-            val: f32,
+            pub const base_tag = Tag.float_32;
+
+            base: Payload = .{ .tag = base_tag },
+            data: f32,
         };
 
         pub const Float_64 = struct {
-            base: Payload = .{ .tag = .float_64 },
-            val: f64,
+            pub const base_tag = Tag.float_64;
+
+            base: Payload = .{ .tag = base_tag },
+            data: f64,
         };
 
         pub const Float_128 = struct {
-            base: Payload = .{ .tag = .float_128 },
-            val: f128,
+            pub const base_tag = Tag.float_128;
+
+            base: Payload = .{ .tag = base_tag },
+            data: f128,
         };
 
         pub const ErrorSet = struct {
-            base: Payload = .{ .tag = .error_set },
+            pub const base_tag = Tag.error_set;
 
-            // TODO revisit this when we have the concept of the error tag type
-            fields: std.StringHashMapUnmanaged(u16),
-            decl: *Module.Decl,
+            base: Payload = .{ .tag = base_tag },
+            data: struct {
+                // TODO revisit this when we have the concept of the error tag type
+                fields: std.StringHashMapUnmanaged(u16),
+                decl: *Module.Decl,
+            },
         };
 
         pub const Error = struct {
             base: Payload = .{ .tag = .@"error" },
-
-            // TODO revisit this when we have the concept of the error tag type
-            /// `name` is owned by `Module` and will be valid for the entire
-            /// duration of the compilation.
-            name: []const u8,
-            value: u16,
+            data: struct {
+                // TODO revisit this when we have the concept of the error tag type
+                /// `name` is owned by `Module` and will be valid for the entire
+                /// duration of the compilation.
+                name: []const u8,
+                value: u16,
+            },
         };
     };
 
@@ -1914,15 +2032,24 @@ pub const Value = extern union {
 
 test "hash same value different representation" {
     const zero_1 = Value.initTag(.zero);
-    var payload_1 = Value.Payload.Int_u64{ .int = 0 };
+    var payload_1 = Value.Payload.U64{
+        .base = .{ .tag = .int_u64 },
+        .data = 0,
+    };
     const zero_2 = Value.initPayload(&payload_1.base);
     std.testing.expectEqual(zero_1.hash(), zero_2.hash());
 
-    var payload_2 = Value.Payload.Int_i64{ .int = 0 };
+    var payload_2 = Value.Payload.I64{
+        .base = .{ .tag = .int_i64 },
+        .data = 0,
+    };
     const zero_3 = Value.initPayload(&payload_2.base);
     std.testing.expectEqual(zero_2.hash(), zero_3.hash());
 
-    var payload_3 = Value.Payload.IntBigNegative{ .limbs = &[_]std.math.big.Limb{0} };
+    var payload_3 = Value.Payload.BigInt{
+        .base = .{ .tag = .int_big_negative },
+        .data = &[_]std.math.big.Limb{0},
+    };
     const zero_4 = Value.initPayload(&payload_3.base);
     std.testing.expectEqual(zero_3.hash(), zero_4.hash());
 }
diff --git a/src/zir.zig b/src/zir.zig
index bd9ab2c538..21bd4f8435 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1990,15 +1990,15 @@ const EmitZIR = struct {
 
     fn resolveInst(self: *EmitZIR, new_body: ZirBody, inst: *ir.Inst) !*Inst {
         if (inst.cast(ir.Inst.Constant)) |const_inst| {
-            const new_inst = if (const_inst.val.cast(Value.Payload.Function)) |func_pl| blk: {
-                const owner_decl = func_pl.func.owner_decl;
+            const new_inst = if (const_inst.val.castTag(.function)) |func_pl| blk: {
+                const owner_decl = func_pl.data.owner_decl;
                 break :blk try self.emitDeclVal(inst.src, mem.spanZ(owner_decl.name));
-            } else if (const_inst.val.cast(Value.Payload.DeclRef)) |declref| blk: {
-                const decl_ref = try self.emitDeclRef(inst.src, declref.decl);
+            } else if (const_inst.val.castTag(.decl_ref)) |declref| blk: {
+                const decl_ref = try self.emitDeclRef(inst.src, declref.data);
                 try new_body.instructions.append(decl_ref);
                 break :blk decl_ref;
-            } else if (const_inst.val.cast(Value.Payload.Variable)) |var_pl| blk: {
-                const owner_decl = var_pl.variable.owner_decl;
+            } else if (const_inst.val.castTag(.variable)) |var_pl| blk: {
+                const owner_decl = var_pl.data.owner_decl;
                 break :blk try self.emitDeclVal(inst.src, mem.spanZ(owner_decl.name));
             } else blk: {
                 break :blk (try self.emitTypedValue(inst.src, .{ .ty = inst.ty, .val = const_inst.val })).inst;
@@ -2150,13 +2150,13 @@ const EmitZIR = struct {
 
     fn emitTypedValue(self: *EmitZIR, src: usize, typed_value: TypedValue) Allocator.Error!*Decl {
         const allocator = &self.arena.allocator;
-        if (typed_value.val.cast(Value.Payload.DeclRef)) |decl_ref| {
-            const decl = decl_ref.decl;
+        if (typed_value.val.castTag(.decl_ref)) |decl_ref| {
+            const decl = decl_ref.data;
             return try self.emitUnnamedDecl(try self.emitDeclRef(src, decl));
-        } else if (typed_value.val.cast(Value.Payload.Variable)) |variable| {
+        } else if (typed_value.val.castTag(.variable)) |variable| {
             return self.emitTypedValue(src, .{
                 .ty = typed_value.ty,
-                .val = variable.variable.init,
+                .val = variable.data.init,
             });
         }
         if (typed_value.val.isUndef()) {
@@ -2215,7 +2215,7 @@ const EmitZIR = struct {
                 return self.emitType(src, ty);
             },
             .Fn => {
-                const module_fn = typed_value.val.cast(Value.Payload.Function).?.func;
+                const module_fn = typed_value.val.castTag(.function).?.data;
                 return self.emitFn(module_fn, src, typed_value.ty);
             },
             .Array => {
@@ -2248,7 +2248,7 @@ const EmitZIR = struct {
             else
                 return self.emitPrimitive(src, .@"false"),
             .EnumLiteral => {
-                const enum_literal = @fieldParentPtr(Value.Payload.Bytes, "base", typed_value.val.ptr_otherwise);
+                const enum_literal = typed_value.val.castTag(.enum_literal).?;
                 const inst = try self.arena.allocator.create(Inst.Str);
                 inst.* = .{
                     .base = .{
@@ -2748,9 +2748,8 @@ const EmitZIR = struct {
                         .signed => .@"true",
                         .unsigned => .@"false",
                     });
-                    const bits_payload = try self.arena.allocator.create(Value.Payload.Int_u64);
-                    bits_payload.* = .{ .int = info.bits };
-                    const bits = try self.emitComptimeIntVal(src, Value.initPayload(&bits_payload.base));
+                    const bits_val = try Value.Tag.int_u64.create(&self.arena.allocator, info.bits);
+                    const bits = try self.emitComptimeIntVal(src, bits_val);
                     const inttype_inst = try self.arena.allocator.create(Inst.IntType);
                     inttype_inst.* = .{
                         .base = .{
@@ -2800,7 +2799,10 @@ const EmitZIR = struct {
                     return self.emitUnnamedDecl(&inst.base);
                 },
                 .Array => {
-                    var len_pl = Value.Payload.Int_u64{ .int = ty.arrayLen() };
+                    var len_pl = Value.Payload.U64{
+                        .base = .{ .tag = .int_u64 },
+                        .data = ty.arrayLen(),
+                    };
                     const len = Value.initPayload(&len_pl.base);
 
                     const inst = if (ty.sentinel()) |sentinel| blk: {
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 8960f3ba4d..2e3cced839 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -364,12 +364,9 @@ fn analyzeInstRef(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!
     const ptr_type = try mod.simplePtrType(scope, inst.base.src, operand.ty, false, .One);
 
     if (operand.value()) |val| {
-        const ref_payload = try scope.arena().create(Value.Payload.RefVal);
-        ref_payload.* = .{ .val = val };
-
         return mod.constInst(scope, inst.base.src, .{
             .ty = ptr_type,
-            .val = Value.initPayload(&ref_payload.base),
+            .val = try Value.Tag.ref_val.create(scope.arena(), val),
         });
     }
 
@@ -480,12 +477,9 @@ fn analyzeInstStr(mod: *Module, scope: *Scope, str_inst: *zir.Inst.Str) InnerErr
     errdefer new_decl_arena.deinit();
     const arena_bytes = try new_decl_arena.allocator.dupe(u8, str_inst.positionals.bytes);
 
-    const bytes_payload = try scope.arena().create(Value.Payload.Bytes);
-    bytes_payload.* = .{ .data = arena_bytes };
-
     const new_decl = try mod.createAnonymousDecl(scope, &new_decl_arena, .{
         .ty = try Type.Tag.array_u8_sentinel_0.create(scope.arena(), arena_bytes.len),
-        .val = Value.initPayload(&bytes_payload.base),
+        .val = try Value.Tag.bytes.create(scope.arena(), arena_bytes),
     });
     return mod.analyzeDeclRef(scope, str_inst.base.src, new_decl);
 }
@@ -779,11 +773,9 @@ fn analyzeInstFn(mod: *Module, scope: *Scope, fn_inst: *zir.Inst.Fn) InnerError!
         .analysis = .{ .queued = fn_zir },
         .owner_decl = scope.decl().?,
     };
-    const fn_payload = try scope.arena().create(Value.Payload.Function);
-    fn_payload.* = .{ .func = new_func };
     return mod.constInst(scope, fn_inst.base.src, .{
         .ty = fn_type,
-        .val = Value.initPayload(&fn_payload.base),
+        .val = try Value.Tag.function.create(scope.arena(), new_func),
     });
 }
 
@@ -838,14 +830,17 @@ fn analyzeInstErrorSet(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorSet) In
 
     const payload = try scope.arena().create(Value.Payload.ErrorSet);
     payload.* = .{
-        .fields = .{},
-        .decl = undefined, // populated below
+        .base = .{ .tag = .error_set },
+        .data = .{
+            .fields = .{},
+            .decl = undefined, // populated below
+        },
     };
-    try payload.fields.ensureCapacity(&new_decl_arena.allocator, @intCast(u32, inst.positionals.fields.len));
+    try payload.data.fields.ensureCapacity(&new_decl_arena.allocator, @intCast(u32, inst.positionals.fields.len));
 
     for (inst.positionals.fields) |field_name| {
         const entry = try mod.getErrorValue(field_name);
-        if (payload.fields.fetchPutAssumeCapacity(entry.key, entry.value)) |prev| {
+        if (payload.data.fields.fetchPutAssumeCapacity(entry.key, entry.value)) |prev| {
             return mod.fail(scope, inst.base.src, "duplicate error: '{}'", .{field_name});
         }
     }
@@ -854,7 +849,7 @@ fn analyzeInstErrorSet(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorSet) In
         .ty = Type.initTag(.type),
         .val = Value.initPayload(&payload.base),
     });
-    payload.decl = new_decl;
+    payload.data.decl = new_decl;
     return mod.analyzeDeclRef(scope, inst.base.src, new_decl);
 }
 
@@ -863,14 +858,10 @@ fn analyzeInstMergeErrorSets(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp)
 }
 
 fn analyzeInstEnumLiteral(mod: *Module, scope: *Scope, inst: *zir.Inst.EnumLiteral) InnerError!*Inst {
-    const payload = try scope.arena().create(Value.Payload.Bytes);
-    payload.* = .{
-        .base = .{ .tag = .enum_literal },
-        .data = try scope.arena().dupe(u8, inst.positionals.name),
-    };
+    const duped_name = try scope.arena().dupe(u8, inst.positionals.name);
     return mod.constInst(scope, inst.base.src, .{
         .ty = Type.initTag(.enum_literal),
-        .val = Value.initPayload(&payload.base),
+        .val = try Value.Tag.enum_literal.create(scope.arena(), duped_name),
     });
 }
 
@@ -989,15 +980,12 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
     switch (elem_ty.zigTypeTag()) {
         .Array => {
             if (mem.eql(u8, field_name, "len")) {
-                const len_payload = try scope.arena().create(Value.Payload.Int_u64);
-                len_payload.* = .{ .int = elem_ty.arrayLen() };
-
-                const ref_payload = try scope.arena().create(Value.Payload.RefVal);
-                ref_payload.* = .{ .val = Value.initPayload(&len_payload.base) };
-
                 return mod.constInst(scope, fieldptr.base.src, .{
                     .ty = Type.initTag(.single_const_pointer_to_comptime_int),
-                    .val = Value.initPayload(&ref_payload.base),
+                    .val = try Value.Tag.ref_val.create(
+                        scope.arena(),
+                        try Value.Tag.int_u64.create(scope.arena(), elem_ty.arrayLen()),
+                    ),
                 });
             } else {
                 return mod.fail(
@@ -1013,15 +1001,12 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
             switch (ptr_child.zigTypeTag()) {
                 .Array => {
                     if (mem.eql(u8, field_name, "len")) {
-                        const len_payload = try scope.arena().create(Value.Payload.Int_u64);
-                        len_payload.* = .{ .int = ptr_child.arrayLen() };
-
-                        const ref_payload = try scope.arena().create(Value.Payload.RefVal);
-                        ref_payload.* = .{ .val = Value.initPayload(&len_payload.base) };
-
                         return mod.constInst(scope, fieldptr.base.src, .{
                             .ty = Type.initTag(.single_const_pointer_to_comptime_int),
-                            .val = Value.initPayload(&ref_payload.base),
+                            .val = try Value.Tag.ref_val.create(
+                                scope.arena(),
+                                try Value.Tag.int_u64.create(scope.arena(), ptr_child.arrayLen()),
+                            ),
                         });
                     } else {
                         return mod.fail(
@@ -1043,21 +1028,12 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
             switch (child_type.zigTypeTag()) {
                 .ErrorSet => {
                     // TODO resolve inferred error sets
-                    const entry = if (val.cast(Value.Payload.ErrorSet)) |payload|
-                        (payload.fields.getEntry(field_name) orelse
+                    const entry = if (val.castTag(.error_set)) |payload|
+                        (payload.data.fields.getEntry(field_name) orelse
                             return mod.fail(scope, fieldptr.base.src, "no error named '{}' in '{}'", .{ field_name, child_type })).*
                     else
                         try mod.getErrorValue(field_name);
 
-                    const error_payload = try scope.arena().create(Value.Payload.Error);
-                    error_payload.* = .{
-                        .name = entry.key,
-                        .value = entry.value,
-                    };
-
-                    const ref_payload = try scope.arena().create(Value.Payload.RefVal);
-                    ref_payload.* = .{ .val = Value.initPayload(&error_payload.base) };
-
                     const result_type = if (child_type.tag() == .anyerror)
                         try Type.Tag.error_set_single.create(scope.arena(), entry.key)
                     else
@@ -1065,7 +1041,13 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
 
                     return mod.constInst(scope, fieldptr.base.src, .{
                         .ty = try mod.simplePtrType(scope, fieldptr.base.src, result_type, false, .One),
-                        .val = Value.initPayload(&ref_payload.base),
+                        .val = try Value.Tag.ref_val.create(
+                            scope.arena(),
+                            try Value.Tag.@"error".create(scope.arena(), .{
+                                .name = entry.key,
+                                .value = entry.value,
+                            }),
+                        ),
                     });
                 },
                 .Struct => {
-- 
cgit v1.2.3


From 91a35e1a922e865ca0e9beb43a2be2c5552fc494 Mon Sep 17 00:00:00 2001
From: Jakub Konka <kubkon@jakubkonka.com>
Date: Sun, 20 Dec 2020 11:45:48 +0100
Subject: Detect native iframework dirs on macOS

This commit adds default search paths for system frameworks
on macOS while also adding `-isysroot` for OS versions at least BigSur.
Since BigSur (11.0.1), neither headers nor libs exist in standard
root locations (`/usr/include`, `/System/Library/Frameworks`). Instead, they
are now exclusively part of the installed developer toolchain (either
via XCode.app or CLT), and specifying `-isysroot` allows us to keep
using universal search paths such as `/System/Library/Frameworks` while
only changing the include flag from `-iframework` to
`-iframeworkwithsysroot`.
---
 lib/std/zig/system.zig | 29 +++++++++++++++++++++++++++--
 src/Compilation.zig    |  7 +------
 src/main.zig           | 26 +++++++++++++++++++++++++-
 3 files changed, 53 insertions(+), 9 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig
index 574ccfb32d..09fc15c8ba 100644
--- a/lib/std/zig/system.zig
+++ b/lib/std/zig/system.zig
@@ -22,6 +22,7 @@ pub const getSDKPath = macos.getSDKPath;
 pub const NativePaths = struct {
     include_dirs: ArrayList([:0]u8),
     lib_dirs: ArrayList([:0]u8),
+    framework_dirs: ArrayList([:0]u8),
     rpaths: ArrayList([:0]u8),
     warnings: ArrayList([:0]u8),
 
@@ -29,6 +30,7 @@ pub const NativePaths = struct {
         var self: NativePaths = .{
             .include_dirs = ArrayList([:0]u8).init(allocator),
             .lib_dirs = ArrayList([:0]u8).init(allocator),
+            .framework_dirs = ArrayList([:0]u8).init(allocator),
             .rpaths = ArrayList([:0]u8).init(allocator),
             .warnings = ArrayList([:0]u8).init(allocator),
         };
@@ -88,6 +90,19 @@ pub const NativePaths = struct {
             return self;
         }
 
+        if (comptime Target.current.isDarwin()) {
+            try self.addIncludeDir("/usr/include");
+            try self.addIncludeDir("/usr/local/include");
+
+            try self.addLibDir("/usr/lib");
+            try self.addLibDir("/usr/local/lib");
+
+            try self.addFrameworkDir("/Library/Frameworks");
+            try self.addFrameworkDir("/System/Library/Frameworks");
+
+            return self;
+        }
+
         if (!is_windows) {
             const triple = try Target.current.linuxTriple(allocator);
             const qual = Target.current.cpu.arch.ptrBitWidth();
@@ -122,6 +137,7 @@ pub const NativePaths = struct {
     pub fn deinit(self: *NativePaths) void {
         deinitArray(&self.include_dirs);
         deinitArray(&self.lib_dirs);
+        deinitArray(&self.framework_dirs);
         deinitArray(&self.rpaths);
         deinitArray(&self.warnings);
         self.* = undefined;
@@ -158,6 +174,16 @@ pub const NativePaths = struct {
         return self.appendArray(&self.warnings, s);
     }
 
+    pub fn addFrameworkDir(self: *NativePaths, s: []const u8) !void {
+        return self.appendArray(&self.framework_dirs, s);
+    }
+
+    pub fn addFrameworkDirFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void {
+        const item = try std.fmt.allocPrint0(self.framework_dirs.allocator, fmt, args);
+        errdefer self.framework_dirs.allocator.free(item);
+        try self.framework_dirs.append(item);
+    }
+
     pub fn addWarningFmt(self: *NativePaths, comptime fmt: []const u8, args: anytype) !void {
         const item = try std.fmt.allocPrint0(self.warnings.allocator, fmt, args);
         errdefer self.warnings.allocator.free(item);
@@ -237,8 +263,7 @@ pub const NativeTargetInfo = struct {
                     //   `---` `` ``--> Sub-version (Starting from Windows 10 onwards)
                     //     \    `--> Service pack (Always zero in the constants defined)
                     //      `--> OS version (Major & minor)
-                    const os_ver: u16 =
-                        @intCast(u16, version_info.dwMajorVersion & 0xff) << 8 |
+                    const os_ver: u16 = @intCast(u16, version_info.dwMajorVersion & 0xff) << 8 |
                         @intCast(u16, version_info.dwMinorVersion & 0xff);
                     const sp_ver: u8 = 0;
                     const sub_ver: u8 = if (os_ver >= 0x0A00) subver: {
diff --git a/src/Compilation.zig b/src/Compilation.zig
index cd3db84ec2..4fd0e54aa8 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2079,12 +2079,6 @@ pub fn addCCArgs(
         try argv.append("-ffunction-sections");
     }
 
-    try argv.ensureCapacity(argv.items.len + comp.bin_file.options.framework_dirs.len * 2);
-    for (comp.bin_file.options.framework_dirs) |framework_dir| {
-        argv.appendAssumeCapacity("-iframework");
-        argv.appendAssumeCapacity(framework_dir);
-    }
-
     if (comp.bin_file.options.link_libcpp) {
         const libcxx_include_path = try std.fs.path.join(arena, &[_][]const u8{
             comp.zig_lib_directory.path.?, "libcxx", "include",
@@ -2894,6 +2888,7 @@ fn buildOutputFromZig(
         .directory = null, // Put it in the cache directory.
         .basename = bin_basename,
     };
+
     const sub_compilation = try Compilation.create(comp.gpa, .{
         .global_cache_directory = comp.global_cache_directory,
         .local_cache_directory = comp.global_cache_directory,
diff --git a/src/main.zig b/src/main.zig
index 247f8327f0..096fa86c75 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1436,11 +1436,35 @@ fn buildOutputType(
         for (paths.warnings.items) |warning| {
             warn("{}", .{warning});
         }
+
+        const has_sysroot = if (comptime std.Target.current.isDarwin()) outer: {
+            const at_least_big_sur = target_info.target.os.getVersionRange().semver.min.major >= 11;
+            if (at_least_big_sur) {
+                const sdk_path = try std.zig.system.getSDKPath(arena);
+                try clang_argv.ensureCapacity(clang_argv.items.len + 2);
+                clang_argv.appendAssumeCapacity("-isysroot");
+                clang_argv.appendAssumeCapacity(sdk_path);
+                break :outer true;
+            }
+            break :outer false;
+        } else false;
+
         try clang_argv.ensureCapacity(clang_argv.items.len + paths.include_dirs.items.len * 2);
+        const isystem_flag = if (has_sysroot) "-iwithsysroot" else "-isystem";
         for (paths.include_dirs.items) |include_dir| {
-            clang_argv.appendAssumeCapacity("-isystem");
+            clang_argv.appendAssumeCapacity(isystem_flag);
             clang_argv.appendAssumeCapacity(include_dir);
         }
+
+        try clang_argv.ensureCapacity(clang_argv.items.len + paths.framework_dirs.items.len * 2);
+        try framework_dirs.ensureCapacity(framework_dirs.items.len + paths.framework_dirs.items.len);
+        const iframework_flag = if (has_sysroot) "-iframeworkwithsysroot" else "-iframework";
+        for (paths.framework_dirs.items) |framework_dir| {
+            clang_argv.appendAssumeCapacity(iframework_flag);
+            clang_argv.appendAssumeCapacity(framework_dir);
+            framework_dirs.appendAssumeCapacity(framework_dir);
+        }
+
         for (paths.lib_dirs.items) |lib_dir| {
             try lib_dirs.append(lib_dir);
         }
-- 
cgit v1.2.3


From 763d8073774ae0b77014187a75b2167522c44079 Mon Sep 17 00:00:00 2001
From: Jakub Konka <kubkon@jakubkonka.com>
Date: Sun, 20 Dec 2020 11:52:25 +0100
Subject: Duplicate OSAtomic.h between aarch64 and x86_64

The reason this is required is for two reasons: 1) the libc
targeting `aarch64` macOS is slightly newer than that targeting
`x86_64`, and 2) `OSAtomic.h` uses relative imports rather than
system-wide imports for accompanying headers which clearly is an
oversight on Apple's part. Until such time when `libkern` headers
between `x86_64` and `aarch64` are identical, this will require a
manual intervention to duplicate the relevant headers between the
respective architectures.
---
 .../include/aarch64-macos-gnu/libkern/OSAtomic.h   |  47 +++++
 .../libkern/OSSpinLockDeprecated.h                 | 212 +++++++++++++++++++++
 lib/libc/include/any-macos-any/libkern/OSAtomic.h  |  47 -----
 .../any-macos-any/libkern/OSSpinLockDeprecated.h   | 212 ---------------------
 .../include/x86_64-macos-gnu/libkern/OSAtomic.h    |  47 +++++
 .../libkern/OSSpinLockDeprecated.h                 | 212 +++++++++++++++++++++
 src/Compilation.zig                                |   1 -
 7 files changed, 518 insertions(+), 260 deletions(-)
 create mode 100644 lib/libc/include/aarch64-macos-gnu/libkern/OSAtomic.h
 create mode 100644 lib/libc/include/aarch64-macos-gnu/libkern/OSSpinLockDeprecated.h
 delete mode 100644 lib/libc/include/any-macos-any/libkern/OSAtomic.h
 delete mode 100644 lib/libc/include/any-macos-any/libkern/OSSpinLockDeprecated.h
 create mode 100644 lib/libc/include/x86_64-macos-gnu/libkern/OSAtomic.h
 create mode 100644 lib/libc/include/x86_64-macos-gnu/libkern/OSSpinLockDeprecated.h

(limited to 'src/Compilation.zig')

diff --git a/lib/libc/include/aarch64-macos-gnu/libkern/OSAtomic.h b/lib/libc/include/aarch64-macos-gnu/libkern/OSAtomic.h
new file mode 100644
index 0000000000..37ef16ce44
--- /dev/null
+++ b/lib/libc/include/aarch64-macos-gnu/libkern/OSAtomic.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OSATOMIC_H_
+#define _OSATOMIC_H_
+
+/*! @header
+ * These are deprecated legacy interfaces for atomic and synchronization
+ * operations.
+ *
+ * Define OSATOMIC_USE_INLINED=1 to get inline implementations of the
+ * OSAtomic interfaces in terms of the <stdatomic.h> primitives.
+ *
+ * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of the
+ * OSSpinLock interfaces in terms of the <os/lock.h> primitives.
+ *
+ * These are intended as a transition convenience, direct use of those
+ * primitives should be preferred.
+ */
+
+#include <sys/cdefs.h>
+
+#include "OSAtomicDeprecated.h"
+#include "OSSpinLockDeprecated.h"
+#include "OSAtomicQueue.h"
+
+#endif /* _OSATOMIC_H_ */
\ No newline at end of file
diff --git a/lib/libc/include/aarch64-macos-gnu/libkern/OSSpinLockDeprecated.h b/lib/libc/include/aarch64-macos-gnu/libkern/OSSpinLockDeprecated.h
new file mode 100644
index 0000000000..a654a7bbc2
--- /dev/null
+++ b/lib/libc/include/aarch64-macos-gnu/libkern/OSSpinLockDeprecated.h
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OSSPINLOCK_DEPRECATED_H_
+#define _OSSPINLOCK_DEPRECATED_H_
+
+/*! @header
+ * These are deprecated legacy interfaces for userspace spinlocks.
+ *
+ * These interfaces should no longer be used, particularily in situations where
+ * threads of differing priorities may contend on the same spinlock.
+ *
+ * The interfaces in <os/lock.h> should be used instead in cases where a very
+ * low-level lock primitive is required. In general however, using higher level
+ * synchronization primitives such as those provided by the pthread or dispatch
+ * subsystems should be preferred.
+ *
+ * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of these
+ * interfaces in terms of the <os/lock.h> primitives. This is intended as a
+ * transition convenience, direct use of those primitives is preferred.
+ */
+
+#ifndef OSSPINLOCK_DEPRECATED
+#define OSSPINLOCK_DEPRECATED 1
+#define OSSPINLOCK_DEPRECATED_MSG(_r) "Use " #_r "() from <os/lock.h> instead"
+#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r) \
+	__OS_AVAILABILITY_MSG(macosx, deprecated=10.12, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(ios, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(tvos, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(watchos, deprecated=3.0, OSSPINLOCK_DEPRECATED_MSG(_r))
+#else
+#undef OSSPINLOCK_DEPRECATED
+#define OSSPINLOCK_DEPRECATED 0
+#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r)
+#endif
+
+#if !(defined(OSSPINLOCK_USE_INLINED) && OSSPINLOCK_USE_INLINED)
+
+#include    <sys/cdefs.h>
+#include    <stddef.h>
+#include    <stdint.h>
+#include    <stdbool.h>
+#include    <Availability.h>
+
+__BEGIN_DECLS
+
+/*! @abstract The default value for an <code>OSSpinLock</code>.
+    @discussion
+	The convention is that unlocked is zero, locked is nonzero.
+ */
+#define	OS_SPINLOCK_INIT    0
+
+
+/*! @abstract Data type for a spinlock.
+    @discussion
+	You should always initialize a spinlock to {@link OS_SPINLOCK_INIT} before
+	using it.
+ */
+typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
+
+
+/*! @abstract Locks a spinlock if it would not block
+    @result
+	Returns <code>false</code> if the lock was already held by another thread,
+	<code>true</code> if it took the lock successfully.
+ */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+bool    OSSpinLockTry( volatile OSSpinLock *__lock );
+
+
+/*! @abstract Locks a spinlock
+    @discussion
+	Although the lock operation spins, it employs various strategies to back
+	off if the lock is held.
+ */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+void    OSSpinLockLock( volatile OSSpinLock *__lock );
+
+
+/*! @abstract Unlocks a spinlock */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+void    OSSpinLockUnlock( volatile OSSpinLock *__lock );
+
+__END_DECLS
+
+#else /* OSSPINLOCK_USE_INLINED */
+
+/*
+ * Inline implementations of the legacy OSSpinLock interfaces in terms of the
+ * of the <os/lock.h> primitives. Direct use of those primitives is preferred.
+ *
+ * NOTE: the locked value of os_unfair_lock is implementation defined and
+ * subject to change, code that relies on the specific locked value used by the
+ * legacy OSSpinLock interface WILL break when using these inline
+ * implementations in terms of os_unfair_lock.
+ */
+
+#if !OSSPINLOCK_USE_INLINED_TRANSPARENT
+
+#include <os/lock.h>
+
+__BEGIN_DECLS
+
+#if __has_attribute(always_inline)
+#define OSSPINLOCK_INLINE static __inline
+#else
+#define OSSPINLOCK_INLINE static __inline __attribute__((__always_inline__))
+#endif
+
+#define OS_SPINLOCK_INIT 0
+typedef int32_t OSSpinLock;
+
+#if  __has_extension(c_static_assert)
+_Static_assert(sizeof(OSSpinLock) == sizeof(os_unfair_lock),
+		"Incompatible os_unfair_lock type");
+#endif
+
+OSSPINLOCK_INLINE
+void
+OSSpinLockLock(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_lock(lock);
+}
+
+OSSPINLOCK_INLINE
+bool
+OSSpinLockTry(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_trylock(lock);
+}
+
+OSSPINLOCK_INLINE
+void
+OSSpinLockUnlock(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_unlock(lock);
+}
+
+#undef OSSPINLOCK_INLINE
+
+__END_DECLS
+
+#else /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
+
+#include    <sys/cdefs.h>
+#include    <stddef.h>
+#include    <stdint.h>
+#include    <stdbool.h>
+#include    <Availability.h>
+
+#define OS_NOSPIN_LOCK_AVAILABILITY \
+		__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) \
+		__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0)
+
+__BEGIN_DECLS
+
+#define OS_SPINLOCK_INIT 0
+typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
+typedef volatile OSSpinLock *_os_nospin_lock_t
+		OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_t);
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
+OS_NOSPIN_LOCK_AVAILABILITY
+void _os_nospin_lock_lock(_os_nospin_lock_t lock);
+#undef OSSpinLockLock
+#define OSSpinLockLock(lock) _os_nospin_lock_lock(lock)
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
+OS_NOSPIN_LOCK_AVAILABILITY
+bool _os_nospin_lock_trylock(_os_nospin_lock_t lock);
+#undef OSSpinLockTry
+#define OSSpinLockTry(lock) _os_nospin_lock_trylock(lock)
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
+OS_NOSPIN_LOCK_AVAILABILITY
+void _os_nospin_lock_unlock(_os_nospin_lock_t lock);
+#undef OSSpinLockUnlock
+#define OSSpinLockUnlock(lock) _os_nospin_lock_unlock(lock)
+
+__END_DECLS
+
+#endif /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
+
+#endif /* OSSPINLOCK_USE_INLINED */
+
+#endif /* _OSSPINLOCK_DEPRECATED_H_ */
\ No newline at end of file
diff --git a/lib/libc/include/any-macos-any/libkern/OSAtomic.h b/lib/libc/include/any-macos-any/libkern/OSAtomic.h
deleted file mode 100644
index 37ef16ce44..0000000000
--- a/lib/libc/include/any-macos-any/libkern/OSAtomic.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#ifndef _OSATOMIC_H_
-#define _OSATOMIC_H_
-
-/*! @header
- * These are deprecated legacy interfaces for atomic and synchronization
- * operations.
- *
- * Define OSATOMIC_USE_INLINED=1 to get inline implementations of the
- * OSAtomic interfaces in terms of the <stdatomic.h> primitives.
- *
- * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of the
- * OSSpinLock interfaces in terms of the <os/lock.h> primitives.
- *
- * These are intended as a transition convenience, direct use of those
- * primitives should be preferred.
- */
-
-#include <sys/cdefs.h>
-
-#include "OSAtomicDeprecated.h"
-#include "OSSpinLockDeprecated.h"
-#include "OSAtomicQueue.h"
-
-#endif /* _OSATOMIC_H_ */
\ No newline at end of file
diff --git a/lib/libc/include/any-macos-any/libkern/OSSpinLockDeprecated.h b/lib/libc/include/any-macos-any/libkern/OSSpinLockDeprecated.h
deleted file mode 100644
index a654a7bbc2..0000000000
--- a/lib/libc/include/any-macos-any/libkern/OSSpinLockDeprecated.h
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#ifndef _OSSPINLOCK_DEPRECATED_H_
-#define _OSSPINLOCK_DEPRECATED_H_
-
-/*! @header
- * These are deprecated legacy interfaces for userspace spinlocks.
- *
- * These interfaces should no longer be used, particularily in situations where
- * threads of differing priorities may contend on the same spinlock.
- *
- * The interfaces in <os/lock.h> should be used instead in cases where a very
- * low-level lock primitive is required. In general however, using higher level
- * synchronization primitives such as those provided by the pthread or dispatch
- * subsystems should be preferred.
- *
- * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of these
- * interfaces in terms of the <os/lock.h> primitives. This is intended as a
- * transition convenience, direct use of those primitives is preferred.
- */
-
-#ifndef OSSPINLOCK_DEPRECATED
-#define OSSPINLOCK_DEPRECATED 1
-#define OSSPINLOCK_DEPRECATED_MSG(_r) "Use " #_r "() from <os/lock.h> instead"
-#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r) \
-	__OS_AVAILABILITY_MSG(macosx, deprecated=10.12, OSSPINLOCK_DEPRECATED_MSG(_r)) \
-	__OS_AVAILABILITY_MSG(ios, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
-	__OS_AVAILABILITY_MSG(tvos, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
-	__OS_AVAILABILITY_MSG(watchos, deprecated=3.0, OSSPINLOCK_DEPRECATED_MSG(_r))
-#else
-#undef OSSPINLOCK_DEPRECATED
-#define OSSPINLOCK_DEPRECATED 0
-#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r)
-#endif
-
-#if !(defined(OSSPINLOCK_USE_INLINED) && OSSPINLOCK_USE_INLINED)
-
-#include    <sys/cdefs.h>
-#include    <stddef.h>
-#include    <stdint.h>
-#include    <stdbool.h>
-#include    <Availability.h>
-
-__BEGIN_DECLS
-
-/*! @abstract The default value for an <code>OSSpinLock</code>.
-    @discussion
-	The convention is that unlocked is zero, locked is nonzero.
- */
-#define	OS_SPINLOCK_INIT    0
-
-
-/*! @abstract Data type for a spinlock.
-    @discussion
-	You should always initialize a spinlock to {@link OS_SPINLOCK_INIT} before
-	using it.
- */
-typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
-
-
-/*! @abstract Locks a spinlock if it would not block
-    @result
-	Returns <code>false</code> if the lock was already held by another thread,
-	<code>true</code> if it took the lock successfully.
- */
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
-__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
-bool    OSSpinLockTry( volatile OSSpinLock *__lock );
-
-
-/*! @abstract Locks a spinlock
-    @discussion
-	Although the lock operation spins, it employs various strategies to back
-	off if the lock is held.
- */
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
-__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
-void    OSSpinLockLock( volatile OSSpinLock *__lock );
-
-
-/*! @abstract Unlocks a spinlock */
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
-__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
-void    OSSpinLockUnlock( volatile OSSpinLock *__lock );
-
-__END_DECLS
-
-#else /* OSSPINLOCK_USE_INLINED */
-
-/*
- * Inline implementations of the legacy OSSpinLock interfaces in terms of the
- * of the <os/lock.h> primitives. Direct use of those primitives is preferred.
- *
- * NOTE: the locked value of os_unfair_lock is implementation defined and
- * subject to change, code that relies on the specific locked value used by the
- * legacy OSSpinLock interface WILL break when using these inline
- * implementations in terms of os_unfair_lock.
- */
-
-#if !OSSPINLOCK_USE_INLINED_TRANSPARENT
-
-#include <os/lock.h>
-
-__BEGIN_DECLS
-
-#if __has_attribute(always_inline)
-#define OSSPINLOCK_INLINE static __inline
-#else
-#define OSSPINLOCK_INLINE static __inline __attribute__((__always_inline__))
-#endif
-
-#define OS_SPINLOCK_INIT 0
-typedef int32_t OSSpinLock;
-
-#if  __has_extension(c_static_assert)
-_Static_assert(sizeof(OSSpinLock) == sizeof(os_unfair_lock),
-		"Incompatible os_unfair_lock type");
-#endif
-
-OSSPINLOCK_INLINE
-void
-OSSpinLockLock(volatile OSSpinLock *__lock)
-{
-	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
-	return os_unfair_lock_lock(lock);
-}
-
-OSSPINLOCK_INLINE
-bool
-OSSpinLockTry(volatile OSSpinLock *__lock)
-{
-	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
-	return os_unfair_lock_trylock(lock);
-}
-
-OSSPINLOCK_INLINE
-void
-OSSpinLockUnlock(volatile OSSpinLock *__lock)
-{
-	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
-	return os_unfair_lock_unlock(lock);
-}
-
-#undef OSSPINLOCK_INLINE
-
-__END_DECLS
-
-#else /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
-
-#include    <sys/cdefs.h>
-#include    <stddef.h>
-#include    <stdint.h>
-#include    <stdbool.h>
-#include    <Availability.h>
-
-#define OS_NOSPIN_LOCK_AVAILABILITY \
-		__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) \
-		__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0)
-
-__BEGIN_DECLS
-
-#define OS_SPINLOCK_INIT 0
-typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
-typedef volatile OSSpinLock *_os_nospin_lock_t
-		OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_t);
-
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
-OS_NOSPIN_LOCK_AVAILABILITY
-void _os_nospin_lock_lock(_os_nospin_lock_t lock);
-#undef OSSpinLockLock
-#define OSSpinLockLock(lock) _os_nospin_lock_lock(lock)
-
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
-OS_NOSPIN_LOCK_AVAILABILITY
-bool _os_nospin_lock_trylock(_os_nospin_lock_t lock);
-#undef OSSpinLockTry
-#define OSSpinLockTry(lock) _os_nospin_lock_trylock(lock)
-
-OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
-OS_NOSPIN_LOCK_AVAILABILITY
-void _os_nospin_lock_unlock(_os_nospin_lock_t lock);
-#undef OSSpinLockUnlock
-#define OSSpinLockUnlock(lock) _os_nospin_lock_unlock(lock)
-
-__END_DECLS
-
-#endif /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
-
-#endif /* OSSPINLOCK_USE_INLINED */
-
-#endif /* _OSSPINLOCK_DEPRECATED_H_ */
\ No newline at end of file
diff --git a/lib/libc/include/x86_64-macos-gnu/libkern/OSAtomic.h b/lib/libc/include/x86_64-macos-gnu/libkern/OSAtomic.h
new file mode 100644
index 0000000000..37ef16ce44
--- /dev/null
+++ b/lib/libc/include/x86_64-macos-gnu/libkern/OSAtomic.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OSATOMIC_H_
+#define _OSATOMIC_H_
+
+/*! @header
+ * These are deprecated legacy interfaces for atomic and synchronization
+ * operations.
+ *
+ * Define OSATOMIC_USE_INLINED=1 to get inline implementations of the
+ * OSAtomic interfaces in terms of the <stdatomic.h> primitives.
+ *
+ * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of the
+ * OSSpinLock interfaces in terms of the <os/lock.h> primitives.
+ *
+ * These are intended as a transition convenience, direct use of those
+ * primitives should be preferred.
+ */
+
+#include <sys/cdefs.h>
+
+#include "OSAtomicDeprecated.h"
+#include "OSSpinLockDeprecated.h"
+#include "OSAtomicQueue.h"
+
+#endif /* _OSATOMIC_H_ */
\ No newline at end of file
diff --git a/lib/libc/include/x86_64-macos-gnu/libkern/OSSpinLockDeprecated.h b/lib/libc/include/x86_64-macos-gnu/libkern/OSSpinLockDeprecated.h
new file mode 100644
index 0000000000..a654a7bbc2
--- /dev/null
+++ b/lib/libc/include/x86_64-macos-gnu/libkern/OSSpinLockDeprecated.h
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OSSPINLOCK_DEPRECATED_H_
+#define _OSSPINLOCK_DEPRECATED_H_
+
+/*! @header
+ * These are deprecated legacy interfaces for userspace spinlocks.
+ *
+ * These interfaces should no longer be used, particularily in situations where
+ * threads of differing priorities may contend on the same spinlock.
+ *
+ * The interfaces in <os/lock.h> should be used instead in cases where a very
+ * low-level lock primitive is required. In general however, using higher level
+ * synchronization primitives such as those provided by the pthread or dispatch
+ * subsystems should be preferred.
+ *
+ * Define OSSPINLOCK_USE_INLINED=1 to get inline implementations of these
+ * interfaces in terms of the <os/lock.h> primitives. This is intended as a
+ * transition convenience, direct use of those primitives is preferred.
+ */
+
+#ifndef OSSPINLOCK_DEPRECATED
+#define OSSPINLOCK_DEPRECATED 1
+#define OSSPINLOCK_DEPRECATED_MSG(_r) "Use " #_r "() from <os/lock.h> instead"
+#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r) \
+	__OS_AVAILABILITY_MSG(macosx, deprecated=10.12, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(ios, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(tvos, deprecated=10.0, OSSPINLOCK_DEPRECATED_MSG(_r)) \
+	__OS_AVAILABILITY_MSG(watchos, deprecated=3.0, OSSPINLOCK_DEPRECATED_MSG(_r))
+#else
+#undef OSSPINLOCK_DEPRECATED
+#define OSSPINLOCK_DEPRECATED 0
+#define OSSPINLOCK_DEPRECATED_REPLACE_WITH(_r)
+#endif
+
+#if !(defined(OSSPINLOCK_USE_INLINED) && OSSPINLOCK_USE_INLINED)
+
+#include    <sys/cdefs.h>
+#include    <stddef.h>
+#include    <stdint.h>
+#include    <stdbool.h>
+#include    <Availability.h>
+
+__BEGIN_DECLS
+
+/*! @abstract The default value for an <code>OSSpinLock</code>.
+    @discussion
+	The convention is that unlocked is zero, locked is nonzero.
+ */
+#define	OS_SPINLOCK_INIT    0
+
+
+/*! @abstract Data type for a spinlock.
+    @discussion
+	You should always initialize a spinlock to {@link OS_SPINLOCK_INIT} before
+	using it.
+ */
+typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
+
+
+/*! @abstract Locks a spinlock if it would not block
+    @result
+	Returns <code>false</code> if the lock was already held by another thread,
+	<code>true</code> if it took the lock successfully.
+ */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+bool    OSSpinLockTry( volatile OSSpinLock *__lock );
+
+
+/*! @abstract Locks a spinlock
+    @discussion
+	Although the lock operation spins, it employs various strategies to back
+	off if the lock is held.
+ */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+void    OSSpinLockLock( volatile OSSpinLock *__lock );
+
+
+/*! @abstract Unlocks a spinlock */
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
+__OSX_AVAILABLE_STARTING(__MAC_10_4, __IPHONE_2_0)
+void    OSSpinLockUnlock( volatile OSSpinLock *__lock );
+
+__END_DECLS
+
+#else /* OSSPINLOCK_USE_INLINED */
+
+/*
+ * Inline implementations of the legacy OSSpinLock interfaces in terms of the
+ * of the <os/lock.h> primitives. Direct use of those primitives is preferred.
+ *
+ * NOTE: the locked value of os_unfair_lock is implementation defined and
+ * subject to change, code that relies on the specific locked value used by the
+ * legacy OSSpinLock interface WILL break when using these inline
+ * implementations in terms of os_unfair_lock.
+ */
+
+#if !OSSPINLOCK_USE_INLINED_TRANSPARENT
+
+#include <os/lock.h>
+
+__BEGIN_DECLS
+
+#if __has_attribute(always_inline)
+#define OSSPINLOCK_INLINE static __inline
+#else
+#define OSSPINLOCK_INLINE static __inline __attribute__((__always_inline__))
+#endif
+
+#define OS_SPINLOCK_INIT 0
+typedef int32_t OSSpinLock;
+
+#if  __has_extension(c_static_assert)
+_Static_assert(sizeof(OSSpinLock) == sizeof(os_unfair_lock),
+		"Incompatible os_unfair_lock type");
+#endif
+
+OSSPINLOCK_INLINE
+void
+OSSpinLockLock(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_lock(lock);
+}
+
+OSSPINLOCK_INLINE
+bool
+OSSpinLockTry(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_trylock(lock);
+}
+
+OSSPINLOCK_INLINE
+void
+OSSpinLockUnlock(volatile OSSpinLock *__lock)
+{
+	os_unfair_lock_t lock = (os_unfair_lock_t)__lock;
+	return os_unfair_lock_unlock(lock);
+}
+
+#undef OSSPINLOCK_INLINE
+
+__END_DECLS
+
+#else /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
+
+#include    <sys/cdefs.h>
+#include    <stddef.h>
+#include    <stdint.h>
+#include    <stdbool.h>
+#include    <Availability.h>
+
+#define OS_NOSPIN_LOCK_AVAILABILITY \
+		__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) \
+		__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0)
+
+__BEGIN_DECLS
+
+#define OS_SPINLOCK_INIT 0
+typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock);
+typedef volatile OSSpinLock *_os_nospin_lock_t
+		OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_t);
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock)
+OS_NOSPIN_LOCK_AVAILABILITY
+void _os_nospin_lock_lock(_os_nospin_lock_t lock);
+#undef OSSpinLockLock
+#define OSSpinLockLock(lock) _os_nospin_lock_lock(lock)
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_trylock)
+OS_NOSPIN_LOCK_AVAILABILITY
+bool _os_nospin_lock_trylock(_os_nospin_lock_t lock);
+#undef OSSpinLockTry
+#define OSSpinLockTry(lock) _os_nospin_lock_trylock(lock)
+
+OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_unlock)
+OS_NOSPIN_LOCK_AVAILABILITY
+void _os_nospin_lock_unlock(_os_nospin_lock_t lock);
+#undef OSSpinLockUnlock
+#define OSSpinLockUnlock(lock) _os_nospin_lock_unlock(lock)
+
+__END_DECLS
+
+#endif /* OSSPINLOCK_USE_INLINED_TRANSPARENT */
+
+#endif /* OSSPINLOCK_USE_INLINED */
+
+#endif /* _OSSPINLOCK_DEPRECATED_H_ */
\ No newline at end of file
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 4fd0e54aa8..33376d46d2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2888,7 +2888,6 @@ fn buildOutputFromZig(
         .directory = null, // Put it in the cache directory.
         .basename = bin_basename,
     };
-
     const sub_compilation = try Compilation.create(comp.gpa, .{
         .global_cache_directory = comp.global_cache_directory,
         .local_cache_directory = comp.global_cache_directory,
-- 
cgit v1.2.3


From 1c13ca5a05978011283ff55a586443b10b69fc85 Mon Sep 17 00:00:00 2001
From: LemonBoy <thatlemon@gmail.com>
Date: Thu, 26 Nov 2020 13:19:30 +0100
Subject: stage2: Use {s} instead of {} when formatting strings

---
 lib/std/meta/trait.zig    |  14 +++
 src/Cache.zig             |   4 +-
 src/Compilation.zig       |  94 +++++++++---------
 src/DepTokenizer.zig      |   6 +-
 src/Module.zig            |  16 ++--
 src/astgen.zig            |   8 +-
 src/codegen.zig           |  38 ++++----
 src/codegen/c.zig         |  15 +--
 src/codegen/llvm.zig      | 125 ++++++++++++++++++++++++
 src/glibc.zig             |  26 ++---
 src/libc_installation.zig |  32 +++----
 src/link.zig              |   8 +-
 src/link/C.zig            |   5 +-
 src/link/Coff.zig         |  10 +-
 src/link/Elf.zig          |  18 ++--
 src/link/MachO.zig        |  22 ++---
 src/link/Wasm.zig         |   6 +-
 src/main.zig              | 236 +++++++++++++++++++++++-----------------------
 src/mingw.zig             |   4 +-
 src/musl.zig              |   8 +-
 src/print_env.zig         |   2 +-
 src/print_targets.zig     |   4 +-
 src/stage1.zig            |   4 +-
 src/translate_c.zig       |  76 +++++++--------
 src/value.zig             |   4 +-
 src/zir.zig               |  44 ++++-----
 src/zir_sema.zig          |  34 +++----
 27 files changed, 503 insertions(+), 360 deletions(-)
 create mode 100644 src/codegen/llvm.zig

(limited to 'src/Compilation.zig')

diff --git a/lib/std/meta/trait.zig b/lib/std/meta/trait.zig
index 8c8b26cf45..8e54293533 100644
--- a/lib/std/meta/trait.zig
+++ b/lib/std/meta/trait.zig
@@ -298,6 +298,20 @@ pub fn isNumber(comptime T: type) bool {
     };
 }
 
+pub fn isIntegerNumber(comptime T: type) bool {
+    return switch (@typeInfo(T)) {
+        .Int, .ComptimeInt => true,
+        else => false,
+    };
+}
+
+pub fn isFloatingNumber(comptime T: type) bool {
+    return switch (@typeInfo(T)) {
+        .Float, .ComptimeFloat => true,
+        else => false,
+    };
+}
+
 test "std.meta.trait.isNumber" {
     const NotANumber = struct {
         number: u8,
diff --git a/src/Cache.zig b/src/Cache.zig
index 3d33226f27..03a0d61157 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -549,7 +549,7 @@ pub const Manifest = struct {
             .target, .target_must_resolve, .prereq => {},
             else => |err| {
                 try err.printError(error_buf.writer());
-                std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
+                std.log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
                 return error.InvalidDepFile;
             },
         }
@@ -561,7 +561,7 @@ pub const Manifest = struct {
                 .prereq => |bytes| try self.addFilePost(bytes),
                 else => |err| {
                     try err.printError(error_buf.writer());
-                    std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
+                    std.log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
                     return error.InvalidDepFile;
                 },
             }
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 33376d46d2..3a9c1ea993 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1475,7 +1475,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     // lifetime annotations in the ZIR.
                     var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa);
                     defer decl.typed_value.most_recent.arena.?.* = decl_arena.state;
-                    log.debug("analyze liveness of {}\n", .{decl.name});
+                    log.debug("analyze liveness of {s}\n", .{decl.name});
                     try liveness.analyze(module.gpa, &decl_arena.allocator, func.analysis.success);
                 }
 
@@ -1492,7 +1492,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
-                                "unable to codegen: {}",
+                                "unable to codegen: {s}",
                                 .{@errorName(err)},
                             ));
                             decl.analysis = .codegen_failure_retryable;
@@ -1535,7 +1535,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                     module.gpa,
                     decl.src(),
-                    "unable to update line number: {}",
+                    "unable to update line number: {s}",
                     .{@errorName(err)},
                 ));
                 decl.analysis = .codegen_failure_retryable;
@@ -1544,50 +1544,50 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
         .glibc_crt_file => |crt_file| {
             glibc.buildCRTFile(self, crt_file) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build glibc CRT file: {}", .{@errorName(err)});
+                fatal("unable to build glibc CRT file: {s}", .{@errorName(err)});
             };
         },
         .glibc_shared_objects => {
             glibc.buildSharedObjects(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build glibc shared objects: {}", .{@errorName(err)});
+                fatal("unable to build glibc shared objects: {s}", .{@errorName(err)});
             };
         },
         .musl_crt_file => |crt_file| {
             musl.buildCRTFile(self, crt_file) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build musl CRT file: {}", .{@errorName(err)});
+                fatal("unable to build musl CRT file: {s}", .{@errorName(err)});
             };
         },
         .mingw_crt_file => |crt_file| {
             mingw.buildCRTFile(self, crt_file) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build mingw-w64 CRT file: {}", .{@errorName(err)});
+                fatal("unable to build mingw-w64 CRT file: {s}", .{@errorName(err)});
             };
         },
         .windows_import_lib => |index| {
             const link_lib = self.bin_file.options.system_libs.items()[index].key;
             mingw.buildImportLib(self, link_lib) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to generate DLL import .lib file: {}", .{@errorName(err)});
+                fatal("unable to generate DLL import .lib file: {s}", .{@errorName(err)});
             };
         },
         .libunwind => {
             libunwind.buildStaticLib(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build libunwind: {}", .{@errorName(err)});
+                fatal("unable to build libunwind: {s}", .{@errorName(err)});
             };
         },
         .libcxx => {
             libcxx.buildLibCXX(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build libcxx: {}", .{@errorName(err)});
+                fatal("unable to build libcxx: {s}", .{@errorName(err)});
             };
         },
         .libcxxabi => {
             libcxx.buildLibCXXABI(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build libcxxabi: {}", .{@errorName(err)});
+                fatal("unable to build libcxxabi: {s}", .{@errorName(err)});
             };
         },
         .libtsan => {
@@ -1611,20 +1611,20 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
         .libssp => {
             self.buildOutputFromZig("ssp.zig", .Lib, &self.libssp_static_lib) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build libssp: {}", .{@errorName(err)});
+                fatal("unable to build libssp: {s}", .{@errorName(err)});
             };
         },
         .zig_libc => {
             self.buildOutputFromZig("c.zig", .Lib, &self.libc_static_lib) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build zig's multitarget libc: {}", .{@errorName(err)});
+                fatal("unable to build zig's multitarget libc: {s}", .{@errorName(err)});
             };
         },
         .generate_builtin_zig => {
             // This Job is only queued up if there is a zig module.
             self.updateBuiltinZigFile(self.bin_file.options.module.?) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to update builtin.zig file: {}", .{@errorName(err)});
+                fatal("unable to update builtin.zig file: {s}", .{@errorName(err)});
             };
         },
         .stage1_module => {
@@ -1704,11 +1704,11 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{
             tmp_dir_sub_path, cimport_basename,
         });
-        const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path});
+        const out_dep_path = try std.fmt.allocPrint(arena, "{s}.d", .{out_h_path});
 
         try zig_cache_tmp_dir.writeFile(cimport_basename, c_src);
         if (comp.verbose_cimport) {
-            log.info("C import source: {}", .{out_h_path});
+            log.info("C import source: {s}", .{out_h_path});
         }
 
         var argv = std.ArrayList([]const u8).init(comp.gpa);
@@ -1755,7 +1755,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         defer tree.deinit();
 
         if (comp.verbose_cimport) {
-            log.info("C import .d file: {}", .{out_dep_path});
+            log.info("C import .d file: {s}", .{out_dep_path});
         }
 
         const dep_basename = std.fs.path.basename(out_dep_path);
@@ -1775,7 +1775,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         try bos.flush();
 
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest for C import: {}", .{@errorName(err)});
+            log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)});
         };
 
         break :digest digest;
@@ -1785,7 +1785,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         "o", &digest, cimport_zig_basename,
     });
     if (comp.verbose_cimport) {
-        log.info("C import output: {}\n", .{out_zig_path});
+        log.info("C import output: {s}\n", .{out_zig_path});
     }
     return CImportResult{
         .out_zig_path = out_zig_path,
@@ -1946,7 +1946,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
             child.stderr_behavior = .Inherit;
 
             const term = child.spawnAndWait() catch |err| {
-                return comp.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) });
+                return comp.failCObj(c_object, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
             };
             switch (term) {
                 .Exited => |code| {
@@ -1974,7 +1974,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
             const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024);
 
             const term = child.wait() catch |err| {
-                return comp.failCObj(c_object, "unable to spawn {}: {}", .{ argv.items[0], @errorName(err) });
+                return comp.failCObj(c_object, "unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
             };
 
             switch (term) {
@@ -1982,12 +1982,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
                     if (code != 0) {
                         // TODO parse clang stderr and turn it into an error message
                         // and then call failCObjWithOwnedErrorMsg
-                        log.err("clang failed with stderr: {}", .{stderr});
+                        log.err("clang failed with stderr: {s}", .{stderr});
                         return comp.failCObj(c_object, "clang exited with code {}", .{code});
                     }
                 },
                 else => {
-                    log.err("clang terminated with stderr: {}", .{stderr});
+                    log.err("clang terminated with stderr: {s}", .{stderr});
                     return comp.failCObj(c_object, "clang terminated unexpectedly", .{});
                 },
             }
@@ -1999,7 +1999,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
             try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
             // Just to save disk space, we delete the file because it is never needed again.
             zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| {
-                log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) });
+                log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) });
             };
         }
 
@@ -2015,7 +2015,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
         try std.fs.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename);
 
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) });
+            log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ c_object.src.src_path, @errorName(err) });
         };
         break :blk digest;
     };
@@ -2034,7 +2034,7 @@ pub fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) er
     const s = std.fs.path.sep_str;
     const rand_int = std.crypto.random.int(u64);
     if (comp.local_cache_directory.path) |p| {
-        return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
+        return std.fmt.allocPrint(arena, "{s}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix });
     } else {
         return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix });
     }
@@ -2144,7 +2144,7 @@ pub fn addCCArgs(
             }
             const mcmodel = comp.bin_file.options.machine_code_model;
             if (mcmodel != .default) {
-                try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={}", .{@tagName(mcmodel)}));
+                try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={s}", .{@tagName(mcmodel)}));
             }
 
             switch (target.os.tag) {
@@ -2497,22 +2497,22 @@ fn detectLibCIncludeDirs(
         const s = std.fs.path.sep_str;
         const arch_include_dir = try std.fmt.allocPrint(
             arena,
-            "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-{}",
+            "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}",
             .{ zig_lib_dir, arch_name, os_name, abi_name },
         );
         const generic_include_dir = try std.fmt.allocPrint(
             arena,
-            "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{}",
+            "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}",
             .{ zig_lib_dir, generic_name },
         );
         const arch_os_include_dir = try std.fmt.allocPrint(
             arena,
-            "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-any",
+            "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any",
             .{ zig_lib_dir, @tagName(target.cpu.arch), os_name },
         );
         const generic_os_include_dir = try std.fmt.allocPrint(
             arena,
-            "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{}-any",
+            "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any",
             .{ zig_lib_dir, os_name },
         );
 
@@ -2631,9 +2631,9 @@ fn updateBuiltinZigFile(comp: *Compilation, mod: *Module) !void {
 
 pub fn dump_argv(argv: []const []const u8) void {
     for (argv[0 .. argv.len - 1]) |arg| {
-        std.debug.print("{} ", .{arg});
+        std.debug.print("{s} ", .{arg});
     }
-    std.debug.print("{}\n", .{argv[argv.len - 1]});
+    std.debug.print("{s}\n", .{argv[argv.len - 1]});
 }
 
 pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8 {
@@ -2653,15 +2653,15 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const arch = Target.current.cpu.arch;
         \\/// Deprecated
         \\pub const endian = Target.current.cpu.arch.endian();
-        \\pub const output_mode = OutputMode.{};
-        \\pub const link_mode = LinkMode.{};
+        \\pub const output_mode = OutputMode.{s};
+        \\pub const link_mode = LinkMode.{s};
         \\pub const is_test = {};
         \\pub const single_threaded = {};
-        \\pub const abi = Abi.{};
+        \\pub const abi = Abi.{s};
         \\pub const cpu: Cpu = Cpu{{
-        \\    .arch = .{},
-        \\    .model = &Target.{}.cpu.{},
-        \\    .features = Target.{}.featureSet(&[_]Target.{}.Feature{{
+        \\    .arch = .{s},
+        \\    .model = &Target.{s}.cpu.{s},
+        \\    .features = Target.{s}.featureSet(&[_]Target.{s}.Feature{{
         \\
     , .{
         @tagName(comp.bin_file.options.output_mode),
@@ -2692,7 +2692,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\    }}),
         \\}};
         \\pub const os = Os{{
-        \\    .tag = .{},
+        \\    .tag = .{s},
         \\    .version_range = .{{
     ,
         .{@tagName(target.os.tag)},
@@ -2778,8 +2778,8 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         (comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
 
     try buffer.writer().print(
-        \\pub const object_format = ObjectFormat.{};
-        \\pub const mode = Mode.{};
+        \\pub const object_format = ObjectFormat.{s};
+        \\pub const mode = Mode.{s};
         \\pub const link_libc = {};
         \\pub const link_libcpp = {};
         \\pub const have_error_return_tracing = {};
@@ -2787,7 +2787,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const position_independent_code = {};
         \\pub const position_independent_executable = {};
         \\pub const strip_debug_info = {};
-        \\pub const code_model = CodeModel.{};
+        \\pub const code_model = CodeModel.{s};
         \\
     , .{
         @tagName(comp.bin_file.options.object_format),
@@ -3013,7 +3013,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("stage1 {} new_digest={} error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
+            log.debug("stage1 {} new_digest={} error: {s}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -3189,7 +3189,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     // Update the small file with the digest. If it fails we can continue; it only
     // means that the next invocation will have an unnecessary cache miss.
     const stage1_flags_byte = @bitCast(u8, mod.stage1_flags);
-    log.debug("stage1 {} final digest={} flags={x}", .{
+    log.debug("stage1 {s} final digest={} flags={x}", .{
         mod.root_pkg.root_src_path, digest, stage1_flags_byte,
     });
     var digest_plus_flags: [digest.len + 2]u8 = undefined;
@@ -3202,11 +3202,11 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup,
     });
     Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest_plus_flags) catch |err| {
-        log.warn("failed to save stage1 hash digest file: {}", .{@errorName(err)});
+        log.warn("failed to save stage1 hash digest file: {s}", .{@errorName(err)});
     };
     // Failure here only means an unnecessary cache miss.
     man.writeManifest() catch |err| {
-        log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+        log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
     };
     // We hang on to this lock so that the output file path can be used without
     // other processes clobbering it.
diff --git a/src/DepTokenizer.zig b/src/DepTokenizer.zig
index 99db6e4b3c..c42583a786 100644
--- a/src/DepTokenizer.zig
+++ b/src/DepTokenizer.zig
@@ -366,7 +366,7 @@ pub const Token = union(enum) {
             .incomplete_quoted_prerequisite,
             .incomplete_target,
             => |index_and_bytes| {
-                try writer.print("{} '", .{self.errStr()});
+                try writer.print("{s} '", .{self.errStr()});
                 if (self == .incomplete_target) {
                     const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
                     try tmp.resolve(writer);
@@ -383,7 +383,7 @@ pub const Token = union(enum) {
             => |index_and_char| {
                 try writer.writeAll("illegal char ");
                 try printUnderstandableChar(writer, index_and_char.char);
-                try writer.print(" at position {}: {}", .{ index_and_char.index, self.errStr() });
+                try writer.print(" at position {}: {s}", .{ index_and_char.index, self.errStr() });
             },
         }
     }
@@ -943,7 +943,7 @@ fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
 
 fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
     var buf: [80]u8 = undefined;
-    var text = try std.fmt.bufPrint(buf[0..], "{} {} bytes ", .{ label, bytes.len });
+    var text = try std.fmt.bufPrint(buf[0..], "{s} {} bytes ", .{ label, bytes.len });
     try out.writeAll(text);
     var i: usize = text.len;
     const end = 79;
diff --git a/src/Module.zig b/src/Module.zig
index 5ea78d06d1..e0042d6c4d 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -953,7 +953,7 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
                 self.failed_decls.putAssumeCapacityNoClobber(decl, try Compilation.ErrorMsg.create(
                     self.gpa,
                     decl.src(),
-                    "unable to analyze: {}",
+                    "unable to analyze: {s}",
                     .{@errorName(err)},
                 ));
                 decl.analysis = .sema_failure_retryable;
@@ -1475,7 +1475,7 @@ fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module {
             if (zir_module.error_msg) |src_err_msg| {
                 self.failed_files.putAssumeCapacityNoClobber(
                     &root_scope.base,
-                    try Compilation.ErrorMsg.create(self.gpa, src_err_msg.byte_offset, "{}", .{src_err_msg.msg}),
+                    try Compilation.ErrorMsg.create(self.gpa, src_err_msg.byte_offset, "{s}", .{src_err_msg.msg}),
                 );
                 root_scope.status = .unloaded_parse_failure;
                 return error.AnalysisFail;
@@ -1581,7 +1581,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                 decl.src_index = decl_i;
                 if (deleted_decls.remove(decl) == null) {
                     decl.analysis = .sema_failure;
-                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{}'", .{decl.name});
+                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{s}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
                     try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
                 } else {
@@ -1623,7 +1623,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                 decl.src_index = decl_i;
                 if (deleted_decls.remove(decl) == null) {
                     decl.analysis = .sema_failure;
-                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{}'", .{decl.name});
+                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{s}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
                     try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
                 } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
@@ -1991,7 +1991,7 @@ pub fn analyzeExport(
         self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create(
             self.gpa,
             src,
-            "exported symbol collision: {}",
+            "exported symbol collision: {s}",
             .{symbol_name},
         ));
         // TODO: add a note
@@ -2007,7 +2007,7 @@ pub fn analyzeExport(
             self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create(
                 self.gpa,
                 src,
-                "unable to export: {}",
+                "unable to export: {s}",
                 .{@errorName(err)},
             ));
             new_export.status = .failed_retryable;
@@ -2277,7 +2277,7 @@ pub fn createAnonymousDecl(
 ) !*Decl {
     const name_index = self.getNextAnonNameIndex();
     const scope_decl = scope.decl().?;
-    const name = try std.fmt.allocPrint(self.gpa, "{}__anon_{}", .{ scope_decl.name, name_index });
+    const name = try std.fmt.allocPrint(self.gpa, "{s}__anon_{}", .{ scope_decl.name, name_index });
     defer self.gpa.free(name);
     const name_hash = scope.namespace().fullyQualifiedNameHash(name);
     const src_hash: std.zig.SrcHash = undefined;
@@ -2384,7 +2384,7 @@ pub fn analyzeDeref(self: *Module, scope: *Scope, src: usize, ptr: *Inst, ptr_sr
 
 pub fn analyzeDeclRefByName(self: *Module, scope: *Scope, src: usize, decl_name: []const u8) InnerError!*Inst {
     const decl = self.lookupDeclName(scope, decl_name) orelse
-        return self.fail(scope, src, "decl '{}' not found", .{decl_name});
+        return self.fail(scope, src, "decl '{s}' not found", .{decl_name});
     return self.analyzeDeclRef(scope, src, decl);
 }
 
diff --git a/src/astgen.zig b/src/astgen.zig
index 3670cb260e..b4e1760368 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -1955,7 +1955,7 @@ fn identifier(mod: *Module, scope: *Scope, rl: ResultLoc, ident: *ast.Node.OneTo
                 error.Overflow => return mod.failNode(
                     scope,
                     &ident.base,
-                    "primitive integer type '{}' exceeds maximum bit width of 65535",
+                    "primitive integer type '{s}' exceeds maximum bit width of 65535",
                     .{ident_name},
                 ),
                 error.InvalidCharacter => break :integer,
@@ -2010,7 +2010,7 @@ fn identifier(mod: *Module, scope: *Scope, rl: ResultLoc, ident: *ast.Node.OneTo
         return rlWrapPtr(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.DeclValInModule, .{ .decl = decl }, .{}));
     }
 
-    return mod.failNode(scope, &ident.base, "use of undeclared identifier '{}'", .{ident_name});
+    return mod.failNode(scope, &ident.base, "use of undeclared identifier '{s}'", .{ident_name});
 }
 
 fn stringLiteral(mod: *Module, scope: *Scope, str_lit: *ast.Node.OneToken) InnerError!*zir.Inst {
@@ -2204,7 +2204,7 @@ fn ensureBuiltinParamCount(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinC
         return;
 
     const s = if (count == 1) "" else "s";
-    return mod.failTok(scope, call.builtin_token, "expected {} parameter{}, found {}", .{ count, s, call.params_len });
+    return mod.failTok(scope, call.builtin_token, "expected {} parameter{s}, found {}", .{ count, s, call.params_len });
 }
 
 fn simpleCast(
@@ -2383,7 +2383,7 @@ fn builtinCall(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.Built
     } else if (mem.eql(u8, builtin_name, "@compileError")) {
         return compileError(mod, scope, call);
     } else {
-        return mod.failTok(scope, call.builtin_token, "invalid builtin function: '{}'", .{builtin_name});
+        return mod.failTok(scope, call.builtin_token, "invalid builtin function: '{s}'", .{builtin_name});
     }
 }
 
diff --git a/src/codegen.zig b/src/codegen.zig
index 9e6de711d4..6530b687e5 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -228,7 +228,7 @@ pub fn generateSymbol(
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
                     src,
-                    "TODO implement generateSymbol for type '{}'",
+                    "TODO implement generateSymbol for type '{s}'",
                     .{@tagName(t)},
                 ),
             };
@@ -2029,7 +2029,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                             });
                             break :blk 0x84;
                         },
-                        else => return self.fail(inst.base.src, "TODO implement condbr {} when condition is {}", .{ self.target.cpu.arch, @tagName(cond) }),
+                        else => return self.fail(inst.base.src, "TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
                     };
                     self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode });
                     const reloc = Reloc{ .rel32 = self.code.items.len };
@@ -2376,11 +2376,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 .arm, .armeb => {
                     for (inst.inputs) |input, i| {
                         if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input});
+                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input});
                         }
                         const reg_name = input[1 .. input.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         const arg = try self.resolveInst(inst.args[i]);
                         try self.genSetReg(inst.base.src, reg, arg);
                     }
@@ -2393,11 +2393,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                     if (inst.output) |output| {
                         if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output});
+                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output});
                         }
                         const reg_name = output[2 .. output.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         return MCValue{ .register = reg };
                     } else {
                         return MCValue.none;
@@ -2406,11 +2406,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 .aarch64 => {
                     for (inst.inputs) |input, i| {
                         if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input});
+                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input});
                         }
                         const reg_name = input[1 .. input.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         const arg = try self.resolveInst(inst.args[i]);
                         try self.genSetReg(inst.base.src, reg, arg);
                     }
@@ -2425,11 +2425,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                     if (inst.output) |output| {
                         if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output});
+                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output});
                         }
                         const reg_name = output[2 .. output.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         return MCValue{ .register = reg };
                     } else {
                         return MCValue.none;
@@ -2438,11 +2438,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 .riscv64 => {
                     for (inst.inputs) |input, i| {
                         if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input});
+                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input});
                         }
                         const reg_name = input[1 .. input.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         const arg = try self.resolveInst(inst.args[i]);
                         try self.genSetReg(inst.base.src, reg, arg);
                     }
@@ -2455,11 +2455,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                     if (inst.output) |output| {
                         if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output});
+                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output});
                         }
                         const reg_name = output[2 .. output.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         return MCValue{ .register = reg };
                     } else {
                         return MCValue.none;
@@ -2468,11 +2468,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 .x86_64, .i386 => {
                     for (inst.inputs) |input, i| {
                         if (input.len < 3 or input[0] != '{' or input[input.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{}'", .{input});
+                            return self.fail(inst.base.src, "unrecognized asm input constraint: '{s}'", .{input});
                         }
                         const reg_name = input[1 .. input.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         const arg = try self.resolveInst(inst.args[i]);
                         try self.genSetReg(inst.base.src, reg, arg);
                     }
@@ -2485,11 +2485,11 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                     if (inst.output) |output| {
                         if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
-                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{}'", .{output});
+                            return self.fail(inst.base.src, "unrecognized asm output constraint: '{s}'", .{output});
                         }
                         const reg_name = output[2 .. output.len - 1];
                         const reg = parseRegName(reg_name) orelse
-                            return self.fail(inst.base.src, "unrecognized register: '{}'", .{reg_name});
+                            return self.fail(inst.base.src, "unrecognized register: '{s}'", .{reg_name});
                         return MCValue{ .register = reg };
                     } else {
                         return MCValue.none;
@@ -3417,7 +3417,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                                             next_int_reg += 1;
                                         }
                                     },
-                                    else => return self.fail(src, "TODO implement function parameters of type {}", .{@tagName(ty.zigTypeTag())}),
+                                    else => return self.fail(src, "TODO implement function parameters of type {s}", .{@tagName(ty.zigTypeTag())}),
                                 }
                             }
                             result.stack_byte_count = next_stack_offset;
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index c6c29942d9..b97e1590e3 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -235,7 +235,7 @@ fn renderFunctionSignature(
                 try writer.writeAll(", ");
             }
             try renderType(ctx, writer, tv.ty.fnParamType(index));
-            try writer.print(" arg{}", .{index});
+            try writer.print(" arg{d}", .{index});
         }
     }
     try writer.writeByte(')');
@@ -481,8 +481,9 @@ fn genBinOp(ctx: *Context, file: *C, inst: *Inst.BinOp, operator: []const u8) !?
     const rhs = try ctx.resolveInst(inst.rhs);
     const writer = file.main.writer();
     const name = try ctx.name();
-    try renderTypeAndName(ctx, writer, inst.base.ty, name, .Const);
-    try writer.print(" = {s} {s} {s};\n", .{ lhs, operator, rhs });
+    try writer.writeAll(indentation ++ "const ");
+    try renderType(ctx, writer, inst.base.ty);
+    try writer.print(" {s} = {s} " ++ operator ++ " {s};\n", .{ name, lhs, rhs });
     return name;
 }
 
@@ -587,7 +588,7 @@ fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
             const arg = as.args[index];
             try writer.writeAll("register ");
             try renderType(ctx, writer, arg.ty);
-            try writer.print(" {}_constant __asm__(\"{}\") = ", .{ reg, reg });
+            try writer.print(" {s}_constant __asm__(\"{s}\") = ", .{ reg, reg });
             // TODO merge constant handling into inst_map as well
             if (arg.castTag(.constant)) |c| {
                 try renderValue(ctx, writer, arg.ty, c.val);
@@ -597,13 +598,13 @@ fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
                 if (!gop.found_existing) {
                     return ctx.fail(ctx.decl.src(), "Internal error in C backend: asm argument not found in inst_map", .{});
                 }
-                try writer.print("{};\n    ", .{gop.entry.value});
+                try writer.print("{s};\n    ", .{gop.entry.value});
             }
         } else {
             return ctx.fail(ctx.decl.src(), "TODO non-explicit inline asm regs", .{});
         }
     }
-    try writer.print("__asm {} (\"{}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source });
+    try writer.print("__asm {s} (\"{s}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source });
     if (as.output) |o| {
         return ctx.fail(ctx.decl.src(), "TODO inline asm output", .{});
     }
@@ -619,7 +620,7 @@ fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
                 if (index > 0) {
                     try writer.writeAll(", ");
                 }
-                try writer.print("\"\"({}_constant)", .{reg});
+                try writer.print("\"\"({s}_constant)", .{reg});
             } else {
                 // This is blocked by the earlier test
                 unreachable;
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
new file mode 100644
index 0000000000..3a1ebada3b
--- /dev/null
+++ b/src/codegen/llvm.zig
@@ -0,0 +1,125 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+
+pub fn targetTriple(allocator: *Allocator, target: std.Target) ![]u8 {
+    const llvm_arch = switch (target.cpu.arch) {
+        .arm => "arm",
+        .armeb => "armeb",
+        .aarch64 => "aarch64",
+        .aarch64_be => "aarch64_be",
+        .aarch64_32 => "aarch64_32",
+        .arc => "arc",
+        .avr => "avr",
+        .bpfel => "bpfel",
+        .bpfeb => "bpfeb",
+        .hexagon => "hexagon",
+        .mips => "mips",
+        .mipsel => "mipsel",
+        .mips64 => "mips64",
+        .mips64el => "mips64el",
+        .msp430 => "msp430",
+        .powerpc => "powerpc",
+        .powerpc64 => "powerpc64",
+        .powerpc64le => "powerpc64le",
+        .r600 => "r600",
+        .amdgcn => "amdgcn",
+        .riscv32 => "riscv32",
+        .riscv64 => "riscv64",
+        .sparc => "sparc",
+        .sparcv9 => "sparcv9",
+        .sparcel => "sparcel",
+        .s390x => "s390x",
+        .tce => "tce",
+        .tcele => "tcele",
+        .thumb => "thumb",
+        .thumbeb => "thumbeb",
+        .i386 => "i386",
+        .x86_64 => "x86_64",
+        .xcore => "xcore",
+        .nvptx => "nvptx",
+        .nvptx64 => "nvptx64",
+        .le32 => "le32",
+        .le64 => "le64",
+        .amdil => "amdil",
+        .amdil64 => "amdil64",
+        .hsail => "hsail",
+        .hsail64 => "hsail64",
+        .spir => "spir",
+        .spir64 => "spir64",
+        .kalimba => "kalimba",
+        .shave => "shave",
+        .lanai => "lanai",
+        .wasm32 => "wasm32",
+        .wasm64 => "wasm64",
+        .renderscript32 => "renderscript32",
+        .renderscript64 => "renderscript64",
+        .ve => "ve",
+        .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII,
+    };
+    // TODO Add a sub-arch for some architectures depending on CPU features.
+
+    const llvm_os = switch (target.os.tag) {
+        .freestanding => "unknown",
+        .ananas => "ananas",
+        .cloudabi => "cloudabi",
+        .dragonfly => "dragonfly",
+        .freebsd => "freebsd",
+        .fuchsia => "fuchsia",
+        .ios => "ios",
+        .kfreebsd => "kfreebsd",
+        .linux => "linux",
+        .lv2 => "lv2",
+        .macos => "macosx",
+        .netbsd => "netbsd",
+        .openbsd => "openbsd",
+        .solaris => "solaris",
+        .windows => "windows",
+        .haiku => "haiku",
+        .minix => "minix",
+        .rtems => "rtems",
+        .nacl => "nacl",
+        .cnk => "cnk",
+        .aix => "aix",
+        .cuda => "cuda",
+        .nvcl => "nvcl",
+        .amdhsa => "amdhsa",
+        .ps4 => "ps4",
+        .elfiamcu => "elfiamcu",
+        .tvos => "tvos",
+        .watchos => "watchos",
+        .mesa3d => "mesa3d",
+        .contiki => "contiki",
+        .amdpal => "amdpal",
+        .hermit => "hermit",
+        .hurd => "hurd",
+        .wasi => "wasi",
+        .emscripten => "emscripten",
+        .uefi => "windows",
+        .other => "unknown",
+    };
+
+    const llvm_abi = switch (target.abi) {
+        .none => "unknown",
+        .gnu => "gnu",
+        .gnuabin32 => "gnuabin32",
+        .gnuabi64 => "gnuabi64",
+        .gnueabi => "gnueabi",
+        .gnueabihf => "gnueabihf",
+        .gnux32 => "gnux32",
+        .code16 => "code16",
+        .eabi => "eabi",
+        .eabihf => "eabihf",
+        .android => "android",
+        .musl => "musl",
+        .musleabi => "musleabi",
+        .musleabihf => "musleabihf",
+        .msvc => "msvc",
+        .itanium => "itanium",
+        .cygnus => "cygnus",
+        .coreclr => "coreclr",
+        .simulator => "simulator",
+        .macabi => "macabi",
+    };
+
+    return std.fmt.allocPrint(allocator, "{s}-unknown-{s}-{s}", .{ llvm_arch, llvm_os, llvm_abi });
+}
diff --git a/src/glibc.zig b/src/glibc.zig
index 0c27872720..f69dd11ada 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -72,7 +72,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
     errdefer version_table.deinit(gpa);
 
     var glibc_dir = zig_lib_dir.openDir("libc" ++ path.sep_str ++ "glibc", .{}) catch |err| {
-        std.log.err("unable to open glibc dir: {}", .{@errorName(err)});
+        std.log.err("unable to open glibc dir: {s}", .{@errorName(err)});
         return error.ZigInstallationCorrupt;
     };
     defer glibc_dir.close();
@@ -81,7 +81,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
     const vers_txt_contents = glibc_dir.readFileAlloc(gpa, "vers.txt", max_txt_size) catch |err| switch (err) {
         error.OutOfMemory => return error.OutOfMemory,
         else => {
-            std.log.err("unable to read vers.txt: {}", .{@errorName(err)});
+            std.log.err("unable to read vers.txt: {s}", .{@errorName(err)});
             return error.ZigInstallationCorrupt;
         },
     };
@@ -91,7 +91,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
     const fns_txt_contents = glibc_dir.readFileAlloc(arena, "fns.txt", max_txt_size) catch |err| switch (err) {
         error.OutOfMemory => return error.OutOfMemory,
         else => {
-            std.log.err("unable to read fns.txt: {}", .{@errorName(err)});
+            std.log.err("unable to read fns.txt: {s}", .{@errorName(err)});
             return error.ZigInstallationCorrupt;
         },
     };
@@ -99,7 +99,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
     const abi_txt_contents = glibc_dir.readFileAlloc(gpa, "abi.txt", max_txt_size) catch |err| switch (err) {
         error.OutOfMemory => return error.OutOfMemory,
         else => {
-            std.log.err("unable to read abi.txt: {}", .{@errorName(err)});
+            std.log.err("unable to read abi.txt: {s}", .{@errorName(err)});
             return error.ZigInstallationCorrupt;
         },
     };
@@ -116,7 +116,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
             }
             const adjusted_line = line[prefix.len..];
             const ver = std.builtin.Version.parse(adjusted_line) catch |err| {
-                std.log.err("vers.txt:{}: unable to parse glibc version '{}': {}", .{ line_i, line, @errorName(err) });
+                std.log.err("vers.txt:{}: unable to parse glibc version '{s}': {s}", .{ line_i, line, @errorName(err) });
                 return error.ZigInstallationCorrupt;
             };
             try all_versions.append(arena, ver);
@@ -136,7 +136,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
                 return error.ZigInstallationCorrupt;
             };
             const lib = findLib(lib_name) orelse {
-                std.log.err("fns.txt:{}: unknown library name: {}", .{ line_i, lib_name });
+                std.log.err("fns.txt:{}: unknown library name: {s}", .{ line_i, lib_name });
                 return error.ZigInstallationCorrupt;
             };
             try all_functions.append(arena, .{
@@ -170,15 +170,15 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
                         return error.ZigInstallationCorrupt;
                     };
                     const arch_tag = std.meta.stringToEnum(std.Target.Cpu.Arch, arch_name) orelse {
-                        std.log.err("abi.txt:{}: unrecognized arch: '{}'", .{ line_i, arch_name });
+                        std.log.err("abi.txt:{}: unrecognized arch: '{s}'", .{ line_i, arch_name });
                         return error.ZigInstallationCorrupt;
                     };
                     if (!mem.eql(u8, os_name, "linux")) {
-                        std.log.err("abi.txt:{}: expected OS 'linux', found '{}'", .{ line_i, os_name });
+                        std.log.err("abi.txt:{}: expected OS 'linux', found '{s}'", .{ line_i, os_name });
                         return error.ZigInstallationCorrupt;
                     }
                     const abi_tag = std.meta.stringToEnum(std.Target.Abi, abi_name) orelse {
-                        std.log.err("abi.txt:{}: unrecognized ABI: '{}'", .{ line_i, abi_name });
+                        std.log.err("abi.txt:{}: unrecognized ABI: '{s}'", .{ line_i, abi_name });
                         return error.ZigInstallationCorrupt;
                     };
 
@@ -211,7 +211,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
                     }
                     const version_index = std.fmt.parseInt(u8, version_index_string, 10) catch |err| {
                         // If this happens with legit data, increase the size of the integer type in the struct.
-                        std.log.err("abi.txt:{}: unable to parse version: {}", .{ line_i, @errorName(err) });
+                        std.log.err("abi.txt:{}: unable to parse version: {s}", .{ line_i, @errorName(err) });
                         return error.ZigInstallationCorrupt;
                     };
 
@@ -531,7 +531,7 @@ fn add_include_dirs(comp: *Compilation, arena: *Allocator, args: *std.ArrayList(
     try args.append(try path.join(arena, &[_][]const u8{ comp.zig_lib_directory.path.?, lib_libc ++ "glibc" }));
 
     try args.append("-I");
-    try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-{}-{}", .{
+    try args.append(try std.fmt.allocPrint(arena, "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", .{
         comp.zig_lib_directory.path.?, @tagName(arch), @tagName(target.os.tag), @tagName(target.abi),
     }));
 
@@ -539,7 +539,7 @@ fn add_include_dirs(comp: *Compilation, arena: *Allocator, args: *std.ArrayList(
     try args.append(try lib_path(comp, arena, lib_libc ++ "include" ++ s ++ "generic-glibc"));
 
     try args.append("-I");
-    try args.append(try std.fmt.allocPrint(arena, "{}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{}-linux-any", .{
+    try args.append(try std.fmt.allocPrint(arena, "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-linux-any", .{
         comp.zig_lib_directory.path.?, @tagName(arch),
     }));
 
@@ -881,7 +881,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void {
         if (o_directory.handle.createFile(ok_basename, .{})) |file| {
             file.close();
         } else |err| {
-            std.log.warn("glibc shared objects: failed to mark completion: {}", .{@errorName(err)});
+            std.log.warn("glibc shared objects: failed to mark completion: {s}", .{@errorName(err)});
         }
     }
 
diff --git a/src/libc_installation.zig b/src/libc_installation.zig
index 0b1029eeb8..cc96146e0b 100644
--- a/src/libc_installation.zig
+++ b/src/libc_installation.zig
@@ -83,7 +83,7 @@ pub const LibCInstallation = struct {
         }
         inline for (fields) |field, i| {
             if (!found_keys[i].found) {
-                log.err("missing field: {}\n", .{field.name});
+                log.err("missing field: {s}\n", .{field.name});
                 return error.ParseError;
             }
         }
@@ -96,18 +96,18 @@ pub const LibCInstallation = struct {
             return error.ParseError;
         }
         if (self.crt_dir == null and !is_darwin) {
-            log.err("crt_dir may not be empty for {}\n", .{@tagName(Target.current.os.tag)});
+            log.err("crt_dir may not be empty for {s}\n", .{@tagName(Target.current.os.tag)});
             return error.ParseError;
         }
         if (self.msvc_lib_dir == null and is_windows and !is_gnu) {
-            log.err("msvc_lib_dir may not be empty for {}-{}\n", .{
+            log.err("msvc_lib_dir may not be empty for {s}-{s}\n", .{
                 @tagName(Target.current.os.tag),
                 @tagName(Target.current.abi),
             });
             return error.ParseError;
         }
         if (self.kernel32_lib_dir == null and is_windows and !is_gnu) {
-            log.err("kernel32_lib_dir may not be empty for {}-{}\n", .{
+            log.err("kernel32_lib_dir may not be empty for {s}-{s}\n", .{
                 @tagName(Target.current.os.tag),
                 @tagName(Target.current.abi),
             });
@@ -128,25 +128,25 @@ pub const LibCInstallation = struct {
         try out.print(
             \\# The directory that contains `stdlib.h`.
             \\# On POSIX-like systems, include directories be found with: `cc -E -Wp,-v -xc /dev/null`
-            \\include_dir={}
+            \\include_dir={s}
             \\
             \\# The system-specific include directory. May be the same as `include_dir`.
             \\# On Windows it's the directory that includes `vcruntime.h`.
             \\# On POSIX it's the directory that includes `sys/errno.h`.
-            \\sys_include_dir={}
+            \\sys_include_dir={s}
             \\
             \\# The directory that contains `crt1.o` or `crt2.o`.
             \\# On POSIX, can be found with `cc -print-file-name=crt1.o`.
             \\# Not needed when targeting MacOS.
-            \\crt_dir={}
+            \\crt_dir={s}
             \\
             \\# The directory that contains `vcruntime.lib`.
             \\# Only needed when targeting MSVC on Windows.
-            \\msvc_lib_dir={}
+            \\msvc_lib_dir={s}
             \\
             \\# The directory that contains `kernel32.lib`.
             \\# Only needed when targeting MSVC on Windows.
-            \\kernel32_lib_dir={}
+            \\kernel32_lib_dir={s}
             \\
         , .{
             include_dir,
@@ -338,7 +338,7 @@ pub const LibCInstallation = struct {
 
         for (searches) |search| {
             result_buf.shrink(0);
-            try result_buf.outStream().print("{}\\Include\\{}\\ucrt", .{ search.path, search.version });
+            try result_buf.outStream().print("{s}\\Include\\{s}\\ucrt", .{ search.path, search.version });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
                 error.FileNotFound,
@@ -384,7 +384,7 @@ pub const LibCInstallation = struct {
 
         for (searches) |search| {
             result_buf.shrink(0);
-            try result_buf.outStream().print("{}\\Lib\\{}\\ucrt\\{}", .{ search.path, search.version, arch_sub_dir });
+            try result_buf.outStream().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ search.path, search.version, arch_sub_dir });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
                 error.FileNotFound,
@@ -439,7 +439,7 @@ pub const LibCInstallation = struct {
         for (searches) |search| {
             result_buf.shrink(0);
             const stream = result_buf.outStream();
-            try stream.print("{}\\Lib\\{}\\um\\{}", .{ search.path, search.version, arch_sub_dir });
+            try stream.print("{s}\\Lib\\{s}\\um\\{s}", .{ search.path, search.version, arch_sub_dir });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
                 error.FileNotFound,
@@ -520,7 +520,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {
     const allocator = args.allocator;
 
     const cc_exe = std.os.getenvZ("CC") orelse default_cc_exe;
-    const arg1 = try std.fmt.allocPrint(allocator, "-print-file-name={}", .{args.search_basename});
+    const arg1 = try std.fmt.allocPrint(allocator, "-print-file-name={s}", .{args.search_basename});
     defer allocator.free(arg1);
     const argv = [_][]const u8{ cc_exe, arg1 };
 
@@ -584,17 +584,17 @@ fn printVerboseInvocation(
     if (!verbose) return;
 
     if (search_basename) |s| {
-        std.debug.warn("Zig attempted to find the file '{}' by executing this command:\n", .{s});
+        std.debug.warn("Zig attempted to find the file '{s}' by executing this command:\n", .{s});
     } else {
         std.debug.warn("Zig attempted to find the path to native system libc headers by executing this command:\n", .{});
     }
     for (argv) |arg, i| {
         if (i != 0) std.debug.warn(" ", .{});
-        std.debug.warn("{}", .{arg});
+        std.debug.warn("{s}", .{arg});
     }
     std.debug.warn("\n", .{});
     if (stderr) |s| {
-        std.debug.warn("Output:\n==========\n{}\n==========\n", .{s});
+        std.debug.warn("Output:\n==========\n{s}\n==========\n", .{s});
     }
 }
 
diff --git a/src/link.zig b/src/link.zig
index 468f23bffd..92702c7973 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -560,9 +560,9 @@ pub const File = struct {
         const full_out_path_z = try arena.dupeZ(u8, full_out_path);
 
         if (base.options.verbose_link) {
-            std.debug.print("ar rcs {}", .{full_out_path_z});
+            std.debug.print("ar rcs {s}", .{full_out_path_z});
             for (object_files.items) |arg| {
-                std.debug.print(" {}", .{arg});
+                std.debug.print(" {s}", .{arg});
             }
             std.debug.print("\n", .{});
         }
@@ -574,11 +574,11 @@ pub const File = struct {
 
         if (!base.options.disable_lld_caching) {
             Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
-                log.warn("failed to save archive hash digest file: {}", .{@errorName(err)});
+                log.warn("failed to save archive hash digest file: {s}", .{@errorName(err)});
             };
 
             man.writeManifest() catch |err| {
-                log.warn("failed to write cache manifest when archiving: {}", .{@errorName(err)});
+                log.warn("failed to write cache manifest when archiving: {s}", .{@errorName(err)});
             };
 
             base.lock = man.toOwnedLock();
diff --git a/src/link/C.zig b/src/link/C.zig
index 1059e52115..3ac9db717f 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -111,8 +111,11 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     if (self.header.buf.items.len > 0) {
         try writer.writeByte('\n');
     }
+    if (self.header.items.len > 0) {
+        try writer.print("{s}\n", .{self.header.items});
+    }
     if (self.constants.items.len > 0) {
-        try writer.print("{}\n", .{self.constants.items});
+        try writer.print("{s}\n", .{self.constants.items});
     }
     if (self.main.items.len > 1) {
         const last_two = self.main.items[self.main.items.len - 2 ..];
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 096fa2cd0b..a3cb5cc4c7 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -686,7 +686,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
         if (need_realloc) {
             const curr_vaddr = self.getDeclVAddr(decl);
             const vaddr = try self.growTextBlock(&decl.link.coff, code.len, required_alignment);
-            log.debug("growing {} from 0x{x} to 0x{x}\n", .{ decl.name, curr_vaddr, vaddr });
+            log.debug("growing {s} from 0x{x} to 0x{x}\n", .{ decl.name, curr_vaddr, vaddr });
             if (vaddr != curr_vaddr) {
                 log.debug("  (writing new offset table entry)\n", .{});
                 self.offset_table.items[decl.link.coff.offset_table_index] = vaddr;
@@ -697,7 +697,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
         }
     } else {
         const vaddr = try self.allocateTextBlock(&decl.link.coff, code.len, required_alignment);
-        log.debug("allocated text block for {} at 0x{x} (size: {Bi})\n", .{ mem.spanZ(decl.name), vaddr, code.len });
+        log.debug("allocated text block for {s} at 0x{x} (size: {Bi})\n", .{ mem.spanZ(decl.name), vaddr, code.len });
         errdefer self.freeTextBlock(&decl.link.coff);
         self.offset_table.items[decl.link.coff.offset_table_index] = vaddr;
         try self.writeOffsetTableEntry(decl.link.coff.offset_table_index);
@@ -880,7 +880,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("COFF LLD new_digest={} error: {}", .{ digest, @errorName(err) });
+            log.debug("COFF LLD new_digest={} error: {s}", .{ digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -1236,11 +1236,11 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         // Update the file with the digest. If it fails we can continue; it only
         // means that the next invocation will have an unnecessary cache miss.
         Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
-            log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
+            log.warn("failed to save linking hash digest file: {s}", .{@errorName(err)});
         };
         // Again failure here only means an unnecessary cache miss.
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+            log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
         };
         // We hang on to this lock so that the output file path can be used without
         // other processes clobbering it.
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 4b2b95fc72..2db15ae280 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1362,7 +1362,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("ELF LLD new_digest={} error: {}", .{ digest, @errorName(err) });
+            log.debug("ELF LLD new_digest={} error: {s}", .{ digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -1396,7 +1396,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
 
     if (self.base.options.output_mode == .Exe) {
         try argv.append("-z");
-        try argv.append(try std.fmt.allocPrint(arena, "stack-size={}", .{stack_size}));
+        try argv.append(try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size}));
     }
 
     if (self.base.options.image_base_override) |image_base| {
@@ -1438,7 +1438,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
     if (getLDMOption(target)) |ldm| {
         // Any target ELF will use the freebsd osabi if suffixed with "_fbsd".
         const arg = if (target.os.tag == .freebsd)
-            try std.fmt.allocPrint(arena, "{}_fbsd", .{ldm})
+            try std.fmt.allocPrint(arena, "{s}_fbsd", .{ldm})
         else
             ldm;
         try argv.append("-m");
@@ -1599,7 +1599,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
             // (the check for that needs to be earlier), but they could be full paths to .so files, in which
             // case we want to avoid prepending "-l".
             const ext = Compilation.classifyFileExt(link_lib);
-            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
+            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
             argv.appendAssumeCapacity(arg);
         }
 
@@ -1733,11 +1733,11 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         // Update the file with the digest. If it fails we can continue; it only
         // means that the next invocation will have an unnecessary cache miss.
         Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
-            log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
+            log.warn("failed to save linking hash digest file: {s}", .{@errorName(err)});
         };
         // Again failure here only means an unnecessary cache miss.
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+            log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
         };
         // We hang on to this lock so that the output file path can be used without
         // other processes clobbering it.
@@ -2082,10 +2082,10 @@ pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void {
     try self.offset_table.ensureCapacity(self.base.allocator, self.offset_table.items.len + 1);
 
     if (self.local_symbol_free_list.popOrNull()) |i| {
-        log.debug("reusing symbol index {} for {}\n", .{ i, decl.name });
+        log.debug("reusing symbol index {} for {s}\n", .{ i, decl.name });
         decl.link.elf.local_sym_index = i;
     } else {
-        log.debug("allocating symbol index {} for {}\n", .{ self.local_symbols.items.len, decl.name });
+        log.debug("allocating symbol index {} for {s}\n", .{ self.local_symbols.items.len, decl.name });
         decl.link.elf.local_sym_index = @intCast(u32, self.local_symbols.items.len);
         _ = self.local_symbols.addOneAssumeCapacity();
     }
@@ -2182,7 +2182,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
         if (zir_dumps.len != 0) {
             for (zir_dumps) |fn_name| {
                 if (mem.eql(u8, mem.spanZ(decl.name), fn_name)) {
-                    std.debug.print("\n{}\n", .{decl.name});
+                    std.debug.print("\n{s}\n", .{decl.name});
                     typed_value.val.castTag(.function).?.data.dump(module.*);
                 }
             }
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 6abbae2c26..d4a61c8149 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -520,7 +520,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("MachO LLD new_digest={} error: {}", .{ digest, @errorName(err) });
+            log.debug("MachO LLD new_digest={} error: {s}", .{ digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -706,7 +706,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
             // (the check for that needs to be earlier), but they could be full paths to .dylib files, in which
             // case we want to avoid prepending "-l".
             const ext = Compilation.classifyFileExt(link_lib);
-            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{}", .{link_lib});
+            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
             argv.appendAssumeCapacity(arg);
         }
 
@@ -759,15 +759,15 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
                 self.base.allocator.free(result.stderr);
             }
             if (result.stdout.len != 0) {
-                log.warn("unexpected LD stdout: {}", .{result.stdout});
+                log.warn("unexpected LD stdout: {s}", .{result.stdout});
             }
             if (result.stderr.len != 0) {
-                log.warn("unexpected LD stderr: {}", .{result.stderr});
+                log.warn("unexpected LD stderr: {s}", .{result.stderr});
             }
             if (result.term != .Exited or result.term.Exited != 0) {
                 // TODO parse this output and surface with the Compilation API rather than
                 // directly outputting to stderr here.
-                log.err("{}", .{result.stderr});
+                log.err("{s}", .{result.stderr});
                 return error.LDReportedFailure;
             }
         } else {
@@ -980,11 +980,11 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
         // Update the file with the digest. If it fails we can continue; it only
         // means that the next invocation will have an unnecessary cache miss.
         Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
-            log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
+            log.warn("failed to save linking hash digest file: {s}", .{@errorName(err)});
         };
         // Again failure here only means an unnecessary cache miss.
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+            log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
         };
         // We hang on to this lock so that the output file path can be used without
         // other processes clobbering it.
@@ -1088,10 +1088,10 @@ pub fn allocateDeclIndexes(self: *MachO, decl: *Module.Decl) !void {
     try self.offset_table.ensureCapacity(self.base.allocator, self.offset_table.items.len + 1);
 
     if (self.local_symbol_free_list.popOrNull()) |i| {
-        log.debug("reusing symbol index {} for {}", .{ i, decl.name });
+        log.debug("reusing symbol index {d} for {s}", .{ i, decl.name });
         decl.link.macho.local_sym_index = i;
     } else {
-        log.debug("allocating symbol index {} for {}", .{ self.local_symbols.items.len, decl.name });
+        log.debug("allocating symbol index {d} for {s}", .{ self.local_symbols.items.len, decl.name });
         decl.link.macho.local_sym_index = @intCast(u32, self.local_symbols.items.len);
         _ = self.local_symbols.addOneAssumeCapacity();
     }
@@ -1165,7 +1165,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
         const need_realloc = code.len > capacity or !mem.isAlignedGeneric(u64, symbol.n_value, required_alignment);
         if (need_realloc) {
             const vaddr = try self.growTextBlock(&decl.link.macho, code.len, required_alignment);
-            log.debug("growing {} from 0x{x} to 0x{x}", .{ decl.name, symbol.n_value, vaddr });
+            log.debug("growing {s} from 0x{x} to 0x{x}", .{ decl.name, symbol.n_value, vaddr });
             if (vaddr != symbol.n_value) {
                 symbol.n_value = vaddr;
                 log.debug(" (writing new offset table entry)", .{});
@@ -1188,7 +1188,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
         const decl_name = mem.spanZ(decl.name);
         const name_str_index = try self.makeString(decl_name);
         const addr = try self.allocateTextBlock(&decl.link.macho, code.len, required_alignment);
-        log.debug("allocated text block for {} at 0x{x}", .{ decl_name, addr });
+        log.debug("allocated text block for {s} at 0x{x}", .{ decl_name, addr });
         errdefer self.freeTextBlock(&decl.link.macho);
 
         symbol.* = .{
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 374fa2230b..cbb3e83147 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -321,7 +321,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("WASM LLD new_digest={} error: {}", .{ digest, @errorName(err) });
+            log.debug("WASM LLD new_digest={} error: {s}", .{ digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -463,11 +463,11 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
         // Update the file with the digest. If it fails we can continue; it only
         // means that the next invocation will have an unnecessary cache miss.
         Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
-            log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+            log.warn("failed to save linking hash digest symlink: {s}", .{@errorName(err)});
         };
         // Again failure here only means an unnecessary cache miss.
         man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+            log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
         };
         // We hang on to this lock so that the output file path can be used without
         // other processes clobbering it.
diff --git a/src/main.zig b/src/main.zig
index 98b19cc3ea..7d03ea956f 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -118,7 +118,7 @@ pub fn main() anyerror!void {
 
 pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !void {
     if (args.len <= 1) {
-        std.log.info("{}", .{usage});
+        std.log.info("{s}", .{usage});
         fatal("expected command argument", .{});
     }
 
@@ -204,8 +204,8 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
     } else if (mem.eql(u8, cmd, "help") or mem.eql(u8, cmd, "-h") or mem.eql(u8, cmd, "--help")) {
         try io.getStdOut().writeAll(usage);
     } else {
-        std.log.info("{}", .{usage});
-        fatal("unknown command: {}", .{args[1]});
+        std.log.info("{s}", .{usage});
+        fatal("unknown command: {s}", .{args[1]});
     }
 }
 
@@ -615,7 +615,7 @@ fn buildOutputType(
                             fatal("unexpected end-of-parameter mark: --", .{});
                         }
                     } else if (mem.eql(u8, arg, "--pkg-begin")) {
-                        if (i + 2 >= args.len) fatal("Expected 2 arguments after {}", .{arg});
+                        if (i + 2 >= args.len) fatal("Expected 2 arguments after {s}", .{arg});
                         i += 1;
                         const pkg_name = args[i];
                         i += 1;
@@ -635,7 +635,7 @@ fn buildOutputType(
                         cur_pkg = cur_pkg.parent orelse
                             fatal("encountered --pkg-end with no matching --pkg-begin", .{});
                     } else if (mem.eql(u8, arg, "--main-pkg-path")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         main_pkg_path = args[i];
                     } else if (mem.eql(u8, arg, "-cflags")) {
@@ -653,10 +653,10 @@ fn buildOutputType(
                         i += 1;
                         const next_arg = args[i];
                         color = std.meta.stringToEnum(Color, next_arg) orelse {
-                            fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg});
+                            fatal("expected [auto|on|off] after --color, found '{s}'", .{next_arg});
                         };
                     } else if (mem.eql(u8, arg, "--subsystem")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         if (mem.eql(u8, args[i], "console")) {
                             subsystem = .Console;
@@ -689,51 +689,51 @@ fn buildOutputType(
                             });
                         }
                     } else if (mem.eql(u8, arg, "-O")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         optimize_mode_string = args[i];
                     } else if (mem.eql(u8, arg, "--stack")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         stack_size_override = std.fmt.parseUnsigned(u64, args[i], 0) catch |err| {
                             fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
                         };
                     } else if (mem.eql(u8, arg, "--image-base")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         image_base_override = std.fmt.parseUnsigned(u64, args[i], 0) catch |err| {
                             fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
                         };
                     } else if (mem.eql(u8, arg, "--name")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         provided_name = args[i];
                     } else if (mem.eql(u8, arg, "-rpath")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try rpath_list.append(args[i]);
                     } else if (mem.eql(u8, arg, "--library-directory") or mem.eql(u8, arg, "-L")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try lib_dirs.append(args[i]);
                     } else if (mem.eql(u8, arg, "-F")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try framework_dirs.append(args[i]);
                     } else if (mem.eql(u8, arg, "-framework")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try frameworks.append(args[i]);
                     } else if (mem.eql(u8, arg, "-T") or mem.eql(u8, arg, "--script")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         linker_script = args[i];
                     } else if (mem.eql(u8, arg, "--version-script")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         version_script = args[i];
                     } else if (mem.eql(u8, arg, "--library") or mem.eql(u8, arg, "-l")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         // We don't know whether this library is part of libc or libc++ until we resolve the target.
                         // So we simply append to the list for now.
                         i += 1;
@@ -743,7 +743,7 @@ fn buildOutputType(
                         mem.eql(u8, arg, "-I") or
                         mem.eql(u8, arg, "-dirafter"))
                     {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try clang_argv.append(arg);
                         try clang_argv.append(args[i]);
@@ -753,19 +753,19 @@ fn buildOutputType(
                         }
                         i += 1;
                         version = std.builtin.Version.parse(args[i]) catch |err| {
-                            fatal("unable to parse --version '{}': {}", .{ args[i], @errorName(err) });
+                            fatal("unable to parse --version '{s}': {s}", .{ args[i], @errorName(err) });
                         };
                         have_version = true;
                     } else if (mem.eql(u8, arg, "-target")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         target_arch_os_abi = args[i];
                     } else if (mem.eql(u8, arg, "-mcpu")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         target_mcpu = args[i];
                     } else if (mem.eql(u8, arg, "-mcmodel")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         machine_code_model = parseCodeModel(args[i]);
                     } else if (mem.startsWith(u8, arg, "-ofmt=")) {
@@ -777,35 +777,35 @@ fn buildOutputType(
                     } else if (mem.startsWith(u8, arg, "-O")) {
                         optimize_mode_string = arg["-O".len..];
                     } else if (mem.eql(u8, arg, "--dynamic-linker")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         target_dynamic_linker = args[i];
                     } else if (mem.eql(u8, arg, "--libc")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         libc_paths_file = args[i];
                     } else if (mem.eql(u8, arg, "--test-filter")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         test_filter = args[i];
                     } else if (mem.eql(u8, arg, "--test-name-prefix")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         test_name_prefix = args[i];
                     } else if (mem.eql(u8, arg, "--test-cmd")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         try test_exec_args.append(args[i]);
                     } else if (mem.eql(u8, arg, "--cache-dir")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         override_local_cache_dir = args[i];
                     } else if (mem.eql(u8, arg, "--global-cache-dir")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         override_global_cache_dir = args[i];
                     } else if (mem.eql(u8, arg, "--override-lib-dir")) {
-                        if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg});
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         override_lib_dir = args[i];
                     } else if (mem.eql(u8, arg, "-fcompiler-rt")) {
@@ -968,7 +968,7 @@ fn buildOutputType(
                     {
                         try clang_argv.append(arg);
                     } else {
-                        fatal("unrecognized parameter: '{}'", .{arg});
+                        fatal("unrecognized parameter: '{s}'", .{arg});
                     }
                 } else switch (Compilation.classifyFileExt(arg)) {
                     .object, .static_library, .shared_library => {
@@ -982,19 +982,19 @@ fn buildOutputType(
                     },
                     .zig, .zir => {
                         if (root_src_file) |other| {
-                            fatal("found another zig file '{}' after root source file '{}'", .{ arg, other });
+                            fatal("found another zig file '{s}' after root source file '{s}'", .{ arg, other });
                         } else {
                             root_src_file = arg;
                         }
                     },
                     .unknown => {
-                        fatal("unrecognized file extension of parameter '{}'", .{arg});
+                        fatal("unrecognized file extension of parameter '{s}'", .{arg});
                     },
                 }
             }
             if (optimize_mode_string) |s| {
                 optimize_mode = std.meta.stringToEnum(std.builtin.Mode, s) orelse
-                    fatal("unrecognized optimization mode: '{}'", .{s});
+                    fatal("unrecognized optimization mode: '{s}'", .{s});
             }
         },
         .cc, .cpp => {
@@ -1018,7 +1018,7 @@ fn buildOutputType(
             var it = ClangArgIterator.init(arena, all_args);
             while (it.has_next) {
                 it.next() catch |err| {
-                    fatal("unable to parse command line parameters: {}", .{@errorName(err)});
+                    fatal("unable to parse command line parameters: {s}", .{@errorName(err)});
                 };
                 switch (it.zig_equivalent) {
                     .target => target_arch_os_abi = it.only_arg, // example: -target riscv64-linux-unknown
@@ -1038,7 +1038,7 @@ fn buildOutputType(
                             },
                             .zig, .zir => {
                                 if (root_src_file) |other| {
-                                    fatal("found another zig file '{}' after root source file '{}'", .{ it.only_arg, other });
+                                    fatal("found another zig file '{s}' after root source file '{s}'", .{ it.only_arg, other });
                                 } else {
                                     root_src_file = it.only_arg;
                                 }
@@ -1153,7 +1153,7 @@ fn buildOutputType(
                 if (mem.eql(u8, arg, "-soname")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     const name = linker_args.items[i];
                     soname = .{ .yes = name };
@@ -1185,7 +1185,7 @@ fn buildOutputType(
                 } else if (mem.eql(u8, arg, "-rpath")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     try rpath_list.append(linker_args.items[i]);
                 } else if (mem.eql(u8, arg, "-I") or
@@ -1194,7 +1194,7 @@ fn buildOutputType(
                 {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     target_dynamic_linker = linker_args.items[i];
                 } else if (mem.eql(u8, arg, "-E") or
@@ -1205,7 +1205,7 @@ fn buildOutputType(
                 } else if (mem.eql(u8, arg, "--version-script")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     version_script = linker_args.items[i];
                 } else if (mem.startsWith(u8, arg, "-O")) {
@@ -1227,7 +1227,7 @@ fn buildOutputType(
                 } else if (mem.eql(u8, arg, "-z")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     const z_arg = linker_args.items[i];
                     if (mem.eql(u8, z_arg, "nodelete")) {
@@ -1235,44 +1235,44 @@ fn buildOutputType(
                     } else if (mem.eql(u8, z_arg, "defs")) {
                         linker_z_defs = true;
                     } else {
-                        warn("unsupported linker arg: -z {}", .{z_arg});
+                        warn("unsupported linker arg: -z {s}", .{z_arg});
                     }
                 } else if (mem.eql(u8, arg, "--major-image-version")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     version.major = std.fmt.parseUnsigned(u32, linker_args.items[i], 10) catch |err| {
-                        fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                     };
                     have_version = true;
                 } else if (mem.eql(u8, arg, "--minor-image-version")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     version.minor = std.fmt.parseUnsigned(u32, linker_args.items[i], 10) catch |err| {
-                        fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                     };
                     have_version = true;
                 } else if (mem.eql(u8, arg, "--stack")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     stack_size_override = std.fmt.parseUnsigned(u64, linker_args.items[i], 0) catch |err| {
-                        fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                     };
                 } else if (mem.eql(u8, arg, "--image-base")) {
                     i += 1;
                     if (i >= linker_args.items.len) {
-                        fatal("expected linker arg after '{}'", .{arg});
+                        fatal("expected linker arg after '{s}'", .{arg});
                     }
                     image_base_override = std.fmt.parseUnsigned(u64, linker_args.items[i], 0) catch |err| {
-                        fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                     };
                 } else {
-                    warn("unsupported linker arg: {}", .{arg});
+                    warn("unsupported linker arg: {s}", .{arg});
                 }
             }
 
@@ -1328,7 +1328,7 @@ fn buildOutputType(
     }
 
     if (arg_mode == .translate_c and c_source_files.items.len != 1) {
-        fatal("translate-c expects exactly 1 source file (found {})", .{c_source_files.items.len});
+        fatal("translate-c expects exactly 1 source file (found {d})", .{c_source_files.items.len});
     }
 
     if (root_src_file == null and arg_mode == .zig_test) {
@@ -1373,25 +1373,25 @@ fn buildOutputType(
             help: {
                 var help_text = std.ArrayList(u8).init(arena);
                 for (diags.arch.?.allCpuModels()) |cpu| {
-                    help_text.writer().print(" {}\n", .{cpu.name}) catch break :help;
+                    help_text.writer().print(" {s}\n", .{cpu.name}) catch break :help;
                 }
-                std.log.info("Available CPUs for architecture '{}': {}", .{
+                std.log.info("Available CPUs for architecture '{s}': {s}", .{
                     @tagName(diags.arch.?), help_text.items,
                 });
             }
-            fatal("Unknown CPU: '{}'", .{diags.cpu_name.?});
+            fatal("Unknown CPU: '{s}'", .{diags.cpu_name.?});
         },
         error.UnknownCpuFeature => {
             help: {
                 var help_text = std.ArrayList(u8).init(arena);
                 for (diags.arch.?.allFeaturesList()) |feature| {
-                    help_text.writer().print(" {}: {}\n", .{ feature.name, feature.description }) catch break :help;
+                    help_text.writer().print(" {s}: {s}\n", .{ feature.name, feature.description }) catch break :help;
                 }
-                std.log.info("Available CPU features for architecture '{}': {}", .{
+                std.log.info("Available CPU features for architecture '{s}': {s}", .{
                     @tagName(diags.arch.?), help_text.items,
                 });
             }
-            fatal("Unknown CPU feature: '{}'", .{diags.unknown_feature_name});
+            fatal("Unknown CPU feature: '{s}'", .{diags.unknown_feature_name});
         },
         else => |e| return e,
     };
@@ -1431,10 +1431,10 @@ fn buildOutputType(
 
     if (cross_target.isNativeOs() and (system_libs.items.len != 0 or want_native_include_dirs)) {
         const paths = std.zig.system.NativePaths.detect(arena) catch |err| {
-            fatal("unable to detect native system paths: {}", .{@errorName(err)});
+            fatal("unable to detect native system paths: {s}", .{@errorName(err)});
         };
         for (paths.warnings.items) |warning| {
-            warn("{}", .{warning});
+            warn("{s}", .{warning});
         }
 
         const has_sysroot = if (comptime std.Target.current.isDarwin()) outer: {
@@ -1492,7 +1492,7 @@ fn buildOutputType(
         } else if (mem.eql(u8, ofmt, "raw")) {
             break :blk .raw;
         } else {
-            fatal("unsupported object format: {}", .{ofmt});
+            fatal("unsupported object format: {s}", .{ofmt});
         }
     };
 
@@ -1562,7 +1562,7 @@ fn buildOutputType(
             }
             if (fs.path.dirname(full_path)) |dirname| {
                 const handle = fs.cwd().openDir(dirname, .{}) catch |err| {
-                    fatal("unable to open output directory '{}': {}", .{ dirname, @errorName(err) });
+                    fatal("unable to open output directory '{s}': {s}", .{ dirname, @errorName(err) });
                 };
                 cleanup_emit_bin_dir = handle;
                 break :b Compilation.EmitLoc{
@@ -1585,19 +1585,19 @@ fn buildOutputType(
         },
     };
 
-    const default_h_basename = try std.fmt.allocPrint(arena, "{}.h", .{root_name});
+    const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name});
     var emit_h_resolved = try emit_h.resolve(default_h_basename);
     defer emit_h_resolved.deinit();
 
-    const default_asm_basename = try std.fmt.allocPrint(arena, "{}.s", .{root_name});
+    const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name});
     var emit_asm_resolved = try emit_asm.resolve(default_asm_basename);
     defer emit_asm_resolved.deinit();
 
-    const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{}.ll", .{root_name});
+    const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name});
     var emit_llvm_ir_resolved = try emit_llvm_ir.resolve(default_llvm_ir_basename);
     defer emit_llvm_ir_resolved.deinit();
 
-    const default_analysis_basename = try std.fmt.allocPrint(arena, "{}-analysis.json", .{root_name});
+    const default_analysis_basename = try std.fmt.allocPrint(arena, "{s}-analysis.json", .{root_name});
     var emit_analysis_resolved = try emit_analysis.resolve(default_analysis_basename);
     defer emit_analysis_resolved.deinit();
 
@@ -1609,10 +1609,10 @@ fn buildOutputType(
         .yes_default_path => blk: {
             if (root_src_file) |rsf| {
                 if (mem.endsWith(u8, rsf, ".zir")) {
-                    break :blk try std.fmt.allocPrint(arena, "{}.out.zir", .{root_name});
+                    break :blk try std.fmt.allocPrint(arena, "{s}.out.zir", .{root_name});
                 }
             }
-            break :blk try std.fmt.allocPrint(arena, "{}.zir", .{root_name});
+            break :blk try std.fmt.allocPrint(arena, "{s}.zir", .{root_name});
         },
         .yes => |p| p,
     };
@@ -1642,7 +1642,7 @@ fn buildOutputType(
         }
     else
         introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {
-            fatal("unable to find zig installation directory: {}", .{@errorName(err)});
+            fatal("unable to find zig installation directory: {s}", .{@errorName(err)});
         };
     defer zig_lib_directory.handle.close();
 
@@ -1655,7 +1655,7 @@ fn buildOutputType(
 
     if (libc_paths_file) |paths_file| {
         libc_installation = LibCInstallation.parse(gpa, paths_file) catch |err| {
-            fatal("unable to parse libc paths file: {}", .{@errorName(err)});
+            fatal("unable to parse libc paths file: {s}", .{@errorName(err)});
         };
     }
 
@@ -1791,7 +1791,7 @@ fn buildOutputType(
         .disable_lld_caching = !have_enable_cache,
         .subsystem = subsystem,
     }) catch |err| {
-        fatal("unable to create compilation: {}", .{@errorName(err)});
+        fatal("unable to create compilation: {s}", .{@errorName(err)});
     };
     var comp_destroyed = false;
     defer if (!comp_destroyed) comp.destroy();
@@ -1914,12 +1914,12 @@ fn buildOutputType(
                                 if (!watch) return cleanExit();
                             } else {
                                 const cmd = try argvCmd(arena, argv.items);
-                                fatal("the following test command failed with exit code {}:\n{}", .{ code, cmd });
+                                fatal("the following test command failed with exit code {}:\n{s}", .{ code, cmd });
                             }
                         },
                         else => {
                             const cmd = try argvCmd(arena, argv.items);
-                            fatal("the following test command crashed:\n{}", .{cmd});
+                            fatal("the following test command crashed:\n{s}", .{cmd});
                         },
                     }
                 },
@@ -1936,7 +1936,7 @@ fn buildOutputType(
         try stderr.print("(zig) ", .{});
         try comp.makeBinFileExecutable();
         if (stdin.readUntilDelimiterOrEof(&repl_buf, '\n') catch |err| {
-            try stderr.print("\nUnable to parse command: {}\n", .{@errorName(err)});
+            try stderr.print("\nUnable to parse command: {s}\n", .{@errorName(err)});
             continue;
         }) |line| {
             const actual_line = mem.trimRight(u8, line, "\r\n ");
@@ -1954,7 +1954,7 @@ fn buildOutputType(
             } else if (mem.eql(u8, actual_line, "help")) {
                 try stderr.writeAll(repl_help);
             } else {
-                try stderr.print("unknown command: {}\n", .{actual_line});
+                try stderr.print("unknown command: {s}\n", .{actual_line});
             }
         } else {
             break;
@@ -2012,14 +2012,14 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
     assert(comp.c_source_files.len == 1);
     const c_source_file = comp.c_source_files[0];
 
-    const translated_zig_basename = try std.fmt.allocPrint(arena, "{}.zig", .{comp.bin_file.options.root_name});
+    const translated_zig_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.bin_file.options.root_name});
 
     var man: Cache.Manifest = comp.obtainCObjectCacheManifest();
     defer if (enable_cache) man.deinit();
 
     man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects
     _ = man.addFile(c_source_file.src_path, null) catch |err| {
-        fatal("unable to process '{}': {}", .{ c_source_file.src_path, @errorName(err) });
+        fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });
     };
 
     const digest = if (try man.hit()) man.final() else digest: {
@@ -2034,7 +2034,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
                 break :blk null;
 
             const c_src_basename = fs.path.basename(c_source_file.src_path);
-            const dep_basename = try std.fmt.allocPrint(arena, "{}.d", .{c_src_basename});
+            const dep_basename = try std.fmt.allocPrint(arena, "{s}.d", .{c_src_basename});
             const out_dep_path = try comp.tmpFilePath(arena, dep_basename);
             break :blk out_dep_path;
         };
@@ -2069,7 +2069,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
             error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}),
             error.SemanticAnalyzeFail => {
                 for (clang_errors) |clang_err| {
-                    std.debug.print("{}:{}:{}: {}\n", .{
+                    std.debug.print("{s}:{}:{}: {s}\n", .{
                         if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)",
                         clang_err.line + 1,
                         clang_err.column + 1,
@@ -2087,7 +2087,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
             try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
             // Just to save disk space, we delete the file because it is never needed again.
             zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| {
-                warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) });
+                warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) });
             };
         }
 
@@ -2102,7 +2102,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
         _ = try std.zig.render(comp.gpa, bos.writer(), tree);
         try bos.flush();
 
-        man.writeManifest() catch |err| warn("failed to write cache manifest: {}", .{@errorName(err)});
+        man.writeManifest() catch |err| warn("failed to write cache manifest: {s}", .{@errorName(err)});
 
         break :digest digest;
     };
@@ -2111,7 +2111,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
         const full_zig_path = try comp.local_cache_directory.join(arena, &[_][]const u8{
             "o", &digest, translated_zig_basename,
         });
-        try io.getStdOut().writer().print("{}\n", .{full_zig_path});
+        try io.getStdOut().writer().print("{s}\n", .{full_zig_path});
         return cleanExit();
     } else {
         const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename });
@@ -2148,10 +2148,10 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void {
                     try stdout.writeAll(usage_libc);
                     return cleanExit();
                 } else {
-                    fatal("unrecognized parameter: '{}'", .{arg});
+                    fatal("unrecognized parameter: '{s}'", .{arg});
                 }
             } else if (input_file != null) {
-                fatal("unexpected extra parameter: '{}'", .{arg});
+                fatal("unexpected extra parameter: '{s}'", .{arg});
             } else {
                 input_file = arg;
             }
@@ -2159,7 +2159,7 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void {
     }
     if (input_file) |libc_file| {
         var libc = LibCInstallation.parse(gpa, libc_file) catch |err| {
-            fatal("unable to parse libc file: {}", .{@errorName(err)});
+            fatal("unable to parse libc file: {s}", .{@errorName(err)});
         };
         defer libc.deinit(gpa);
     } else {
@@ -2167,7 +2167,7 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void {
             .allocator = gpa,
             .verbose = true,
         }) catch |err| {
-            fatal("unable to detect native libc: {}", .{@errorName(err)});
+            fatal("unable to detect native libc: {s}", .{@errorName(err)});
         };
         defer libc.deinit(gpa);
 
@@ -2205,16 +2205,16 @@ pub fn cmdInit(
                     try io.getStdOut().writeAll(usage_init);
                     return cleanExit();
                 } else {
-                    fatal("unrecognized parameter: '{}'", .{arg});
+                    fatal("unrecognized parameter: '{s}'", .{arg});
                 }
             } else {
-                fatal("unexpected extra parameter: '{}'", .{arg});
+                fatal("unexpected extra parameter: '{s}'", .{arg});
             }
         }
     }
     const self_exe_path = try fs.selfExePathAlloc(arena);
     var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {
-        fatal("unable to find zig installation directory: {}\n", .{@errorName(err)});
+        fatal("unable to find zig installation directory: {s}\n", .{@errorName(err)});
     };
     defer zig_lib_directory.handle.close();
 
@@ -2232,7 +2232,7 @@ pub fn cmdInit(
 
     const max_bytes = 10 * 1024 * 1024;
     const build_zig_contents = template_dir.readFileAlloc(arena, "build.zig", max_bytes) catch |err| {
-        fatal("unable to read template file 'build.zig': {}", .{@errorName(err)});
+        fatal("unable to read template file 'build.zig': {s}", .{@errorName(err)});
     };
     var modified_build_zig_contents = std.ArrayList(u8).init(arena);
     try modified_build_zig_contents.ensureCapacity(build_zig_contents.len);
@@ -2244,13 +2244,13 @@ pub fn cmdInit(
         }
     }
     const main_zig_contents = template_dir.readFileAlloc(arena, "src" ++ s ++ "main.zig", max_bytes) catch |err| {
-        fatal("unable to read template file 'main.zig': {}", .{@errorName(err)});
+        fatal("unable to read template file 'main.zig': {s}", .{@errorName(err)});
     };
     if (fs.cwd().access("build.zig", .{})) |_| {
         fatal("existing build.zig file would be overwritten", .{});
     } else |err| switch (err) {
         error.FileNotFound => {},
-        else => fatal("unable to test existence of build.zig: {}\n", .{@errorName(err)}),
+        else => fatal("unable to test existence of build.zig: {s}\n", .{@errorName(err)}),
     }
     var src_dir = try fs.cwd().makeOpenPath("src", .{});
     defer src_dir.close();
@@ -2311,23 +2311,23 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
                 const arg = args[i];
                 if (mem.startsWith(u8, arg, "-")) {
                     if (mem.eql(u8, arg, "--build-file")) {
-                        if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg});
+                        if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
                         i += 1;
                         build_file = args[i];
                         continue;
                     } else if (mem.eql(u8, arg, "--override-lib-dir")) {
-                        if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg});
+                        if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
                         i += 1;
                         override_lib_dir = args[i];
                         try child_argv.appendSlice(&[_][]const u8{ arg, args[i] });
                         continue;
                     } else if (mem.eql(u8, arg, "--cache-dir")) {
-                        if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg});
+                        if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
                         i += 1;
                         override_local_cache_dir = args[i];
                         continue;
                     } else if (mem.eql(u8, arg, "--global-cache-dir")) {
-                        if (i + 1 >= args.len) fatal("expected argument after '{}'", .{arg});
+                        if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
                         i += 1;
                         override_global_cache_dir = args[i];
                         continue;
@@ -2344,7 +2344,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             }
         else
             introspect.findZigLibDirFromSelfExe(arena, self_exe_path) catch |err| {
-                fatal("unable to find zig installation directory: {}", .{@errorName(err)});
+                fatal("unable to find zig installation directory: {s}", .{@errorName(err)});
             };
         defer zig_lib_directory.handle.close();
 
@@ -2385,7 +2385,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
                 } else |err| switch (err) {
                     error.FileNotFound => {
                         dirname = fs.path.dirname(dirname) orelse {
-                            std.log.info("{}", .{
+                            std.log.info("{s}", .{
                                 \\Initialize a 'build.zig' template file with `zig init-lib` or `zig init-exe`,
                                 \\or see `zig --help` for more options.
                             });
@@ -2467,7 +2467,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
             .self_exe_path = self_exe_path,
             .thread_pool = &thread_pool,
         }) catch |err| {
-            fatal("unable to create compilation: {}", .{@errorName(err)});
+            fatal("unable to create compilation: {s}", .{@errorName(err)});
         };
         defer comp.destroy();
 
@@ -2493,11 +2493,11 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
         .Exited => |code| {
             if (code == 0) return cleanExit();
             const cmd = try argvCmd(arena, child_argv);
-            fatal("the following build command failed with exit code {}:\n{}", .{ code, cmd });
+            fatal("the following build command failed with exit code {}:\n{s}", .{ code, cmd });
         },
         else => {
             const cmd = try argvCmd(arena, child_argv);
-            fatal("the following build command crashed:\n{}", .{cmd});
+            fatal("the following build command crashed:\n{s}", .{cmd});
         },
     }
 }
@@ -2564,14 +2564,14 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
                     i += 1;
                     const next_arg = args[i];
                     color = std.meta.stringToEnum(Color, next_arg) orelse {
-                        fatal("expected [auto|on|off] after --color, found '{}'", .{next_arg});
+                        fatal("expected [auto|on|off] after --color, found '{s}'", .{next_arg});
                     };
                 } else if (mem.eql(u8, arg, "--stdin")) {
                     stdin_flag = true;
                 } else if (mem.eql(u8, arg, "--check")) {
                     check_flag = true;
                 } else {
-                    fatal("unrecognized parameter: '{}'", .{arg});
+                    fatal("unrecognized parameter: '{s}'", .{arg});
                 }
             } else {
                 try input_files.append(arg);
@@ -2590,7 +2590,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
         defer gpa.free(source_code);
 
         const tree = std.zig.parse(gpa, source_code) catch |err| {
-            fatal("error parsing stdin: {}", .{err});
+            fatal("error parsing stdin: {s}", .{err});
         };
         defer tree.deinit();
 
@@ -2629,7 +2629,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
     for (input_files.items) |file_path| {
         // Get the real path here to avoid Windows failing on relative file paths with . or .. in them.
         const real_path = fs.realpathAlloc(gpa, file_path) catch |err| {
-            fatal("unable to open '{}': {}", .{ file_path, err });
+            fatal("unable to open '{s}': {s}", .{ file_path, @errorName(err) });
         };
         defer gpa.free(real_path);
 
@@ -2668,7 +2668,7 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_
     fmtPathFile(fmt, file_path, check_mode, dir, sub_path) catch |err| switch (err) {
         error.IsDir, error.AccessDenied => return fmtPathDir(fmt, file_path, check_mode, dir, sub_path),
         else => {
-            warn("unable to format '{}': {}", .{ file_path, err });
+            warn("unable to format '{s}': {s}", .{ file_path, @errorName(err) });
             fmt.any_error = true;
             return;
         },
@@ -2702,7 +2702,7 @@ fn fmtPathDir(
                 try fmtPathDir(fmt, full_path, check_mode, dir, entry.name);
             } else {
                 fmtPathFile(fmt, full_path, check_mode, dir, entry.name) catch |err| {
-                    warn("unable to format '{}': {}", .{ full_path, err });
+                    warn("unable to format '{s}': {s}", .{ full_path, @errorName(err) });
                     fmt.any_error = true;
                     return;
                 };
@@ -2761,7 +2761,7 @@ fn fmtPathFile(
         const anything_changed = try std.zig.render(fmt.gpa, io.null_out_stream, tree);
         if (anything_changed) {
             const stdout = io.getStdOut().writer();
-            try stdout.print("{}\n", .{file_path});
+            try stdout.print("{s}\n", .{file_path});
             fmt.any_error = true;
         }
     } else {
@@ -2779,7 +2779,7 @@ fn fmtPathFile(
         try af.file.writeAll(fmt.out_buffer.items);
         try af.finish();
         const stdout = io.getStdOut().writer();
-        try stdout.print("{}\n", .{file_path});
+        try stdout.print("{s}\n", .{file_path});
     }
 }
 
@@ -2812,7 +2812,7 @@ fn printErrMsgToFile(
     const text = text_buf.items;
 
     const stream = file.outStream();
-    try stream.print("{}:{}:{}: error: {}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
+    try stream.print("{s}:{}:{}: error: {s}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
 
     if (!color_on) return;
 
@@ -2984,7 +2984,7 @@ pub const ClangArgIterator = struct {
             const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
             const resp_file_path = arg[1..];
             const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch |err| {
-                fatal("unable to read response file '{}': {}", .{ resp_file_path, @errorName(err) });
+                fatal("unable to read response file '{s}': {s}", .{ resp_file_path, @errorName(err) });
             };
             defer allocator.free(resp_contents);
             // TODO is there a specification for this file format? Let's find it and make this parsing more robust
@@ -3057,7 +3057,7 @@ pub const ClangArgIterator = struct {
                 const prefix_len = clang_arg.matchStartsWith(arg);
                 if (prefix_len == arg.len) {
                     if (self.next_index >= self.argv.len) {
-                        fatal("Expected parameter after '{}'", .{arg});
+                        fatal("Expected parameter after '{s}'", .{arg});
                     }
                     self.only_arg = self.argv[self.next_index];
                     self.incrementArgIndex();
@@ -3078,7 +3078,7 @@ pub const ClangArgIterator = struct {
                 if (prefix_len != 0) {
                     self.only_arg = arg[prefix_len..];
                     if (self.next_index >= self.argv.len) {
-                        fatal("Expected parameter after '{}'", .{arg});
+                        fatal("Expected parameter after '{s}'", .{arg});
                     }
                     self.second_arg = self.argv[self.next_index];
                     self.incrementArgIndex();
@@ -3089,7 +3089,7 @@ pub const ClangArgIterator = struct {
             },
             .separate => if (clang_arg.matchEql(arg) > 0) {
                 if (self.next_index >= self.argv.len) {
-                    fatal("Expected parameter after '{}'", .{arg});
+                    fatal("Expected parameter after '{s}'", .{arg});
                 }
                 self.only_arg = self.argv[self.next_index];
                 self.incrementArgIndex();
@@ -3115,7 +3115,7 @@ pub const ClangArgIterator = struct {
             },
         }
         else {
-            fatal("Unknown Clang option: '{}'", .{arg});
+            fatal("Unknown Clang option: '{s}'", .{arg});
         }
     }
 
@@ -3143,7 +3143,7 @@ pub const ClangArgIterator = struct {
 
 fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {
     return std.meta.stringToEnum(std.builtin.CodeModel, arg) orelse
-        fatal("unsupported machine code model: '{}'", .{arg});
+        fatal("unsupported machine code model: '{s}'", .{arg});
 }
 
 /// Raise the open file descriptor limit. Ask and ye shall receive.
@@ -3263,7 +3263,7 @@ fn detectNativeTargetInfo(gpa: *Allocator, cross_target: std.zig.CrossTarget) !s
         // CPU model & feature detection is todo so here we rely on LLVM.
         // https://github.com/ziglang/zig/issues/4591
         if (!build_options.have_llvm)
-            fatal("CPU features detection is not yet available for {} without LLVM extensions", .{@tagName(arch)});
+            fatal("CPU features detection is not yet available for {s} without LLVM extensions", .{@tagName(arch)});
 
         const llvm = @import("llvm_bindings.zig");
         const llvm_cpu_name = llvm.GetHostCPUName();
diff --git a/src/mingw.zig b/src/mingw.zig
index 246b0f33dc..d55cc28b2b 100644
--- a/src/mingw.zig
+++ b/src/mingw.zig
@@ -381,7 +381,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
 
     const term = child.wait() catch |err| {
         // TODO surface a proper error here
-        log.err("unable to spawn {}: {}", .{ args[0], @errorName(err) });
+        log.err("unable to spawn {s}: {s}", .{ args[0], @errorName(err) });
         return error.ClangPreprocessorFailed;
     };
 
@@ -395,7 +395,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
         },
         else => {
             // TODO surface a proper error here
-            log.err("clang terminated unexpectedly with stderr: {}", .{stderr});
+            log.err("clang terminated unexpectedly with stderr: {s}", .{stderr});
             return error.ClangPreprocessorFailed;
         },
     }
diff --git a/src/musl.zig b/src/musl.zig
index f67fe90add..a865e78623 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -155,21 +155,21 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 if (!is_arch_specific) {
                     // Look for an arch specific override.
                     override_path.shrinkRetainingCapacity(0);
-                    try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.s", .{
+                    try override_path.writer().print("{s}" ++ s ++ "{s}" ++ s ++ "{s}.s", .{
                         dirname, arch_name, noextbasename,
                     });
                     if (source_table.contains(override_path.items))
                         continue;
 
                     override_path.shrinkRetainingCapacity(0);
-                    try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.S", .{
+                    try override_path.writer().print("{s}" ++ s ++ "{s}" ++ s ++ "{s}.S", .{
                         dirname, arch_name, noextbasename,
                     });
                     if (source_table.contains(override_path.items))
                         continue;
 
                     override_path.shrinkRetainingCapacity(0);
-                    try override_path.writer().print("{}" ++ s ++ "{}" ++ s ++ "{}.c", .{
+                    try override_path.writer().print("{s}" ++ s ++ "{s}" ++ s ++ "{s}.c", .{
                         dirname, arch_name, noextbasename,
                     });
                     if (source_table.contains(override_path.items))
@@ -322,7 +322,7 @@ fn add_cc_args(
     const target = comp.getTarget();
     const arch_name = target_util.archMuslName(target.cpu.arch);
     const os_name = @tagName(target.os.tag);
-    const triple = try std.fmt.allocPrint(arena, "{}-{}-musl", .{ arch_name, os_name });
+    const triple = try std.fmt.allocPrint(arena, "{s}-{s}-musl", .{ arch_name, os_name });
     const o_arg = if (want_O3) "-O3" else "-Os";
 
     try args.appendSlice(&[_][]const u8{
diff --git a/src/print_env.zig b/src/print_env.zig
index 8aa692e644..bcf4a983ab 100644
--- a/src/print_env.zig
+++ b/src/print_env.zig
@@ -9,7 +9,7 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: std.fs.File.Wri
     defer gpa.free(self_exe_path);
 
     var zig_lib_directory = introspect.findZigLibDirFromSelfExe(gpa, self_exe_path) catch |err| {
-        fatal("unable to find zig installation directory: {}\n", .{@errorName(err)});
+        fatal("unable to find zig installation directory: {s}\n", .{@errorName(err)});
     };
     defer gpa.free(zig_lib_directory.path.?);
     defer zig_lib_directory.handle.close();
diff --git a/src/print_targets.zig b/src/print_targets.zig
index 724cb7a9ac..cf55eee516 100644
--- a/src/print_targets.zig
+++ b/src/print_targets.zig
@@ -18,7 +18,7 @@ pub fn cmdTargets(
     native_target: Target,
 ) !void {
     var zig_lib_directory = introspect.findZigLibDir(allocator) catch |err| {
-        fatal("unable to find zig installation directory: {}\n", .{@errorName(err)});
+        fatal("unable to find zig installation directory: {s}\n", .{@errorName(err)});
     };
     defer zig_lib_directory.handle.close();
     defer allocator.free(zig_lib_directory.path.?);
@@ -61,7 +61,7 @@ pub fn cmdTargets(
     try jws.objectField("libc");
     try jws.beginArray();
     for (target.available_libcs) |libc| {
-        const tmp = try std.fmt.allocPrint(allocator, "{}-{}-{}", .{
+        const tmp = try std.fmt.allocPrint(allocator, "{s}-{s}-{s}", .{
             @tagName(libc.arch), @tagName(libc.os), @tagName(libc.abi),
         });
         defer allocator.free(tmp);
diff --git a/src/stage1.zig b/src/stage1.zig
index 1d50a71ad1..cf3a252ce8 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -37,14 +37,14 @@ pub export fn main(argc: c_int, argv: [*][*:0]u8) c_int {
     defer arena_instance.deinit();
     const arena = &arena_instance.allocator;
 
-    const args = arena.alloc([]const u8, @intCast(usize, argc)) catch fatal("{}", .{"OutOfMemory"});
+    const args = arena.alloc([]const u8, @intCast(usize, argc)) catch fatal("{s}", .{"OutOfMemory"});
     for (args) |*arg, i| {
         arg.* = mem.spanZ(argv[i]);
     }
     if (std.builtin.mode == .Debug) {
         stage2.mainArgs(gpa, arena, args) catch unreachable;
     } else {
-        stage2.mainArgs(gpa, arena, args) catch |err| fatal("{}", .{@errorName(err)});
+        stage2.mainArgs(gpa, arena, args) catch |err| fatal("{s}", .{@errorName(err)});
     }
     return 0;
 }
diff --git a/src/translate_c.zig b/src/translate_c.zig
index 1b2aa4b219..c609597770 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -136,7 +136,7 @@ const Scope = struct {
             var proposed_name = name_copy;
             while (scope.contains(proposed_name)) {
                 scope.mangle_count += 1;
-                proposed_name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, scope.mangle_count });
+                proposed_name = try std.fmt.allocPrint(c.arena, "{s}_{}", .{ name, scope.mangle_count });
             }
             try scope.variables.append(.{ .name = name_copy, .alias = proposed_name });
             return proposed_name;
@@ -290,7 +290,7 @@ pub const Context = struct {
 
         const line = c.source_manager.getSpellingLineNumber(spelling_loc);
         const column = c.source_manager.getSpellingColumnNumber(spelling_loc);
-        return std.fmt.allocPrint(c.arena, "{}:{}:{}", .{ filename, line, column });
+        return std.fmt.allocPrint(c.arena, "{s}:{d}:{d}", .{ filename, line, column });
     }
 
     fn createCall(c: *Context, fn_expr: *ast.Node, params_len: ast.NodeIndex) !*ast.Node.Call {
@@ -530,7 +530,7 @@ fn declVisitor(c: *Context, decl: *const clang.Decl) Error!void {
         },
         else => {
             const decl_name = try c.str(decl.getDeclKindName());
-            try emitWarning(c, decl.getLocation(), "ignoring {} declaration", .{decl_name});
+            try emitWarning(c, decl.getLocation(), "ignoring {s} declaration", .{decl_name});
         },
     }
 }
@@ -625,7 +625,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const clang.FunctionDecl) Error!void {
         const param_name = if (param.name_token) |name_tok|
             tokenSlice(c, name_tok)
         else
-            return failDecl(c, fn_decl_loc, fn_name, "function {} parameter has no name", .{fn_name});
+            return failDecl(c, fn_decl_loc, fn_name, "function {s} parameter has no name", .{fn_name});
 
         const c_param = fn_decl.getParamDecl(param_id);
         const qual_type = c_param.getOriginalType();
@@ -634,7 +634,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const clang.FunctionDecl) Error!void {
         const mangled_param_name = try block_scope.makeMangledName(c, param_name);
 
         if (!is_const) {
-            const bare_arg_name = try std.fmt.allocPrint(c.arena, "arg_{}", .{mangled_param_name});
+            const bare_arg_name = try std.fmt.allocPrint(c.arena, "arg_{s}", .{mangled_param_name});
             const arg_name = try block_scope.makeMangledName(c, bare_arg_name);
 
             const mut_tok = try appendToken(c, .Keyword_var, "var");
@@ -727,7 +727,7 @@ fn visitVarDecl(c: *Context, var_decl: *const clang.VarDecl, mangled_name: ?[]co
 
     // TODO https://github.com/ziglang/zig/issues/3756
     // TODO https://github.com/ziglang/zig/issues/1802
-    const checked_name = if (isZigPrimitiveType(var_name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ var_name, c.getMangle() }) else var_name;
+    const checked_name = if (isZigPrimitiveType(var_name)) try std.fmt.allocPrint(c.arena, "{s}_{d}", .{ var_name, c.getMangle() }) else var_name;
     const var_decl_loc = var_decl.getLocation();
 
     const qual_type = var_decl.getTypeSourceInfo_getType();
@@ -808,7 +808,7 @@ fn visitVarDecl(c: *Context, var_decl: *const clang.VarDecl, mangled_name: ?[]co
             _ = try appendToken(rp.c, .LParen, "(");
             const expr = try transCreateNodeStringLiteral(
                 rp.c,
-                try std.fmt.allocPrint(rp.c.arena, "\"{}\"", .{str_ptr[0..str_len]}),
+                try std.fmt.allocPrint(rp.c.arena, "\"{s}\"", .{str_ptr[0..str_len]}),
             );
             _ = try appendToken(rp.c, .RParen, ")");
 
@@ -887,7 +887,7 @@ fn transTypeDef(c: *Context, typedef_decl: *const clang.TypedefNameDecl, top_lev
 
     // TODO https://github.com/ziglang/zig/issues/3756
     // TODO https://github.com/ziglang/zig/issues/1802
-    const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name;
+    const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.arena, "{s}_{d}", .{ typedef_name, c.getMangle() }) else typedef_name;
     if (checkForBuiltinTypedef(checked_name)) |builtin| {
         return transTypeDefAsBuiltin(c, typedef_decl, builtin);
     }
@@ -958,11 +958,11 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
         container_kind_name = "struct";
         container_kind = .Keyword_struct;
     } else {
-        try emitWarning(c, record_loc, "record {} is not a struct or union", .{bare_name});
+        try emitWarning(c, record_loc, "record {s} is not a struct or union", .{bare_name});
         return null;
     }
 
-    const name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ container_kind_name, bare_name });
+    const name = try std.fmt.allocPrint(c.arena, "{s}_{s}", .{ container_kind_name, bare_name });
     _ = try c.decl_table.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), name);
 
     const visib_tok = if (!is_unnamed) try appendToken(c, .Keyword_pub, "pub") else null;
@@ -1003,7 +1003,7 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
                 _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {});
                 const opaque_type = try transCreateNodeOpaqueType(c);
                 semicolon = try appendToken(c, .Semicolon, ";");
-                try emitWarning(c, field_loc, "{} demoted to opaque type - has bitfield", .{container_kind_name});
+                try emitWarning(c, field_loc, "{s} demoted to opaque type - has bitfield", .{container_kind_name});
                 break :blk opaque_type;
             }
 
@@ -1011,7 +1011,7 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
                 _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {});
                 const opaque_type = try transCreateNodeOpaqueType(c);
                 semicolon = try appendToken(c, .Semicolon, ";");
-                try emitWarning(c, field_loc, "{} demoted to opaque type - has variable length array", .{container_kind_name});
+                try emitWarning(c, field_loc, "{s} demoted to opaque type - has variable length array", .{container_kind_name});
                 break :blk opaque_type;
             }
 
@@ -1030,7 +1030,7 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
                     _ = try c.opaque_demotes.put(c.gpa, @ptrToInt(record_decl.getCanonicalDecl()), {});
                     const opaque_type = try transCreateNodeOpaqueType(c);
                     semicolon = try appendToken(c, .Semicolon, ";");
-                    try emitWarning(c, record_loc, "{} demoted to opaque type - unable to translate type of field {}", .{ container_kind_name, raw_name });
+                    try emitWarning(c, record_loc, "{s} demoted to opaque type - unable to translate type of field {s}", .{ container_kind_name, raw_name });
                     break :blk opaque_type;
                 },
                 else => |e| return e,
@@ -1114,7 +1114,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const clang.EnumDecl) Error!?*ast.Node
         is_unnamed = true;
     }
 
-    const name = try std.fmt.allocPrint(c.arena, "enum_{}", .{bare_name});
+    const name = try std.fmt.allocPrint(c.arena, "enum_{s}", .{bare_name});
     _ = try c.decl_table.put(c.gpa, @ptrToInt(enum_decl.getCanonicalDecl()), name);
 
     const visib_tok = if (!is_unnamed) try appendToken(c, .Keyword_pub, "pub") else null;
@@ -1385,7 +1385,7 @@ fn transStmt(
                 rp,
                 error.UnsupportedTranslation,
                 stmt.getBeginLoc(),
-                "TODO implement translation of stmt class {}",
+                "TODO implement translation of stmt class {s}",
                 .{@tagName(sc)},
             );
         },
@@ -1684,7 +1684,7 @@ fn transDeclStmtOne(
             rp,
             error.UnsupportedTranslation,
             decl.getLocation(),
-            "TODO implement translation of DeclStmt kind {}",
+            "TODO implement translation of DeclStmt kind {s}",
             .{@tagName(kind)},
         ),
     }
@@ -1782,7 +1782,7 @@ fn transImplicitCastExpr(
             rp,
             error.UnsupportedTranslation,
             @ptrCast(*const clang.Stmt, expr).getBeginLoc(),
-            "TODO implement translation of CastKind {}",
+            "TODO implement translation of CastKind {s}",
             .{@tagName(kind)},
         ),
     }
@@ -2043,7 +2043,7 @@ fn transStringLiteral(
             rp,
             error.UnsupportedTranslation,
             @ptrCast(*const clang.Stmt, stmt).getBeginLoc(),
-            "TODO: support string literal kind {}",
+            "TODO: support string literal kind {s}",
             .{kind},
         ),
     }
@@ -2168,7 +2168,6 @@ fn transCCast(
         // @boolToInt returns either a comptime_int or a u1
         // TODO: if dst_type is 1 bit & signed (bitfield) we need @bitCast
         // instead of @as
-
         const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1);
         builtin_node.params()[0] = expr;
         builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")");
@@ -2455,7 +2454,7 @@ fn transInitListExpr(
         );
     } else {
         const type_name = rp.c.str(qual_type.getTypeClassName());
-        return revertAndWarn(rp, error.UnsupportedType, source_loc, "unsupported initlist type: '{}'", .{type_name});
+        return revertAndWarn(rp, error.UnsupportedType, source_loc, "unsupported initlist type: '{s}'", .{type_name});
     }
 }
 
@@ -4433,7 +4432,8 @@ fn transCreateNodeBoolLiteral(c: *Context, value: bool) !*ast.Node {
 }
 
 fn transCreateNodeInt(c: *Context, int: anytype) !*ast.Node {
-    const token = try appendTokenFmt(c, .IntegerLiteral, "{}", .{int});
+    const fmt_s = if (comptime std.meta.trait.isIntegerNumber(@TypeOf(int))) "{d}" else "{s}";
+    const token = try appendTokenFmt(c, .IntegerLiteral, fmt_s, .{int});
     const node = try c.arena.create(ast.Node.OneToken);
     node.* = .{
         .base = .{ .tag = .IntegerLiteral },
@@ -4442,8 +4442,8 @@ fn transCreateNodeInt(c: *Context, int: anytype) !*ast.Node {
     return &node.base;
 }
 
-fn transCreateNodeFloat(c: *Context, int: anytype) !*ast.Node {
-    const token = try appendTokenFmt(c, .FloatLiteral, "{}", .{int});
+fn transCreateNodeFloat(c: *Context, str: []const u8) !*ast.Node {
+    const token = try appendTokenFmt(c, .FloatLiteral, "{s}", .{str});
     const node = try c.arena.create(ast.Node.OneToken);
     node.* = .{
         .base = .{ .tag = .FloatLiteral },
@@ -4916,7 +4916,7 @@ fn transType(rp: RestorePoint, ty: *const clang.Type, source_loc: clang.SourceLo
         },
         else => {
             const type_name = rp.c.str(ty.getTypeClassName());
-            return revertAndWarn(rp, error.UnsupportedType, source_loc, "unsupported type: '{}'", .{type_name});
+            return revertAndWarn(rp, error.UnsupportedType, source_loc, "unsupported type: '{s}'", .{type_name});
         },
     }
 }
@@ -4999,7 +4999,7 @@ fn transCC(
             rp,
             error.UnsupportedType,
             source_loc,
-            "unsupported calling convention: {}",
+            "unsupported calling convention: {s}",
             .{@tagName(clang_cc)},
         ),
     }
@@ -5117,7 +5117,7 @@ fn finishTransFnProto(
                 _ = try appendToken(rp.c, .LParen, "(");
                 const expr = try transCreateNodeStringLiteral(
                     rp.c,
-                    try std.fmt.allocPrint(rp.c.arena, "\"{}\"", .{str_ptr[0..str_len]}),
+                    try std.fmt.allocPrint(rp.c.arena, "\"{s}\"", .{str_ptr[0..str_len]}),
                 );
                 _ = try appendToken(rp.c, .RParen, ")");
 
@@ -5214,7 +5214,7 @@ fn revertAndWarn(
 
 fn emitWarning(c: *Context, loc: clang.SourceLocation, comptime format: []const u8, args: anytype) !void {
     const args_prefix = .{c.locStr(loc)};
-    _ = try appendTokenFmt(c, .LineComment, "// {}: warning: " ++ format, args_prefix ++ args);
+    _ = try appendTokenFmt(c, .LineComment, "// {s}: warning: " ++ format, args_prefix ++ args);
 }
 
 pub fn failDecl(c: *Context, loc: clang.SourceLocation, name: []const u8, comptime format: []const u8, args: anytype) !void {
@@ -5228,7 +5228,7 @@ pub fn failDecl(c: *Context, loc: clang.SourceLocation, name: []const u8, compti
     const msg_tok = try appendTokenFmt(c, .StringLiteral, "\"" ++ format ++ "\"", args);
     const rparen_tok = try appendToken(c, .RParen, ")");
     const semi_tok = try appendToken(c, .Semicolon, ";");
-    _ = try appendTokenFmt(c, .LineComment, "// {}", .{c.locStr(loc)});
+    _ = try appendTokenFmt(c, .LineComment, "// {s}", .{c.locStr(loc)});
 
     const msg_node = try c.arena.create(ast.Node.OneToken);
     msg_node.* = .{
@@ -5258,7 +5258,7 @@ pub fn failDecl(c: *Context, loc: clang.SourceLocation, name: []const u8, compti
 
 fn appendToken(c: *Context, token_id: Token.Id, bytes: []const u8) !ast.TokenIndex {
     std.debug.assert(token_id != .Identifier); // use appendIdentifier
-    return appendTokenFmt(c, token_id, "{}", .{bytes});
+    return appendTokenFmt(c, token_id, "{s}", .{bytes});
 }
 
 fn appendTokenFmt(c: *Context, token_id: Token.Id, comptime format: []const u8, args: anytype) !ast.TokenIndex {
@@ -5329,7 +5329,7 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node {
 }
 
 fn transCreateNodeIdentifierUnchecked(c: *Context, name: []const u8) !*ast.Node {
-    const token_index = try appendTokenFmt(c, .Identifier, "{}", .{name});
+    const token_index = try appendTokenFmt(c, .Identifier, "{s}", .{name});
     const identifier = try c.arena.create(ast.Node.OneToken);
     identifier.* = .{
         .base = .{ .tag = .Identifier },
@@ -5390,7 +5390,7 @@ fn transPreprocessorEntities(c: *Context, unit: *clang.ASTUnit) Error!void {
                 const name = try c.str(raw_name);
                 // TODO https://github.com/ziglang/zig/issues/3756
                 // TODO https://github.com/ziglang/zig/issues/1802
-                const mangled_name = if (isZigPrimitiveType(name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, c.getMangle() }) else name;
+                const mangled_name = if (isZigPrimitiveType(name)) try std.fmt.allocPrint(c.arena, "{s}_{d}", .{ name, c.getMangle() }) else name;
                 if (scope.containsNow(mangled_name)) {
                     continue;
                 }
@@ -5468,7 +5468,7 @@ fn transMacroDefine(c: *Context, m: *MacroCtx) ParseError!void {
     const init_node = try parseCExpr(c, m, scope);
     const last = m.next().?;
     if (last != .Eof and last != .Nl)
-        return m.fail(c, "unable to translate C expr: unexpected token .{}", .{@tagName(last)});
+        return m.fail(c, "unable to translate C expr: unexpected token .{s}", .{@tagName(last)});
 
     const semicolon_token = try appendToken(c, .Semicolon, ";");
     const node = try ast.Node.VarDecl.create(c.arena, .{
@@ -5540,7 +5540,7 @@ fn transMacroFnDefine(c: *Context, m: *MacroCtx) ParseError!void {
     const expr = try parseCExpr(c, m, scope);
     const last = m.next().?;
     if (last != .Eof and last != .Nl)
-        return m.fail(c, "unable to translate C expr: unexpected token .{}", .{@tagName(last)});
+        return m.fail(c, "unable to translate C expr: unexpected token .{s}", .{@tagName(last)});
     _ = try appendToken(c, .Semicolon, ";");
     const type_of_arg = if (!expr.tag.isBlock()) expr else blk: {
         const stmts = expr.blockStatements();
@@ -5623,11 +5623,11 @@ fn parseCNumLit(c: *Context, m: *MacroCtx) ParseError!*ast.Node {
                 switch (lit_bytes[1]) {
                     '0'...'7' => {
                         // Octal
-                        lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes});
+                        lit_bytes = try std.fmt.allocPrint(c.arena, "0o{s}", .{lit_bytes});
                     },
                     'X' => {
                         // Hexadecimal with capital X, valid in C but not in Zig
-                        lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]});
+                        lit_bytes = try std.fmt.allocPrint(c.arena, "0x{s}", .{lit_bytes[2..]});
                     },
                     else => {},
                 }
@@ -5659,7 +5659,7 @@ fn parseCNumLit(c: *Context, m: *MacroCtx) ParseError!*ast.Node {
         },
         .FloatLiteral => |suffix| {
             if (lit_bytes[0] == '.')
-                lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes});
+                lit_bytes = try std.fmt.allocPrint(c.arena, "0{s}", .{lit_bytes});
             if (suffix == .none) {
                 return transCreateNodeFloat(c, lit_bytes);
             }
@@ -5937,7 +5937,7 @@ fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*
 
             const next_id = m.next().?;
             if (next_id != .RParen) {
-                try m.fail(c, "unable to translate C expr: expected ')' instead got: {}", .{@tagName(next_id)});
+                try m.fail(c, "unable to translate C expr: expected ')' instead got: {s}", .{@tagName(next_id)});
                 return error.ParseError;
             }
             var saw_l_paren = false;
@@ -5995,7 +5995,7 @@ fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*
             return &group_node.base;
         },
         else => {
-            try m.fail(c, "unable to translate C expr: unexpected token .{}", .{@tagName(tok)});
+            try m.fail(c, "unable to translate C expr: unexpected token .{s}", .{@tagName(tok)});
             return error.ParseError;
         },
     }
diff --git a/src/value.zig b/src/value.zig
index 8180df2f9e..10f58fa44f 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -464,7 +464,7 @@ pub const Value = extern union {
             .ty => return val.castTag(.ty).?.data.format("", options, out_stream),
             .int_type => {
                 const int_type = val.castTag(.int_type).?.data;
-                return out_stream.print("{}{}", .{
+                return out_stream.print("{s}{d}", .{
                     if (int_type.signed) "s" else "u",
                     int_type.bits,
                 });
@@ -507,7 +507,7 @@ pub const Value = extern union {
                 }
                 return out_stream.writeAll("}");
             },
-            .@"error" => return out_stream.print("error.{}", .{val.castTag(.@"error").?.data.name}),
+            .@"error" => return out_stream.print("error.{s}", .{val.castTag(.@"error").?.data.name}),
             .inferred_alloc => return out_stream.writeAll("(inferred allocation value)"),
         };
     }
diff --git a/src/zir.zig b/src/zir.zig
index ce8498d7e8..94e2b24b0c 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1150,7 +1150,7 @@ pub const Module = struct {
 
         for (self.decls) |decl, i| {
             write.next_instr_index = 0;
-            try stream.print("@{} ", .{decl.name});
+            try stream.print("@{s} ", .{decl.name});
             try write.writeInstToStream(stream, decl.inst);
             try stream.writeByte('\n');
         }
@@ -1206,13 +1206,13 @@ const Writer = struct {
             if (@typeInfo(arg_field.field_type) == .Optional) {
                 if (@field(inst.kw_args, arg_field.name)) |non_optional| {
                     if (need_comma) try stream.writeAll(", ");
-                    try stream.print("{}=", .{arg_field.name});
+                    try stream.print("{s}=", .{arg_field.name});
                     try self.writeParamToStream(stream, &non_optional);
                     need_comma = true;
                 }
             } else {
                 if (need_comma) try stream.writeAll(", ");
-                try stream.print("{}=", .{arg_field.name});
+                try stream.print("{s}=", .{arg_field.name});
                 try self.writeParamToStream(stream, &@field(inst.kw_args, arg_field.name));
                 need_comma = true;
             }
@@ -1334,16 +1334,16 @@ const Writer = struct {
             if (info.index) |i| {
                 try stream.print("%{}", .{info.index});
             } else {
-                try stream.print("@{}", .{info.name});
+                try stream.print("@{s}", .{info.name});
             }
         } else if (inst.cast(Inst.DeclVal)) |decl_val| {
-            try stream.print("@{}", .{decl_val.positionals.name});
+            try stream.print("@{s}", .{decl_val.positionals.name});
         } else if (inst.cast(Inst.DeclValInModule)) |decl_val| {
-            try stream.print("@{}", .{decl_val.positionals.decl.name});
+            try stream.print("@{s}", .{decl_val.positionals.decl.name});
         } else {
             // This should be unreachable in theory, but since ZIR is used for debugging the compiler
             // we output some debug text instead.
-            try stream.print("?{}?", .{@tagName(inst.tag)});
+            try stream.print("?{s}?", .{@tagName(inst.tag)});
         }
     }
 };
@@ -1424,7 +1424,7 @@ const Parser = struct {
                 const decl = try parseInstruction(self, &body_context, ident);
                 const ident_index = body_context.instructions.items.len;
                 if (try body_context.name_map.fetchPut(ident, decl.inst)) |_| {
-                    return self.fail("redefinition of identifier '{}'", .{ident});
+                    return self.fail("redefinition of identifier '{s}'", .{ident});
                 }
                 try body_context.instructions.append(decl.inst);
                 continue;
@@ -1510,7 +1510,7 @@ const Parser = struct {
                     const decl = try parseInstruction(self, null, ident);
                     const ident_index = self.decls.items.len;
                     if (try self.global_name_map.fetchPut(ident, decl.inst)) |_| {
-                        return self.fail("redefinition of identifier '{}'", .{ident});
+                        return self.fail("redefinition of identifier '{s}'", .{ident});
                     }
                     try self.decls.append(self.allocator, decl);
                 },
@@ -1538,7 +1538,7 @@ const Parser = struct {
         for (bytes) |byte| {
             if (self.source[self.i] != byte) {
                 self.i = start;
-                return self.fail("expected '{}'", .{bytes});
+                return self.fail("expected '{s}'", .{bytes});
             }
             self.i += 1;
         }
@@ -1585,7 +1585,7 @@ const Parser = struct {
                 return parseInstructionGeneric(self, field.name, tag.Type(), tag, body_ctx, name, contents_start);
             }
         }
-        return self.fail("unknown instruction '{}'", .{fn_name});
+        return self.fail("unknown instruction '{s}'", .{fn_name});
     }
 
     fn parseInstructionGeneric(
@@ -1621,7 +1621,7 @@ const Parser = struct {
                 self.i += 1;
                 skipSpace(self);
             } else if (self.source[self.i] == ')') {
-                return self.fail("expected positional parameter '{}'", .{arg_field.name});
+                return self.fail("expected positional parameter '{s}'", .{arg_field.name});
             }
             @field(inst_specific.positionals, arg_field.name) = try parseParameterGeneric(
                 self,
@@ -1648,7 +1648,7 @@ const Parser = struct {
                     break;
                 }
             } else {
-                return self.fail("unrecognized keyword parameter: '{}'", .{name});
+                return self.fail("unrecognized keyword parameter: '{s}'", .{name});
             }
             skipSpace(self);
         }
@@ -1672,7 +1672,7 @@ const Parser = struct {
                 ' ', '\n', ',', ')' => {
                     const enum_name = self.source[start..self.i];
                     return std.meta.stringToEnum(T, enum_name) orelse {
-                        return self.fail("tag '{}' not a member of enum '{}'", .{ enum_name, @typeName(T) });
+                        return self.fail("tag '{s}' not a member of enum '{s}'", .{ enum_name, @typeName(T) });
                     };
                 },
                 0 => return self.failByte(0),
@@ -1710,7 +1710,7 @@ const Parser = struct {
             BigIntConst => return self.parseIntegerLiteral(),
             usize => {
                 const big_int = try self.parseIntegerLiteral();
-                return big_int.to(usize) catch |err| return self.fail("integer literal: {}", .{@errorName(err)});
+                return big_int.to(usize) catch |err| return self.fail("integer literal: {s}", .{@errorName(err)});
             },
             TypedValue => return self.fail("'const' is a special instruction; not legal in ZIR text", .{}),
             *IrModule.Decl => return self.fail("'declval_in_module' is a special instruction; not legal in ZIR text", .{}),
@@ -1759,7 +1759,7 @@ const Parser = struct {
             },
             else => @compileError("Unimplemented: ir parseParameterGeneric for type " ++ @typeName(T)),
         }
-        return self.fail("TODO parse parameter {}", .{@typeName(T)});
+        return self.fail("TODO parse parameter {s}", .{@typeName(T)});
     }
 
     fn parseParameterInst(self: *Parser, body_ctx: ?*Body) !*Inst {
@@ -1788,7 +1788,7 @@ const Parser = struct {
             const src = name_start - 1;
             if (local_ref) {
                 self.i = src;
-                return self.fail("unrecognized identifier: {}", .{bad_name});
+                return self.fail("unrecognized identifier: {s}", .{bad_name});
             } else {
                 const declval = try self.arena.allocator.create(Inst.DeclVal);
                 declval.* = .{
@@ -1873,7 +1873,7 @@ pub fn dumpFn(old_module: IrModule, module_fn: *IrModule.Fn) void {
 
     const fn_ty = module_fn.owner_decl.typed_value.most_recent.typed_value.ty;
     _ = ctx.emitFn(module_fn, 0, fn_ty) catch |err| {
-        std.debug.print("unable to dump function: {}\n", .{err});
+        std.debug.print("unable to dump function: {s}\n", .{@errorName(err)});
         return;
     };
     var module = Module{
@@ -2203,7 +2203,7 @@ const EmitZIR = struct {
                         };
                         return self.emitStringLiteral(src, bytes);
                     },
-                    else => |t| std.debug.panic("TODO implement emitTypedValue for pointer to {}", .{@tagName(t)}),
+                    else => |t| std.debug.panic("TODO implement emitTypedValue for pointer to {s}", .{@tagName(t)}),
                 }
             },
             .ComptimeInt => return self.emitComptimeIntVal(src, typed_value.val),
@@ -2274,7 +2274,7 @@ const EmitZIR = struct {
                 };
                 return self.emitUnnamedDecl(&inst.base);
             },
-            else => |t| std.debug.panic("TODO implement emitTypedValue for {}", .{@tagName(t)}),
+            else => |t| std.debug.panic("TODO implement emitTypedValue for {s}", .{@tagName(t)}),
         }
     }
 
@@ -2947,7 +2947,7 @@ pub fn dumpZir(allocator: *Allocator, kind: []const u8, decl_name: [*:0]const u8
     try write.inst_table.ensureCapacity(@intCast(u32, instructions.len));
 
     const stderr = std.io.getStdErr().outStream();
-    try stderr.print("{} {s} {{ // unanalyzed\n", .{ kind, decl_name });
+    try stderr.print("{s} {s} {{ // unanalyzed\n", .{ kind, decl_name });
 
     for (instructions) |inst| {
         const my_i = write.next_instr_index;
@@ -2967,5 +2967,5 @@ pub fn dumpZir(allocator: *Allocator, kind: []const u8, decl_name: [*:0]const u8
         try stderr.writeByte('\n');
     }
 
-    try stderr.print("}} // {} {s}\n\n", .{ kind, decl_name });
+    try stderr.print("}} // {s} {s}\n\n", .{ kind, decl_name });
 }
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 629dfa0c9a..72a1b04238 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -274,7 +274,7 @@ pub fn resolveInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
     const entry = if (old_inst.cast(zir.Inst.DeclVal)) |declval| blk: {
         const decl_name = declval.positionals.name;
         const entry = zir_module.contents.module.findDecl(decl_name) orelse
-            return mod.fail(scope, old_inst.src, "decl '{}' not found", .{decl_name});
+            return mod.fail(scope, old_inst.src, "decl '{s}' not found", .{decl_name});
         break :blk entry;
     } else blk: {
         // If this assert trips, the instruction that was referenced did not get
@@ -564,14 +564,14 @@ fn analyzeInstStr(mod: *Module, scope: *Scope, str_inst: *zir.Inst.Str) InnerErr
 fn analyzeInstExport(mod: *Module, scope: *Scope, export_inst: *zir.Inst.Export) InnerError!*Inst {
     const symbol_name = try resolveConstString(mod, scope, export_inst.positionals.symbol_name);
     const exported_decl = mod.lookupDeclName(scope, export_inst.positionals.decl_name) orelse
-        return mod.fail(scope, export_inst.base.src, "decl '{}' not found", .{export_inst.positionals.decl_name});
+        return mod.fail(scope, export_inst.base.src, "decl '{s}' not found", .{export_inst.positionals.decl_name});
     try mod.analyzeExport(scope, export_inst.base.src, symbol_name, exported_decl);
     return mod.constVoid(scope, export_inst.base.src);
 }
 
 fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!*Inst {
     const msg = try resolveConstString(mod, scope, inst.positionals.operand);
-    return mod.fail(scope, inst.base.src, "{}", .{msg});
+    return mod.fail(scope, inst.base.src, "{s}", .{msg});
 }
 
 fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*Inst {
@@ -918,7 +918,7 @@ fn analyzeInstErrorSet(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorSet) In
     for (inst.positionals.fields) |field_name| {
         const entry = try mod.getErrorValue(field_name);
         if (payload.data.fields.fetchPutAssumeCapacity(entry.key, entry.value)) |prev| {
-            return mod.fail(scope, inst.base.src, "duplicate error: '{}'", .{field_name});
+            return mod.fail(scope, inst.base.src, "duplicate error: '{s}'", .{field_name});
         }
     }
     // TODO create name in format "error:line:column"
@@ -1068,7 +1068,7 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                 return mod.fail(
                     scope,
                     fieldptr.positionals.field_name.src,
-                    "no member named '{}' in '{}'",
+                    "no member named '{s}' in '{}'",
                     .{ field_name, elem_ty },
                 );
             }
@@ -1089,7 +1089,7 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                         return mod.fail(
                             scope,
                             fieldptr.positionals.field_name.src,
-                            "no member named '{}' in '{}'",
+                            "no member named '{s}' in '{}'",
                             .{ field_name, elem_ty },
                         );
                     }
@@ -1107,7 +1107,7 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                     // TODO resolve inferred error sets
                     const entry = if (val.castTag(.error_set)) |payload|
                         (payload.data.fields.getEntry(field_name) orelse
-                            return mod.fail(scope, fieldptr.base.src, "no error named '{}' in '{}'", .{ field_name, child_type })).*
+                            return mod.fail(scope, fieldptr.base.src, "no error named '{s}' in '{}'", .{ field_name, child_type })).*
                     else
                         try mod.getErrorValue(field_name);
 
@@ -1135,9 +1135,9 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                     }
 
                     if (&container_scope.file_scope.base == mod.root_scope) {
-                        return mod.fail(scope, fieldptr.base.src, "root source file has no member called '{}'", .{field_name});
+                        return mod.fail(scope, fieldptr.base.src, "root source file has no member called '{s}'", .{field_name});
                     } else {
-                        return mod.fail(scope, fieldptr.base.src, "container '{}' has no member called '{}'", .{ child_type, field_name });
+                        return mod.fail(scope, fieldptr.base.src, "container '{}' has no member called '{s}'", .{ child_type, field_name });
                     }
                 },
                 else => return mod.fail(scope, fieldptr.base.src, "type '{}' does not support field access", .{child_type}),
@@ -1503,14 +1503,14 @@ fn analyzeInstImport(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerErr
 
     const file_scope = mod.analyzeImport(scope, inst.base.src, operand) catch |err| switch (err) {
         error.ImportOutsidePkgPath => {
-            return mod.fail(scope, inst.base.src, "import of file outside package path: '{}'", .{operand});
+            return mod.fail(scope, inst.base.src, "import of file outside package path: '{s}'", .{operand});
         },
         error.FileNotFound => {
-            return mod.fail(scope, inst.base.src, "unable to find '{}'", .{operand});
+            return mod.fail(scope, inst.base.src, "unable to find '{s}'", .{operand});
         },
         else => {
             // TODO user friendly error to string
-            return mod.fail(scope, inst.base.src, "unable to open '{}': {}", .{ operand, @errorName(err) });
+            return mod.fail(scope, inst.base.src, "unable to open '{s}': {s}", .{ operand, @errorName(err) });
         },
     };
     return mod.constType(scope, inst.base.src, file_scope.root_container.ty);
@@ -1637,7 +1637,7 @@ fn analyzeInstArithmetic(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) Inn
     const is_float = scalar_tag == .Float or scalar_tag == .ComptimeFloat;
 
     if (!is_int and !(is_float and floatOpAllowed(inst.base.tag))) {
-        return mod.fail(scope, inst.base.src, "invalid operands to binary expression: '{}' and '{}'", .{ @tagName(lhs.ty.zigTypeTag()), @tagName(rhs.ty.zigTypeTag()) });
+        return mod.fail(scope, inst.base.src, "invalid operands to binary expression: '{s}' and '{s}'", .{ @tagName(lhs.ty.zigTypeTag()), @tagName(rhs.ty.zigTypeTag()) });
     }
 
     if (casted_lhs.value()) |lhs_val| {
@@ -1656,7 +1656,7 @@ fn analyzeInstArithmetic(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) Inn
     const ir_tag = switch (inst.base.tag) {
         .add => Inst.Tag.add,
         .sub => Inst.Tag.sub,
-        else => return mod.fail(scope, inst.base.src, "TODO implement arithmetic for operand '{}''", .{@tagName(inst.base.tag)}),
+        else => return mod.fail(scope, inst.base.src, "TODO implement arithmetic for operand '{s}''", .{@tagName(inst.base.tag)}),
     };
 
     return mod.addBinOp(b, inst.base.src, scalar_type, ir_tag, casted_lhs, casted_rhs);
@@ -1689,7 +1689,7 @@ fn analyzeInstComptimeOp(mod: *Module, scope: *Scope, res_type: Type, inst: *zir
                 mod.floatSub(scope, res_type, inst.base.src, lhs_val, rhs_val);
             break :blk val;
         },
-        else => return mod.fail(scope, inst.base.src, "TODO Implement arithmetic operand '{}'", .{@tagName(inst.base.tag)}),
+        else => return mod.fail(scope, inst.base.src, "TODO Implement arithmetic operand '{s}'", .{@tagName(inst.base.tag)}),
     };
 
     return mod.constInst(scope, inst.base.src, .{
@@ -1781,7 +1781,7 @@ fn analyzeInstCmp(
         return mod.fail(scope, inst.base.src, "TODO implement equality comparison between a union's tag value and an enum literal", .{});
     } else if (lhs_ty_tag == .ErrorSet and rhs_ty_tag == .ErrorSet) {
         if (!is_equality_cmp) {
-            return mod.fail(scope, inst.base.src, "{} operator not allowed for errors", .{@tagName(op)});
+            return mod.fail(scope, inst.base.src, "{s} operator not allowed for errors", .{@tagName(op)});
         }
         return mod.fail(scope, inst.base.src, "TODO implement equality comparison between errors", .{});
     } else if (lhs.ty.isNumeric() and rhs.ty.isNumeric()) {
@@ -1962,7 +1962,7 @@ fn analyzeDeclVal(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclVal) InnerErr
     const decl_name = inst.positionals.name;
     const zir_module = scope.namespace().cast(Scope.ZIRModule).?;
     const src_decl = zir_module.contents.module.findDecl(decl_name) orelse
-        return mod.fail(scope, inst.base.src, "use of undeclared identifier '{}'", .{decl_name});
+        return mod.fail(scope, inst.base.src, "use of undeclared identifier '{s}'", .{decl_name});
 
     const decl = try resolveCompleteZirDecl(mod, scope, src_decl.decl);
 
-- 
cgit v1.2.3


From 04f37dcd8e9c32d7ce8712939fb9c8dfa6cc6bbe Mon Sep 17 00:00:00 2001
From: LemonBoy <thatlemon@gmail.com>
Date: Thu, 26 Nov 2020 22:20:44 +0100
Subject: stage2: Use {z} instead of {s} in generated Zig code

---
 src/Compilation.zig | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 3a9c1ea993..c688cf7980 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2653,15 +2653,15 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const arch = Target.current.cpu.arch;
         \\/// Deprecated
         \\pub const endian = Target.current.cpu.arch.endian();
-        \\pub const output_mode = OutputMode.{s};
-        \\pub const link_mode = LinkMode.{s};
+        \\pub const output_mode = OutputMode.{z};
+        \\pub const link_mode = LinkMode.{z};
         \\pub const is_test = {};
         \\pub const single_threaded = {};
-        \\pub const abi = Abi.{s};
+        \\pub const abi = Abi.{z};
         \\pub const cpu: Cpu = Cpu{{
-        \\    .arch = .{s},
-        \\    .model = &Target.{s}.cpu.{s},
-        \\    .features = Target.{s}.featureSet(&[_]Target.{s}.Feature{{
+        \\    .arch = .{z},
+        \\    .model = &Target.{z}.cpu.{z},
+        \\    .features = Target.{z}.featureSet(&[_]Target.{z}.Feature{{
         \\
     , .{
         @tagName(comp.bin_file.options.output_mode),
@@ -2692,7 +2692,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\    }}),
         \\}};
         \\pub const os = Os{{
-        \\    .tag = .{s},
+        \\    .tag = .{z},
         \\    .version_range = .{{
     ,
         .{@tagName(target.os.tag)},
@@ -2778,8 +2778,8 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         (comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
 
     try buffer.writer().print(
-        \\pub const object_format = ObjectFormat.{s};
-        \\pub const mode = Mode.{s};
+        \\pub const object_format = ObjectFormat.{z};
+        \\pub const mode = Mode.{z};
         \\pub const link_libc = {};
         \\pub const link_libcpp = {};
         \\pub const have_error_return_tracing = {};
@@ -2787,7 +2787,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const position_independent_code = {};
         \\pub const position_independent_executable = {};
         \\pub const strip_debug_info = {};
-        \\pub const code_model = CodeModel.{s};
+        \\pub const code_model = CodeModel.{z};
         \\
     , .{
         @tagName(comp.bin_file.options.object_format),
-- 
cgit v1.2.3


From 974c008a0ee0e0d7933e37d5ea930f712d494f6a Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 2 Jan 2021 19:03:14 -0700
Subject: convert more {} to {d} and {s}

---
 lib/std/SemanticVersion.zig     |  2 +-
 lib/std/c/ast.zig               |  8 ++++----
 lib/std/fs/wasi.zig             |  2 +-
 lib/std/os.zig                  |  4 ++--
 lib/std/special/c.zig           |  2 +-
 lib/std/special/compiler_rt.zig |  2 +-
 lib/std/testing.zig             |  2 +-
 lib/std/zig/parser_test.zig     |  4 ++--
 lib/std/zig/system/macos.zig    |  4 ++--
 src/Compilation.zig             | 12 ++++++------
 src/DepTokenizer.zig            |  6 +++---
 src/Module.zig                  | 22 +++++++++++-----------
 src/astgen.zig                  |  6 +++---
 src/codegen/c.zig               |  6 +++---
 src/glibc.zig                   | 28 ++++++++++++++--------------
 src/link.zig                    |  2 +-
 src/link/Elf.zig                |  6 +++---
 src/liveness.zig                |  3 ++-
 src/main.zig                    | 14 +++++++-------
 src/translate_c.zig             | 18 +++++++++---------
 src/type.zig                    |  8 ++++----
 src/zir.zig                     | 19 +++++++++----------
 src/zir_sema.zig                | 14 +++++++-------
 23 files changed, 97 insertions(+), 97 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/SemanticVersion.zig b/lib/std/SemanticVersion.zig
index eeb6916ad6..dc9e6d8572 100644
--- a/lib/std/SemanticVersion.zig
+++ b/lib/std/SemanticVersion.zig
@@ -163,7 +163,7 @@ pub fn format(
     out_stream: anytype,
 ) !void {
     if (fmt.len != 0) @compileError("Unknown format string: '" ++ fmt ++ "'");
-    try std.fmt.format(out_stream, "{}.{}.{}", .{ self.major, self.minor, self.patch });
+    try std.fmt.format(out_stream, "{d}.{d}.{d}", .{ self.major, self.minor, self.patch });
     if (self.pre) |pre| try std.fmt.format(out_stream, "-{s}", .{pre});
     if (self.build) |build| try std.fmt.format(out_stream, "+{s}", .{build});
 }
diff --git a/lib/std/c/ast.zig b/lib/std/c/ast.zig
index c1e20f799d..207fe8eac8 100644
--- a/lib/std/c/ast.zig
+++ b/lib/std/c/ast.zig
@@ -115,10 +115,10 @@ pub const Error = union(enum) {
         pub fn render(self: *const ExpectedToken, tree: *Tree, stream: anytype) !void {
             const found_token = tree.tokens.at(self.token);
             if (found_token.id == .Invalid) {
-                return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
+                return stream.print("expected '{s}', found invalid bytes", .{self.expected_id.symbol()});
             } else {
                 const token_name = found_token.id.symbol();
-                return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
+                return stream.print("expected '{s}', found '{s}'", .{ self.expected_id.symbol(), token_name });
             }
         }
     };
@@ -131,7 +131,7 @@ pub const Error = union(enum) {
             try stream.write("invalid type specifier '");
             try type_spec.spec.print(tree, stream);
             const token_name = tree.tokens.at(self.token).id.symbol();
-            return stream.print("{}'", .{token_name});
+            return stream.print("{s}'", .{token_name});
         }
     };
 
@@ -140,7 +140,7 @@ pub const Error = union(enum) {
         name: TokenIndex,
 
         pub fn render(self: *const ExpectedToken, tree: *Tree, stream: anytype) !void {
-            return stream.print("must use '{}' tag to refer to type '{}'", .{ tree.slice(kw), tree.slice(name) });
+            return stream.print("must use '{s}' tag to refer to type '{s}'", .{ tree.slice(kw), tree.slice(name) });
         }
     };
 
diff --git a/lib/std/fs/wasi.zig b/lib/std/fs/wasi.zig
index 761f6e8466..900adc5e2d 100644
--- a/lib/std/fs/wasi.zig
+++ b/lib/std/fs/wasi.zig
@@ -38,7 +38,7 @@ pub const PreopenType = union(PreopenTypeTag) {
     pub fn format(self: Self, comptime fmt: []const u8, options: std.fmt.FormatOptions, out_stream: anytype) !void {
         try out_stream.print("PreopenType{{ ", .{});
         switch (self) {
-            PreopenType.Dir => |path| try out_stream.print(".Dir = '{}'", .{path}),
+            PreopenType.Dir => |path| try out_stream.print(".Dir = '{z}'", .{path}),
         }
         return out_stream.print(" }}", .{});
     }
diff --git a/lib/std/os.zig b/lib/std/os.zig
index fb187252a1..1bde12a6fb 100644
--- a/lib/std/os.zig
+++ b/lib/std/os.zig
@@ -4256,7 +4256,7 @@ pub fn getFdPath(fd: fd_t, out_buffer: *[MAX_PATH_BYTES]u8) RealPathError![]u8 {
         },
         .linux => {
             var procfs_buf: ["/proc/self/fd/-2147483648".len:0]u8 = undefined;
-            const proc_path = std.fmt.bufPrint(procfs_buf[0..], "/proc/self/fd/{}\x00", .{fd}) catch unreachable;
+            const proc_path = std.fmt.bufPrint(procfs_buf[0..], "/proc/self/fd/{d}\x00", .{fd}) catch unreachable;
 
             const target = readlinkZ(std.meta.assumeSentinel(proc_path.ptr, 0), out_buffer) catch |err| {
                 switch (err) {
@@ -4487,7 +4487,7 @@ pub const UnexpectedError = error{
 /// and you get an unexpected error.
 pub fn unexpectedErrno(err: usize) UnexpectedError {
     if (unexpected_error_tracing) {
-        std.debug.warn("unexpected errno: {}\n", .{err});
+        std.debug.warn("unexpected errno: {d}\n", .{err});
         std.debug.dumpCurrentStackTrace(null);
     }
     return error.Unexpected;
diff --git a/lib/std/special/c.zig b/lib/std/special/c.zig
index c92ffa21ac..51cbafc133 100644
--- a/lib/std/special/c.zig
+++ b/lib/std/special/c.zig
@@ -172,7 +172,7 @@ test "strncmp" {
 pub fn panic(msg: []const u8, error_return_trace: ?*builtin.StackTrace) noreturn {
     if (builtin.is_test) {
         @setCold(true);
-        std.debug.panic("{}", .{msg});
+        std.debug.panic("{s}", .{msg});
     }
     if (builtin.os.tag != .freestanding and builtin.os.tag != .other) {
         std.os.abort();
diff --git a/lib/std/special/compiler_rt.zig b/lib/std/special/compiler_rt.zig
index e0743cd4b7..a06c568b2e 100644
--- a/lib/std/special/compiler_rt.zig
+++ b/lib/std/special/compiler_rt.zig
@@ -324,7 +324,7 @@ pub usingnamespace @import("compiler_rt/atomics.zig");
 pub fn panic(msg: []const u8, error_return_trace: ?*builtin.StackTrace) noreturn {
     @setCold(true);
     if (is_test) {
-        std.debug.panic("{}", .{msg});
+        std.debug.panic("{s}", .{msg});
     } else {
         unreachable;
     }
diff --git a/lib/std/testing.zig b/lib/std/testing.zig
index 80cc7f930d..4f4a46dbf7 100644
--- a/lib/std/testing.zig
+++ b/lib/std/testing.zig
@@ -258,7 +258,7 @@ pub fn expectEqualSlices(comptime T: type, expected: []const T, actual: []const
     // If the child type is u8 and no weird bytes, we could print it as strings
     // Even for the length difference, it would be useful to see the values of the slices probably.
     if (expected.len != actual.len) {
-        std.debug.panic("slice lengths differ. expected {}, found {}", .{ expected.len, actual.len });
+        std.debug.panic("slice lengths differ. expected {d}, found {d}", .{ expected.len, actual.len });
     }
     var i: usize = 0;
     while (i < expected.len) : (i += 1) {
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index a736006bc6..63d04f9a80 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -3742,7 +3742,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
     for (tree.errors) |*parse_error| {
         const token = tree.token_locs[parse_error.loc()];
         const loc = tree.tokenLocation(0, parse_error.loc());
-        try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });
+        try stderr.print("(memory buffer):{d}:{d}: error: ", .{ loc.line + 1, loc.column + 1 });
         try tree.renderError(parse_error, stderr);
         try stderr.print("\n{s}\n", .{source[loc.line_start..loc.line_end]});
         {
@@ -3800,7 +3800,7 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
             error.OutOfMemory => {
                 if (failing_allocator.allocated_bytes != failing_allocator.freed_bytes) {
                     warn(
-                        "\nfail_index: {}/{}\nallocated bytes: {}\nfreed bytes: {}\nallocations: {}\ndeallocations: {}\n",
+                        "\nfail_index: {d}/{d}\nallocated bytes: {d}\nfreed bytes: {d}\nallocations: {d}\ndeallocations: {d}\n",
                         .{
                             fail_index,
                             needed_alloc_count,
diff --git a/lib/std/zig/system/macos.zig b/lib/std/zig/system/macos.zig
index fbc0b5149b..7c4e255cc5 100644
--- a/lib/std/zig/system/macos.zig
+++ b/lib/std/zig/system/macos.zig
@@ -450,7 +450,7 @@ test "version_from_build" {
     for (known) |pair| {
         var buf: [32]u8 = undefined;
         const ver = try version_from_build(pair[0]);
-        const sver = try std.fmt.bufPrint(buf[0..], "{}.{}.{}", .{ ver.major, ver.minor, ver.patch });
+        const sver = try std.fmt.bufPrint(buf[0..], "{d}.{d}.{d}", .{ ver.major, ver.minor, ver.patch });
         std.testing.expect(std.mem.eql(u8, sver, pair[1]));
     }
 }
@@ -468,7 +468,7 @@ pub fn getSDKPath(allocator: *mem.Allocator) ![]u8 {
         allocator.free(result.stdout);
     }
     if (result.stderr.len != 0) {
-        std.log.err("unexpected 'xcrun --show-sdk-path' stderr: {}", .{result.stderr});
+        std.log.err("unexpected 'xcrun --show-sdk-path' stderr: {s}", .{result.stderr});
     }
     if (result.term.Exited != 0) {
         return error.ProcessTerminated;
diff --git a/src/Compilation.zig b/src/Compilation.zig
index c688cf7980..de115b9b40 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1512,7 +1512,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                             module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
                                 module.gpa,
                                 decl.src(),
-                                "unable to generate C header: {}",
+                                "unable to generate C header: {s}",
                                 .{@errorName(err)},
                             ));
                             decl.analysis = .codegen_failure_retryable;
@@ -1593,7 +1593,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
         .libtsan => {
             libtsan.buildTsan(self) catch |err| {
                 // TODO Expose this as a normal compile error rather than crashing here.
-                fatal("unable to build TSAN library: {}", .{@errorName(err)});
+                fatal("unable to build TSAN library: {s}", .{@errorName(err)});
             };
         },
         .compiler_rt_lib => {
@@ -1983,7 +1983,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
                         // TODO parse clang stderr and turn it into an error message
                         // and then call failCObjWithOwnedErrorMsg
                         log.err("clang failed with stderr: {s}", .{stderr});
-                        return comp.failCObj(c_object, "clang exited with code {}", .{code});
+                        return comp.failCObj(c_object, "clang exited with code {d}", .{code});
                     }
                 },
                 else => {
@@ -3013,7 +3013,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
             id_symlink_basename,
             &prev_digest_buf,
         ) catch |err| blk: {
-            log.debug("stage1 {} new_digest={} error: {s}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
+            log.debug("stage1 {s} new_digest={} error: {s}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
@@ -3021,7 +3021,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
             if (!mem.eql(u8, prev_digest[0..digest.len], &digest))
                 break :hit;
 
-            log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
+            log.debug("stage1 {s} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
             var flags_bytes: [1]u8 = undefined;
             _ = std.fmt.hexToBytes(&flags_bytes, prev_digest[digest.len..]) catch {
                 log.warn("bad cache stage1 digest: '{s}'", .{prev_digest});
@@ -3044,7 +3044,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
             mod.stage1_flags = @bitCast(@TypeOf(mod.stage1_flags), flags_bytes[0]);
             return;
         }
-        log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
+        log.debug("stage1 {s} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
         man.unhit(prev_hash_state, input_file_count);
     }
 
diff --git a/src/DepTokenizer.zig b/src/DepTokenizer.zig
index c42583a786..0872513b51 100644
--- a/src/DepTokenizer.zig
+++ b/src/DepTokenizer.zig
@@ -373,7 +373,7 @@ pub const Token = union(enum) {
                 } else {
                     try printCharValues(writer, index_and_bytes.bytes);
                 }
-                try writer.print("' at position {}", .{index_and_bytes.index});
+                try writer.print("' at position {d}", .{index_and_bytes.index});
             },
             .invalid_target,
             .bad_target_escape,
@@ -383,7 +383,7 @@ pub const Token = union(enum) {
             => |index_and_char| {
                 try writer.writeAll("illegal char ");
                 try printUnderstandableChar(writer, index_and_char.char);
-                try writer.print(" at position {}: {s}", .{ index_and_char.index, self.errStr() });
+                try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
             },
         }
     }
@@ -943,7 +943,7 @@ fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
 
 fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
     var buf: [80]u8 = undefined;
-    var text = try std.fmt.bufPrint(buf[0..], "{s} {} bytes ", .{ label, bytes.len });
+    var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
     try out.writeAll(text);
     var i: usize = text.len;
     const end = 79;
diff --git a/src/Module.zig b/src/Module.zig
index 0d03703dcb..d1719059c4 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -248,7 +248,7 @@ pub const Decl = struct {
 
     pub fn dump(self: *Decl) void {
         const loc = std.zig.findLineColumn(self.scope.source.bytes, self.src);
-        std.debug.print("{}:{}:{} name={} status={}", .{
+        std.debug.print("{s}:{d}:{d} name={s} status={s}", .{
             self.scope.sub_file_path,
             loc.line + 1,
             loc.column + 1,
@@ -308,7 +308,7 @@ pub const Fn = struct {
 
     /// For debugging purposes.
     pub fn dump(self: *Fn, mod: Module) void {
-        std.debug.print("Module.Function(name={}) ", .{self.owner_decl.name});
+        std.debug.print("Module.Function(name={s}) ", .{self.owner_decl.name});
         switch (self.analysis) {
             .queued => {
                 std.debug.print("queued\n", .{});
@@ -632,7 +632,7 @@ pub const Scope = struct {
 
         pub fn dumpSrc(self: *File, src: usize) void {
             const loc = std.zig.findLineColumn(self.source.bytes, src);
-            std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
+            std.debug.print("{s}:{d}:{d}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
         }
 
         pub fn getSource(self: *File, module: *Module) ![:0]const u8 {
@@ -730,7 +730,7 @@ pub const Scope = struct {
 
         pub fn dumpSrc(self: *ZIRModule, src: usize) void {
             const loc = std.zig.findLineColumn(self.source.bytes, src);
-            std.debug.print("{}:{}:{}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
+            std.debug.print("{s}:{d}:{d}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
         }
 
         pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 {
@@ -1641,7 +1641,7 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
             }
         } else if (src_decl.castTag(.Comptime)) |comptime_node| {
             const name_index = self.getNextAnonNameIndex();
-            const name = try std.fmt.allocPrint(self.gpa, "__comptime_{}", .{name_index});
+            const name = try std.fmt.allocPrint(self.gpa, "__comptime_{d}", .{name_index});
             defer self.gpa.free(name);
 
             const name_hash = container_scope.fullyQualifiedNameHash(name);
@@ -2277,7 +2277,7 @@ pub fn createAnonymousDecl(
 ) !*Decl {
     const name_index = self.getNextAnonNameIndex();
     const scope_decl = scope.decl().?;
-    const name = try std.fmt.allocPrint(self.gpa, "{s}__anon_{}", .{ scope_decl.name, name_index });
+    const name = try std.fmt.allocPrint(self.gpa, "{s}__anon_{d}", .{ scope_decl.name, name_index });
     defer self.gpa.free(name);
     const name_hash = scope.namespace().fullyQualifiedNameHash(name);
     const src_hash: std.zig.SrcHash = undefined;
@@ -2555,7 +2555,7 @@ pub fn cmpNumeric(
 
     if (lhs_ty_tag == .Vector and rhs_ty_tag == .Vector) {
         if (lhs.ty.arrayLen() != rhs.ty.arrayLen()) {
-            return self.fail(scope, src, "vector length mismatch: {} and {}", .{
+            return self.fail(scope, src, "vector length mismatch: {d} and {d}", .{
                 lhs.ty.arrayLen(),
                 rhs.ty.arrayLen(),
             });
@@ -2700,7 +2700,7 @@ pub fn cmpNumeric(
     const dest_type = if (dest_float_type) |ft| ft else blk: {
         const max_bits = std.math.max(lhs_bits, rhs_bits);
         const casted_bits = std.math.cast(u16, max_bits) catch |err| switch (err) {
-            error.Overflow => return self.fail(scope, src, "{} exceeds maximum integer bit count", .{max_bits}),
+            error.Overflow => return self.fail(scope, src, "{d} exceeds maximum integer bit count", .{max_bits}),
         };
         break :blk try self.makeIntType(scope, dest_int_is_signed, casted_bits);
     };
@@ -3319,7 +3319,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void {
     const source = zir_module.getSource(self) catch @panic("dumpInst failed to get source");
     const loc = std.zig.findLineColumn(source, inst.src);
     if (inst.tag == .constant) {
-        std.debug.print("constant ty={} val={} src={}:{}:{}\n", .{
+        std.debug.print("constant ty={} val={} src={s}:{d}:{d}\n", .{
             inst.ty,
             inst.castTag(.constant).?.val,
             zir_module.subFilePath(),
@@ -3327,7 +3327,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void {
             loc.column + 1,
         });
     } else if (inst.deaths == 0) {
-        std.debug.print("{} ty={} src={}:{}:{}\n", .{
+        std.debug.print("{s} ty={} src={s}:{d}:{d}\n", .{
             @tagName(inst.tag),
             inst.ty,
             zir_module.subFilePath(),
@@ -3335,7 +3335,7 @@ pub fn dumpInst(self: *Module, scope: *Scope, inst: *Inst) void {
             loc.column + 1,
         });
     } else {
-        std.debug.print("{} ty={} deaths={b} src={}:{}:{}\n", .{
+        std.debug.print("{s} ty={} deaths={b} src={s}:{d}:{d}\n", .{
             @tagName(inst.tag),
             inst.ty,
             inst.deaths,
diff --git a/src/astgen.zig b/src/astgen.zig
index b3cb648dcb..a50fa026ca 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -385,7 +385,7 @@ fn breakExpr(mod: *Module, parent_scope: *Scope, node: *ast.Node.ControlFlowExpr
             .local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
             else => if (node.getLabel()) |break_label| {
                 const label_name = try identifierTokenString(mod, parent_scope, break_label);
-                return mod.failTok(parent_scope, break_label, "label not found: '{}'", .{label_name});
+                return mod.failTok(parent_scope, break_label, "label not found: '{s}'", .{label_name});
             } else {
                 return mod.failTok(parent_scope, src, "break expression outside loop", .{});
             },
@@ -427,7 +427,7 @@ fn continueExpr(mod: *Module, parent_scope: *Scope, node: *ast.Node.ControlFlowE
             .local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
             else => if (node.getLabel()) |break_label| {
                 const label_name = try identifierTokenString(mod, parent_scope, break_label);
-                return mod.failTok(parent_scope, break_label, "label not found: '{}'", .{label_name});
+                return mod.failTok(parent_scope, break_label, "label not found: '{s}'", .{label_name});
             } else {
                 return mod.failTok(parent_scope, src, "continue expression outside loop", .{});
             },
@@ -2204,7 +2204,7 @@ fn ensureBuiltinParamCount(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinC
         return;
 
     const s = if (count == 1) "" else "s";
-    return mod.failTok(scope, call.builtin_token, "expected {} parameter{s}, found {}", .{ count, s, call.params_len });
+    return mod.failTok(scope, call.builtin_token, "expected {d} parameter{s}, found {d}", .{ count, s, call.params_len });
 }
 
 fn simpleCast(
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 37654b8ce0..684a03eb79 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -383,7 +383,7 @@ const Context = struct {
     }
 
     fn name(self: *Context) ![]u8 {
-        const val = try std.fmt.allocPrint(&self.arena.allocator, "__temp_{}", .{self.unnamed_index});
+        const val = try std.fmt.allocPrint(&self.arena.allocator, "__temp_{d}", .{self.unnamed_index});
         self.unnamed_index += 1;
         return val;
     }
@@ -420,7 +420,7 @@ fn genAlloc(ctx: *Context, file: *C, alloc: *Inst.NoOp) !?[]u8 {
 }
 
 fn genArg(ctx: *Context) !?[]u8 {
-    const name = try std.fmt.allocPrint(&ctx.arena.allocator, "arg{}", .{ctx.argdex});
+    const name = try std.fmt.allocPrint(&ctx.arena.allocator, "arg{d}", .{ctx.argdex});
     ctx.argdex += 1;
     return name;
 }
@@ -528,7 +528,7 @@ fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
                     try renderValue(ctx, writer, arg.ty, val);
                 } else {
                     const val = try ctx.resolveInst(arg);
-                    try writer.print("{}", .{val});
+                    try writer.print("{s}", .{val});
                 }
             }
         }
diff --git a/src/glibc.zig b/src/glibc.zig
index f69dd11ada..e7b7b1b1cf 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -111,12 +111,12 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
         while (it.next()) |line| : (line_i += 1) {
             const prefix = "GLIBC_";
             if (!mem.startsWith(u8, line, prefix)) {
-                std.log.err("vers.txt:{}: expected 'GLIBC_' prefix", .{line_i});
+                std.log.err("vers.txt:{d}: expected 'GLIBC_' prefix", .{line_i});
                 return error.ZigInstallationCorrupt;
             }
             const adjusted_line = line[prefix.len..];
             const ver = std.builtin.Version.parse(adjusted_line) catch |err| {
-                std.log.err("vers.txt:{}: unable to parse glibc version '{s}': {s}", .{ line_i, line, @errorName(err) });
+                std.log.err("vers.txt:{d}: unable to parse glibc version '{s}': {s}", .{ line_i, line, @errorName(err) });
                 return error.ZigInstallationCorrupt;
             };
             try all_versions.append(arena, ver);
@@ -128,15 +128,15 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
         while (file_it.next()) |line| : (line_i += 1) {
             var line_it = mem.tokenize(line, " ");
             const fn_name = line_it.next() orelse {
-                std.log.err("fns.txt:{}: expected function name", .{line_i});
+                std.log.err("fns.txt:{d}: expected function name", .{line_i});
                 return error.ZigInstallationCorrupt;
             };
             const lib_name = line_it.next() orelse {
-                std.log.err("fns.txt:{}: expected library name", .{line_i});
+                std.log.err("fns.txt:{d}: expected library name", .{line_i});
                 return error.ZigInstallationCorrupt;
             };
             const lib = findLib(lib_name) orelse {
-                std.log.err("fns.txt:{}: unknown library name: {s}", .{ line_i, lib_name });
+                std.log.err("fns.txt:{d}: unknown library name: {s}", .{ line_i, lib_name });
                 return error.ZigInstallationCorrupt;
             };
             try all_functions.append(arena, .{
@@ -158,27 +158,27 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
                 while (line_it.next()) |target_string| {
                     var component_it = mem.tokenize(target_string, "-");
                     const arch_name = component_it.next() orelse {
-                        std.log.err("abi.txt:{}: expected arch name", .{line_i});
+                        std.log.err("abi.txt:{d}: expected arch name", .{line_i});
                         return error.ZigInstallationCorrupt;
                     };
                     const os_name = component_it.next() orelse {
-                        std.log.err("abi.txt:{}: expected OS name", .{line_i});
+                        std.log.err("abi.txt:{d}: expected OS name", .{line_i});
                         return error.ZigInstallationCorrupt;
                     };
                     const abi_name = component_it.next() orelse {
-                        std.log.err("abi.txt:{}: expected ABI name", .{line_i});
+                        std.log.err("abi.txt:{d}: expected ABI name", .{line_i});
                         return error.ZigInstallationCorrupt;
                     };
                     const arch_tag = std.meta.stringToEnum(std.Target.Cpu.Arch, arch_name) orelse {
-                        std.log.err("abi.txt:{}: unrecognized arch: '{s}'", .{ line_i, arch_name });
+                        std.log.err("abi.txt:{d}: unrecognized arch: '{s}'", .{ line_i, arch_name });
                         return error.ZigInstallationCorrupt;
                     };
                     if (!mem.eql(u8, os_name, "linux")) {
-                        std.log.err("abi.txt:{}: expected OS 'linux', found '{s}'", .{ line_i, os_name });
+                        std.log.err("abi.txt:{d}: expected OS 'linux', found '{s}'", .{ line_i, os_name });
                         return error.ZigInstallationCorrupt;
                     }
                     const abi_tag = std.meta.stringToEnum(std.Target.Abi, abi_name) orelse {
-                        std.log.err("abi.txt:{}: unrecognized ABI: '{s}'", .{ line_i, abi_name });
+                        std.log.err("abi.txt:{d}: unrecognized ABI: '{s}'", .{ line_i, abi_name });
                         return error.ZigInstallationCorrupt;
                     };
 
@@ -193,7 +193,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
             };
             for (ver_list_base) |*ver_list| {
                 const line = file_it.next() orelse {
-                    std.log.err("abi.txt:{}: missing version number line", .{line_i});
+                    std.log.err("abi.txt:{d}: missing version number line", .{line_i});
                     return error.ZigInstallationCorrupt;
                 };
                 line_i += 1;
@@ -206,12 +206,12 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
                 while (line_it.next()) |version_index_string| {
                     if (ver_list.len >= ver_list.versions.len) {
                         // If this happens with legit data, increase the array len in the type.
-                        std.log.err("abi.txt:{}: too many versions", .{line_i});
+                        std.log.err("abi.txt:{d}: too many versions", .{line_i});
                         return error.ZigInstallationCorrupt;
                     }
                     const version_index = std.fmt.parseInt(u8, version_index_string, 10) catch |err| {
                         // If this happens with legit data, increase the size of the integer type in the struct.
-                        std.log.err("abi.txt:{}: unable to parse version: {s}", .{ line_i, @errorName(err) });
+                        std.log.err("abi.txt:{d}: unable to parse version: {s}", .{ line_i, @errorName(err) });
                         return error.ZigInstallationCorrupt;
                     };
 
diff --git a/src/link.zig b/src/link.zig
index 92702c7973..3dbfb3b922 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -523,7 +523,7 @@ pub const File = struct {
                 id_symlink_basename,
                 &prev_digest_buf,
             ) catch |err| b: {
-                log.debug("archive new_digest={} readFile error: {}", .{ digest, @errorName(err) });
+                log.debug("archive new_digest={} readFile error: {s}", .{ digest, @errorName(err) });
                 break :b prev_digest_buf[0..0];
             };
             if (mem.eql(u8, prev_digest, &digest)) {
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 5d0e897008..116d7c9859 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2082,10 +2082,10 @@ pub fn allocateDeclIndexes(self: *Elf, decl: *Module.Decl) !void {
     try self.offset_table.ensureCapacity(self.base.allocator, self.offset_table.items.len + 1);
 
     if (self.local_symbol_free_list.popOrNull()) |i| {
-        log.debug("reusing symbol index {} for {s}\n", .{ i, decl.name });
+        log.debug("reusing symbol index {d} for {s}\n", .{ i, decl.name });
         decl.link.elf.local_sym_index = i;
     } else {
-        log.debug("allocating symbol index {} for {s}\n", .{ self.local_symbols.items.len, decl.name });
+        log.debug("allocating symbol index {d} for {s}\n", .{ self.local_symbols.items.len, decl.name });
         decl.link.elf.local_sym_index = @intCast(u32, self.local_symbols.items.len);
         _ = self.local_symbols.addOneAssumeCapacity();
     }
@@ -2432,7 +2432,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
             if (needed_size > self.allocatedSize(debug_line_sect.sh_offset)) {
                 const new_offset = self.findFreeSpace(needed_size, 1);
                 const existing_size = last_src_fn.off;
-                log.debug("moving .debug_line section: {} bytes from 0x{x} to 0x{x}\n", .{
+                log.debug("moving .debug_line section: {d} bytes from 0x{x} to 0x{x}\n", .{
                     existing_size,
                     debug_line_sect.sh_offset,
                     new_offset,
diff --git a/src/liveness.zig b/src/liveness.zig
index 0d759f8312..b0aafa28f1 100644
--- a/src/liveness.zig
+++ b/src/liveness.zig
@@ -1,6 +1,7 @@
 const std = @import("std");
 const ir = @import("ir.zig");
 const trace = @import("tracy.zig").trace;
+const log = std.log.scoped(.liveness);
 
 /// Perform Liveness Analysis over the `Body`. Each `Inst` will have its `deaths` field populated.
 pub fn analyze(
@@ -248,5 +249,5 @@ fn analyzeInst(
         @panic("Handle liveness analysis for instructions with many parameters");
     }
 
-    std.log.scoped(.liveness).debug("analyze {}: 0b{b}\n", .{ base.tag, base.deaths });
+    log.debug("analyze {}: 0b{b}\n", .{ base.tag, base.deaths });
 }
diff --git a/src/main.zig b/src/main.zig
index 7d03ea956f..b1243badff 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -626,7 +626,7 @@ fn buildOutputType(
                             fs.path.dirname(pkg_path),
                             fs.path.basename(pkg_path),
                         ) catch |err| {
-                            fatal("Failed to add package at path {}: {}", .{ pkg_path, @errorName(err) });
+                            fatal("Failed to add package at path {s}: {s}", .{ pkg_path, @errorName(err) });
                         };
                         new_cur_pkg.parent = cur_pkg;
                         try cur_pkg.add(gpa, pkg_name, new_cur_pkg);
@@ -696,13 +696,13 @@ fn buildOutputType(
                         if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         stack_size_override = std.fmt.parseUnsigned(u64, args[i], 0) catch |err| {
-                            fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                            fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                         };
                     } else if (mem.eql(u8, arg, "--image-base")) {
                         if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         image_base_override = std.fmt.parseUnsigned(u64, args[i], 0) catch |err| {
-                            fatal("unable to parse '{}': {}", .{ arg, @errorName(err) });
+                            fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                         };
                     } else if (mem.eql(u8, arg, "--name")) {
                         if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
@@ -1914,7 +1914,7 @@ fn buildOutputType(
                                 if (!watch) return cleanExit();
                             } else {
                                 const cmd = try argvCmd(arena, argv.items);
-                                fatal("the following test command failed with exit code {}:\n{s}", .{ code, cmd });
+                                fatal("the following test command failed with exit code {d}:\n{s}", .{ code, cmd });
                             }
                         },
                         else => {
@@ -2069,7 +2069,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
             error.ASTUnitFailure => fatal("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{}),
             error.SemanticAnalyzeFail => {
                 for (clang_errors) |clang_err| {
-                    std.debug.print("{s}:{}:{}: {s}\n", .{
+                    std.debug.print("{s}:{d}:{d}: {s}\n", .{
                         if (clang_err.filename_ptr) |p| p[0..clang_err.filename_len] else "(no file)",
                         clang_err.line + 1,
                         clang_err.column + 1,
@@ -2493,7 +2493,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
         .Exited => |code| {
             if (code == 0) return cleanExit();
             const cmd = try argvCmd(arena, child_argv);
-            fatal("the following build command failed with exit code {}:\n{s}", .{ code, cmd });
+            fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
         },
         else => {
             const cmd = try argvCmd(arena, child_argv);
@@ -2812,7 +2812,7 @@ fn printErrMsgToFile(
     const text = text_buf.items;
 
     const stream = file.outStream();
-    try stream.print("{s}:{}:{}: error: {s}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
+    try stream.print("{s}:{d}:{d}: error: {s}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
 
     if (!color_on) return;
 
diff --git a/src/translate_c.zig b/src/translate_c.zig
index c609597770..1b6455fd4d 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -136,7 +136,7 @@ const Scope = struct {
             var proposed_name = name_copy;
             while (scope.contains(proposed_name)) {
                 scope.mangle_count += 1;
-                proposed_name = try std.fmt.allocPrint(c.arena, "{s}_{}", .{ name, scope.mangle_count });
+                proposed_name = try std.fmt.allocPrint(c.arena, "{s}_{d}", .{ name, scope.mangle_count });
             }
             try scope.variables.append(.{ .name = name_copy, .alias = proposed_name });
             return proposed_name;
@@ -440,7 +440,7 @@ pub fn translate(
     mem.copy(*ast.Node, root_node.decls(), context.root_decls.items);
 
     if (false) {
-        std.debug.warn("debug source:\n{}\n==EOF==\ntokens:\n", .{source_buffer.items});
+        std.debug.warn("debug source:\n{s}\n==EOF==\ntokens:\n", .{source_buffer.items});
         for (context.token_ids.items) |token| {
             std.debug.warn("{}\n", .{token});
         }
@@ -945,7 +945,7 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
     // Record declarations such as `struct {...} x` have no name but they're not
     // anonymous hence here isAnonymousStructOrUnion is not needed
     if (bare_name.len == 0) {
-        bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{c.getMangle()});
+        bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{d}", .{c.getMangle()});
         is_unnamed = true;
     }
 
@@ -1019,7 +1019,7 @@ fn transRecordDecl(c: *Context, record_decl: *const clang.RecordDecl) Error!?*as
             var raw_name = try c.str(@ptrCast(*const clang.NamedDecl, field_decl).getName_bytes_begin());
             if (field_decl.isAnonymousStructOrUnion() or raw_name.len == 0) {
                 // Context.getMangle() is not used here because doing so causes unpredictable field names for anonymous fields.
-                raw_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{unnamed_field_count});
+                raw_name = try std.fmt.allocPrint(c.arena, "unnamed_{d}", .{unnamed_field_count});
                 unnamed_field_count += 1;
                 is_anon = true;
             }
@@ -1110,7 +1110,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const clang.EnumDecl) Error!?*ast.Node
     var bare_name = try c.str(@ptrCast(*const clang.NamedDecl, enum_decl).getName_bytes_begin());
     var is_unnamed = false;
     if (bare_name.len == 0) {
-        bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{c.getMangle()});
+        bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{d}", .{c.getMangle()});
         is_unnamed = true;
     }
 
@@ -3956,7 +3956,7 @@ fn qualTypeToLog2IntRef(rp: RestorePoint, qt: clang.QualType, source_loc: clang.
         const node = try rp.c.arena.create(ast.Node.OneToken);
         node.* = .{
             .base = .{ .tag = .IntegerLiteral },
-            .token = try appendTokenFmt(rp.c, .Identifier, "u{}", .{cast_bit_width}),
+            .token = try appendTokenFmt(rp.c, .Identifier, "u{d}", .{cast_bit_width}),
         };
         return &node.base;
     }
@@ -4484,7 +4484,7 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a
             _ = try appendToken(c, .Comma, ",");
         }
         const param_name_tok = param.name_token orelse
-            try appendTokenFmt(c, .Identifier, "arg_{}", .{c.getMangle()});
+            try appendTokenFmt(c, .Identifier, "arg_{d}", .{c.getMangle()});
 
         _ = try appendToken(c, .Colon, ":");
 
@@ -5916,11 +5916,11 @@ fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!*
             // struct Foo will be declared as struct_Foo by transRecordDecl
             const next_id = m.next().?;
             if (next_id != .Identifier) {
-                try m.fail(c, "unable to translate C expr: expected Identifier instead got: {}", .{@tagName(next_id)});
+                try m.fail(c, "unable to translate C expr: expected Identifier instead got: {s}", .{@tagName(next_id)});
                 return error.ParseError;
             }
 
-            const ident_token = try appendTokenFmt(c, .Identifier, "{}_{}", .{ slice, m.slice() });
+            const ident_token = try appendTokenFmt(c, .Identifier, "{s}_{s}", .{ slice, m.slice() });
             const identifier = try c.arena.create(ast.Node.OneToken);
             identifier.* = .{
                 .base = .{ .tag = .Identifier },
diff --git a/src/type.zig b/src/type.zig
index bb2e3b3bc2..9e2cd321f0 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -558,21 +558,21 @@ pub const Type = extern union {
                 },
                 .array_u8 => {
                     const len = ty.castTag(.array_u8).?.data;
-                    return out_stream.print("[{}]u8", .{len});
+                    return out_stream.print("[{d}]u8", .{len});
                 },
                 .array_u8_sentinel_0 => {
                     const len = ty.castTag(.array_u8_sentinel_0).?.data;
-                    return out_stream.print("[{}:0]u8", .{len});
+                    return out_stream.print("[{d}:0]u8", .{len});
                 },
                 .array => {
                     const payload = ty.castTag(.array).?.data;
-                    try out_stream.print("[{}]", .{payload.len});
+                    try out_stream.print("[{d}]", .{payload.len});
                     ty = payload.elem_type;
                     continue;
                 },
                 .array_sentinel => {
                     const payload = ty.castTag(.array_sentinel).?.data;
-                    try out_stream.print("[{}:{}]", .{ payload.len, payload.sentinel });
+                    try out_stream.print("[{d}:{}]", .{ payload.len, payload.sentinel });
                     ty = payload.elem_type;
                     continue;
                 },
diff --git a/src/zir.zig b/src/zir.zig
index 94e2b24b0c..0593cbd8fd 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1257,12 +1257,12 @@ const Writer = struct {
                     self.next_instr_index += 1;
                     try self.inst_table.putNoClobber(inst, .{ .inst = inst, .index = my_i, .name = undefined });
                     try stream.writeByteNTimes(' ', self.indent);
-                    try stream.print("%{} ", .{my_i});
+                    try stream.print("%{d} ", .{my_i});
                     if (inst.cast(Inst.Block)) |block| {
-                        const name = try std.fmt.allocPrint(&self.arena.allocator, "label_{}", .{my_i});
+                        const name = try std.fmt.allocPrint(&self.arena.allocator, "label_{d}", .{my_i});
                         try self.block_table.put(block, name);
                     } else if (inst.cast(Inst.Loop)) |loop| {
-                        const name = try std.fmt.allocPrint(&self.arena.allocator, "loop_{}", .{my_i});
+                        const name = try std.fmt.allocPrint(&self.arena.allocator, "loop_{d}", .{my_i});
                         try self.loop_table.put(loop, name);
                     }
                     self.indent += 2;
@@ -1332,7 +1332,7 @@ const Writer = struct {
     fn writeInstParamToStream(self: *Writer, stream: anytype, inst: *Inst) !void {
         if (self.inst_table.get(inst)) |info| {
             if (info.index) |i| {
-                try stream.print("%{}", .{info.index});
+                try stream.print("%{d}", .{info.index});
             } else {
                 try stream.print("@{s}", .{info.name});
             }
@@ -1660,7 +1660,6 @@ const Parser = struct {
             .contents_hash = std.zig.hashSrc(self.source[contents_start..self.i]),
             .inst = &inst_specific.base,
         };
-        //std.debug.warn("parsed {} = '{}'\n", .{ inst_specific.base.name, inst_specific.base.contents });
 
         return decl;
     }
@@ -1805,7 +1804,7 @@ const Parser = struct {
     }
 
     fn generateName(self: *Parser) ![]u8 {
-        const result = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${}", .{self.unnamed_index});
+        const result = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${d}", .{self.unnamed_index});
         self.unnamed_index += 1;
         return result;
     }
@@ -2865,7 +2864,7 @@ const EmitZIR = struct {
 
     fn autoName(self: *EmitZIR) ![]u8 {
         while (true) {
-            const proposed_name = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${}", .{self.next_auto_name});
+            const proposed_name = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${d}", .{self.next_auto_name});
             self.next_auto_name += 1;
             const gop = try self.names.getOrPut(proposed_name);
             if (!gop.found_existing) {
@@ -2954,15 +2953,15 @@ pub fn dumpZir(allocator: *Allocator, kind: []const u8, decl_name: [*:0]const u8
         write.next_instr_index += 1;
 
         if (inst.cast(Inst.Block)) |block| {
-            const name = try std.fmt.allocPrint(&write.arena.allocator, "label_{}", .{my_i});
+            const name = try std.fmt.allocPrint(&write.arena.allocator, "label_{d}", .{my_i});
             try write.block_table.put(block, name);
         } else if (inst.cast(Inst.Loop)) |loop| {
-            const name = try std.fmt.allocPrint(&write.arena.allocator, "loop_{}", .{my_i});
+            const name = try std.fmt.allocPrint(&write.arena.allocator, "loop_{d}", .{my_i});
             try write.loop_table.put(loop, name);
         }
 
         try write.inst_table.putNoClobber(inst, .{ .inst = inst, .index = my_i, .name = "inst" });
-        try stderr.print("  %{} ", .{my_i});
+        try stderr.print("  %{d} ", .{my_i});
         try write.writeInstToStream(stderr, inst);
         try stderr.writeByte('\n');
     }
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 72a1b04238..f9cd0e1a3d 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -535,7 +535,7 @@ fn analyzeInstParamType(mod: *Module, scope: *Scope, inst: *zir.Inst.ParamType)
     // TODO support C-style var args
     const param_count = fn_ty.fnParamLen();
     if (arg_index >= param_count) {
-        return mod.fail(scope, inst.base.src, "arg index {} out of bounds; '{}' has {} argument(s)", .{
+        return mod.fail(scope, inst.base.src, "arg index {d} out of bounds; '{}' has {d} argument(s)", .{
             arg_index,
             fn_ty,
             param_count,
@@ -580,7 +580,7 @@ fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*
     const param_index = b.instructions.items.len;
     const param_count = fn_ty.fnParamLen();
     if (param_index >= param_count) {
-        return mod.fail(scope, inst.base.src, "parameter index {} outside list of length {}", .{
+        return mod.fail(scope, inst.base.src, "parameter index {d} outside list of length {d}", .{
             param_index,
             param_count,
         });
@@ -790,7 +790,7 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
             return mod.fail(
                 scope,
                 inst.positionals.func.src,
-                "expected at least {} argument(s), found {}",
+                "expected at least {d} argument(s), found {d}",
                 .{ fn_params_len, call_params_len },
             );
         }
@@ -800,7 +800,7 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
         return mod.fail(
             scope,
             inst.positionals.func.src,
-            "expected {} argument(s), found {}",
+            "expected {d} argument(s), found {d}",
             .{ fn_params_len, call_params_len },
         );
     }
@@ -1545,7 +1545,7 @@ fn analyzeInstBitwise(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerE
 
     if (lhs.ty.zigTypeTag() == .Vector and rhs.ty.zigTypeTag() == .Vector) {
         if (lhs.ty.arrayLen() != rhs.ty.arrayLen()) {
-            return mod.fail(scope, inst.base.src, "vector length mismatch: {} and {}", .{
+            return mod.fail(scope, inst.base.src, "vector length mismatch: {d} and {d}", .{
                 lhs.ty.arrayLen(),
                 rhs.ty.arrayLen(),
             });
@@ -1620,7 +1620,7 @@ fn analyzeInstArithmetic(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) Inn
 
     if (lhs.ty.zigTypeTag() == .Vector and rhs.ty.zigTypeTag() == .Vector) {
         if (lhs.ty.arrayLen() != rhs.ty.arrayLen()) {
-            return mod.fail(scope, inst.base.src, "vector length mismatch: {} and {}", .{
+            return mod.fail(scope, inst.base.src, "vector length mismatch: {d} and {d}", .{
                 lhs.ty.arrayLen(),
                 rhs.ty.arrayLen(),
             });
@@ -1791,7 +1791,7 @@ fn analyzeInstCmp(
         return mod.cmpNumeric(scope, inst.base.src, lhs, rhs, op);
     } else if (lhs_ty_tag == .Type and rhs_ty_tag == .Type) {
         if (!is_equality_cmp) {
-            return mod.fail(scope, inst.base.src, "{} operator not allowed for types", .{@tagName(op)});
+            return mod.fail(scope, inst.base.src, "{s} operator not allowed for types", .{@tagName(op)});
         }
         return mod.constBool(scope, inst.base.src, lhs.value().?.eql(rhs.value().?) == (op == .eq));
     }
-- 
cgit v1.2.3


From 9362f382ab7023592cc1d71044217b847b122406 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 2 Jan 2021 12:32:30 -0700
Subject: stage2: implement function call inlining in the frontend

 * remove the -Ddump-zir thing. that's handled through --verbose-ir
 * rework Fn to have an is_inline flag without requiring any more memory
   on the heap per function.
 * implement a rough first version of dumping typed zir (tzir) which is
   a lot more helpful for debugging than what we had before. We don't
   have a way to parse it though.
 * keep track of whether the inline-ness of a function changes because
   if it does we have to go update callsites.
 * add compile error for inline and export used together.

inline function calls and comptime function calls are implemented the
same way. A block instruction is set up to capture the result, and then
a scope is set up that has a flag for is_comptime and some state if the
scope is being inlined.

when analyzing `ret` instructions, zig looks for inlining state in the
scope, and if found, treats `ret` as a `break` instruction instead, with
the target block being the one set up at the inline callsite.

Follow-up items:
 * Complete out the debug TZIR dumping code.
 * Don't redundantly generate ZIR for each inline/comptime function
   call. Instead we should add a new state enum tag to Fn.
 * comptime and inlining branch quotas.
 * Add more test cases.
---
 build.zig                       |   2 -
 src/Compilation.zig             |  14 +-
 src/Module.zig                  | 150 ++++++++++++-----
 src/codegen.zig                 |  10 +-
 src/codegen/c.zig               |   2 +-
 src/codegen/wasm.zig            |   2 +-
 src/config.zig.in               |   1 -
 src/link/Elf.zig                |  10 --
 src/link/MachO/DebugSymbols.zig |  10 --
 src/llvm_backend.zig            |   2 +-
 src/zir.zig                     | 349 ++++++++++++++++++++++++++++------------
 src/zir_sema.zig                | 239 ++++++++++++++++++---------
 test/stage2/zir.zig             |  12 +-
 13 files changed, 549 insertions(+), 254 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/build.zig b/build.zig
index 2b0685d19e..f86b0d3bec 100644
--- a/build.zig
+++ b/build.zig
@@ -220,7 +220,6 @@ pub fn build(b: *Builder) !void {
     }
 
     const log_scopes = b.option([]const []const u8, "log", "Which log scopes to enable") orelse &[0][]const u8{};
-    const zir_dumps = b.option([]const []const u8, "dump-zir", "Which functions to dump ZIR for before codegen") orelse &[0][]const u8{};
 
     const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git.");
     const version = if (opt_version_string) |version| version else v: {
@@ -277,7 +276,6 @@ pub fn build(b: *Builder) !void {
     exe.addBuildOption(std.SemanticVersion, "semver", semver);
 
     exe.addBuildOption([]const []const u8, "log_scopes", log_scopes);
-    exe.addBuildOption([]const []const u8, "zir_dumps", zir_dumps);
     exe.addBuildOption(bool, "enable_tracy", tracy != null);
     exe.addBuildOption(bool, "is_stage1", is_stage1);
     if (tracy) |tracy_path| {
diff --git a/src/Compilation.zig b/src/Compilation.zig
index de115b9b40..a6f39a3154 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1459,10 +1459,10 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 const module = self.bin_file.options.module.?;
                 if (decl.typed_value.most_recent.typed_value.val.castTag(.function)) |payload| {
                     const func = payload.data;
-                    switch (func.analysis) {
+                    switch (func.bits.state) {
                         .queued => module.analyzeFnBody(decl, func) catch |err| switch (err) {
                             error.AnalysisFail => {
-                                assert(func.analysis != .in_progress);
+                                assert(func.bits.state != .in_progress);
                                 continue;
                             },
                             error.OutOfMemory => return error.OutOfMemory,
@@ -1471,12 +1471,16 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                         .sema_failure, .dependency_failure => continue,
                         .success => {},
                     }
-                    // Here we tack on additional allocations to the Decl's arena. The allocations are
-                    // lifetime annotations in the ZIR.
+                    // Here we tack on additional allocations to the Decl's arena. The allocations
+                    // are lifetime annotations in the ZIR.
                     var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa);
                     defer decl.typed_value.most_recent.arena.?.* = decl_arena.state;
                     log.debug("analyze liveness of {s}\n", .{decl.name});
-                    try liveness.analyze(module.gpa, &decl_arena.allocator, func.analysis.success);
+                    try liveness.analyze(module.gpa, &decl_arena.allocator, func.data.body);
+
+                    if (self.verbose_ir) {
+                        func.dump(module.*);
+                    }
                 }
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
diff --git a/src/Module.zig b/src/Module.zig
index 29c19c09a0..db76ecd5db 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -286,23 +286,40 @@ pub const Decl = struct {
 /// Extern functions do not have this data structure; they are represented by
 /// the `Decl` only, with a `Value` tag of `extern_fn`.
 pub const Fn = struct {
-    /// This memory owned by the Decl's TypedValue.Managed arena allocator.
-    analysis: union(enum) {
+    bits: packed struct {
+        /// Get and set this field via `analysis` and `setAnalysis`.
+        state: Analysis.Tag,
+        /// We carry this state into `Fn` instead of leaving it in the AST so that
+        /// analysis of function calls can happen even on functions whose AST has
+        /// been unloaded from memory.
+        is_inline: bool,
+        unused_bits: u4 = 0,
+    },
+    /// Get and set this data via `analysis` and `setAnalysis`.
+    data: union {
+        none: void,
+        zir: *ZIR,
+        body: Body,
+    },
+    owner_decl: *Decl,
+
+    pub const Analysis = union(Tag) {
         queued: *ZIR,
         in_progress,
-        /// There will be a corresponding ErrorMsg in Module.failed_decls
         sema_failure,
-        /// This Fn might be OK but it depends on another Decl which did not successfully complete
-        /// semantic analysis.
         dependency_failure,
         success: Body,
-    },
-    owner_decl: *Decl,
 
-    /// This memory is temporary and points to stack memory for the duration
-    /// of Fn analysis.
-    pub const Analysis = struct {
-        inner_block: Scope.Block,
+        pub const Tag = enum(u3) {
+            queued,
+            in_progress,
+            /// There will be a corresponding ErrorMsg in Module.failed_decls
+            sema_failure,
+            /// This Fn might be OK but it depends on another Decl which did not
+            /// successfully complete semantic analysis.
+            dependency_failure,
+            success,
+        };
     };
 
     /// Contains un-analyzed ZIR instructions generated from Zig source AST.
@@ -311,22 +328,37 @@ pub const Fn = struct {
         arena: std.heap.ArenaAllocator.State,
     };
 
-    /// For debugging purposes.
-    pub fn dump(self: *Fn, mod: Module) void {
-        std.debug.print("Module.Function(name={s}) ", .{self.owner_decl.name});
-        switch (self.analysis) {
-            .queued => {
-                std.debug.print("queued\n", .{});
+    pub fn analysis(self: Fn) Analysis {
+        return switch (self.bits.state) {
+            .queued => .{ .queued = self.data.zir },
+            .success => .{ .success = self.data.body },
+            .in_progress => .in_progress,
+            .sema_failure => .sema_failure,
+            .dependency_failure => .dependency_failure,
+        };
+    }
+
+    pub fn setAnalysis(self: *Fn, anal: Analysis) void {
+        switch (anal) {
+            .queued => |zir_ptr| {
+                self.bits.state = .queued;
+                self.data = .{ .zir = zir_ptr };
             },
-            .in_progress => {
-                std.debug.print("in_progress\n", .{});
+            .success => |body| {
+                self.bits.state = .success;
+                self.data = .{ .body = body };
             },
-            else => {
-                std.debug.print("\n", .{});
-                zir.dumpFn(mod, self);
+            .in_progress, .sema_failure, .dependency_failure => {
+                self.bits.state = anal;
+                self.data = .{ .none = {} };
             },
         }
     }
+
+    /// For debugging purposes.
+    pub fn dump(self: *Fn, mod: Module) void {
+        zir.dumpFn(mod, self);
+    }
 };
 
 pub const Var = struct {
@@ -773,13 +805,33 @@ pub const Scope = struct {
         instructions: ArrayListUnmanaged(*Inst),
         /// Points to the arena allocator of DeclAnalysis
         arena: *Allocator,
-        label: ?Label = null,
+        label: Label = Label.none,
         is_comptime: bool,
 
-        pub const Label = struct {
-            zir_block: *zir.Inst.Block,
-            results: ArrayListUnmanaged(*Inst),
-            block_inst: *Inst.Block,
+        pub const Label = union(enum) {
+            none,
+            /// This `Block` maps a block ZIR instruction to the corresponding
+            /// TZIR instruction for break instruction analysis.
+            breaking: struct {
+                zir_block: *zir.Inst.Block,
+                merges: Merges,
+            },
+            /// This `Block` indicates that an inline function call is happening
+            /// and return instructions should be analyzed as a break instruction
+            /// to this TZIR block instruction.
+            inlining: struct {
+                /// We use this to count from 0 so that arg instructions know
+                /// which parameter index they are, without having to store
+                /// a parameter index with each arg instruction.
+                param_index: usize,
+                casted_args: []*Inst,
+                merges: Merges,
+            },
+
+            pub const Merges = struct {
+                results: ArrayListUnmanaged(*Inst),
+                block_inst: *Inst.Block,
+            };
         };
 
         /// For debugging purposes.
@@ -1189,8 +1241,21 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 break :blk fn_zir;
             };
 
+            const is_inline = blk: {
+                if (fn_proto.getExternExportInlineToken()) |maybe_inline_token| {
+                    if (tree.token_ids[maybe_inline_token] == .Keyword_inline) {
+                        break :blk true;
+                    }
+                }
+                break :blk false;
+            };
+
             new_func.* = .{
-                .analysis = .{ .queued = fn_zir },
+                .bits = .{
+                    .state = .queued,
+                    .is_inline = is_inline,
+                },
+                .data = .{ .zir = fn_zir },
                 .owner_decl = decl,
             };
             fn_payload.* = .{
@@ -1199,11 +1264,16 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             };
 
             var prev_type_has_bits = false;
+            var prev_is_inline = false;
             var type_changed = true;
 
             if (decl.typedValueManaged()) |tvm| {
                 prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits();
                 type_changed = !tvm.typed_value.ty.eql(fn_type);
+                if (tvm.typed_value.val.castTag(.function)) |payload| {
+                    const prev_func = payload.data;
+                    prev_is_inline = prev_func.bits.is_inline;
+                }
 
                 tvm.deinit(self.gpa);
             }
@@ -1221,18 +1291,26 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             decl.analysis = .complete;
             decl.generation = self.generation;
 
-            if (fn_type.hasCodeGenBits()) {
+            if (!is_inline and fn_type.hasCodeGenBits()) {
                 // We don't fully codegen the decl until later, but we do need to reserve a global
                 // offset table index for it. This allows us to codegen decls out of dependency order,
                 // increasing how many computations can be done in parallel.
                 try self.comp.bin_file.allocateDeclIndexes(decl);
                 try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
-            } else if (prev_type_has_bits) {
+            } else if (!prev_is_inline and prev_type_has_bits) {
                 self.comp.bin_file.freeDecl(decl);
             }
 
             if (fn_proto.getExternExportInlineToken()) |maybe_export_token| {
                 if (tree.token_ids[maybe_export_token] == .Keyword_export) {
+                    if (is_inline) {
+                        return self.failTok(
+                            &block_scope.base,
+                            maybe_export_token,
+                            "export of inline function",
+                            .{},
+                        );
+                    }
                     const export_src = tree.token_locs[maybe_export_token].start;
                     const name_loc = tree.token_locs[fn_proto.getNameToken().?];
                     const name = tree.tokenSliceLoc(name_loc);
@@ -1240,7 +1318,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                     try self.analyzeExport(&block_scope.base, export_src, name, decl);
                 }
             }
-            return type_changed;
+            return type_changed or is_inline != prev_is_inline;
         },
         .VarDecl => {
             const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", ast_node);
@@ -1824,15 +1902,15 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
     };
     defer inner_block.instructions.deinit(self.gpa);
 
-    const fn_zir = func.analysis.queued;
+    const fn_zir = func.data.zir;
     defer fn_zir.arena.promote(self.gpa).deinit();
-    func.analysis = .{ .in_progress = {} };
+    func.setAnalysis(.in_progress);
     log.debug("set {s} to in_progress\n", .{decl.name});
 
     try zir_sema.analyzeBody(self, &inner_block.base, fn_zir.body);
 
     const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items);
-    func.analysis = .{ .success = .{ .instructions = instructions } };
+    func.setAnalysis(.{ .success = .{ .instructions = instructions } });
     log.debug("set {s} to success\n", .{decl.name});
 }
 
@@ -2329,7 +2407,7 @@ pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) Inn
     self.ensureDeclAnalyzed(decl) catch |err| {
         if (scope.cast(Scope.Block)) |block| {
             if (block.func) |func| {
-                func.analysis = .dependency_failure;
+                func.setAnalysis(.dependency_failure);
             } else {
                 block.decl.analysis = .dependency_failure;
             }
@@ -3029,7 +3107,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
         .block => {
             const block = scope.cast(Scope.Block).?;
             if (block.func) |func| {
-                func.analysis = .sema_failure;
+                func.setAnalysis(.sema_failure);
             } else {
                 block.decl.analysis = .sema_failure;
                 block.decl.generation = self.generation;
diff --git a/src/codegen.zig b/src/codegen.zig
index 6530b687e5..588c3dec4c 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -532,7 +532,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         self.code.items.len += 4;
 
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.analysis.success);
+                        try self.genBody(self.mod_fn.data.body);
 
                         const stack_end = self.max_end_stack;
                         if (stack_end > math.maxInt(i32))
@@ -576,7 +576,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         });
                     } else {
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.analysis.success);
+                        try self.genBody(self.mod_fn.data.body);
                         try self.dbgSetEpilogueBegin();
                     }
                 },
@@ -593,7 +593,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                         try self.dbgSetPrologueEnd();
 
-                        try self.genBody(self.mod_fn.analysis.success);
+                        try self.genBody(self.mod_fn.data.body);
 
                         // Backpatch stack offset
                         const stack_end = self.max_end_stack;
@@ -638,13 +638,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         writeInt(u32, try self.code.addManyAsArray(4), Instruction.pop(.al, .{ .fp, .pc }).toU32());
                     } else {
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.analysis.success);
+                        try self.genBody(self.mod_fn.data.body);
                         try self.dbgSetEpilogueBegin();
                     }
                 },
                 else => {
                     try self.dbgSetPrologueEnd();
-                    try self.genBody(self.mod_fn.analysis.success);
+                    try self.genBody(self.mod_fn.data.body);
                     try self.dbgSetEpilogueBegin();
                 },
             }
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 684a03eb79..712d663af0 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -275,7 +275,7 @@ pub fn generate(file: *C, module: *Module, decl: *Decl) !void {
         try writer.writeAll(" {");
 
         const func: *Module.Fn = func_payload.data;
-        const instructions = func.analysis.success.instructions;
+        const instructions = func.data.body.instructions;
         if (instructions.len > 0) {
             try writer.writeAll("\n");
             for (instructions) |inst| {
diff --git a/src/codegen/wasm.zig b/src/codegen/wasm.zig
index c7ad59f5d1..1eb4f5bc29 100644
--- a/src/codegen/wasm.zig
+++ b/src/codegen/wasm.zig
@@ -63,7 +63,7 @@ pub fn genCode(buf: *ArrayList(u8), decl: *Decl) !void {
     // TODO: check for and handle death of instructions
     const tv = decl.typed_value.most_recent.typed_value;
     const mod_fn = tv.val.castTag(.function).?.data;
-    for (mod_fn.analysis.success.instructions) |inst| try genInst(buf, decl, inst);
+    for (mod_fn.data.body.instructions) |inst| try genInst(buf, decl, inst);
 
     // Write 'end' opcode
     try writer.writeByte(0x0B);
diff --git a/src/config.zig.in b/src/config.zig.in
index 9d16cf3824..0dbd3f3c91 100644
--- a/src/config.zig.in
+++ b/src/config.zig.in
@@ -2,7 +2,6 @@ pub const have_llvm = true;
 pub const version: [:0]const u8 = "@ZIG_VERSION@";
 pub const semver = try @import("std").SemanticVersion.parse(version);
 pub const log_scopes: []const []const u8 = &[_][]const u8{};
-pub const zir_dumps: []const []const u8 = &[_][]const u8{};
 pub const enable_tracy = false;
 pub const is_stage1 = true;
 pub const skip_non_native = false;
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 116d7c9859..d74236f8c1 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2178,16 +2178,6 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
         else => false,
     };
     if (is_fn) {
-        const zir_dumps = if (std.builtin.is_test) &[0][]const u8{} else build_options.zir_dumps;
-        if (zir_dumps.len != 0) {
-            for (zir_dumps) |fn_name| {
-                if (mem.eql(u8, mem.spanZ(decl.name), fn_name)) {
-                    std.debug.print("\n{s}\n", .{decl.name});
-                    typed_value.val.castTag(.function).?.data.dump(module.*);
-                }
-            }
-        }
-
         // For functions we need to add a prologue to the debug line program.
         try dbg_line_buffer.ensureCapacity(26);
 
diff --git a/src/link/MachO/DebugSymbols.zig b/src/link/MachO/DebugSymbols.zig
index c70fcc5825..11f87d5495 100644
--- a/src/link/MachO/DebugSymbols.zig
+++ b/src/link/MachO/DebugSymbols.zig
@@ -936,16 +936,6 @@ pub fn initDeclDebugBuffers(
     const typed_value = decl.typed_value.most_recent.typed_value;
     switch (typed_value.ty.zigTypeTag()) {
         .Fn => {
-            const zir_dumps = if (std.builtin.is_test) &[0][]const u8{} else build_options.zir_dumps;
-            if (zir_dumps.len != 0) {
-                for (zir_dumps) |fn_name| {
-                    if (mem.eql(u8, mem.spanZ(decl.name), fn_name)) {
-                        std.debug.print("\n{}\n", .{decl.name});
-                        typed_value.val.cast(Value.Payload.Function).?.func.dump(module.*);
-                    }
-                }
-            }
-
             // For functions we need to add a prologue to the debug line program.
             try dbg_line_buffer.ensureCapacity(26);
 
diff --git a/src/llvm_backend.zig b/src/llvm_backend.zig
index 51d1a0840e..5814aa7e7e 100644
--- a/src/llvm_backend.zig
+++ b/src/llvm_backend.zig
@@ -294,7 +294,7 @@ pub const LLVMIRModule = struct {
                 const entry_block = llvm_func.appendBasicBlock("Entry");
                 self.builder.positionBuilderAtEnd(entry_block);
 
-                const instructions = func.analysis.success.instructions;
+                const instructions = func.data.body.instructions;
                 for (instructions) |inst| {
                     switch (inst.tag) {
                         .breakpoint => try self.genBreakpoint(inst.castTag(.breakpoint).?),
diff --git a/src/zir.zig b/src/zir.zig
index 043c54faf0..64b74f24d9 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -793,7 +793,9 @@ pub const Inst = struct {
             fn_type: *Inst,
             body: Module.Body,
         },
-        kw_args: struct {},
+        kw_args: struct {
+            is_inline: bool = false,
+        },
     };
 
     pub const FnType = struct {
@@ -1847,83 +1849,258 @@ pub fn emit(allocator: *Allocator, old_module: *IrModule) !Module {
 /// For debugging purposes, prints a function representation to stderr.
 pub fn dumpFn(old_module: IrModule, module_fn: *IrModule.Fn) void {
     const allocator = old_module.gpa;
-    var ctx: EmitZIR = .{
+    var ctx: DumpTzir = .{
         .allocator = allocator,
-        .decls = .{},
         .arena = std.heap.ArenaAllocator.init(allocator),
         .old_module = &old_module,
-        .next_auto_name = 0,
-        .names = std.StringArrayHashMap(void).init(allocator),
-        .primitive_table = std.AutoHashMap(Inst.Primitive.Builtin, *Decl).init(allocator),
-        .indent = 0,
-        .block_table = std.AutoHashMap(*ir.Inst.Block, *Inst.Block).init(allocator),
-        .loop_table = std.AutoHashMap(*ir.Inst.Loop, *Inst.Loop).init(allocator),
-        .metadata = std.AutoHashMap(*Inst, Module.MetaData).init(allocator),
-        .body_metadata = std.AutoHashMap(*Module.Body, Module.BodyMetaData).init(allocator),
+        .module_fn = module_fn,
+        .indent = 2,
+        .inst_table = DumpTzir.InstTable.init(allocator),
+        .partial_inst_table = DumpTzir.InstTable.init(allocator),
+        .const_table = DumpTzir.InstTable.init(allocator),
     };
-    defer ctx.metadata.deinit();
-    defer ctx.body_metadata.deinit();
-    defer ctx.block_table.deinit();
-    defer ctx.loop_table.deinit();
-    defer ctx.decls.deinit(allocator);
-    defer ctx.names.deinit();
-    defer ctx.primitive_table.deinit();
+    defer ctx.inst_table.deinit();
+    defer ctx.partial_inst_table.deinit();
+    defer ctx.const_table.deinit();
     defer ctx.arena.deinit();
 
-    const fn_ty = module_fn.owner_decl.typed_value.most_recent.typed_value.ty;
-    _ = ctx.emitFn(module_fn, 0, fn_ty) catch |err| {
-        std.debug.print("unable to dump function: {s}\n", .{@errorName(err)});
-        return;
-    };
-    var module = Module{
-        .decls = ctx.decls.items,
-        .arena = ctx.arena,
-        .metadata = ctx.metadata,
-        .body_metadata = ctx.body_metadata,
-    };
-
-    module.dump();
+    switch (module_fn.analysis()) {
+        .queued => std.debug.print("(queued)", .{}),
+        .in_progress => std.debug.print("(in_progress)", .{}),
+        .sema_failure => std.debug.print("(sema_failure)", .{}),
+        .dependency_failure => std.debug.print("(dependency_failure)", .{}),
+        .success => |body| {
+            ctx.dump(body, std.io.getStdErr().writer()) catch @panic("failed to dump TZIR");
+        },
+    }
 }
 
-/// For debugging purposes, prints a function representation to stderr.
-pub fn dumpBlock(old_module: IrModule, module_block: *IrModule.Scope.Block) void {
-    const allocator = old_module.gpa;
-    var ctx: EmitZIR = .{
-        .allocator = allocator,
-        .decls = .{},
-        .arena = std.heap.ArenaAllocator.init(allocator),
-        .old_module = &old_module,
-        .next_auto_name = 0,
-        .names = std.StringArrayHashMap(void).init(allocator),
-        .primitive_table = std.AutoHashMap(Inst.Primitive.Builtin, *Decl).init(allocator),
-        .indent = 0,
-        .block_table = std.AutoHashMap(*ir.Inst.Block, *Inst.Block).init(allocator),
-        .loop_table = std.AutoHashMap(*ir.Inst.Loop, *Inst.Loop).init(allocator),
-        .metadata = std.AutoHashMap(*Inst, Module.MetaData).init(allocator),
-        .body_metadata = std.AutoHashMap(*Module.Body, Module.BodyMetaData).init(allocator),
-    };
-    defer ctx.metadata.deinit();
-    defer ctx.body_metadata.deinit();
-    defer ctx.block_table.deinit();
-    defer ctx.loop_table.deinit();
-    defer ctx.decls.deinit(allocator);
-    defer ctx.names.deinit();
-    defer ctx.primitive_table.deinit();
-    defer ctx.arena.deinit();
+const DumpTzir = struct {
+    allocator: *Allocator,
+    arena: std.heap.ArenaAllocator,
+    old_module: *const IrModule,
+    module_fn: *IrModule.Fn,
+    indent: usize,
+    inst_table: InstTable,
+    partial_inst_table: InstTable,
+    const_table: InstTable,
+    next_index: usize = 0,
+    next_partial_index: usize = 0,
+    next_const_index: usize = 0,
+
+    const InstTable = std.AutoArrayHashMap(*ir.Inst, usize);
+
+    fn dump(dtz: *DumpTzir, body: ir.Body, writer: std.fs.File.Writer) !void {
+        // First pass to pre-populate the table so that we can show even invalid references.
+        // Must iterate the same order we iterate the second time.
+        // We also look for constants and put them in the const_table.
+        for (body.instructions) |inst| {
+            try dtz.inst_table.put(inst, dtz.next_index);
+            dtz.next_index += 1;
+            switch (inst.tag) {
+                .alloc,
+                .retvoid,
+                .unreach,
+                .breakpoint,
+                .dbg_stmt,
+                => {},
 
-    _ = ctx.emitBlock(module_block, 0) catch |err| {
-        std.debug.print("unable to dump function: {}\n", .{err});
-        return;
-    };
-    var module = Module{
-        .decls = ctx.decls.items,
-        .arena = ctx.arena,
-        .metadata = ctx.metadata,
-        .body_metadata = ctx.body_metadata,
-    };
+                .ref,
+                .ret,
+                .bitcast,
+                .not,
+                .isnonnull,
+                .isnull,
+                .iserr,
+                .ptrtoint,
+                .floatcast,
+                .intcast,
+                .load,
+                .unwrap_optional,
+                .wrap_optional,
+                => {
+                    const un_op = inst.cast(ir.Inst.UnOp).?;
+                    try dtz.findConst(un_op.operand);
+                },
 
-    module.dump();
-}
+                .add,
+                .sub,
+                .cmp_lt,
+                .cmp_lte,
+                .cmp_eq,
+                .cmp_gte,
+                .cmp_gt,
+                .cmp_neq,
+                .store,
+                .booland,
+                .boolor,
+                .bitand,
+                .bitor,
+                .xor,
+                => {
+                    const bin_op = inst.cast(ir.Inst.BinOp).?;
+                    try dtz.findConst(bin_op.lhs);
+                    try dtz.findConst(bin_op.rhs);
+                },
+
+                .arg => {},
+
+                // TODO fill out this debug printing
+                .assembly,
+                .block,
+                .br,
+                .brvoid,
+                .call,
+                .condbr,
+                .constant,
+                .loop,
+                .varptr,
+                .switchbr,
+                => {},
+            }
+        }
+
+        std.debug.print("Module.Function(name={s}):\n", .{dtz.module_fn.owner_decl.name});
+
+        for (dtz.const_table.items()) |entry| {
+            const constant = entry.key.castTag(.constant).?;
+            try writer.print("  @{d}: {} = {};\n", .{
+                entry.value, constant.base.ty, constant.val,
+            });
+        }
+
+        return dtz.dumpBody(body, writer);
+    }
+
+    fn dumpBody(dtz: *DumpTzir, body: ir.Body, writer: std.fs.File.Writer) !void {
+        for (body.instructions) |inst| {
+            const my_index = dtz.next_partial_index;
+            try dtz.partial_inst_table.put(inst, my_index);
+            dtz.next_partial_index += 1;
+
+            try writer.writeByteNTimes(' ', dtz.indent);
+            try writer.print("%{d}: {} = {s}(", .{
+                my_index, inst.ty, @tagName(inst.tag),
+            });
+            switch (inst.tag) {
+                .alloc,
+                .retvoid,
+                .unreach,
+                .breakpoint,
+                .dbg_stmt,
+                => try writer.writeAll(")\n"),
+
+                .ref,
+                .ret,
+                .bitcast,
+                .not,
+                .isnonnull,
+                .isnull,
+                .iserr,
+                .ptrtoint,
+                .floatcast,
+                .intcast,
+                .load,
+                .unwrap_optional,
+                .wrap_optional,
+                => {
+                    const un_op = inst.cast(ir.Inst.UnOp).?;
+                    if (dtz.partial_inst_table.get(un_op.operand)) |operand_index| {
+                        try writer.print("%{d})\n", .{operand_index});
+                    } else if (dtz.const_table.get(un_op.operand)) |operand_index| {
+                        try writer.print("@{d})\n", .{operand_index});
+                    } else if (dtz.inst_table.get(un_op.operand)) |operand_index| {
+                        try writer.print("%{d}) // Instruction does not dominate all uses!\n", .{
+                            operand_index,
+                        });
+                    } else {
+                        try writer.writeAll("!BADREF!)\n");
+                    }
+                },
+
+                .add,
+                .sub,
+                .cmp_lt,
+                .cmp_lte,
+                .cmp_eq,
+                .cmp_gte,
+                .cmp_gt,
+                .cmp_neq,
+                .store,
+                .booland,
+                .boolor,
+                .bitand,
+                .bitor,
+                .xor,
+                => {
+                    var lhs_kinky: ?usize = null;
+                    var rhs_kinky: ?usize = null;
+
+                    const bin_op = inst.cast(ir.Inst.BinOp).?;
+                    if (dtz.partial_inst_table.get(bin_op.lhs)) |operand_index| {
+                        try writer.print("%{d}, ", .{operand_index});
+                    } else if (dtz.const_table.get(bin_op.lhs)) |operand_index| {
+                        try writer.print("@{d}, ", .{operand_index});
+                    } else if (dtz.inst_table.get(bin_op.lhs)) |operand_index| {
+                        lhs_kinky = operand_index;
+                        try writer.print("%{d}, ", .{operand_index});
+                    } else {
+                        try writer.writeAll("!BADREF!, ");
+                    }
+                    if (dtz.partial_inst_table.get(bin_op.rhs)) |operand_index| {
+                        try writer.print("%{d}", .{operand_index});
+                    } else if (dtz.const_table.get(bin_op.rhs)) |operand_index| {
+                        try writer.print("@{d}", .{operand_index});
+                    } else if (dtz.inst_table.get(bin_op.rhs)) |operand_index| {
+                        rhs_kinky = operand_index;
+                        try writer.print("%{d}", .{operand_index});
+                    } else {
+                        try writer.writeAll("!BADREF!");
+                    }
+                    if (lhs_kinky != null or rhs_kinky != null) {
+                        try writer.writeAll(") // Instruction does not dominate all uses!");
+                        if (lhs_kinky) |lhs| {
+                            try writer.print(" %{d}", .{lhs});
+                        }
+                        if (rhs_kinky) |rhs| {
+                            try writer.print(" %{d}", .{rhs});
+                        }
+                        try writer.writeAll("\n");
+                    } else {
+                        try writer.writeAll(")\n");
+                    }
+                },
+
+                .arg => {
+                    const arg = inst.castTag(.arg).?;
+                    try writer.print("{s})\n", .{arg.name});
+                },
+
+                // TODO fill out this debug printing
+                .assembly,
+                .block,
+                .br,
+                .brvoid,
+                .call,
+                .condbr,
+                .constant,
+                .loop,
+                .varptr,
+                .switchbr,
+                => {
+                    try writer.writeAll("!TODO!)\n");
+                },
+            }
+        }
+    }
+
+    fn findConst(dtz: *DumpTzir, operand: *ir.Inst) !void {
+        if (operand.tag == .constant) {
+            try dtz.const_table.put(operand, dtz.next_const_index);
+            dtz.next_const_index += 1;
+        }
+    }
+};
 
 const EmitZIR = struct {
     allocator: *Allocator,
@@ -2105,36 +2282,6 @@ const EmitZIR = struct {
         return &declref_inst.base;
     }
 
-    fn emitBlock(self: *EmitZIR, module_block: *IrModule.Scope.Block, src: usize) Allocator.Error!*Decl {
-        var inst_table = std.AutoHashMap(*ir.Inst, *Inst).init(self.allocator);
-        defer inst_table.deinit();
-
-        var instructions = std.ArrayList(*Inst).init(self.allocator);
-        defer instructions.deinit();
-
-        const body: ir.Body = .{ .instructions = module_block.instructions.items };
-        try self.emitBody(body, &inst_table, &instructions);
-
-        const fn_type = try self.emitType(src, Type.initTag(.void));
-
-        const arena_instrs = try self.arena.allocator.alloc(*Inst, instructions.items.len);
-        mem.copy(*Inst, arena_instrs, instructions.items);
-
-        const fn_inst = try self.arena.allocator.create(Inst.Fn);
-        fn_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.Fn.base_tag,
-            },
-            .positionals = .{
-                .fn_type = fn_type.inst,
-                .body = .{ .instructions = arena_instrs },
-            },
-            .kw_args = .{},
-        };
-        return self.emitUnnamedDecl(&fn_inst.base);
-    }
-
     fn emitFn(self: *EmitZIR, module_fn: *IrModule.Fn, src: usize, ty: Type) Allocator.Error!*Decl {
         var inst_table = std.AutoHashMap(*ir.Inst, *Inst).init(self.allocator);
         defer inst_table.deinit();
@@ -2142,7 +2289,7 @@ const EmitZIR = struct {
         var instructions = std.ArrayList(*Inst).init(self.allocator);
         defer instructions.deinit();
 
-        switch (module_fn.analysis) {
+        switch (module_fn.analysis()) {
             .queued => unreachable,
             .in_progress => unreachable,
             .success => |body| {
@@ -2224,7 +2371,9 @@ const EmitZIR = struct {
                 .fn_type = fn_type.inst,
                 .body = .{ .instructions = arena_instrs },
             },
-            .kw_args = .{},
+            .kw_args = .{
+                .is_inline = module_fn.bits.is_inline,
+            },
         };
         return self.emitUnnamedDecl(&fn_inst.base);
     }
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 96a19f20f4..9365996bb6 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -577,7 +577,15 @@ fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) In
 }
 
 fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*Inst {
-    const b = try mod.requireRuntimeBlock(scope, inst.base.src);
+    const b = try mod.requireFunctionBlock(scope, inst.base.src);
+    switch (b.label) {
+        .none, .breaking => {},
+        .inlining => |*inlining| {
+            const param_index = inlining.param_index;
+            inlining.param_index += 1;
+            return inlining.casted_args[param_index];
+        },
+    }
     const fn_ty = b.func.?.owner_decl.typed_value.most_recent.typed_value.ty;
     const param_index = b.instructions.items.len;
     const param_count = fn_ty.fnParamLen();
@@ -636,7 +644,7 @@ fn analyzeInstBlockFlat(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_c
         .decl = parent_block.decl,
         .instructions = .{},
         .arena = parent_block.arena,
-        .label = null,
+        .label = .none,
         .is_comptime = parent_block.is_comptime or is_comptime,
     };
     defer child_block.instructions.deinit(mod.gpa);
@@ -674,41 +682,56 @@ fn analyzeInstBlock(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_compt
         .decl = parent_block.decl,
         .instructions = .{},
         .arena = parent_block.arena,
-        // TODO @as here is working around a stage1 miscompilation bug :(
-        .label = @as(?Scope.Block.Label, Scope.Block.Label{
-            .zir_block = inst,
-            .results = .{},
-            .block_inst = block_inst,
-        }),
+        .label = Scope.Block.Label{
+            .breaking = .{
+                .zir_block = inst,
+                .merges = .{
+                    .results = .{},
+                    .block_inst = block_inst,
+                },
+            },
+        },
         .is_comptime = is_comptime or parent_block.is_comptime,
     };
-    const label = &child_block.label.?;
+    const merges = &child_block.label.breaking.merges;
 
     defer child_block.instructions.deinit(mod.gpa);
-    defer label.results.deinit(mod.gpa);
+    defer merges.results.deinit(mod.gpa);
 
     try analyzeBody(mod, &child_block.base, inst.positionals.body);
 
+    return analyzeBlockBody(mod, scope, &child_block, merges);
+}
+
+fn analyzeBlockBody(
+    mod: *Module,
+    scope: *Scope,
+    child_block: *Scope.Block,
+    merges: *Scope.Block.Label.Merges,
+) InnerError!*Inst {
+    const parent_block = scope.cast(Scope.Block).?;
+
     // Blocks must terminate with noreturn instruction.
     assert(child_block.instructions.items.len != 0);
     assert(child_block.instructions.items[child_block.instructions.items.len - 1].ty.isNoReturn());
 
-    if (label.results.items.len == 0) {
-        // No need for a block instruction. We can put the new instructions directly into the parent block.
+    if (merges.results.items.len == 0) {
+        // No need for a block instruction. We can put the new instructions
+        // directly into the parent block.
         const copied_instructions = try parent_block.arena.dupe(*Inst, child_block.instructions.items);
         try parent_block.instructions.appendSlice(mod.gpa, copied_instructions);
         return copied_instructions[copied_instructions.len - 1];
     }
-    if (label.results.items.len == 1) {
+    if (merges.results.items.len == 1) {
         const last_inst_index = child_block.instructions.items.len - 1;
         const last_inst = child_block.instructions.items[last_inst_index];
         if (last_inst.breakBlock()) |br_block| {
-            if (br_block == block_inst) {
+            if (br_block == merges.block_inst) {
                 // No need for a block instruction. We can put the new instructions directly into the parent block.
                 // Here we omit the break instruction.
                 const copied_instructions = try parent_block.arena.dupe(*Inst, child_block.instructions.items[0..last_inst_index]);
                 try parent_block.instructions.appendSlice(mod.gpa, copied_instructions);
-                return label.results.items[0];
+                return merges.results.items[0];
             }
         }
     }
@@ -717,10 +740,10 @@ fn analyzeInstBlock(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_compt
 
     // Need to set the type and emit the Block instruction. This allows machine code generation
     // to emit a jump instruction to after the block when it encounters the break.
-    try parent_block.instructions.append(mod.gpa, &block_inst.base);
-    block_inst.base.ty = try mod.resolvePeerTypes(scope, label.results.items);
-    block_inst.body = .{ .instructions = try parent_block.arena.dupe(*Inst, child_block.instructions.items) };
-    return &block_inst.base;
+    try parent_block.instructions.append(mod.gpa, &merges.block_inst.base);
+    merges.block_inst.base.ty = try mod.resolvePeerTypes(scope, merges.results.items);
+    merges.block_inst.body = .{ .instructions = try parent_block.arena.dupe(*Inst, child_block.instructions.items) };
+    return &merges.block_inst.base;
 }
 
 fn analyzeInstBreakpoint(mod: *Module, scope: *Scope, inst: *zir.Inst.NoOp) InnerError!*Inst {
@@ -829,14 +852,32 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
     const ret_type = func.ty.fnReturnType();
 
     const b = try mod.requireFunctionBlock(scope, inst.base.src);
-    if (b.is_comptime) {
-        const fn_val = try mod.resolveConstValue(scope, func);
-        const module_fn = switch (fn_val.tag()) {
-            .function => fn_val.castTag(.function).?.data,
-            .extern_fn => return mod.fail(scope, inst.base.src, "comptime call of extern function", .{}),
+    const is_comptime_call = b.is_comptime or inst.kw_args.modifier == .compile_time;
+    const is_inline_call = is_comptime_call or inst.kw_args.modifier == .always_inline or blk: {
+        // This logic will get simplified by
+        // https://github.com/ziglang/zig/issues/6429
+        if (try mod.resolveDefinedValue(scope, func)) |func_val| {
+            const module_fn = switch (func_val.tag()) {
+                .function => func_val.castTag(.function).?.data,
+                else => break :blk false,
+            };
+            break :blk module_fn.bits.is_inline;
+        }
+        break :blk false;
+    };
+    if (is_inline_call) {
+        const func_val = try mod.resolveConstValue(scope, func);
+        const module_fn = switch (func_val.tag()) {
+            .function => func_val.castTag(.function).?.data,
+            .extern_fn => return mod.fail(scope, inst.base.src, "{s} call of extern function", .{
+                @as([]const u8, if (is_comptime_call) "comptime" else "inline"),
+            }),
             else => unreachable,
         };
         const callee_decl = module_fn.owner_decl;
+        // TODO: De-duplicate this with the code in Module.zig that generates
+        // ZIR for the same function and re-use the same ZIR for runtime function
+        // generation and for inline/comptime calls.
         const callee_file_scope = callee_decl.getFileScope();
         const tree = mod.getAstTree(callee_file_scope) catch |err| switch (err) {
             error.OutOfMemory => return error.OutOfMemory,
@@ -859,23 +900,31 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
         };
         defer gen_scope.instructions.deinit(mod.gpa);
 
-        // Add a const instruction for each parameter.
+        // We need an instruction for each parameter, and they must be first in the body.
+        try gen_scope.instructions.resize(mod.gpa, fn_proto.params_len);
         var params_scope = &gen_scope.base;
         for (fn_proto.params()) |param, i| {
             const name_token = param.name_token.?;
             const src = tree.token_locs[name_token].start;
             const param_name = try mod.identifierTokenString(scope, name_token);
-            const arg_val = try mod.resolveConstValue(scope, casted_args[i]);
-            const arg = try astgen.addZIRInstConst(mod, params_scope, src, .{
-                .ty = casted_args[i].ty,
-                .val = arg_val,
-            });
+            const arg = try call_arena.allocator.create(zir.Inst.Arg);
+            arg.* = .{
+                .base = .{
+                    .tag = .arg,
+                    .src = src,
+                },
+                .positionals = .{
+                    .name = param_name,
+                },
+                .kw_args = .{},
+            };
+            gen_scope.instructions.items[i] = &arg.base;
             const sub_scope = try call_arena.allocator.create(Scope.LocalVal);
             sub_scope.* = .{
                 .parent = params_scope,
                 .gen_zir = &gen_scope,
                 .name = param_name,
-                .inst = arg,
+                .inst = &arg.base,
             };
             params_scope = &sub_scope.base;
         }
@@ -896,42 +945,52 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
             zir.dumpZir(mod.gpa, "fn_body_callee", callee_decl.name, gen_scope.instructions.items) catch {};
         }
 
-        // Analyze the ZIR.
-        var inner_block: Scope.Block = .{
+        // Analyze the ZIR. The same ZIR gets analyzed into a runtime function
+        // or an inlined call depending on what union tag the `label` field is
+        // set to in the `Scope.Block`.
+        // This block instruction will be used to capture the return value from the
+        // inlined function.
+        const block_inst = try scope.arena().create(Inst.Block);
+        block_inst.* = .{
+            .base = .{
+                .tag = Inst.Block.base_tag,
+                .ty = ret_type,
+                .src = inst.base.src,
+            },
+            .body = undefined,
+        };
+        var child_block: Scope.Block = .{
             .parent = null,
             .func = module_fn,
-            .decl = callee_decl,
+            // Note that we pass the caller's Decl, not the callee. This causes
+            // compile errors to be attached (correctly) to the caller's Decl.
+            .decl = scope.decl().?,
             .instructions = .{},
-            .arena = &call_arena.allocator,
-            .is_comptime = true,
+            .arena = scope.arena(),
+            .label = Scope.Block.Label{
+                .inlining = .{
+                    .param_index = 0,
+                    .casted_args = casted_args,
+                    .merges = .{
+                        .results = .{},
+                        .block_inst = block_inst,
+                    },
+                },
+            },
+            .is_comptime = is_comptime_call,
         };
-        defer inner_block.instructions.deinit(mod.gpa);
+        const merges = &child_block.label.inlining.merges;
+
+        defer child_block.instructions.deinit(mod.gpa);
+        defer merges.results.deinit(mod.gpa);
 
-        // TODO make sure compile errors that happen from this analyzeBody are reported correctly
-        // and attach to the caller Decl not the callee.
-        try analyzeBody(mod, &inner_block.base, .{
+        // This will have return instructions analyzed as break instructions to
+        // the block_inst above.
+        try analyzeBody(mod, &child_block.base, .{
             .instructions = gen_scope.instructions.items,
         });
 
-        if (mod.comp.verbose_ir) {
-            inner_block.dump(mod.*);
-        }
-
-        assert(inner_block.instructions.items.len == 1);
-        const only_inst = inner_block.instructions.items[0];
-        switch (only_inst.tag) {
-            .ret => {
-                const ret_inst = only_inst.castTag(.ret).?;
-                const operand = ret_inst.operand;
-                const callee_arena = scope.arena();
-                return mod.constInst(scope, inst.base.src, .{
-                    .ty = try operand.ty.copy(callee_arena),
-                    .val = try operand.value().?.copy(callee_arena),
-                });
-            },
-            .retvoid => return mod.constVoid(scope, inst.base.src),
-            else => unreachable,
-        }
+        return analyzeBlockBody(mod, scope, &child_block, merges);
     }
 
     return mod.addCall(b, inst.base.src, ret_type, func, casted_args);
@@ -954,7 +1013,11 @@ fn analyzeInstFn(mod: *Module, scope: *Scope, fn_inst: *zir.Inst.Fn) InnerError!
     };
     const new_func = try scope.arena().create(Module.Fn);
     new_func.* = .{
-        .analysis = .{ .queued = fn_zir },
+        .bits = .{
+            .state = .queued,
+            .is_inline = fn_inst.kw_args.is_inline,
+        },
+        .data = .{ .zir = fn_zir },
         .owner_decl = scope.decl().?,
     };
     return mod.constInst(scope, fn_inst.base.src, .{
@@ -2020,21 +2083,41 @@ fn analyzeInstUnreachable(
 fn analyzeInstRet(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!*Inst {
     const operand = try resolveInst(mod, scope, inst.positionals.operand);
     const b = try mod.requireFunctionBlock(scope, inst.base.src);
-    return mod.addUnOp(b, inst.base.src, Type.initTag(.noreturn), .ret, operand);
+
+    switch (b.label) {
+        .inlining => |*inlining| {
+            // We are inlining a function call; rewrite the `ret` as a `break`.
+            try inlining.merges.results.append(mod.gpa, operand);
+            return mod.addBr(b, inst.base.src, inlining.merges.block_inst, operand);
+        },
+        .none, .breaking => {
+            return mod.addUnOp(b, inst.base.src, Type.initTag(.noreturn), .ret, operand);
+        },
+    }
 }
 
 fn analyzeInstRetVoid(mod: *Module, scope: *Scope, inst: *zir.Inst.NoOp) InnerError!*Inst {
     const b = try mod.requireFunctionBlock(scope, inst.base.src);
-    if (b.func) |func| {
-        // Need to emit a compile error if returning void is not allowed.
-        const void_inst = try mod.constVoid(scope, inst.base.src);
-        const fn_ty = func.owner_decl.typed_value.most_recent.typed_value.ty;
-        const casted_void = try mod.coerce(scope, fn_ty.fnReturnType(), void_inst);
-        if (casted_void.ty.zigTypeTag() != .Void) {
-            return mod.addUnOp(b, inst.base.src, Type.initTag(.noreturn), .ret, casted_void);
-        }
+    switch (b.label) {
+        .inlining => |*inlining| {
+            // We are inlining a function call; rewrite the `retvoid` as a `breakvoid`.
+            const void_inst = try mod.constVoid(scope, inst.base.src);
+            try inlining.merges.results.append(mod.gpa, void_inst);
+            return mod.addBr(b, inst.base.src, inlining.merges.block_inst, void_inst);
+        },
+        .none, .breaking => {
+            if (b.func) |func| {
+                // Need to emit a compile error if returning void is not allowed.
+                const void_inst = try mod.constVoid(scope, inst.base.src);
+                const fn_ty = func.owner_decl.typed_value.most_recent.typed_value.ty;
+                const casted_void = try mod.coerce(scope, fn_ty.fnReturnType(), void_inst);
+                if (casted_void.ty.zigTypeTag() != .Void) {
+                    return mod.addUnOp(b, inst.base.src, Type.initTag(.noreturn), .ret, casted_void);
+                }
+            }
+            return mod.addNoOp(b, inst.base.src, Type.initTag(.noreturn), .retvoid);
+        },
     }
-    return mod.addNoOp(b, inst.base.src, Type.initTag(.noreturn), .retvoid);
 }
 
 fn floatOpAllowed(tag: zir.Inst.Tag) bool {
@@ -2054,12 +2137,16 @@ fn analyzeBreak(
 ) InnerError!*Inst {
     var opt_block = scope.cast(Scope.Block);
     while (opt_block) |block| {
-        if (block.label) |*label| {
-            if (label.zir_block == zir_block) {
-                try label.results.append(mod.gpa, operand);
-                const b = try mod.requireRuntimeBlock(scope, src);
-                return mod.addBr(b, src, label.block_inst, operand);
-            }
+        switch (block.label) {
+            .none => {},
+            .breaking => |*label| {
+                if (label.zir_block == zir_block) {
+                    try label.merges.results.append(mod.gpa, operand);
+                    const b = try mod.requireFunctionBlock(scope, src);
+                    return mod.addBr(b, src, label.merges.block_inst, operand);
+                }
+            },
+            .inlining => unreachable, // Invalid `break` ZIR inside inline function call.
         }
         opt_block = block.parent;
     } else unreachable;
diff --git a/test/stage2/zir.zig b/test/stage2/zir.zig
index da4038e792..c29e636cd4 100644
--- a/test/stage2/zir.zig
+++ b/test/stage2/zir.zig
@@ -30,7 +30,7 @@ pub fn addCases(ctx: *TestContext) !void {
         \\@unnamed$7 = fntype([], @void, cc=C)
         \\@entry = fn(@unnamed$7, {
         \\  %0 = returnvoid() ; deaths=0b1000000000000000
-        \\})
+        \\}, is_inline=0)
         \\
     );
     ctx.transformZIR("elemptr, add, cmp, condbr, return, breakpoint", linux_x64,
@@ -78,7 +78,7 @@ pub fn addCases(ctx: *TestContext) !void {
         \\@unnamed$6 = fntype([], @void, cc=C)
         \\@entry = fn(@unnamed$6, {
         \\  %0 = returnvoid() ; deaths=0b1000000000000000
-        \\})
+        \\}, is_inline=0)
         \\@entry__anon_1 = str("2\x08\x01\n")
         \\@9 = declref("9__anon_0")
         \\@9__anon_0 = str("entry")
@@ -123,17 +123,17 @@ pub fn addCases(ctx: *TestContext) !void {
             \\@entry = fn(@unnamed$7, {
             \\  %0 = call(@a, [], modifier=auto) ; deaths=0b1000000000000001
             \\  %1 = returnvoid() ; deaths=0b1000000000000000
-            \\})
+            \\}, is_inline=0)
             \\@unnamed$9 = fntype([], @void, cc=C)
             \\@a = fn(@unnamed$9, {
             \\  %0 = call(@b, [], modifier=auto) ; deaths=0b1000000000000001
             \\  %1 = returnvoid() ; deaths=0b1000000000000000
-            \\})
+            \\}, is_inline=0)
             \\@unnamed$11 = fntype([], @void, cc=C)
             \\@b = fn(@unnamed$11, {
             \\  %0 = call(@a, [], modifier=auto) ; deaths=0b1000000000000001
             \\  %1 = returnvoid() ; deaths=0b1000000000000000
-            \\})
+            \\}, is_inline=0)
             \\
         );
         // Now we introduce a compile error
@@ -203,7 +203,7 @@ pub fn addCases(ctx: *TestContext) !void {
             \\@unnamed$7 = fntype([], @void, cc=C)
             \\@entry = fn(@unnamed$7, {
             \\  %0 = returnvoid() ; deaths=0b1000000000000000
-            \\})
+            \\}, is_inline=0)
             \\
         );
     }
-- 
cgit v1.2.3


From 006e7f68056af62ae7713d7ef228841d11874735 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 2 Jan 2021 13:40:23 -0700
Subject: stage2: re-use ZIR for comptime and inline calls

Instead of freeing ZIR after semantic analysis, we keep it around so
that it can be used for comptime calls, inline calls, and generic
function calls. ZIR memory is now managed by the Decl arena.

Debug dump() functions are conditionally compiled; only available in
Debug builds of the compiler.

Add a test for an inline function call.
---
 src/Compilation.zig  |   9 ++--
 src/Module.zig       | 136 +++++++++++++++------------------------------------
 src/codegen.zig      |  10 ++--
 src/codegen/c.zig    |   2 +-
 src/codegen/wasm.zig |   2 +-
 src/llvm_backend.zig |   2 +-
 src/zir.zig          |  17 ++++---
 src/zir_sema.zig     |  99 ++-----------------------------------
 test/stage2/test.zig |  25 ++++++++++
 9 files changed, 93 insertions(+), 209 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index a6f39a3154..9a06aee561 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1459,15 +1459,16 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 const module = self.bin_file.options.module.?;
                 if (decl.typed_value.most_recent.typed_value.val.castTag(.function)) |payload| {
                     const func = payload.data;
-                    switch (func.bits.state) {
+                    switch (func.state) {
                         .queued => module.analyzeFnBody(decl, func) catch |err| switch (err) {
                             error.AnalysisFail => {
-                                assert(func.bits.state != .in_progress);
+                                assert(func.state != .in_progress);
                                 continue;
                             },
                             error.OutOfMemory => return error.OutOfMemory,
                         },
                         .in_progress => unreachable,
+                        .inline_only => unreachable, // don't queue work for this
                         .sema_failure, .dependency_failure => continue,
                         .success => {},
                     }
@@ -1476,9 +1477,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa);
                     defer decl.typed_value.most_recent.arena.?.* = decl_arena.state;
                     log.debug("analyze liveness of {s}\n", .{decl.name});
-                    try liveness.analyze(module.gpa, &decl_arena.allocator, func.data.body);
+                    try liveness.analyze(module.gpa, &decl_arena.allocator, func.body);
 
-                    if (self.verbose_ir) {
+                    if (std.builtin.mode == .Debug and self.verbose_ir) {
                         func.dump(module.*);
                     }
                 }
diff --git a/src/Module.zig b/src/Module.zig
index db76ecd5db..be6ca0df63 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -286,75 +286,29 @@ pub const Decl = struct {
 /// Extern functions do not have this data structure; they are represented by
 /// the `Decl` only, with a `Value` tag of `extern_fn`.
 pub const Fn = struct {
-    bits: packed struct {
-        /// Get and set this field via `analysis` and `setAnalysis`.
-        state: Analysis.Tag,
-        /// We carry this state into `Fn` instead of leaving it in the AST so that
-        /// analysis of function calls can happen even on functions whose AST has
-        /// been unloaded from memory.
-        is_inline: bool,
-        unused_bits: u4 = 0,
-    },
-    /// Get and set this data via `analysis` and `setAnalysis`.
-    data: union {
-        none: void,
-        zir: *ZIR,
-        body: Body,
-    },
     owner_decl: *Decl,
-
-    pub const Analysis = union(Tag) {
-        queued: *ZIR,
+    /// Contains un-analyzed ZIR instructions generated from Zig source AST.
+    /// Even after we finish analysis, the ZIR is kept in memory, so that
+    /// comptime and inline function calls can happen.
+    zir: zir.Module.Body,
+    /// undefined unless analysis state is `success`.
+    body: Body,
+    state: Analysis,
+
+    pub const Analysis = enum {
+        queued,
+        /// This function intentionally only has ZIR generated because it is marked
+        /// inline, which means no runtime version of the function will be generated.
+        inline_only,
         in_progress,
+        /// There will be a corresponding ErrorMsg in Module.failed_decls
         sema_failure,
+        /// This Fn might be OK but it depends on another Decl which did not
+        /// successfully complete semantic analysis.
         dependency_failure,
-        success: Body,
-
-        pub const Tag = enum(u3) {
-            queued,
-            in_progress,
-            /// There will be a corresponding ErrorMsg in Module.failed_decls
-            sema_failure,
-            /// This Fn might be OK but it depends on another Decl which did not
-            /// successfully complete semantic analysis.
-            dependency_failure,
-            success,
-        };
+        success,
     };
 
-    /// Contains un-analyzed ZIR instructions generated from Zig source AST.
-    pub const ZIR = struct {
-        body: zir.Module.Body,
-        arena: std.heap.ArenaAllocator.State,
-    };
-
-    pub fn analysis(self: Fn) Analysis {
-        return switch (self.bits.state) {
-            .queued => .{ .queued = self.data.zir },
-            .success => .{ .success = self.data.body },
-            .in_progress => .in_progress,
-            .sema_failure => .sema_failure,
-            .dependency_failure => .dependency_failure,
-        };
-    }
-
-    pub fn setAnalysis(self: *Fn, anal: Analysis) void {
-        switch (anal) {
-            .queued => |zir_ptr| {
-                self.bits.state = .queued;
-                self.data = .{ .zir = zir_ptr };
-            },
-            .success => |body| {
-                self.bits.state = .success;
-                self.data = .{ .body = body };
-            },
-            .in_progress, .sema_failure, .dependency_failure => {
-                self.bits.state = anal;
-                self.data = .{ .none = {} };
-            },
-        }
-    }
-
     /// For debugging purposes.
     pub fn dump(self: *Fn, mod: Module) void {
         zir.dumpFn(mod, self);
@@ -1124,7 +1078,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .param_types = param_types,
             }, .{});
 
-            if (self.comp.verbose_ir) {
+            if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
                 zir.dumpZir(self.gpa, "fn_type", decl.name, fn_type_scope.instructions.items) catch {};
             }
 
@@ -1175,14 +1129,11 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             const new_func = try decl_arena.allocator.create(Fn);
             const fn_payload = try decl_arena.allocator.create(Value.Payload.Function);
 
-            const fn_zir = blk: {
-                // This scope's arena memory is discarded after the ZIR generation
-                // pass completes, and semantic analysis of it completes.
-                var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
-                errdefer gen_scope_arena.deinit();
+            const fn_zir: zir.Module.Body = blk: {
+                // We put the ZIR inside the Decl arena.
                 var gen_scope: Scope.GenZIR = .{
                     .decl = decl,
-                    .arena = &gen_scope_arena.allocator,
+                    .arena = &decl_arena.allocator,
                     .parent = decl.scope,
                 };
                 defer gen_scope.instructions.deinit(self.gpa);
@@ -1194,7 +1145,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                     const name_token = param.name_token.?;
                     const src = tree.token_locs[name_token].start;
                     const param_name = try self.identifierTokenString(&gen_scope.base, name_token);
-                    const arg = try gen_scope_arena.allocator.create(zir.Inst.Arg);
+                    const arg = try decl_arena.allocator.create(zir.Inst.Arg);
                     arg.* = .{
                         .base = .{
                             .tag = .arg,
@@ -1206,7 +1157,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                         .kw_args = .{},
                     };
                     gen_scope.instructions.items[i] = &arg.base;
-                    const sub_scope = try gen_scope_arena.allocator.create(Scope.LocalVal);
+                    const sub_scope = try decl_arena.allocator.create(Scope.LocalVal);
                     sub_scope.* = .{
                         .parent = params_scope,
                         .gen_zir = &gen_scope,
@@ -1227,18 +1178,13 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                     _ = try astgen.addZIRNoOp(self, &gen_scope.base, src, .returnvoid);
                 }
 
-                if (self.comp.verbose_ir) {
+                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
                     zir.dumpZir(self.gpa, "fn_body", decl.name, gen_scope.instructions.items) catch {};
                 }
 
-                const fn_zir = try gen_scope_arena.allocator.create(Fn.ZIR);
-                fn_zir.* = .{
-                    .body = .{
-                        .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items),
-                    },
-                    .arena = gen_scope_arena.state,
+                break :blk .{
+                    .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items),
                 };
-                break :blk fn_zir;
             };
 
             const is_inline = blk: {
@@ -1249,13 +1195,12 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 }
                 break :blk false;
             };
+            const anal_state = ([2]Fn.Analysis{ .queued, .inline_only })[@boolToInt(is_inline)];
 
             new_func.* = .{
-                .bits = .{
-                    .state = .queued,
-                    .is_inline = is_inline,
-                },
-                .data = .{ .zir = fn_zir },
+                .state = anal_state,
+                .zir = fn_zir,
+                .body = undefined,
                 .owner_decl = decl,
             };
             fn_payload.* = .{
@@ -1272,7 +1217,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 type_changed = !tvm.typed_value.ty.eql(fn_type);
                 if (tvm.typed_value.val.castTag(.function)) |payload| {
                     const prev_func = payload.data;
-                    prev_is_inline = prev_func.bits.is_inline;
+                    prev_is_inline = prev_func.state == .inline_only;
                 }
 
                 tvm.deinit(self.gpa);
@@ -1391,7 +1336,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
 
                 const src = tree.token_locs[init_node.firstToken()].start;
                 const init_inst = try astgen.expr(self, &gen_scope.base, init_result_loc, init_node);
-                if (self.comp.verbose_ir) {
+                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
                     zir.dumpZir(self.gpa, "var_init", decl.name, gen_scope.instructions.items) catch {};
                 }
 
@@ -1435,7 +1380,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                     .val = Value.initTag(.type_type),
                 });
                 const var_type = try astgen.expr(self, &type_scope.base, .{ .ty = type_type }, type_node);
-                if (self.comp.verbose_ir) {
+                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
                     zir.dumpZir(self.gpa, "var_type", decl.name, type_scope.instructions.items) catch {};
                 }
 
@@ -1511,7 +1456,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             defer gen_scope.instructions.deinit(self.gpa);
 
             _ = try astgen.comptimeExpr(self, &gen_scope.base, .none, comptime_decl.expr);
-            if (self.comp.verbose_ir) {
+            if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
                 zir.dumpZir(self.gpa, "comptime_block", decl.name, gen_scope.instructions.items) catch {};
             }
 
@@ -1902,15 +1847,14 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
     };
     defer inner_block.instructions.deinit(self.gpa);
 
-    const fn_zir = func.data.zir;
-    defer fn_zir.arena.promote(self.gpa).deinit();
-    func.setAnalysis(.in_progress);
+    func.state = .in_progress;
     log.debug("set {s} to in_progress\n", .{decl.name});
 
-    try zir_sema.analyzeBody(self, &inner_block.base, fn_zir.body);
+    try zir_sema.analyzeBody(self, &inner_block.base, func.zir);
 
     const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items);
-    func.setAnalysis(.{ .success = .{ .instructions = instructions } });
+    func.state = .success;
+    func.body = .{ .instructions = instructions };
     log.debug("set {s} to success\n", .{decl.name});
 }
 
@@ -2407,7 +2351,7 @@ pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) Inn
     self.ensureDeclAnalyzed(decl) catch |err| {
         if (scope.cast(Scope.Block)) |block| {
             if (block.func) |func| {
-                func.setAnalysis(.dependency_failure);
+                func.state = .dependency_failure;
             } else {
                 block.decl.analysis = .dependency_failure;
             }
@@ -3107,7 +3051,7 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
         .block => {
             const block = scope.cast(Scope.Block).?;
             if (block.func) |func| {
-                func.setAnalysis(.sema_failure);
+                func.state = .sema_failure;
             } else {
                 block.decl.analysis = .sema_failure;
                 block.decl.generation = self.generation;
diff --git a/src/codegen.zig b/src/codegen.zig
index 588c3dec4c..58be73a31c 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -532,7 +532,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         self.code.items.len += 4;
 
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.data.body);
+                        try self.genBody(self.mod_fn.body);
 
                         const stack_end = self.max_end_stack;
                         if (stack_end > math.maxInt(i32))
@@ -576,7 +576,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         });
                     } else {
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.data.body);
+                        try self.genBody(self.mod_fn.body);
                         try self.dbgSetEpilogueBegin();
                     }
                 },
@@ -593,7 +593,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                         try self.dbgSetPrologueEnd();
 
-                        try self.genBody(self.mod_fn.data.body);
+                        try self.genBody(self.mod_fn.body);
 
                         // Backpatch stack offset
                         const stack_end = self.max_end_stack;
@@ -638,13 +638,13 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         writeInt(u32, try self.code.addManyAsArray(4), Instruction.pop(.al, .{ .fp, .pc }).toU32());
                     } else {
                         try self.dbgSetPrologueEnd();
-                        try self.genBody(self.mod_fn.data.body);
+                        try self.genBody(self.mod_fn.body);
                         try self.dbgSetEpilogueBegin();
                     }
                 },
                 else => {
                     try self.dbgSetPrologueEnd();
-                    try self.genBody(self.mod_fn.data.body);
+                    try self.genBody(self.mod_fn.body);
                     try self.dbgSetEpilogueBegin();
                 },
             }
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 712d663af0..1a89e22d48 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -275,7 +275,7 @@ pub fn generate(file: *C, module: *Module, decl: *Decl) !void {
         try writer.writeAll(" {");
 
         const func: *Module.Fn = func_payload.data;
-        const instructions = func.data.body.instructions;
+        const instructions = func.body.instructions;
         if (instructions.len > 0) {
             try writer.writeAll("\n");
             for (instructions) |inst| {
diff --git a/src/codegen/wasm.zig b/src/codegen/wasm.zig
index 1eb4f5bc29..036243dcca 100644
--- a/src/codegen/wasm.zig
+++ b/src/codegen/wasm.zig
@@ -63,7 +63,7 @@ pub fn genCode(buf: *ArrayList(u8), decl: *Decl) !void {
     // TODO: check for and handle death of instructions
     const tv = decl.typed_value.most_recent.typed_value;
     const mod_fn = tv.val.castTag(.function).?.data;
-    for (mod_fn.data.body.instructions) |inst| try genInst(buf, decl, inst);
+    for (mod_fn.body.instructions) |inst| try genInst(buf, decl, inst);
 
     // Write 'end' opcode
     try writer.writeByte(0x0B);
diff --git a/src/llvm_backend.zig b/src/llvm_backend.zig
index 5814aa7e7e..97406797b6 100644
--- a/src/llvm_backend.zig
+++ b/src/llvm_backend.zig
@@ -294,7 +294,7 @@ pub const LLVMIRModule = struct {
                 const entry_block = llvm_func.appendBasicBlock("Entry");
                 self.builder.positionBuilderAtEnd(entry_block);
 
-                const instructions = func.data.body.instructions;
+                const instructions = func.body.instructions;
                 for (instructions) |inst| {
                     switch (inst.tag) {
                         .breakpoint => try self.genBreakpoint(inst.castTag(.breakpoint).?),
diff --git a/src/zir.zig b/src/zir.zig
index 64b74f24d9..56ddee919c 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1864,13 +1864,15 @@ pub fn dumpFn(old_module: IrModule, module_fn: *IrModule.Fn) void {
     defer ctx.const_table.deinit();
     defer ctx.arena.deinit();
 
-    switch (module_fn.analysis()) {
+    switch (module_fn.state) {
         .queued => std.debug.print("(queued)", .{}),
+        .inline_only => std.debug.print("(inline_only)", .{}),
         .in_progress => std.debug.print("(in_progress)", .{}),
         .sema_failure => std.debug.print("(sema_failure)", .{}),
         .dependency_failure => std.debug.print("(dependency_failure)", .{}),
-        .success => |body| {
-            ctx.dump(body, std.io.getStdErr().writer()) catch @panic("failed to dump TZIR");
+        .success => {
+            const writer = std.io.getStdErr().writer();
+            ctx.dump(module_fn.body, writer) catch @panic("failed to dump TZIR");
         },
     }
 }
@@ -2289,11 +2291,12 @@ const EmitZIR = struct {
         var instructions = std.ArrayList(*Inst).init(self.allocator);
         defer instructions.deinit();
 
-        switch (module_fn.analysis()) {
+        switch (module_fn.state) {
             .queued => unreachable,
             .in_progress => unreachable,
-            .success => |body| {
-                try self.emitBody(body, &inst_table, &instructions);
+            .inline_only => unreachable,
+            .success => {
+                try self.emitBody(module_fn.body, &inst_table, &instructions);
             },
             .sema_failure => {
                 const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?;
@@ -2372,7 +2375,7 @@ const EmitZIR = struct {
                 .body = .{ .instructions = arena_instrs },
             },
             .kw_args = .{
-                .is_inline = module_fn.bits.is_inline,
+                .is_inline = module_fn.state == .inline_only,
             },
         };
         return self.emitUnnamedDecl(&fn_inst.base);
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 9365996bb6..e8d995dd5e 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -25,8 +25,6 @@ const trace = @import("tracy.zig").trace;
 const Scope = Module.Scope;
 const InnerError = Module.InnerError;
 const Decl = Module.Decl;
-const astgen = @import("astgen.zig");
-const ast = std.zig.ast;
 
 pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!*Inst {
     switch (old_inst.tag) {
@@ -861,7 +859,7 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
                 .function => func_val.castTag(.function).?.data,
                 else => break :blk false,
             };
-            break :blk module_fn.bits.is_inline;
+            break :blk module_fn.state == .inline_only;
         }
         break :blk false;
     };
@@ -874,76 +872,6 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
             }),
             else => unreachable,
         };
-        const callee_decl = module_fn.owner_decl;
-        // TODO: De-duplicate this with the code in Module.zig that generates
-        // ZIR for the same function and re-use the same ZIR for runtime function
-        // generation and for inline/comptime calls.
-        const callee_file_scope = callee_decl.getFileScope();
-        const tree = mod.getAstTree(callee_file_scope) catch |err| switch (err) {
-            error.OutOfMemory => return error.OutOfMemory,
-            error.AnalysisFail => return error.AnalysisFail,
-            // TODO: make sure this gets retried and not cached
-            else => return mod.fail(scope, inst.base.src, "failed to load {s}: {s}", .{
-                callee_file_scope.sub_file_path, @errorName(err),
-            }),
-        };
-        const ast_node = tree.root_node.decls()[callee_decl.src_index];
-        const fn_proto = ast_node.castTag(.FnProto).?;
-
-        var call_arena = std.heap.ArenaAllocator.init(mod.gpa);
-        defer call_arena.deinit();
-
-        var gen_scope: Scope.GenZIR = .{
-            .decl = callee_decl,
-            .arena = &call_arena.allocator,
-            .parent = callee_decl.scope,
-        };
-        defer gen_scope.instructions.deinit(mod.gpa);
-
-        // We need an instruction for each parameter, and they must be first in the body.
-        try gen_scope.instructions.resize(mod.gpa, fn_proto.params_len);
-        var params_scope = &gen_scope.base;
-        for (fn_proto.params()) |param, i| {
-            const name_token = param.name_token.?;
-            const src = tree.token_locs[name_token].start;
-            const param_name = try mod.identifierTokenString(scope, name_token);
-            const arg = try call_arena.allocator.create(zir.Inst.Arg);
-            arg.* = .{
-                .base = .{
-                    .tag = .arg,
-                    .src = src,
-                },
-                .positionals = .{
-                    .name = param_name,
-                },
-                .kw_args = .{},
-            };
-            gen_scope.instructions.items[i] = &arg.base;
-            const sub_scope = try call_arena.allocator.create(Scope.LocalVal);
-            sub_scope.* = .{
-                .parent = params_scope,
-                .gen_zir = &gen_scope,
-                .name = param_name,
-                .inst = &arg.base,
-            };
-            params_scope = &sub_scope.base;
-        }
-
-        const body_node = fn_proto.getBodyNode().?; // We handle extern functions above.
-        const body_block = body_node.cast(ast.Node.Block).?;
-
-        try astgen.blockExpr(mod, params_scope, body_block);
-
-        if (gen_scope.instructions.items.len == 0 or
-            !gen_scope.instructions.items[gen_scope.instructions.items.len - 1].tag.isNoReturn())
-        {
-            const src = tree.token_locs[body_block.rbrace].start;
-            _ = try astgen.addZIRNoOp(mod, &gen_scope.base, src, .returnvoid);
-        }
-
-        if (mod.comp.verbose_ir) {
-            zir.dumpZir(mod.gpa, "fn_body_callee", callee_decl.name, gen_scope.instructions.items) catch {};
-        }
 
         // Analyze the ZIR. The same ZIR gets analyzed into a runtime function
         // or an inlined call depending on what union tag the `label` field is
@@ -986,9 +914,7 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
 
         // This will have return instructions analyzed as break instructions to
         // the block_inst above.
-        try analyzeBody(mod, &child_block.base, .{
-            .instructions = gen_scope.instructions.items,
-        });
+        try analyzeBody(mod, &child_block.base, module_fn.zir);
 
         return analyzeBlockBody(mod, scope, &child_block, merges);
     }
@@ -998,26 +924,11 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
 
 fn analyzeInstFn(mod: *Module, scope: *Scope, fn_inst: *zir.Inst.Fn) InnerError!*Inst {
     const fn_type = try resolveType(mod, scope, fn_inst.positionals.fn_type);
-    const fn_zir = blk: {
-        var fn_arena = std.heap.ArenaAllocator.init(mod.gpa);
-        errdefer fn_arena.deinit();
-
-        const fn_zir = try scope.arena().create(Module.Fn.ZIR);
-        fn_zir.* = .{
-            .body = .{
-                .instructions = fn_inst.positionals.body.instructions,
-            },
-            .arena = fn_arena.state,
-        };
-        break :blk fn_zir;
-    };
     const new_func = try scope.arena().create(Module.Fn);
     new_func.* = .{
-        .bits = .{
-            .state = .queued,
-            .is_inline = fn_inst.kw_args.is_inline,
-        },
-        .data = .{ .zir = fn_zir },
+        .state = if (fn_inst.kw_args.is_inline) .inline_only else .queued,
+        .zir = fn_inst.positionals.body,
+        .body = undefined,
         .owner_decl = scope.decl().?,
     };
     return mod.constInst(scope, fn_inst.base.src, .{
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index 18c54b367e..79f5c3a73e 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -342,6 +342,7 @@ pub fn addCases(ctx: *TestContext) !void {
         ,
             "",
         );
+        // comptime function call
         case.addCompareOutput(
             \\export fn _start() noreturn {
             \\    exit();
@@ -365,6 +366,30 @@ pub fn addCases(ctx: *TestContext) !void {
         ,
             "",
         );
+        // Inline function call
+        case.addCompareOutput(
+            \\export fn _start() noreturn {
+            \\    var x: usize = 3;
+            \\    const y = add(1, 2, x);
+            \\    exit(y - 6);
+            \\}
+            \\
+            \\inline fn add(a: usize, b: usize, c: usize) usize {
+            \\    return a + b + c;
+            \\}
+            \\
+            \\fn exit(code: usize) noreturn {
+            \\    asm volatile ("syscall"
+            \\        :
+            \\        : [number] "{rax}" (231),
+            \\          [arg1] "{rdi}" (code)
+            \\        : "rcx", "r11", "memory"
+            \\    );
+            \\    unreachable;
+            \\}
+        ,
+            "",
+        );
     }
 
     {
-- 
cgit v1.2.3


From 7e5aacab69b74de23ce57d2c7f3af0061c3343cf Mon Sep 17 00:00:00 2001
From: Timon Kruiper <timonkruiper@gmail.com>
Date: Sun, 3 Jan 2021 16:44:53 +0100
Subject: stage2: add some missing deallocations in Compilation.zig

---
 src/Compilation.zig | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9a06aee561..2ba76d0f5e 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1173,6 +1173,7 @@ pub fn destroy(self: *Compilation) void {
 
     const gpa = self.gpa;
     self.work_queue.deinit();
+    self.c_object_work_queue.deinit();
 
     {
         var it = self.crt_files.iterator();
@@ -1202,6 +1203,10 @@ pub fn destroy(self: *Compilation) void {
         crt_file.deinit(gpa);
     }
 
+    if (self.glibc_so_files) |*glibc_file| {
+        glibc_file.deinit(gpa);
+    }
+
     for (self.c_object_table.items()) |entry| {
         entry.key.destroy(gpa);
     }
-- 
cgit v1.2.3


From 2fe8a482159762724df93225bd70abbd0c2c5930 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 4 Jan 2021 14:59:18 -0700
Subject: ci: omit stage2 backend from stage1 on Windows

to avoid out-of-memory on the CI runs.
---
 CMakeLists.txt                   | 7 +++++++
 build.zig                        | 3 ++-
 ci/azure/windows_msvc_script.bat | 5 +++--
 src/Compilation.zig              | 6 ++++++
 src/config.zig.in                | 1 +
 5 files changed, 19 insertions(+), 3 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 291a2f7839..2ed25b93d2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,6 +89,7 @@ set(ZIG_TARGET_MCPU "baseline" CACHE STRING "-mcpu parameter to output binaries
 set(ZIG_EXECUTABLE "" CACHE STRING "(when cross compiling) path to already-built zig binary")
 set(ZIG_PREFER_LLVM_CONFIG off CACHE BOOL "(when cross compiling) use llvm-config to find target llvm dependencies if needed")
 set(ZIG_SINGLE_THREADED off CACHE BOOL "limit the zig compiler to use only 1 thread")
+set(ZIG_OMIT_STAGE2 off CACHE BOOL "omit the stage2 backend from stage1")
 
 find_package(llvm)
 find_package(clang)
@@ -587,6 +588,12 @@ if(MSVC)
     endif()
 endif()
 
+if(ZIG_OMIT_STAGE2)
+  set(ZIG_OMIT_STAGE2_BOOL "true")
+else()
+  set(ZIG_OMIT_STAGE2_BOOL "false")
+endif()
+
 configure_file (
     "${CMAKE_SOURCE_DIR}/src/stage1/config.h.in"
     "${ZIG_CONFIG_H_OUT}"
diff --git a/build.zig b/build.zig
index ce412beef0..77d1b573fd 100644
--- a/build.zig
+++ b/build.zig
@@ -125,7 +125,6 @@ pub fn build(b: *Builder) !void {
             try addCmakeCfgOptionsToExe(b, cfg, tracy, test_stage2);
         } else {
             // Here we are -Denable-llvm but no cmake integration.
-
             try addStaticLlvmOptionsToExe(exe);
             try addStaticLlvmOptionsToExe(test_stage2);
         }
@@ -194,6 +193,7 @@ pub fn build(b: *Builder) !void {
     exe.addBuildOption([]const []const u8, "log_scopes", log_scopes);
     exe.addBuildOption(bool, "enable_tracy", tracy != null);
     exe.addBuildOption(bool, "is_stage1", is_stage1);
+    exe.addBuildOption(bool, "omit_stage2", false);
     if (tracy) |tracy_path| {
         const client_cpp = fs.path.join(
             b.allocator,
@@ -216,6 +216,7 @@ pub fn build(b: *Builder) !void {
 
     test_stage2.addBuildOption(bool, "skip_non_native", skip_non_native);
     test_stage2.addBuildOption(bool, "is_stage1", is_stage1);
+    test_stage2.addBuildOption(bool, "omit_stage2", false);
     test_stage2.addBuildOption(bool, "have_llvm", enable_llvm);
     test_stage2.addBuildOption(bool, "enable_qemu", is_qemu_enabled);
     test_stage2.addBuildOption(bool, "enable_wine", is_wine_enabled);
diff --git a/ci/azure/windows_msvc_script.bat b/ci/azure/windows_msvc_script.bat
index cd5e3d5bca..9d28eccd0b 100644
--- a/ci/azure/windows_msvc_script.bat
+++ b/ci/azure/windows_msvc_script.bat
@@ -23,11 +23,12 @@ git.exe fetch --tags
 
 mkdir %ZIGBUILDDIR%
 cd %ZIGBUILDDIR%
-cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b
+cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release -DZIG_OMIT_STAGE2=ON || exit /b
 msbuild /maxcpucount /p:Configuration=Release INSTALL.vcxproj || exit /b
 
 "%ZIGINSTALLDIR%\bin\zig.exe" build test-behavior -Dskip-non-native || exit /b
-"%ZIGINSTALLDIR%\bin\zig.exe" build test-stage2 -Dskip-non-native || exit /b
+REM Disabled to prevent OOM
+REM "%ZIGINSTALLDIR%\bin\zig.exe" build test-stage2 -Dskip-non-native || exit /b
 "%ZIGINSTALLDIR%\bin\zig.exe" build test-fmt -Dskip-non-native || exit /b
 "%ZIGINSTALLDIR%\bin\zig.exe" build test-std -Dskip-non-native || exit /b
 "%ZIGINSTALLDIR%\bin\zig.exe" build test-compiler-rt -Dskip-non-native || exit /b
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 2ba76d0f5e..9912520437 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1461,6 +1461,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             => continue,
 
             .complete, .codegen_failure_retryable => {
+                if (build_options.omit_stage2)
+                    @panic("sadly stage2 is omitted from this build to save memory on the CI server");
                 const module = self.bin_file.options.module.?;
                 if (decl.typed_value.most_recent.typed_value.val.castTag(.function)) |payload| {
                     const func = payload.data;
@@ -1532,6 +1534,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             },
         },
         .analyze_decl => |decl| {
+            if (build_options.omit_stage2)
+                @panic("sadly stage2 is omitted from this build to save memory on the CI server");
             const module = self.bin_file.options.module.?;
             module.ensureDeclAnalyzed(decl) catch |err| switch (err) {
                 error.OutOfMemory => return error.OutOfMemory,
@@ -1539,6 +1543,8 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             };
         },
         .update_line_number => |decl| {
+            if (build_options.omit_stage2)
+                @panic("sadly stage2 is omitted from this build to save memory on the CI server");
             const module = self.bin_file.options.module.?;
             self.bin_file.updateDeclLineNumber(module, decl) catch |err| {
                 try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
diff --git a/src/config.zig.in b/src/config.zig.in
index 0dbd3f3c91..b672581ea4 100644
--- a/src/config.zig.in
+++ b/src/config.zig.in
@@ -5,3 +5,4 @@ pub const log_scopes: []const []const u8 = &[_][]const u8{};
 pub const enable_tracy = false;
 pub const is_stage1 = true;
 pub const skip_non_native = false;
+pub const omit_stage2: bool = @ZIG_OMIT_STAGE2_BOOL@;
-- 
cgit v1.2.3


From 89286376c627c708e90697cb249a54feb7c827d6 Mon Sep 17 00:00:00 2001
From: Alex Cameron <ascottcameron@gmail.com>
Date: Tue, 29 Dec 2020 16:22:22 +1100
Subject: std: Rename ArrayList shrink => shrinkAndFree

---
 lib/std/array_hash_map.zig |  2 +-
 lib/std/array_list.zig     |  8 ++++----
 lib/std/fs.zig             |  2 +-
 lib/std/io/reader.zig      |  6 +++---
 lib/std/json.zig           |  2 +-
 lib/std/math/big/int.zig   |  2 +-
 lib/std/net.zig            |  4 ++--
 src/Compilation.zig        |  2 +-
 src/libc_installation.zig  |  6 +++---
 src/translate_c.zig        | 10 +++++-----
 10 files changed, 22 insertions(+), 22 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/array_hash_map.zig b/lib/std/array_hash_map.zig
index 7c591bed1b..ddc15666bb 100644
--- a/lib/std/array_hash_map.zig
+++ b/lib/std/array_hash_map.zig
@@ -323,7 +323,7 @@ pub fn ArrayHashMapUnmanaged(
         }
 
         pub fn clearAndFree(self: *Self, allocator: *Allocator) void {
-            self.entries.shrink(allocator, 0);
+            self.entries.shrinkAndFree(allocator, 0);
             if (self.index_header) |header| {
                 header.free(allocator);
                 self.index_header = null;
diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig
index 3114d1b744..53580585ad 100644
--- a/lib/std/array_list.zig
+++ b/lib/std/array_list.zig
@@ -269,7 +269,7 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
 
         /// Reduce allocated capacity to `new_len`.
         /// May invalidate element pointers.
-        pub fn shrink(self: *Self, new_len: usize) void {
+        pub fn shrinkAndFree(self: *Self, new_len: usize) void {
             assert(new_len <= self.items.len);
 
             self.items = self.allocator.realloc(self.allocatedSlice(), new_len) catch |e| switch (e) {
@@ -585,7 +585,7 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
         }
 
         /// Reduce allocated capacity to `new_len`.
-        pub fn shrink(self: *Self, allocator: *Allocator, new_len: usize) void {
+        pub fn shrinkAndFree(self: *Self, allocator: *Allocator, new_len: usize) void {
             assert(new_len <= self.items.len);
 
             self.items = allocator.realloc(self.allocatedSlice(), new_len) catch |e| switch (e) {
@@ -1153,7 +1153,7 @@ test "std.ArrayList/ArrayListUnmanaged.shrink still sets length on error.OutOfMe
         try list.append(2);
         try list.append(3);
 
-        list.shrink(1);
+        list.shrinkAndFree(1);
         testing.expect(list.items.len == 1);
     }
     {
@@ -1163,7 +1163,7 @@ test "std.ArrayList/ArrayListUnmanaged.shrink still sets length on error.OutOfMe
         try list.append(a, 2);
         try list.append(a, 3);
 
-        list.shrink(a, 1);
+        list.shrinkAndFree(a, 1);
         testing.expect(list.items.len == 1);
     }
 }
diff --git a/lib/std/fs.zig b/lib/std/fs.zig
index 5d11d9f394..89984cda07 100644
--- a/lib/std/fs.zig
+++ b/lib/std/fs.zig
@@ -2186,7 +2186,7 @@ pub const Walker = struct {
             var top = &self.stack.items[self.stack.items.len - 1];
             const dirname_len = top.dirname_len;
             if (try top.dir_it.next()) |base| {
-                self.name_buffer.shrink(dirname_len);
+                self.name_buffer.shrinkAndFree(dirname_len);
                 try self.name_buffer.append(path.sep);
                 try self.name_buffer.appendSlice(base.name);
                 if (base.kind == .Directory) {
diff --git a/lib/std/io/reader.zig b/lib/std/io/reader.zig
index 90c1cdac98..705eb9e816 100644
--- a/lib/std/io/reader.zig
+++ b/lib/std/io/reader.zig
@@ -76,12 +76,12 @@ pub fn Reader(
                 start_index += bytes_read;
 
                 if (start_index - original_len > max_append_size) {
-                    array_list.shrink(original_len + max_append_size);
+                    array_list.shrinkAndFree(original_len + max_append_size);
                     return error.StreamTooLong;
                 }
 
                 if (bytes_read != dest_slice.len) {
-                    array_list.shrink(start_index);
+                    array_list.shrinkAndFree(start_index);
                     return;
                 }
 
@@ -111,7 +111,7 @@ pub fn Reader(
             delimiter: u8,
             max_size: usize,
         ) !void {
-            array_list.shrink(0);
+            array_list.shrinkAndFree(0);
             while (true) {
                 var byte: u8 = try self.readByte();
 
diff --git a/lib/std/json.zig b/lib/std/json.zig
index f5be3a2094..87808a7350 100644
--- a/lib/std/json.zig
+++ b/lib/std/json.zig
@@ -1897,7 +1897,7 @@ pub const Parser = struct {
 
     pub fn reset(p: *Parser) void {
         p.state = .Simple;
-        p.stack.shrink(0);
+        p.stack.shrinkAndFree(0);
     }
 
     pub fn parse(p: *Parser, input: []const u8) !ValueTree {
diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig
index a151163bdc..504083dcce 100644
--- a/lib/std/math/big/int.zig
+++ b/lib/std/math/big/int.zig
@@ -607,7 +607,7 @@ pub const Mutable = struct {
     /// it will have the same length as it had when the function was called.
     pub fn gcd(rma: *Mutable, x: Const, y: Const, limbs_buffer: *std.ArrayList(Limb)) !void {
         const prev_len = limbs_buffer.items.len;
-        defer limbs_buffer.shrink(prev_len);
+        defer limbs_buffer.shrinkAndFree(prev_len);
         const x_copy = if (rma.limbs.ptr == x.limbs.ptr) blk: {
             const start = limbs_buffer.items.len;
             try limbs_buffer.appendSlice(x.limbs);
diff --git a/lib/std/net.zig b/lib/std/net.zig
index b8f48b2020..da35bd88b0 100644
--- a/lib/std/net.zig
+++ b/lib/std/net.zig
@@ -1200,13 +1200,13 @@ fn linuxLookupNameFromDnsSearch(
 
     var tok_it = mem.tokenize(search, " \t");
     while (tok_it.next()) |tok| {
-        canon.shrink(canon_name.len + 1);
+        canon.shrinkAndFree(canon_name.len + 1);
         try canon.appendSlice(tok);
         try linuxLookupNameFromDns(addrs, canon, canon.items, family, rc, port);
         if (addrs.items.len != 0) return;
     }
 
-    canon.shrink(canon_name.len);
+    canon.shrinkAndFree(canon_name.len);
     return linuxLookupNameFromDns(addrs, canon, name, family, rc, port);
 }
 
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9912520437..d42508c995 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -735,7 +735,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             }
             assert(mem.endsWith(u8, buf.items, ","));
             buf.items[buf.items.len - 1] = 0;
-            buf.shrink(buf.items.len);
+            buf.shrinkAndFree(buf.items.len);
             break :blk buf.items[0 .. buf.items.len - 1 :0].ptr;
         } else null;
 
diff --git a/src/libc_installation.zig b/src/libc_installation.zig
index cc96146e0b..bc317869e8 100644
--- a/src/libc_installation.zig
+++ b/src/libc_installation.zig
@@ -337,7 +337,7 @@ pub const LibCInstallation = struct {
         defer result_buf.deinit();
 
         for (searches) |search| {
-            result_buf.shrink(0);
+            result_buf.shrinkAndFree(0);
             try result_buf.outStream().print("{s}\\Include\\{s}\\ucrt", .{ search.path, search.version });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
@@ -383,7 +383,7 @@ pub const LibCInstallation = struct {
         };
 
         for (searches) |search| {
-            result_buf.shrink(0);
+            result_buf.shrinkAndFree(0);
             try result_buf.outStream().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ search.path, search.version, arch_sub_dir });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
@@ -437,7 +437,7 @@ pub const LibCInstallation = struct {
         };
 
         for (searches) |search| {
-            result_buf.shrink(0);
+            result_buf.shrinkAndFree(0);
             const stream = result_buf.outStream();
             try stream.print("{s}\\Lib\\{s}\\um\\{s}", .{ search.path, search.version, arch_sub_dir });
 
diff --git a/src/translate_c.zig b/src/translate_c.zig
index 7aebcf069b..9369c6d4b8 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -2846,7 +2846,7 @@ fn transCase(
 
     // take all pending statements
     try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items);
-    block_scope.statements.shrink(0);
+    block_scope.statements.shrinkAndFree(0);
 
     const pending_node = try switch_scope.pending_block.complete(rp.c);
     switch_scope.pending_block.deinit();
@@ -2884,7 +2884,7 @@ fn transDefault(
 
     // take all pending statements
     try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items);
-    block_scope.statements.shrink(0);
+    block_scope.statements.shrinkAndFree(0);
 
     const pending_node = try switch_scope.pending_block.complete(rp.c);
     switch_scope.pending_block.deinit();
@@ -4773,9 +4773,9 @@ const RestorePoint = struct {
     src_buf_index: usize,
 
     fn activate(self: RestorePoint) void {
-        self.c.token_ids.shrink(self.c.gpa, self.token_index);
-        self.c.token_locs.shrink(self.c.gpa, self.token_index);
-        self.c.source_buffer.shrink(self.src_buf_index);
+        self.c.token_ids.shrinkAndFree(self.c.gpa, self.token_index);
+        self.c.token_locs.shrinkAndFree(self.c.gpa, self.token_index);
+        self.c.source_buffer.shrinkAndFree(self.src_buf_index);
     }
 };
 
-- 
cgit v1.2.3


From 7b8cede61fc20c137aca4e02425536bfc9a5a400 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 5 Jan 2021 11:08:34 -0700
Subject: stage2: rework the C backend

 * std.ArrayList gains `moveToUnmanaged` and dead code
   `ArrayListUnmanaged.appendWrite` is deleted.
 * emit_h state is attached to Module rather than Compilation.
 * remove the implementation of emit-h because it did not properly
   integrate with incremental compilation. I will re-implement it
   in a follow-up commit.
 * Compilation: use the .codegen_failure tag rather than
   .dependency_failure tag for when `bin_file.updateDecl` fails.

C backend:
 * Use a CValue tagged union instead of strings for C values.
 * Cleanly separate state into Object and DeclGen:
   - Object is present only when generating a .c file
   - DeclGen is present for both generating a .c and .h
 * Move some functions into their respective Object/DeclGen namespace.
 * Forward decls are managed by the incremental compilation frontend; C
   backend no longer renders function signatures based on callsites.
   For simplicity, all functions always get forward decls.
 * Constants are managed by the incremental compilation frontend. C
   backend no longer has a "constants" section.
 * Participate in incremental compilation. Each Decl gets an ArrayList
   for its generated C code and it is updated when the Decl is updated.
   During flush(), all these are joined together in the output file.
 * The new CValue tagged union is used to clean up using of assigning to
   locals without an additional pointer local.
 * Fix bug with bitcast of non-pointers making the memcpy destination
   immutable.
---
 lib/std/array_list.zig |  22 +-
 src/Compilation.zig    |  58 +--
 src/Module.zig         |   6 +-
 src/codegen/c.zig      | 950 +++++++++++++++++++++++++------------------------
 src/link.zig           |  19 +-
 src/link/C.zig         | 195 ++++++----
 src/link/C/zig.h       |  45 +++
 src/link/cbe.h         |  44 ---
 src/test.zig           |  17 +-
 test/stage2/cbe.zig    |   2 -
 10 files changed, 703 insertions(+), 655 deletions(-)
 create mode 100644 src/link/C/zig.h
 delete mode 100644 src/link/cbe.h

(limited to 'src/Compilation.zig')

diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig
index 3114d1b744..51f5b8dc09 100644
--- a/lib/std/array_list.zig
+++ b/lib/std/array_list.zig
@@ -100,10 +100,20 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
 
         /// Initializes an ArrayListUnmanaged with the `items` and `capacity` fields
         /// of this ArrayList. This ArrayList retains ownership of underlying memory.
+        /// Deprecated: use `moveToUnmanaged` which has different semantics.
         pub fn toUnmanaged(self: Self) ArrayListAlignedUnmanaged(T, alignment) {
             return .{ .items = self.items, .capacity = self.capacity };
         }
 
+        /// Initializes an ArrayListUnmanaged with the `items` and `capacity` fields
+        /// of this ArrayList. Empties this ArrayList.
+        pub fn moveToUnmanaged(self: *Self) ArrayListAlignedUnmanaged(T, alignment) {
+            const allocator = self.allocator;
+            const result = .{ .items = self.items, .capacity = self.capacity };
+            self.* = init(allocator);
+            return result;
+        }
+
         /// The caller owns the returned memory. Empties this ArrayList.
         pub fn toOwnedSlice(self: *Self) Slice {
             const allocator = self.allocator;
@@ -551,14 +561,6 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
             mem.copy(T, self.items[oldlen..], items);
         }
 
-        /// Same as `append` except it returns the number of bytes written, which is always the same
-        /// as `m.len`. The purpose of this function existing is to match `std.io.OutStream` API.
-        /// This function may be called only when `T` is `u8`.
-        fn appendWrite(self: *Self, allocator: *Allocator, m: []const u8) !usize {
-            try self.appendSlice(allocator, m);
-            return m.len;
-        }
-
         /// Append a value to the list `n` times.
         /// Allocates more memory as necessary.
         pub fn appendNTimes(self: *Self, allocator: *Allocator, value: T, n: usize) !void {
@@ -1129,13 +1131,13 @@ test "std.ArrayList/ArrayListUnmanaged: ArrayList(T) of struct T" {
     }
 }
 
-test "std.ArrayList(u8) implements outStream" {
+test "std.ArrayList(u8) implements writer" {
     var buffer = ArrayList(u8).init(std.testing.allocator);
     defer buffer.deinit();
 
     const x: i32 = 42;
     const y: i32 = 1234;
-    try buffer.outStream().print("x: {}\ny: {}\n", .{ x, y });
+    try buffer.writer().print("x: {}\ny: {}\n", .{ x, y });
 
     testing.expectEqualSlices(u8, "x: 42\ny: 1234\n", buffer.items);
 }
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9912520437..c654833270 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -138,8 +138,6 @@ emit_llvm_ir: ?EmitLoc,
 emit_analysis: ?EmitLoc,
 emit_docs: ?EmitLoc,
 
-c_header: ?c_link.Header,
-
 work_queue_wait_group: WaitGroup,
 
 pub const InnerError = Module.InnerError;
@@ -866,9 +864,13 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                 .root_pkg = root_pkg,
                 .root_scope = root_scope,
                 .zig_cache_artifact_directory = zig_cache_artifact_directory,
+                .emit_h = options.emit_h,
             };
             break :blk module;
-        } else null;
+        } else blk: {
+            if (options.emit_h != null) return error.NoZigModuleForCHeader;
+            break :blk null;
+        };
         errdefer if (module) |zm| zm.deinit();
 
         const error_return_tracing = !strip and switch (options.optimize_mode) {
@@ -996,7 +998,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .local_cache_directory = options.local_cache_directory,
             .global_cache_directory = options.global_cache_directory,
             .bin_file = bin_file,
-            .c_header = if (!use_llvm and options.emit_h != null) c_link.Header.init(gpa, options.emit_h) else null,
             .emit_asm = options.emit_asm,
             .emit_llvm_ir = options.emit_llvm_ir,
             .emit_analysis = options.emit_analysis,
@@ -1218,10 +1219,6 @@ pub fn destroy(self: *Compilation) void {
     }
     self.failed_c_objects.deinit(gpa);
 
-    if (self.c_header) |*header| {
-        header.deinit();
-    }
-
     self.cache_parent.manifest_dir.close();
     if (self.owned_link_dir) |*dir| dir.close();
 
@@ -1325,20 +1322,6 @@ pub fn update(self: *Compilation) !void {
             module.root_scope.unload(self.gpa);
         }
     }
-
-    // If we've chosen to emit a C header, flush the header to the disk.
-    if (self.c_header) |header| {
-        const header_path = header.emit_loc.?;
-        // If a directory has been provided, write the header there. Otherwise, just write it to the
-        // cache directory.
-        const header_dir = if (header_path.directory) |dir|
-            dir.handle
-        else
-            self.local_cache_directory.handle;
-        const header_file = try header_dir.createFile(header_path.basename, .{});
-        defer header_file.close();
-        try header.flush(header_file.writer());
-    }
 }
 
 /// Having the file open for writing is problematic as far as executing the
@@ -1497,7 +1480,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     switch (err) {
                         error.OutOfMemory => return error.OutOfMemory,
                         error.AnalysisFail => {
-                            decl.analysis = .dependency_failure;
+                            decl.analysis = .codegen_failure;
                         },
                         else => {
                             try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
@@ -1512,25 +1495,6 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     }
                     return;
                 };
-
-                if (self.c_header) |*header| {
-                    c_codegen.generateHeader(self, module, header, decl) catch |err| switch (err) {
-                        error.OutOfMemory => return error.OutOfMemory,
-                        error.AnalysisFail => {
-                            decl.analysis = .dependency_failure;
-                        },
-                        else => {
-                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
-                                module.gpa,
-                                decl.src(),
-                                "unable to generate C header: {s}",
-                                .{@errorName(err)},
-                            ));
-                            decl.analysis = .codegen_failure_retryable;
-                        },
-                    };
-                }
             },
         },
         .analyze_decl => |decl| {
@@ -2998,9 +2962,9 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
     man.hash.add(comp.bin_file.options.function_sections);
     man.hash.add(comp.bin_file.options.is_test);
     man.hash.add(comp.bin_file.options.emit != null);
-    man.hash.add(comp.c_header != null);
-    if (comp.c_header) |header| {
-        man.hash.addEmitLoc(header.emit_loc.?);
+    man.hash.add(mod.emit_h != null);
+    if (mod.emit_h) |emit_h| {
+        man.hash.addEmitLoc(emit_h);
     }
     man.hash.addOptionalEmitLoc(comp.emit_asm);
     man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
@@ -3105,10 +3069,10 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         });
         break :blk try directory.join(arena, &[_][]const u8{bin_basename});
     } else "";
-    if (comp.c_header != null) {
+    if (comp.emit_h != null) {
         log.warn("-femit-h is not available in the stage1 backend; no .h file will be produced", .{});
     }
-    const emit_h_path = try stage1LocPath(arena, if (comp.c_header) |header| header.emit_loc else null, directory);
+    const emit_h_path = try stage1LocPath(arena, mod.emit_h, directory);
     const emit_asm_path = try stage1LocPath(arena, comp.emit_asm, directory);
     const emit_llvm_ir_path = try stage1LocPath(arena, comp.emit_llvm_ir, directory);
     const emit_analysis_path = try stage1LocPath(arena, comp.emit_analysis, directory);
diff --git a/src/Module.zig b/src/Module.zig
index 6a4575394a..f1cec82680 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -94,6 +94,8 @@ stage1_flags: packed struct {
     reserved: u2 = 0,
 } = .{},
 
+emit_h: ?Compilation.EmitLoc,
+
 pub const Export = struct {
     options: std.builtin.ExportOptions,
     /// Byte offset into the file that contains the export directive.
@@ -1943,14 +1945,14 @@ fn allocateNewDecl(
             .coff => .{ .coff = link.File.Coff.TextBlock.empty },
             .elf => .{ .elf = link.File.Elf.TextBlock.empty },
             .macho => .{ .macho = link.File.MachO.TextBlock.empty },
-            .c => .{ .c = {} },
+            .c => .{ .c = link.File.C.DeclBlock.empty },
             .wasm => .{ .wasm = {} },
         },
         .fn_link = switch (self.comp.bin_file.tag) {
             .coff => .{ .coff = {} },
             .elf => .{ .elf = link.File.Elf.SrcFn.empty },
             .macho => .{ .macho = link.File.MachO.SrcFn.empty },
-            .c => .{ .c = {} },
+            .c => .{ .c = link.File.C.FnBlock.empty },
             .wasm => .{ .wasm = null },
         },
         .generation = 0,
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 1a89e22d48..d87801ae2e 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -1,495 +1,526 @@
 const std = @import("std");
+const mem = std.mem;
+const log = std.log.scoped(.c);
+const Writer = std.ArrayList(u8).Writer;
 
 const link = @import("../link.zig");
 const Module = @import("../Module.zig");
 const Compilation = @import("../Compilation.zig");
-
 const Inst = @import("../ir.zig").Inst;
 const Value = @import("../value.zig").Value;
 const Type = @import("../type.zig").Type;
-
 const C = link.File.C;
 const Decl = Module.Decl;
-const mem = std.mem;
-const log = std.log.scoped(.c);
+const trace = @import("../tracy.zig").trace;
 
-const Writer = std.ArrayList(u8).Writer;
+const Mutability = enum { Const, Mut };
 
-/// Maps a name from Zig source to C. Currently, this will always give the same
-/// output for any given input, sometimes resulting in broken identifiers.
-fn map(allocator: *std.mem.Allocator, name: []const u8) ![]const u8 {
-    return allocator.dupe(u8, name);
-}
+pub const CValue = union(enum) {
+    none: void,
+    /// Index into local_names
+    local: usize,
+    /// Index into local_names, but take the address.
+    local_ref: usize,
+    /// A constant instruction, to be rendered inline.
+    constant: *Inst,
+    /// Index into the parameters
+    arg: usize,
+    /// By-value
+    decl: *Decl,
 
-const Mutability = enum { Const, Mut };
+    pub fn printed(value: CValue, object: *Object) Printed {
+        return .{
+            .value = value,
+            .object = object,
+        };
+    }
+
+    pub const Printed = struct {
+        value: CValue,
+        object: *Object,
+
+        /// TODO this got unwieldly, I want to remove the ability to print this way
+        pub fn format(
+            self: Printed,
+            comptime fmt: []const u8,
+            options: std.fmt.FormatOptions,
+            writer: anytype,
+        ) error{OutOfMemory}!void {
+            if (fmt.len != 0) @compileError("Unknown format string: '" ++ fmt ++ "'");
+            switch (self.value) {
+                .none => unreachable,
+                .local => |i| return std.fmt.format(writer, "t{d}", .{i}),
+                .local_ref => |i| return std.fmt.format(writer, "&t{d}", .{i}),
+                .constant => |inst| {
+                    const o = self.object;
+                    o.dg.renderValue(writer, inst.ty, inst.value().?) catch |err| switch (err) {
+                        error.OutOfMemory => return error.OutOfMemory,
+                        error.AnalysisFail => return,
+                    };
+                },
+                .arg => |i| return std.fmt.format(writer, "a{d}", .{i}),
+                .decl => |decl| return writer.writeAll(mem.span(decl.name)),
+            }
+        }
+    };
+};
 
-fn renderTypeAndName(
-    ctx: *Context,
-    writer: Writer,
-    ty: Type,
-    name: []const u8,
-    mutability: Mutability,
-) error{ OutOfMemory, AnalysisFail }!void {
-    var suffix = std.ArrayList(u8).init(&ctx.arena.allocator);
-
-    var render_ty = ty;
-    while (render_ty.zigTypeTag() == .Array) {
-        const sentinel_bit = @boolToInt(render_ty.sentinel() != null);
-        const c_len = render_ty.arrayLen() + sentinel_bit;
-        try suffix.writer().print("[{d}]", .{c_len});
-        render_ty = render_ty.elemType();
+pub const CValueMap = std.AutoHashMap(*Inst, CValue);
+
+/// This data is available when outputting .c code for a Module.
+/// It is not available when generating .h file.
+pub const Object = struct {
+    dg: DeclGen,
+    gpa: *mem.Allocator,
+    code: std.ArrayList(u8),
+    value_map: CValueMap,
+    next_arg_index: usize = 0,
+    next_local_index: usize = 0,
+
+    fn resolveInst(o: *Object, inst: *Inst) !CValue {
+        if (inst.value()) |_| {
+            return CValue{ .constant = inst };
+        }
+        return o.value_map.get(inst).?; // Instruction does not dominate all uses!
     }
 
-    try renderType(ctx, writer, render_ty);
+    fn allocLocalValue(o: *Object) CValue {
+        const result = o.next_local_index;
+        o.next_local_index += 1;
+        return .{ .local = result };
+    }
 
-    const const_prefix = switch (mutability) {
-        .Const => "const ",
-        .Mut => "",
-    };
-    try writer.print(" {s}{s}{s}", .{ const_prefix, name, suffix.items });
-}
+    fn allocLocal(o: *Object, ty: Type, mutability: Mutability) !CValue {
+        const local_value = o.allocLocalValue();
+        try o.renderTypeAndName(o.code.writer(), ty, local_value, mutability);
+        return local_value;
+    }
 
-fn renderType(
-    ctx: *Context,
-    writer: Writer,
-    t: Type,
-) error{ OutOfMemory, AnalysisFail }!void {
-    switch (t.zigTypeTag()) {
-        .NoReturn => {
-            try writer.writeAll("zig_noreturn void");
-        },
-        .Void => try writer.writeAll("void"),
-        .Bool => try writer.writeAll("bool"),
-        .Int => {
-            switch (t.tag()) {
-                .u8 => try writer.writeAll("uint8_t"),
-                .i8 => try writer.writeAll("int8_t"),
-                .u16 => try writer.writeAll("uint16_t"),
-                .i16 => try writer.writeAll("int16_t"),
-                .u32 => try writer.writeAll("uint32_t"),
-                .i32 => try writer.writeAll("int32_t"),
-                .u64 => try writer.writeAll("uint64_t"),
-                .i64 => try writer.writeAll("int64_t"),
-                .usize => try writer.writeAll("uintptr_t"),
-                .isize => try writer.writeAll("intptr_t"),
-                .c_short => try writer.writeAll("short"),
-                .c_ushort => try writer.writeAll("unsigned short"),
-                .c_int => try writer.writeAll("int"),
-                .c_uint => try writer.writeAll("unsigned int"),
-                .c_long => try writer.writeAll("long"),
-                .c_ulong => try writer.writeAll("unsigned long"),
-                .c_longlong => try writer.writeAll("long long"),
-                .c_ulonglong => try writer.writeAll("unsigned long long"),
-                .int_signed, .int_unsigned => {
-                    const info = t.intInfo(ctx.target);
-                    const sign_prefix = switch (info.signedness) {
-                        .signed => "i",
-                        .unsigned => "",
-                    };
-                    inline for (.{ 8, 16, 32, 64, 128 }) |nbits| {
-                        if (info.bits <= nbits) {
-                            try writer.print("{s}int{d}_t", .{ sign_prefix, nbits });
-                            break;
-                        }
+    fn indent(o: *Object) !void {
+        const indent_size = 4;
+        const indent_level = 1;
+        const indent_amt = indent_size * indent_level;
+        try o.code.writer().writeByteNTimes(' ', indent_amt);
+    }
+
+    fn renderTypeAndName(
+        o: *Object,
+        writer: Writer,
+        ty: Type,
+        name: CValue,
+        mutability: Mutability,
+    ) error{ OutOfMemory, AnalysisFail }!void {
+        var suffix = std.ArrayList(u8).init(o.gpa);
+        defer suffix.deinit();
+
+        var render_ty = ty;
+        while (render_ty.zigTypeTag() == .Array) {
+            const sentinel_bit = @boolToInt(render_ty.sentinel() != null);
+            const c_len = render_ty.arrayLen() + sentinel_bit;
+            try suffix.writer().print("[{d}]", .{c_len});
+            render_ty = render_ty.elemType();
+        }
+
+        try o.dg.renderType(writer, render_ty);
+
+        const const_prefix = switch (mutability) {
+            .Const => "const ",
+            .Mut => "",
+        };
+        try writer.print(" {s}{}{s}", .{ const_prefix, name.printed(o), suffix.items });
+    }
+};
+
+/// This data is available both when outputting .c code and when outputting an .h file.
+const DeclGen = struct {
+    module: *Module,
+    decl: *Decl,
+    fwd_decl: std.ArrayList(u8),
+    error_msg: ?*Compilation.ErrorMsg,
+
+    fn fail(dg: *DeclGen, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } {
+        dg.error_msg = try Compilation.ErrorMsg.create(dg.module.gpa, src, format, args);
+        return error.AnalysisFail;
+    }
+
+    fn renderValue(
+        dg: *DeclGen,
+        writer: Writer,
+        t: Type,
+        val: Value,
+    ) error{ OutOfMemory, AnalysisFail }!void {
+        switch (t.zigTypeTag()) {
+            .Int => {
+                if (t.isSignedInt())
+                    return writer.print("{d}", .{val.toSignedInt()});
+                return writer.print("{d}", .{val.toUnsignedInt()});
+            },
+            .Pointer => switch (val.tag()) {
+                .undef, .zero => try writer.writeAll("0"),
+                .one => try writer.writeAll("1"),
+                .decl_ref => {
+                    const decl = val.castTag(.decl_ref).?.data;
+
+                    // Determine if we must pointer cast.
+                    const decl_tv = decl.typed_value.most_recent.typed_value;
+                    if (t.eql(decl_tv.ty)) {
+                        try writer.print("&{s}", .{decl.name});
                     } else {
-                        return ctx.fail(ctx.decl.src(), "TODO: C backend: implement integer types larger than 128 bits", .{});
+                        try writer.writeAll("(");
+                        try dg.renderType(writer, t);
+                        try writer.print(")&{s}", .{decl.name});
                     }
                 },
+                .function => {
+                    const func = val.castTag(.function).?.data;
+                    try writer.print("{s}", .{func.owner_decl.name});
+                },
+                .extern_fn => {
+                    const decl = val.castTag(.extern_fn).?.data;
+                    try writer.print("{s}", .{decl.name});
+                },
+                else => |e| return dg.fail(
+                    dg.decl.src(),
+                    "TODO: C backend: implement Pointer value {s}",
+                    .{@tagName(e)},
+                ),
+            },
+            .Array => {
+                // First try specific tag representations for more efficiency.
+                switch (val.tag()) {
+                    .undef, .empty_struct_value, .empty_array => try writer.writeAll("{}"),
+                    .bytes => {
+                        const bytes = val.castTag(.bytes).?.data;
+                        // TODO: make our own C string escape instead of using {Z}
+                        try writer.print("\"{Z}\"", .{bytes});
+                    },
+                    else => {
+                        // Fall back to generic implementation.
+                        var arena = std.heap.ArenaAllocator.init(dg.module.gpa);
+                        defer arena.deinit();
+
+                        try writer.writeAll("{");
+                        var index: usize = 0;
+                        const len = t.arrayLen();
+                        const elem_ty = t.elemType();
+                        while (index < len) : (index += 1) {
+                            if (index != 0) try writer.writeAll(",");
+                            const elem_val = try val.elemValue(&arena.allocator, index);
+                            try dg.renderValue(writer, elem_ty, elem_val);
+                        }
+                        if (t.sentinel()) |sentinel_val| {
+                            if (index != 0) try writer.writeAll(",");
+                            try dg.renderValue(writer, elem_ty, sentinel_val);
+                        }
+                        try writer.writeAll("}");
+                    },
+                }
+            },
+            else => |e| return dg.fail(dg.decl.src(), "TODO: C backend: implement value {s}", .{
+                @tagName(e),
+            }),
+        }
+    }
+
+    fn renderFunctionSignature(dg: *DeclGen, w: Writer) !void {
+        const tv = dg.decl.typed_value.most_recent.typed_value;
+        // Determine whether the function is globally visible.
+        const is_global = blk: {
+            switch (tv.val.tag()) {
+                .extern_fn => break :blk true,
+                .function => {
+                    const func = tv.val.castTag(.function).?.data;
+                    break :blk dg.module.decl_exports.contains(func.owner_decl);
+                },
                 else => unreachable,
             }
-        },
-        .Pointer => {
-            if (t.isSlice()) {
-                return ctx.fail(ctx.decl.src(), "TODO: C backend: implement slices", .{});
-            } else {
-                try renderType(ctx, writer, t.elemType());
-                try writer.writeAll(" *");
-                if (t.isConstPtr()) {
-                    try writer.writeAll("const ");
-                }
-                if (t.isVolatilePtr()) {
-                    try writer.writeAll("volatile ");
+        };
+        if (!is_global) {
+            try w.writeAll("static ");
+        }
+        try dg.renderType(w, tv.ty.fnReturnType());
+        const decl_name = mem.span(dg.decl.name);
+        try w.print(" {s}(", .{decl_name});
+        var param_len = tv.ty.fnParamLen();
+        if (param_len == 0)
+            try w.writeAll("void")
+        else {
+            var index: usize = 0;
+            while (index < param_len) : (index += 1) {
+                if (index > 0) {
+                    try w.writeAll(", ");
                 }
+                try dg.renderType(w, tv.ty.fnParamType(index));
+                try w.print(" a{d}", .{index});
             }
-        },
-        .Array => {
-            try renderType(ctx, writer, t.elemType());
-            try writer.writeAll(" *");
-        },
-        else => |e| return ctx.fail(ctx.decl.src(), "TODO: C backend: implement type {s}", .{
-            @tagName(e),
-        }),
+        }
+        try w.writeByte(')');
     }
-}
 
-fn renderValue(
-    ctx: *Context,
-    writer: Writer,
-    t: Type,
-    val: Value,
-) error{ OutOfMemory, AnalysisFail }!void {
-    switch (t.zigTypeTag()) {
-        .Int => {
-            if (t.isSignedInt())
-                return writer.print("{d}", .{val.toSignedInt()});
-            return writer.print("{d}", .{val.toUnsignedInt()});
-        },
-        .Pointer => switch (val.tag()) {
-            .undef, .zero => try writer.writeAll("0"),
-            .one => try writer.writeAll("1"),
-            .decl_ref => {
-                const decl = val.castTag(.decl_ref).?.data;
-
-                // Determine if we must pointer cast.
-                const decl_tv = decl.typed_value.most_recent.typed_value;
-                if (t.eql(decl_tv.ty)) {
-                    try writer.print("&{s}", .{decl.name});
-                } else {
-                    try writer.writeAll("(");
-                    try renderType(ctx, writer, t);
-                    try writer.print(")&{s}", .{decl.name});
-                }
+    fn renderType(dg: *DeclGen, w: Writer, t: Type) error{ OutOfMemory, AnalysisFail }!void {
+        switch (t.zigTypeTag()) {
+            .NoReturn => {
+                try w.writeAll("zig_noreturn void");
             },
-            .function => {
-                const func = val.castTag(.function).?.data;
-                try writer.print("{s}", .{func.owner_decl.name});
-            },
-            .extern_fn => {
-                const decl = val.castTag(.extern_fn).?.data;
-                try writer.print("{s}", .{decl.name});
+            .Void => try w.writeAll("void"),
+            .Bool => try w.writeAll("bool"),
+            .Int => {
+                switch (t.tag()) {
+                    .u8 => try w.writeAll("uint8_t"),
+                    .i8 => try w.writeAll("int8_t"),
+                    .u16 => try w.writeAll("uint16_t"),
+                    .i16 => try w.writeAll("int16_t"),
+                    .u32 => try w.writeAll("uint32_t"),
+                    .i32 => try w.writeAll("int32_t"),
+                    .u64 => try w.writeAll("uint64_t"),
+                    .i64 => try w.writeAll("int64_t"),
+                    .usize => try w.writeAll("uintptr_t"),
+                    .isize => try w.writeAll("intptr_t"),
+                    .c_short => try w.writeAll("short"),
+                    .c_ushort => try w.writeAll("unsigned short"),
+                    .c_int => try w.writeAll("int"),
+                    .c_uint => try w.writeAll("unsigned int"),
+                    .c_long => try w.writeAll("long"),
+                    .c_ulong => try w.writeAll("unsigned long"),
+                    .c_longlong => try w.writeAll("long long"),
+                    .c_ulonglong => try w.writeAll("unsigned long long"),
+                    .int_signed, .int_unsigned => {
+                        const info = t.intInfo(dg.module.getTarget());
+                        const sign_prefix = switch (info.signedness) {
+                            .signed => "i",
+                            .unsigned => "",
+                        };
+                        inline for (.{ 8, 16, 32, 64, 128 }) |nbits| {
+                            if (info.bits <= nbits) {
+                                try w.print("{s}int{d}_t", .{ sign_prefix, nbits });
+                                break;
+                            }
+                        } else {
+                            return dg.fail(dg.decl.src(), "TODO: C backend: implement integer types larger than 128 bits", .{});
+                        }
+                    },
+                    else => unreachable,
+                }
             },
-            else => |e| return ctx.fail(
-                ctx.decl.src(),
-                "TODO: C backend: implement Pointer value {s}",
-                .{@tagName(e)},
-            ),
-        },
-        .Array => {
-            // First try specific tag representations for more efficiency.
-            switch (val.tag()) {
-                .undef, .empty_struct_value, .empty_array => try writer.writeAll("{}"),
-                .bytes => {
-                    const bytes = val.castTag(.bytes).?.data;
-                    // TODO: make our own C string escape instead of using {Z}
-                    try writer.print("\"{Z}\"", .{bytes});
-                },
-                else => {
-                    // Fall back to generic implementation.
-                    try writer.writeAll("{");
-                    var index: usize = 0;
-                    const len = t.arrayLen();
-                    const elem_ty = t.elemType();
-                    while (index < len) : (index += 1) {
-                        if (index != 0) try writer.writeAll(",");
-                        const elem_val = try val.elemValue(&ctx.arena.allocator, index);
-                        try renderValue(ctx, writer, elem_ty, elem_val);
+            .Pointer => {
+                if (t.isSlice()) {
+                    return dg.fail(dg.decl.src(), "TODO: C backend: implement slices", .{});
+                } else {
+                    try dg.renderType(w, t.elemType());
+                    try w.writeAll(" *");
+                    if (t.isConstPtr()) {
+                        try w.writeAll("const ");
                     }
-                    if (t.sentinel()) |sentinel_val| {
-                        if (index != 0) try writer.writeAll(",");
-                        try renderValue(ctx, writer, elem_ty, sentinel_val);
+                    if (t.isVolatilePtr()) {
+                        try w.writeAll("volatile ");
                     }
-                    try writer.writeAll("}");
-                },
-            }
-        },
-        else => |e| return ctx.fail(ctx.decl.src(), "TODO: C backend: implement value {s}", .{
-            @tagName(e),
-        }),
-    }
-}
-
-fn renderFunctionSignature(
-    ctx: *Context,
-    writer: Writer,
-    decl: *Decl,
-) !void {
-    const tv = decl.typed_value.most_recent.typed_value;
-    // Determine whether the function is globally visible.
-    const is_global = blk: {
-        switch (tv.val.tag()) {
-            .extern_fn => break :blk true,
-            .function => {
-                const func = tv.val.castTag(.function).?.data;
-                break :blk ctx.module.decl_exports.contains(func.owner_decl);
+                }
             },
-            else => unreachable,
-        }
-    };
-    if (!is_global) {
-        try writer.writeAll("static ");
-    }
-    try renderType(ctx, writer, tv.ty.fnReturnType());
-    // Use the child allocator directly, as we know the name can be freed before
-    // the rest of the arena.
-    const decl_name = mem.span(decl.name);
-    const name = try map(ctx.arena.child_allocator, decl_name);
-    defer ctx.arena.child_allocator.free(name);
-    try writer.print(" {s}(", .{name});
-    var param_len = tv.ty.fnParamLen();
-    if (param_len == 0)
-        try writer.writeAll("void")
-    else {
-        var index: usize = 0;
-        while (index < param_len) : (index += 1) {
-            if (index > 0) {
-                try writer.writeAll(", ");
-            }
-            try renderType(ctx, writer, tv.ty.fnParamType(index));
-            try writer.print(" arg{d}", .{index});
+            .Array => {
+                try dg.renderType(w, t.elemType());
+                try w.writeAll(" *");
+            },
+            else => |e| return dg.fail(dg.decl.src(), "TODO: C backend: implement type {s}", .{
+                @tagName(e),
+            }),
         }
     }
-    try writer.writeByte(')');
-}
+};
 
-fn indent(file: *C) !void {
-    const indent_size = 4;
-    const indent_level = 1;
-    const indent_amt = indent_size * indent_level;
-    try file.main.writer().writeByteNTimes(' ', indent_amt);
-}
+pub fn genDecl(o: *Object) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
 
-pub fn generate(file: *C, module: *Module, decl: *Decl) !void {
-    const tv = decl.typed_value.most_recent.typed_value;
-
-    var arena = std.heap.ArenaAllocator.init(file.base.allocator);
-    defer arena.deinit();
-    var inst_map = std.AutoHashMap(*Inst, []u8).init(&arena.allocator);
-    defer inst_map.deinit();
-    var ctx = Context{
-        .decl = decl,
-        .arena = &arena,
-        .inst_map = &inst_map,
-        .target = file.base.options.target,
-        .header = &file.header,
-        .module = module,
-    };
-    defer {
-        file.error_msg = ctx.error_msg;
-        ctx.deinit();
-    }
+    const tv = o.dg.decl.typed_value.most_recent.typed_value;
 
     if (tv.val.castTag(.function)) |func_payload| {
-        const writer = file.main.writer();
-        try renderFunctionSignature(&ctx, writer, decl);
-
-        try writer.writeAll(" {");
+        const fwd_decl_writer = o.dg.fwd_decl.writer();
+        try o.dg.renderFunctionSignature(fwd_decl_writer);
+        try fwd_decl_writer.writeAll(";\n");
 
         const func: *Module.Fn = func_payload.data;
         const instructions = func.body.instructions;
-        if (instructions.len > 0) {
-            try writer.writeAll("\n");
-            for (instructions) |inst| {
-                if (switch (inst.tag) {
-                    .add => try genBinOp(&ctx, file, inst.castTag(.add).?, "+"),
-                    .alloc => try genAlloc(&ctx, file, inst.castTag(.alloc).?),
-                    .arg => try genArg(&ctx),
-                    .assembly => try genAsm(&ctx, file, inst.castTag(.assembly).?),
-                    .block => try genBlock(&ctx, file, inst.castTag(.block).?),
-                    .bitcast => try genBitcast(&ctx, file, inst.castTag(.bitcast).?),
-                    .breakpoint => try genBreakpoint(file, inst.castTag(.breakpoint).?),
-                    .call => try genCall(&ctx, file, inst.castTag(.call).?),
-                    .cmp_eq => try genBinOp(&ctx, file, inst.castTag(.cmp_eq).?, "=="),
-                    .cmp_gt => try genBinOp(&ctx, file, inst.castTag(.cmp_gt).?, ">"),
-                    .cmp_gte => try genBinOp(&ctx, file, inst.castTag(.cmp_gte).?, ">="),
-                    .cmp_lt => try genBinOp(&ctx, file, inst.castTag(.cmp_lt).?, "<"),
-                    .cmp_lte => try genBinOp(&ctx, file, inst.castTag(.cmp_lte).?, "<="),
-                    .cmp_neq => try genBinOp(&ctx, file, inst.castTag(.cmp_neq).?, "!="),
-                    .dbg_stmt => try genDbgStmt(&ctx, inst.castTag(.dbg_stmt).?),
-                    .intcast => try genIntCast(&ctx, file, inst.castTag(.intcast).?),
-                    .load => try genLoad(&ctx, file, inst.castTag(.load).?),
-                    .ret => try genRet(&ctx, file, inst.castTag(.ret).?),
-                    .retvoid => try genRetVoid(file),
-                    .store => try genStore(&ctx, file, inst.castTag(.store).?),
-                    .sub => try genBinOp(&ctx, file, inst.castTag(.sub).?, "-"),
-                    .unreach => try genUnreach(file, inst.castTag(.unreach).?),
-                    else => |e| return ctx.fail(decl.src(), "TODO: C backend: implement codegen for {}", .{e}),
-                }) |name| {
-                    try ctx.inst_map.putNoClobber(inst, name);
-                }
+        const writer = o.code.writer();
+        try o.dg.renderFunctionSignature(writer);
+        if (instructions.len == 0) {
+            try writer.writeAll(" {}\n\n");
+            return;
+        }
+
+        try writer.writeAll(" {");
+
+        try writer.writeAll("\n");
+        for (instructions) |inst| {
+            const result_value = switch (inst.tag) {
+                .add => try genBinOp(o, inst.castTag(.add).?, "+"),
+                .alloc => try genAlloc(o, inst.castTag(.alloc).?),
+                .arg => genArg(o),
+                .assembly => try genAsm(o, inst.castTag(.assembly).?),
+                .block => try genBlock(o, inst.castTag(.block).?),
+                .bitcast => try genBitcast(o, inst.castTag(.bitcast).?),
+                .breakpoint => try genBreakpoint(o, inst.castTag(.breakpoint).?),
+                .call => try genCall(o, inst.castTag(.call).?),
+                .cmp_eq => try genBinOp(o, inst.castTag(.cmp_eq).?, "=="),
+                .cmp_gt => try genBinOp(o, inst.castTag(.cmp_gt).?, ">"),
+                .cmp_gte => try genBinOp(o, inst.castTag(.cmp_gte).?, ">="),
+                .cmp_lt => try genBinOp(o, inst.castTag(.cmp_lt).?, "<"),
+                .cmp_lte => try genBinOp(o, inst.castTag(.cmp_lte).?, "<="),
+                .cmp_neq => try genBinOp(o, inst.castTag(.cmp_neq).?, "!="),
+                .dbg_stmt => try genDbgStmt(o, inst.castTag(.dbg_stmt).?),
+                .intcast => try genIntCast(o, inst.castTag(.intcast).?),
+                .load => try genLoad(o, inst.castTag(.load).?),
+                .ret => try genRet(o, inst.castTag(.ret).?),
+                .retvoid => try genRetVoid(o),
+                .store => try genStore(o, inst.castTag(.store).?),
+                .sub => try genBinOp(o, inst.castTag(.sub).?, "-"),
+                .unreach => try genUnreach(o, inst.castTag(.unreach).?),
+                else => |e| return o.dg.fail(o.dg.decl.src(), "TODO: C backend: implement codegen for {}", .{e}),
+            };
+            switch (result_value) {
+                .none => {},
+                else => try o.value_map.putNoClobber(inst, result_value),
             }
         }
 
         try writer.writeAll("}\n\n");
     } else if (tv.val.tag() == .extern_fn) {
-        return; // handled when referenced
+        const writer = o.code.writer();
+        try o.dg.renderFunctionSignature(writer);
+        try writer.writeAll(";\n");
     } else {
-        const writer = file.constants.writer();
+        const writer = o.code.writer();
         try writer.writeAll("static ");
 
         // TODO ask the Decl if it is const
         // https://github.com/ziglang/zig/issues/7582
 
-        try renderTypeAndName(&ctx, writer, tv.ty, mem.span(decl.name), .Mut);
+        const decl_c_value: CValue = .{ .decl = o.dg.decl };
+        try o.renderTypeAndName(writer, tv.ty, decl_c_value, .Mut);
 
         try writer.writeAll(" = ");
-        try renderValue(&ctx, writer, tv.ty, tv.val);
+        try o.dg.renderValue(writer, tv.ty, tv.val);
         try writer.writeAll(";\n");
     }
 }
 
-pub fn generateHeader(
-    comp: *Compilation,
-    module: *Module,
-    header: *C.Header,
-    decl: *Decl,
-) error{ AnalysisFail, OutOfMemory }!void {
+pub fn genHeader(comp: *Compilation, dg: *DeclGen) error{ AnalysisFail, OutOfMemory }!void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
     switch (decl.typed_value.most_recent.typed_value.ty.zigTypeTag()) {
         .Fn => {
-            var inst_map = std.AutoHashMap(*Inst, []u8).init(comp.gpa);
-            defer inst_map.deinit();
-
-            var arena = std.heap.ArenaAllocator.init(comp.gpa);
-            defer arena.deinit();
-
-            var ctx = Context{
-                .decl = decl,
-                .arena = &arena,
-                .inst_map = &inst_map,
-                .target = comp.getTarget(),
-                .header = header,
-                .module = module,
-            };
-            const writer = header.buf.writer();
-            renderFunctionSignature(&ctx, writer, decl) catch |err| {
-                if (err == error.AnalysisFail) {
-                    try module.failed_decls.put(module.gpa, decl, ctx.error_msg);
-                }
-                return err;
+            dg.renderFunctionSignature() catch |err| switch (err) {
+                error.AnalysisFail => {
+                    try dg.module.failed_decls.put(dg.module.gpa, decl, dg.error_msg.?);
+                    dg.error_msg = null;
+                    return error.AnalysisFail;
+                },
+                else => |e| return e,
             };
-            try writer.writeAll(";\n");
+            try dg.fwd_decl.appendSlice(";\n");
         },
         else => {},
     }
 }
 
-const Context = struct {
-    decl: *Decl,
-    inst_map: *std.AutoHashMap(*Inst, []u8),
-    arena: *std.heap.ArenaAllocator,
-    argdex: usize = 0,
-    unnamed_index: usize = 0,
-    error_msg: *Compilation.ErrorMsg = undefined,
-    target: std.Target,
-    header: *C.Header,
-    module: *Module,
-
-    fn resolveInst(self: *Context, inst: *Inst) ![]u8 {
-        if (inst.value()) |val| {
-            var out = std.ArrayList(u8).init(&self.arena.allocator);
-            try renderValue(self, out.writer(), inst.ty, val);
-            return out.toOwnedSlice();
-        }
-        return self.inst_map.get(inst).?; // Instruction does not dominate all uses!
-    }
-
-    fn name(self: *Context) ![]u8 {
-        const val = try std.fmt.allocPrint(&self.arena.allocator, "__temp_{d}", .{self.unnamed_index});
-        self.unnamed_index += 1;
-        return val;
-    }
-
-    fn fail(self: *Context, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } {
-        self.error_msg = try Compilation.ErrorMsg.create(self.arena.child_allocator, src, format, args);
-        return error.AnalysisFail;
-    }
-
-    fn deinit(self: *Context) void {
-        self.* = undefined;
-    }
-};
-
-fn genAlloc(ctx: *Context, file: *C, alloc: *Inst.NoOp) !?[]u8 {
-    const writer = file.main.writer();
+fn genAlloc(o: *Object, alloc: *Inst.NoOp) !CValue {
+    const writer = o.code.writer();
 
     // First line: the variable used as data storage.
-    try indent(file);
-    const local_name = try ctx.name();
+    try o.indent();
     const elem_type = alloc.base.ty.elemType();
     const mutability: Mutability = if (alloc.base.ty.isConstPtr()) .Const else .Mut;
-    try renderTypeAndName(ctx, writer, elem_type, local_name, mutability);
+    const local = try o.allocLocal(elem_type, mutability);
     try writer.writeAll(";\n");
 
-    // Second line: a pointer to it so that we can refer to it as the allocation.
-    // One line for the variable, one line for the pointer to the variable, which we return.
-    try indent(file);
-    const ptr_local_name = try ctx.name();
-    try renderTypeAndName(ctx, writer, alloc.base.ty, ptr_local_name, .Const);
-    try writer.print(" = &{s};\n", .{local_name});
-
-    return ptr_local_name;
+    return CValue{ .local_ref = local.local };
 }
 
-fn genArg(ctx: *Context) !?[]u8 {
-    const name = try std.fmt.allocPrint(&ctx.arena.allocator, "arg{d}", .{ctx.argdex});
-    ctx.argdex += 1;
-    return name;
+fn genArg(o: *Object) CValue {
+    const i = o.next_arg_index;
+    o.next_arg_index += 1;
+    return .{ .arg = i };
 }
 
-fn genRetVoid(file: *C) !?[]u8 {
-    try indent(file);
-    try file.main.writer().print("return;\n", .{});
-    return null;
+fn genRetVoid(o: *Object) !CValue {
+    try o.indent();
+    try o.code.writer().print("return;\n", .{});
+    return CValue.none;
 }
 
-fn genLoad(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
-    const operand = try ctx.resolveInst(inst.operand);
-    const writer = file.main.writer();
-    try indent(file);
-    const local_name = try ctx.name();
-    try renderTypeAndName(ctx, writer, inst.base.ty, local_name, .Const);
-    try writer.print(" = *{s};\n", .{operand});
-    return local_name;
+fn genLoad(o: *Object, inst: *Inst.UnOp) !CValue {
+    const operand = try o.resolveInst(inst.operand);
+    const writer = o.code.writer();
+    try o.indent();
+    const local = try o.allocLocal(inst.base.ty, .Const);
+    switch (operand) {
+        .local_ref => |i| {
+            const wrapped: CValue = .{ .local = i };
+            try writer.print(" = {};\n", .{wrapped.printed(o)});
+        },
+        else => {
+            try writer.print(" = *{};\n", .{operand.printed(o)});
+        },
+    }
+    return local;
 }
 
-fn genRet(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
-    try indent(file);
-    const writer = file.main.writer();
-    try writer.print("return {s};\n", .{try ctx.resolveInst(inst.operand)});
-    return null;
+fn genRet(o: *Object, inst: *Inst.UnOp) !CValue {
+    const operand = try o.resolveInst(inst.operand);
+    try o.indent();
+    try o.code.writer().print("return {};\n", .{operand.printed(o)});
+    return CValue.none;
 }
 
-fn genIntCast(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
+fn genIntCast(o: *Object, inst: *Inst.UnOp) !CValue {
     if (inst.base.isUnused())
-        return null;
-    try indent(file);
-    const writer = file.main.writer();
-    const name = try ctx.name();
-    const from = try ctx.resolveInst(inst.operand);
+        return CValue.none;
 
-    try renderTypeAndName(ctx, writer, inst.base.ty, name, .Const);
+    const from = try o.resolveInst(inst.operand);
+
+    try o.indent();
+    const writer = o.code.writer();
+    const local = try o.allocLocal(inst.base.ty, .Const);
     try writer.writeAll(" = (");
-    try renderType(ctx, writer, inst.base.ty);
-    try writer.print("){s};\n", .{from});
-    return name;
+    try o.dg.renderType(writer, inst.base.ty);
+    try writer.print("){};\n", .{from.printed(o)});
+    return local;
 }
 
-fn genStore(ctx: *Context, file: *C, inst: *Inst.BinOp) !?[]u8 {
+fn genStore(o: *Object, inst: *Inst.BinOp) !CValue {
     // *a = b;
-    try indent(file);
-    const writer = file.main.writer();
-    const dest_ptr_name = try ctx.resolveInst(inst.lhs);
-    const src_val_name = try ctx.resolveInst(inst.rhs);
-    try writer.print("*{s} = {s};\n", .{ dest_ptr_name, src_val_name });
-    return null;
+    const dest_ptr = try o.resolveInst(inst.lhs);
+    const src_val = try o.resolveInst(inst.rhs);
+
+    try o.indent();
+    const writer = o.code.writer();
+    switch (dest_ptr) {
+        .local_ref => |i| {
+            const dest: CValue = .{ .local = i };
+            try writer.print("{} = {};\n", .{ dest.printed(o), src_val.printed(o) });
+        },
+        else => {
+            try writer.print("*{} = {};\n", .{ dest_ptr.printed(o), src_val.printed(o) });
+        },
+    }
+    return CValue.none;
 }
 
-fn genBinOp(ctx: *Context, file: *C, inst: *Inst.BinOp, operator: []const u8) !?[]u8 {
+fn genBinOp(o: *Object, inst: *Inst.BinOp, operator: []const u8) !CValue {
     if (inst.base.isUnused())
-        return null;
-    try indent(file);
-    const lhs = try ctx.resolveInst(inst.lhs);
-    const rhs = try ctx.resolveInst(inst.rhs);
-    const writer = file.main.writer();
-    const name = try ctx.name();
-    try renderTypeAndName(ctx, writer, inst.base.ty, name, .Const);
-    try writer.print(" = {s} {s} {s};\n", .{ lhs, operator, rhs });
-    return name;
+        return CValue.none;
+
+    const lhs = try o.resolveInst(inst.lhs);
+    const rhs = try o.resolveInst(inst.rhs);
+
+    try o.indent();
+    const writer = o.code.writer();
+    const local = try o.allocLocal(inst.base.ty, .Const);
+    try writer.print(" = {} {s} {};\n", .{ lhs.printed(o), operator, rhs.printed(o) });
+    return local;
 }
 
-fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
-    try indent(file);
-    const writer = file.main.writer();
-    const header = file.header.buf.writer();
+fn genCall(o: *Object, inst: *Inst.Call) !CValue {
     if (inst.func.castTag(.constant)) |func_inst| {
         const fn_decl = if (func_inst.val.castTag(.extern_fn)) |extern_fn|
             extern_fn.data
@@ -501,23 +532,19 @@ fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
         const fn_ty = fn_decl.typed_value.most_recent.typed_value.ty;
         const ret_ty = fn_ty.fnReturnType();
         const unused_result = inst.base.isUnused();
-        var result_name: ?[]u8 = null;
+        var result_local: CValue = .none;
+
+        try o.indent();
+        const writer = o.code.writer();
         if (unused_result) {
             if (ret_ty.hasCodeGenBits()) {
                 try writer.print("(void)", .{});
             }
         } else {
-            const local_name = try ctx.name();
-            try renderTypeAndName(ctx, writer, ret_ty, local_name, .Const);
+            result_local = try o.allocLocal(ret_ty, .Const);
             try writer.writeAll(" = ");
-            result_name = local_name;
         }
         const fn_name = mem.spanZ(fn_decl.name);
-        if (file.called.get(fn_name) == null) {
-            try file.called.put(fn_name, {});
-            try renderFunctionSignature(ctx, header, fn_decl);
-            try header.writeAll(";\n");
-        }
         try writer.print("{s}(", .{fn_name});
         if (inst.args.len != 0) {
             for (inst.args) |arg, i| {
@@ -525,87 +552,88 @@ fn genCall(ctx: *Context, file: *C, inst: *Inst.Call) !?[]u8 {
                     try writer.writeAll(", ");
                 }
                 if (arg.value()) |val| {
-                    try renderValue(ctx, writer, arg.ty, val);
+                    try o.dg.renderValue(writer, arg.ty, val);
                 } else {
-                    const val = try ctx.resolveInst(arg);
-                    try writer.print("{s}", .{val});
+                    const val = try o.resolveInst(arg);
+                    try writer.print("{}", .{val.printed(o)});
                 }
             }
         }
         try writer.writeAll(");\n");
-        return result_name;
+        return result_local;
     } else {
-        return ctx.fail(ctx.decl.src(), "TODO: C backend: implement function pointers", .{});
+        return o.dg.fail(o.dg.decl.src(), "TODO: C backend: implement function pointers", .{});
     }
 }
 
-fn genDbgStmt(ctx: *Context, inst: *Inst.NoOp) !?[]u8 {
+fn genDbgStmt(o: *Object, inst: *Inst.NoOp) !CValue {
     // TODO emit #line directive here with line number and filename
-    return null;
+    return CValue.none;
 }
 
-fn genBlock(ctx: *Context, file: *C, inst: *Inst.Block) !?[]u8 {
-    return ctx.fail(ctx.decl.src(), "TODO: C backend: implement blocks", .{});
+fn genBlock(o: *Object, inst: *Inst.Block) !CValue {
+    return o.dg.fail(o.dg.decl.src(), "TODO: C backend: implement blocks", .{});
 }
 
-fn genBitcast(ctx: *Context, file: *C, inst: *Inst.UnOp) !?[]u8 {
-    const writer = file.main.writer();
-    try indent(file);
-    const local_name = try ctx.name();
-    const operand = try ctx.resolveInst(inst.operand);
-    try renderTypeAndName(ctx, writer, inst.base.ty, local_name, .Const);
+fn genBitcast(o: *Object, inst: *Inst.UnOp) !CValue {
+    const operand = try o.resolveInst(inst.operand);
+
+    const writer = o.code.writer();
+    try o.indent();
     if (inst.base.ty.zigTypeTag() == .Pointer and inst.operand.ty.zigTypeTag() == .Pointer) {
+        const local = try o.allocLocal(inst.base.ty, .Const);
         try writer.writeAll(" = (");
-        try renderType(ctx, writer, inst.base.ty);
-        try writer.print("){s};\n", .{operand});
-    } else {
-        try writer.writeAll(";\n");
-        try indent(file);
-        try writer.print("memcpy(&{s}, &{s}, sizeof {s});\n", .{ local_name, operand, local_name });
+        try o.dg.renderType(writer, inst.base.ty);
+        try writer.print("){};\n", .{operand.printed(o)});
+        return local;
     }
-    return local_name;
+
+    const local = try o.allocLocal(inst.base.ty, .Mut);
+    try writer.writeAll(";\n");
+    try o.indent();
+    try writer.print("memcpy(&{}, &{}, sizeof {});\n", .{
+        local.printed(o), operand.printed(o), local.printed(o),
+    });
+    return local;
 }
 
-fn genBreakpoint(file: *C, inst: *Inst.NoOp) !?[]u8 {
-    try indent(file);
-    try file.main.writer().writeAll("zig_breakpoint();\n");
-    return null;
+fn genBreakpoint(o: *Object, inst: *Inst.NoOp) !CValue {
+    try o.indent();
+    try o.code.writer().writeAll("zig_breakpoint();\n");
+    return CValue.none;
 }
 
-fn genUnreach(file: *C, inst: *Inst.NoOp) !?[]u8 {
-    try indent(file);
-    try file.main.writer().writeAll("zig_unreachable();\n");
-    return null;
+fn genUnreach(o: *Object, inst: *Inst.NoOp) !CValue {
+    try o.indent();
+    try o.code.writer().writeAll("zig_unreachable();\n");
+    return CValue.none;
 }
 
-fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
-    try indent(file);
-    const writer = file.main.writer();
+fn genAsm(o: *Object, as: *Inst.Assembly) !CValue {
+    if (as.base.isUnused() and !as.is_volatile)
+        return CValue.none;
+
+    const writer = o.code.writer();
     for (as.inputs) |i, index| {
         if (i[0] == '{' and i[i.len - 1] == '}') {
             const reg = i[1 .. i.len - 1];
             const arg = as.args[index];
+            const arg_c_value = try o.resolveInst(arg);
+            try o.indent();
             try writer.writeAll("register ");
-            try renderType(ctx, writer, arg.ty);
-            try writer.print(" {s}_constant __asm__(\"{s}\") = ", .{ reg, reg });
-            // TODO merge constant handling into inst_map as well
-            if (arg.castTag(.constant)) |c| {
-                try renderValue(ctx, writer, arg.ty, c.val);
-                try writer.writeAll(";\n    ");
-            } else {
-                const gop = try ctx.inst_map.getOrPut(arg);
-                if (!gop.found_existing) {
-                    return ctx.fail(ctx.decl.src(), "Internal error in C backend: asm argument not found in inst_map", .{});
-                }
-                try writer.print("{s};\n    ", .{gop.entry.value});
-            }
+            try o.dg.renderType(writer, arg.ty);
+            try writer.print(" {s}_constant __asm__(\"{s}\") = {};\n", .{
+                reg, reg, arg_c_value.printed(o),
+            });
         } else {
-            return ctx.fail(ctx.decl.src(), "TODO non-explicit inline asm regs", .{});
+            return o.dg.fail(o.dg.decl.src(), "TODO non-explicit inline asm regs", .{});
         }
     }
-    try writer.print("__asm {s} (\"{s}\"", .{ if (as.is_volatile) @as([]const u8, "volatile") else "", as.asm_source });
-    if (as.output) |o| {
-        return ctx.fail(ctx.decl.src(), "TODO inline asm output", .{});
+    try o.indent();
+    const volatile_string: []const u8 = if (as.is_volatile) "volatile " else "";
+    try writer.print("__asm {s}(\"{s}\"", .{ volatile_string, as.asm_source });
+    if (as.output) |_| {
+        return o.dg.fail(o.dg.decl.src(), "TODO inline asm output", .{});
     }
     if (as.inputs.len > 0) {
         if (as.output == null) {
@@ -627,5 +655,9 @@ fn genAsm(ctx: *Context, file: *C, as: *Inst.Assembly) !?[]u8 {
         }
     }
     try writer.writeAll(");\n");
-    return null;
+
+    if (as.base.isUnused())
+        return CValue.none;
+
+    return o.dg.fail(o.dg.decl.src(), "TODO: C backend: inline asm expression result used", .{});
 }
diff --git a/src/link.zig b/src/link.zig
index 488f8bf69b..18b093a07a 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -130,7 +130,7 @@ pub const File = struct {
         elf: Elf.TextBlock,
         coff: Coff.TextBlock,
         macho: MachO.TextBlock,
-        c: void,
+        c: C.DeclBlock,
         wasm: void,
     };
 
@@ -138,7 +138,7 @@ pub const File = struct {
         elf: Elf.SrcFn,
         coff: Coff.SrcFn,
         macho: MachO.SrcFn,
-        c: void,
+        c: C.FnBlock,
         wasm: ?Wasm.FnData,
     };
 
@@ -291,7 +291,7 @@ pub const File = struct {
             .coff => return @fieldParentPtr(Coff, "base", base).updateDecl(module, decl),
             .elf => return @fieldParentPtr(Elf, "base", base).updateDecl(module, decl),
             .macho => return @fieldParentPtr(MachO, "base", base).updateDecl(module, decl),
-            .c => {},
+            .c => return @fieldParentPtr(C, "base", base).updateDecl(module, decl),
             .wasm => return @fieldParentPtr(Wasm, "base", base).updateDecl(module, decl),
         }
     }
@@ -301,7 +301,8 @@ pub const File = struct {
             .coff => return @fieldParentPtr(Coff, "base", base).updateDeclLineNumber(module, decl),
             .elf => return @fieldParentPtr(Elf, "base", base).updateDeclLineNumber(module, decl),
             .macho => return @fieldParentPtr(MachO, "base", base).updateDeclLineNumber(module, decl),
-            .c, .wasm => {},
+            .c => return @fieldParentPtr(C, "base", base).updateDeclLineNumber(module, decl),
+            .wasm => {},
         }
     }
 
@@ -312,7 +313,8 @@ pub const File = struct {
             .coff => return @fieldParentPtr(Coff, "base", base).allocateDeclIndexes(decl),
             .elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl),
             .macho => return @fieldParentPtr(MachO, "base", base).allocateDeclIndexes(decl),
-            .c, .wasm => {},
+            .c => return @fieldParentPtr(C, "base", base).allocateDeclIndexes(decl),
+            .wasm => {},
         }
     }
 
@@ -407,12 +409,13 @@ pub const File = struct {
         }
     }
 
+    /// Called when a Decl is deleted from the Module.
     pub fn freeDecl(base: *File, decl: *Module.Decl) void {
         switch (base.tag) {
             .coff => @fieldParentPtr(Coff, "base", base).freeDecl(decl),
             .elf => @fieldParentPtr(Elf, "base", base).freeDecl(decl),
             .macho => @fieldParentPtr(MachO, "base", base).freeDecl(decl),
-            .c => {},
+            .c => @fieldParentPtr(C, "base", base).freeDecl(decl),
             .wasm => @fieldParentPtr(Wasm, "base", base).freeDecl(decl),
         }
     }
@@ -432,14 +435,14 @@ pub const File = struct {
     pub fn updateDeclExports(
         base: *File,
         module: *Module,
-        decl: *const Module.Decl,
+        decl: *Module.Decl,
         exports: []const *Module.Export,
     ) !void {
         switch (base.tag) {
             .coff => return @fieldParentPtr(Coff, "base", base).updateDeclExports(module, decl, exports),
             .elf => return @fieldParentPtr(Elf, "base", base).updateDeclExports(module, decl, exports),
             .macho => return @fieldParentPtr(MachO, "base", base).updateDeclExports(module, decl, exports),
-            .c => return {},
+            .c => return @fieldParentPtr(C, "base", base).updateDeclExports(module, decl, exports),
             .wasm => return @fieldParentPtr(Wasm, "base", base).updateDeclExports(module, decl, exports),
         }
     }
diff --git a/src/link/C.zig b/src/link/C.zig
index 5f38c9324f..10b98b854c 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -11,45 +11,28 @@ const trace = @import("../tracy.zig").trace;
 const C = @This();
 
 pub const base_tag: link.File.Tag = .c;
+pub const zig_h = @embedFile("C/zig.h");
 
-pub const Header = struct {
-    buf: std.ArrayList(u8),
-    emit_loc: ?Compilation.EmitLoc,
-
-    pub fn init(allocator: *Allocator, emit_loc: ?Compilation.EmitLoc) Header {
-        return .{
-            .buf = std.ArrayList(u8).init(allocator),
-            .emit_loc = emit_loc,
-        };
-    }
-
-    pub fn flush(self: *const Header, writer: anytype) !void {
-        const tracy = trace(@src());
-        defer tracy.end();
+base: link.File,
 
-        try writer.writeAll(@embedFile("cbe.h"));
-        if (self.buf.items.len > 0) {
-            try writer.print("{s}", .{self.buf.items});
-        }
-    }
+/// Per-declaration data. For functions this is the body, and
+/// the forward declaration is stored in the FnBlock.
+pub const DeclBlock = struct {
+    code: std.ArrayListUnmanaged(u8),
 
-    pub fn deinit(self: *Header) void {
-        self.buf.deinit();
-        self.* = undefined;
-    }
+    pub const empty: DeclBlock = .{
+        .code = .{},
+    };
 };
 
-base: link.File,
-
-path: []const u8,
+/// Per-function data.
+pub const FnBlock = struct {
+    fwd_decl: std.ArrayListUnmanaged(u8),
 
-// These are only valid during a flush()!
-header: Header,
-constants: std.ArrayList(u8),
-main: std.ArrayList(u8),
-called: std.StringHashMap(void),
-
-error_msg: *Compilation.ErrorMsg = undefined,
+    pub const empty: FnBlock = .{
+        .fwd_decl = .{},
+    };
+};
 
 pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*C {
     assert(options.object_format == .c);
@@ -57,6 +40,14 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
     if (options.use_llvm) return error.LLVMHasNoCBackend;
     if (options.use_lld) return error.LLDHasNoCBackend;
 
+    const file = try options.emit.?.directory.handle.createFile(sub_path, .{
+        .truncate = true,
+        .mode = link.determineMode(options),
+    });
+    errdefer file.close();
+
+    try file.writeAll(zig_h);
+
     var c_file = try allocator.create(C);
     errdefer allocator.destroy(c_file);
 
@@ -64,25 +55,75 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
         .base = .{
             .tag = .c,
             .options = options,
-            .file = null,
+            .file = file,
             .allocator = allocator,
         },
-        .main = undefined,
-        .header = undefined,
-        .constants = undefined,
-        .called = undefined,
-        .path = sub_path,
     };
 
     return c_file;
 }
 
-pub fn fail(self: *C, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } {
-    self.error_msg = try Compilation.ErrorMsg.create(self.base.allocator, src, format, args);
-    return error.AnalysisFail;
+pub fn deinit(self: *C) void {
+    const module = self.base.options.module orelse return;
+    for (module.decl_table.items()) |entry| {
+        self.freeDecl(entry.value);
+    }
+}
+
+pub fn allocateDeclIndexes(self: *C, decl: *Module.Decl) !void {}
+
+pub fn freeDecl(self: *C, decl: *Module.Decl) void {
+    decl.link.c.code.deinit(self.base.allocator);
+    decl.fn_link.c.fwd_decl.deinit(self.base.allocator);
+}
+
+pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const fwd_decl = &decl.fn_link.c.fwd_decl;
+    const code = &decl.link.c.code;
+    fwd_decl.shrinkRetainingCapacity(0);
+    code.shrinkRetainingCapacity(0);
+
+    var object: codegen.Object = .{
+        .dg = .{
+            .module = module,
+            .error_msg = null,
+            .decl = decl,
+            .fwd_decl = fwd_decl.toManaged(module.gpa),
+        },
+        .gpa = module.gpa,
+        .code = code.toManaged(module.gpa),
+        .value_map = codegen.CValueMap.init(module.gpa),
+    };
+    defer object.value_map.deinit();
+    defer object.code.deinit();
+    defer object.dg.fwd_decl.deinit();
+
+    codegen.genDecl(&object) catch |err| switch (err) {
+        error.AnalysisFail => {},
+        else => |e| return e,
+    };
+    // The code may populate this error without returning error.AnalysisFail.
+    if (object.dg.error_msg) |msg| {
+        try module.failed_decls.put(module.gpa, decl, msg);
+        return;
+    }
+
+    fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();
+    code.* = object.code.moveToUnmanaged();
+
+    // Free excess allocated memory for this Decl.
+    fwd_decl.shrink(module.gpa, fwd_decl.items.len);
+    code.shrink(module.gpa, code.items.len);
 }
 
-pub fn deinit(self: *C) void {}
+pub fn updateDeclLineNumber(self: *C, module: *Module, decl: *Module.Decl) !void {
+    // The C backend does not have the ability to fix line numbers without re-generating
+    // the entire Decl.
+    return self.updateDecl(module, decl);
+}
 
 pub fn flush(self: *C, comp: *Compilation) !void {
     return self.flushModule(comp);
@@ -92,41 +133,45 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
-    self.main = std.ArrayList(u8).init(self.base.allocator);
-    self.header = Header.init(self.base.allocator, null);
-    self.constants = std.ArrayList(u8).init(self.base.allocator);
-    self.called = std.StringHashMap(void).init(self.base.allocator);
-    defer self.main.deinit();
-    defer self.header.deinit();
-    defer self.constants.deinit();
-    defer self.called.deinit();
-
-    const module = self.base.options.module.?;
-    for (self.base.options.module.?.decl_table.entries.items) |kv| {
-        codegen.generate(self, module, kv.value) catch |err| {
-            if (err == error.AnalysisFail) {
-                try module.failed_decls.put(module.gpa, kv.value, self.error_msg);
-            }
-            return err;
-        };
-    }
+    const file = self.base.file.?;
 
-    const file = try self.base.options.emit.?.directory.handle.createFile(self.path, .{ .truncate = true, .read = true, .mode = link.determineMode(self.base.options) });
-    defer file.close();
+    // The header is written upon opening; here we truncate and seek to after the header.
+    // TODO: use writev
+    try file.seekTo(zig_h.len);
+    try file.setEndPos(zig_h.len);
 
-    const writer = file.writer();
-    try self.header.flush(writer);
-    if (self.header.buf.items.len > 0) {
-        try writer.writeByte('\n');
-    }
-    if (self.constants.items.len > 0) {
-        try writer.print("{s}\n", .{self.constants.items});
+    var buffered_writer = std.io.bufferedWriter(file.writer());
+    const writer = buffered_writer.writer();
+
+    const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
+
+    // Forward decls and non-functions first.
+    // TODO: use writev
+    for (module.decl_table.items()) |kv| {
+        const decl = kv.value;
+        const decl_tv = decl.typed_value.most_recent.typed_value;
+        if (decl_tv.val.castTag(.function)) |_| {
+            try writer.writeAll(decl.fn_link.c.fwd_decl.items);
+        } else {
+            try writer.writeAll(decl.link.c.code.items);
+        }
     }
-    if (self.main.items.len > 1) {
-        const last_two = self.main.items[self.main.items.len - 2 ..];
-        if (std.mem.eql(u8, last_two, "\n\n")) {
-            self.main.items.len -= 1;
+
+    // Now the function bodies.
+    for (module.decl_table.items()) |kv| {
+        const decl = kv.value;
+        const decl_tv = decl.typed_value.most_recent.typed_value;
+        if (decl_tv.val.castTag(.function)) |_| {
+            try writer.writeAll(decl.link.c.code.items);
         }
     }
-    try writer.writeAll(self.main.items);
+
+    try buffered_writer.flush();
 }
+
+pub fn updateDeclExports(
+    self: *C,
+    module: *Module,
+    decl: *Module.Decl,
+    exports: []const *Module.Export,
+) !void {}
diff --git a/src/link/C/zig.h b/src/link/C/zig.h
new file mode 100644
index 0000000000..49f97210eb
--- /dev/null
+++ b/src/link/C/zig.h
@@ -0,0 +1,45 @@
+#if __STDC_VERSION__ >= 199901L
+#include <stdbool.h>
+#else
+#define bool unsigned char
+#define true 1
+#define false 0
+#endif
+
+#if __STDC_VERSION__ >= 201112L
+#define zig_noreturn _Noreturn
+#elif __GNUC__
+#define zig_noreturn __attribute__ ((noreturn))
+#elif _MSC_VER
+#define zig_noreturn __declspec(noreturn)
+#else
+#define zig_noreturn
+#endif
+
+#if defined(__GNUC__)
+#define zig_unreachable() __builtin_unreachable()
+#else
+#define zig_unreachable()
+#endif
+
+#if defined(_MSC_VER)
+#define zig_breakpoint __debugbreak()
+#else
+#if defined(__MINGW32__) || defined(__MINGW64__)
+#define zig_breakpoint __debugbreak()
+#elif defined(__clang__)
+#define zig_breakpoint __builtin_debugtrap()
+#elif defined(__GNUC__)
+#define zig_breakpoint __builtin_trap()
+#elif defined(__i386__) || defined(__x86_64__)
+#define zig_breakpoint __asm__ volatile("int $0x03");
+#else
+#define zig_breakpoint raise(SIGTRAP)
+#endif
+#endif
+
+#include <stdint.h>
+#define int128_t __int128
+#define uint128_t unsigned __int128
+#include <string.h>
+
diff --git a/src/link/cbe.h b/src/link/cbe.h
deleted file mode 100644
index 8452af8fbc..0000000000
--- a/src/link/cbe.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#if __STDC_VERSION__ >= 199901L
-#include <stdbool.h>
-#else
-#define bool unsigned char
-#define true 1
-#define false 0
-#endif
-
-#if __STDC_VERSION__ >= 201112L
-#define zig_noreturn _Noreturn
-#elif __GNUC__
-#define zig_noreturn __attribute__ ((noreturn))
-#elif _MSC_VER
-#define zig_noreturn __declspec(noreturn)
-#else
-#define zig_noreturn
-#endif
-
-#if defined(__GNUC__)
-#define zig_unreachable() __builtin_unreachable()
-#else
-#define zig_unreachable()
-#endif
-
-#if defined(_MSC_VER)
-#define zig_breakpoint __debugbreak()
-#else
-#if defined(__MINGW32__) || defined(__MINGW64__)
-#define zig_breakpoint __debugbreak()
-#elif defined(__clang__)
-#define zig_breakpoint __builtin_debugtrap()
-#elif defined(__GNUC__)
-#define zig_breakpoint __builtin_trap()
-#elif defined(__i386__) || defined(__x86_64__)
-#define zig_breakpoint __asm__ volatile("int $0x03");
-#else
-#define zig_breakpoint raise(SIGTRAP)
-#endif
-#endif
-
-#include <stdint.h>
-#define int128_t __int128
-#define uint128_t unsigned __int128
-#include <string.h>
diff --git a/src/test.zig b/src/test.zig
index 67a30f1f32..f630898189 100644
--- a/src/test.zig
+++ b/src/test.zig
@@ -13,7 +13,7 @@ const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_d
 const ThreadPool = @import("ThreadPool.zig");
 const CrossTarget = std.zig.CrossTarget;
 
-const c_header = @embedFile("link/cbe.h");
+const zig_h = link.File.C.zig_h;
 
 test "self-hosted" {
     var ctx = TestContext.init();
@@ -324,11 +324,11 @@ pub const TestContext = struct {
     }
 
     pub fn c(ctx: *TestContext, name: []const u8, target: CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
-        ctx.addC(name, target, .Zig).addCompareObjectFile(src, c_header ++ out);
+        ctx.addC(name, target, .Zig).addCompareObjectFile(src, zig_h ++ out);
     }
 
     pub fn h(ctx: *TestContext, name: []const u8, target: CrossTarget, src: [:0]const u8, comptime out: [:0]const u8) void {
-        ctx.addC(name, target, .Zig).addHeader(src, c_header ++ out);
+        ctx.addC(name, target, .Zig).addHeader(src, zig_h ++ out);
     }
 
     pub fn addCompareOutput(
@@ -700,11 +700,12 @@ pub const TestContext = struct {
                             },
                         }
                     }
-                    if (comp.bin_file.cast(link.File.C)) |c_file| {
-                        std.debug.print("Generated C: \n===============\n{s}\n\n===========\n\n", .{
-                            c_file.main.items,
-                        });
-                    }
+                    // TODO print generated C code
+                    //if (comp.bin_file.cast(link.File.C)) |c_file| {
+                    //    std.debug.print("Generated C: \n===============\n{s}\n\n===========\n\n", .{
+                    //        c_file.main.items,
+                    //    });
+                    //}
                     std.debug.print("Test failed.\n", .{});
                     std.process.exit(1);
                 }
diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig
index a740a8851a..9947c90f13 100644
--- a/test/stage2/cbe.zig
+++ b/test/stage2/cbe.zig
@@ -22,8 +22,6 @@ pub fn addCases(ctx: *TestContext) !void {
         , "hello world!" ++ std.cstr.line_sep);
 
         // Now change the message only
-        // TODO fix C backend not supporting updates
-        // https://github.com/ziglang/zig/issues/7589
         case.addCompareOutput(
             \\extern fn puts(s: [*:0]const u8) c_int;
             \\export fn main() c_int {
-- 
cgit v1.2.3


From 1a2dd85570a6439f82f7a0dd2cbf452198168c5a Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 5 Jan 2021 17:33:31 -0700
Subject: stage2: C backend: re-implement emit-h

and also mark functions as `extern "C"` as appropriate to support c++
compilers.
---
 src/Compilation.zig | 96 +++++++++++++++++++++++++++++++++++++++++------------
 src/Module.zig      | 73 +++++++++++++++++++++++++++++++++++-----
 src/codegen/c.zig   | 60 ++++++++++++++++++---------------
 src/link/C.zig      | 46 +++++++++++++++++++++++--
 src/link/C/zig.h    | 14 +++++---
 test/stage2/cbe.zig | 24 +++++++-------
 6 files changed, 239 insertions(+), 74 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index c654833270..552c550149 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -27,7 +27,6 @@ const Cache = @import("Cache.zig");
 const stage1 = @import("stage1.zig");
 const translate_c = @import("translate_c.zig");
 const c_codegen = @import("codegen/c.zig");
-const c_link = @import("link/C.zig");
 const ThreadPool = @import("ThreadPool.zig");
 const WaitGroup = @import("WaitGroup.zig");
 const libtsan = @import("libtsan.zig");
@@ -162,6 +161,8 @@ pub const CSourceFile = struct {
 const Job = union(enum) {
     /// Write the machine code for a Decl to the output file.
     codegen_decl: *Module.Decl,
+    /// Render the .h file snippet for the Decl.
+    emit_h_decl: *Module.Decl,
     /// The Decl needs to be analyzed and possibly export itself.
     /// It may have already be analyzed, or it may have been determined
     /// to be outdated; in this case perform semantic analysis again.
@@ -1312,9 +1313,14 @@ pub fn update(self: *Compilation) !void {
 
     // This is needed before reading the error flags.
     try self.bin_file.flush(self);
-
     self.link_error_flags = self.bin_file.errorFlags();
 
+    if (!use_stage1) {
+        if (self.bin_file.options.module) |module| {
+            try link.File.C.flushEmitH(module);
+        }
+    }
+
     // If there are any errors, we anticipate the source files being loaded
     // to report error messages. Otherwise we unload all source files to save memory.
     if (self.totalErrorCount() == 0 and !self.keep_source_files_loaded) {
@@ -1340,7 +1346,8 @@ pub fn totalErrorCount(self: *Compilation) usize {
     var total: usize = self.failed_c_objects.items().len;
 
     if (self.bin_file.options.module) |module| {
-        total += module.failed_decls.items().len +
+        total += module.failed_decls.count() +
+            module.emit_h_failed_decls.count() +
             module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
@@ -1379,6 +1386,12 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
             const source = try decl.scope.getSource(module);
             try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
         }
+        for (module.emit_h_failed_decls.items()) |entry| {
+            const decl = entry.key;
+            const err_msg = entry.value;
+            const source = try decl.scope.getSource(module);
+            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+        }
         for (module.failed_exports.items()) |entry| {
             const decl = entry.key.owner_decl;
             const err_msg = entry.value;
@@ -1476,27 +1489,68 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
 
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
 
-                self.bin_file.updateDecl(module, decl) catch |err| {
-                    switch (err) {
-                        error.OutOfMemory => return error.OutOfMemory,
-                        error.AnalysisFail => {
-                            decl.analysis = .codegen_failure;
-                        },
-                        else => {
-                            try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                            module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
-                                module.gpa,
-                                decl.src(),
-                                "unable to codegen: {s}",
-                                .{@errorName(err)},
-                            ));
-                            decl.analysis = .codegen_failure_retryable;
-                        },
-                    }
-                    return;
+                self.bin_file.updateDecl(module, decl) catch |err| switch (err) {
+                    error.OutOfMemory => return error.OutOfMemory,
+                    error.AnalysisFail => {
+                        decl.analysis = .codegen_failure;
+                        continue;
+                    },
+                    else => {
+                        try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
+                        module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                            module.gpa,
+                            decl.src(),
+                            "unable to codegen: {s}",
+                            .{@errorName(err)},
+                        ));
+                        decl.analysis = .codegen_failure_retryable;
+                        continue;
+                    },
                 };
             },
         },
+        .emit_h_decl => |decl| switch (decl.analysis) {
+            .unreferenced => unreachable,
+            .in_progress => unreachable,
+            .outdated => unreachable,
+
+            .sema_failure,
+            .dependency_failure,
+            .sema_failure_retryable,
+            => continue,
+
+            // emit-h only requires semantic analysis of the Decl to be complete,
+            // it does not depend on machine code generation to succeed.
+            .codegen_failure, .codegen_failure_retryable, .complete => {
+                if (build_options.omit_stage2)
+                    @panic("sadly stage2 is omitted from this build to save memory on the CI server");
+                const module = self.bin_file.options.module.?;
+                const emit_loc = module.emit_h.?;
+                const tv = decl.typed_value.most_recent.typed_value;
+                const emit_h = decl.getEmitH(module);
+                const fwd_decl = &emit_h.fwd_decl;
+                fwd_decl.shrinkRetainingCapacity(0);
+
+                var dg: c_codegen.DeclGen = .{
+                    .module = module,
+                    .error_msg = null,
+                    .decl = decl,
+                    .fwd_decl = fwd_decl.toManaged(module.gpa),
+                };
+                defer dg.fwd_decl.deinit();
+
+                c_codegen.genHeader(&dg) catch |err| switch (err) {
+                    error.AnalysisFail => {
+                        try module.emit_h_failed_decls.put(module.gpa, decl, dg.error_msg.?);
+                        continue;
+                    },
+                    else => |e| return e,
+                };
+
+                fwd_decl.* = dg.fwd_decl.moveToUnmanaged();
+                fwd_decl.shrink(module.gpa, fwd_decl.items.len);
+            },
+        },
         .analyze_decl => |decl| {
             if (build_options.omit_stage2)
                 @panic("sadly stage2 is omitted from this build to save memory on the CI server");
diff --git a/src/Module.zig b/src/Module.zig
index f1cec82680..6395a7f3a5 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -57,6 +57,10 @@ decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_has
 /// Note that a Decl can succeed but the Fn it represents can fail. In this case,
 /// a Decl can have a failed_decls entry but have analysis status of success.
 failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
+/// When emit_h is non-null, each Decl gets one more compile error slot for
+/// emit-h failing for that Decl. This table is also how we tell if a Decl has
+/// failed emit-h or succeeded.
+emit_h_failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator.
 failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{},
@@ -116,6 +120,13 @@ pub const Export = struct {
     },
 };
 
+/// When Module emit_h field is non-null, each Decl is allocated via this struct, so that
+/// there can be EmitH state attached to each Decl.
+pub const DeclPlusEmitH = struct {
+    decl: Decl,
+    emit_h: EmitH,
+};
+
 pub const Decl = struct {
     /// This name is relative to the containing namespace of the decl. It uses a null-termination
     /// to save bytes, since there can be a lot of decls in a compilation. The null byte is not allowed
@@ -204,14 +215,21 @@ pub const Decl = struct {
     /// stage1 compiler giving me: `error: struct 'Module.Decl' depends on itself`
     pub const DepsTable = std.ArrayHashMapUnmanaged(*Decl, void, std.array_hash_map.getAutoHashFn(*Decl), std.array_hash_map.getAutoEqlFn(*Decl), false);
 
-    pub fn destroy(self: *Decl, gpa: *Allocator) void {
+    pub fn destroy(self: *Decl, module: *Module) void {
+        const gpa = module.gpa;
         gpa.free(mem.spanZ(self.name));
         if (self.typedValueManaged()) |tvm| {
             tvm.deinit(gpa);
         }
         self.dependants.deinit(gpa);
         self.dependencies.deinit(gpa);
-        gpa.destroy(self);
+        if (module.emit_h != null) {
+            const decl_plus_emit_h = @fieldParentPtr(DeclPlusEmitH, "decl", self);
+            decl_plus_emit_h.emit_h.fwd_decl.deinit(gpa);
+            gpa.destroy(decl_plus_emit_h);
+        } else {
+            gpa.destroy(self);
+        }
     }
 
     pub fn src(self: Decl) usize {
@@ -277,6 +295,12 @@ pub const Decl = struct {
         return self.scope.cast(Scope.Container).?.file_scope;
     }
 
+    pub fn getEmitH(decl: *Decl, module: *Module) *EmitH {
+        assert(module.emit_h != null);
+        const decl_plus_emit_h = @fieldParentPtr(DeclPlusEmitH, "decl", decl);
+        return &decl_plus_emit_h.emit_h;
+    }
+
     fn removeDependant(self: *Decl, other: *Decl) void {
         self.dependants.removeAssertDiscard(other);
     }
@@ -286,6 +310,11 @@ pub const Decl = struct {
     }
 };
 
+/// This state is attached to every Decl when Module emit_h is non-null.
+pub const EmitH = struct {
+    fwd_decl: std.ArrayListUnmanaged(u8) = .{},
+};
+
 /// Fn struct memory is owned by the Decl's TypedValue.Managed arena allocator.
 /// Extern functions do not have this data structure; they are represented by
 /// the `Decl` only, with a `Value` tag of `extern_fn`.
@@ -883,7 +912,7 @@ pub fn deinit(self: *Module) void {
     self.deletion_set.deinit(gpa);
 
     for (self.decl_table.items()) |entry| {
-        entry.value.destroy(gpa);
+        entry.value.destroy(self);
     }
     self.decl_table.deinit(gpa);
 
@@ -892,6 +921,11 @@ pub fn deinit(self: *Module) void {
     }
     self.failed_decls.deinit(gpa);
 
+    for (self.emit_h_failed_decls.items()) |entry| {
+        entry.value.destroy(gpa);
+    }
+    self.emit_h_failed_decls.deinit(gpa);
+
     for (self.failed_files.items()) |entry| {
         entry.value.destroy(gpa);
     }
@@ -1150,6 +1184,10 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 try self.comp.bin_file.allocateDeclIndexes(decl);
                 try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
 
+                if (type_changed and self.emit_h != null) {
+                    try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
+                }
+
                 return type_changed;
             };
 
@@ -1269,6 +1307,9 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 // increasing how many computations can be done in parallel.
                 try self.comp.bin_file.allocateDeclIndexes(decl);
                 try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
+                if (type_changed and self.emit_h != null) {
+                    try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
+                }
             } else if (!prev_is_inline and prev_type_has_bits) {
                 self.comp.bin_file.freeDecl(decl);
             }
@@ -1837,9 +1878,13 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     if (self.failed_decls.remove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
+    if (self.emit_h_failed_decls.remove(decl)) |entry| {
+        entry.value.destroy(self.gpa);
+    }
     self.deleteDeclExports(decl);
     self.comp.bin_file.freeDecl(decl);
-    decl.destroy(self.gpa);
+
+    decl.destroy(self);
 }
 
 /// Delete all the Export objects that are caused by this Decl. Re-analysis of
@@ -1923,16 +1968,28 @@ fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     if (self.failed_decls.remove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
+    if (self.emit_h_failed_decls.remove(decl)) |entry| {
+        entry.value.destroy(self.gpa);
+    }
     decl.analysis = .outdated;
 }
 
 fn allocateNewDecl(
-    self: *Module,
+    mod: *Module,
     scope: *Scope,
     src_index: usize,
     contents_hash: std.zig.SrcHash,
 ) !*Decl {
-    const new_decl = try self.gpa.create(Decl);
+    // If we have emit-h then we must allocate a bigger structure to store the emit-h state.
+    const new_decl: *Decl = if (mod.emit_h != null) blk: {
+        const parent_struct = try mod.gpa.create(DeclPlusEmitH);
+        parent_struct.* = .{
+            .emit_h = .{},
+            .decl = undefined,
+        };
+        break :blk &parent_struct.decl;
+    } else try mod.gpa.create(Decl);
+
     new_decl.* = .{
         .name = "",
         .scope = scope.namespace(),
@@ -1941,14 +1998,14 @@ fn allocateNewDecl(
         .analysis = .unreferenced,
         .deletion_flag = false,
         .contents_hash = contents_hash,
-        .link = switch (self.comp.bin_file.tag) {
+        .link = switch (mod.comp.bin_file.tag) {
             .coff => .{ .coff = link.File.Coff.TextBlock.empty },
             .elf => .{ .elf = link.File.Elf.TextBlock.empty },
             .macho => .{ .macho = link.File.MachO.TextBlock.empty },
             .c => .{ .c = link.File.C.DeclBlock.empty },
             .wasm => .{ .wasm = {} },
         },
-        .fn_link = switch (self.comp.bin_file.tag) {
+        .fn_link = switch (mod.comp.bin_file.tag) {
             .coff => .{ .coff = {} },
             .elf => .{ .elf = link.File.Elf.SrcFn.empty },
             .macho => .{ .macho = link.File.MachO.SrcFn.empty },
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 5e274e0351..3de79e2a3d 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -9,6 +9,7 @@ const Compilation = @import("../Compilation.zig");
 const Inst = @import("../ir.zig").Inst;
 const Value = @import("../value.zig").Value;
 const Type = @import("../type.zig").Type;
+const TypedValue = @import("../TypedValue.zig");
 const C = link.File.C;
 const Decl = Module.Decl;
 const trace = @import("../tracy.zig").trace;
@@ -109,7 +110,7 @@ pub const Object = struct {
 };
 
 /// This data is available both when outputting .c code and when outputting an .h file.
-const DeclGen = struct {
+pub const DeclGen = struct {
     module: *Module,
     decl: *Decl,
     fwd_decl: std.ArrayList(u8),
@@ -199,22 +200,11 @@ const DeclGen = struct {
         }
     }
 
-    fn renderFunctionSignature(dg: *DeclGen, w: Writer) !void {
-        const tv = dg.decl.typed_value.most_recent.typed_value;
-        // Determine whether the function is globally visible.
-        const is_global = blk: {
-            switch (tv.val.tag()) {
-                .extern_fn => break :blk true,
-                .function => {
-                    const func = tv.val.castTag(.function).?.data;
-                    break :blk dg.module.decl_exports.contains(func.owner_decl);
-                },
-                else => unreachable,
-            }
-        };
+    fn renderFunctionSignature(dg: *DeclGen, w: Writer, is_global: bool) !void {
         if (!is_global) {
             try w.writeAll("static ");
         }
+        const tv = dg.decl.typed_value.most_recent.typed_value;
         try dg.renderType(w, tv.ty.fnReturnType());
         const decl_name = mem.span(dg.decl.name);
         try w.print(" {s}(", .{decl_name});
@@ -302,6 +292,17 @@ const DeclGen = struct {
             }),
         }
     }
+
+    fn functionIsGlobal(dg: *DeclGen, tv: TypedValue) bool {
+        switch (tv.val.tag()) {
+            .extern_fn => return true,
+            .function => {
+                const func = tv.val.castTag(.function).?.data;
+                return dg.module.decl_exports.contains(func.owner_decl);
+            },
+            else => unreachable,
+        }
+    }
 };
 
 pub fn genDecl(o: *Object) !void {
@@ -311,15 +312,19 @@ pub fn genDecl(o: *Object) !void {
     const tv = o.dg.decl.typed_value.most_recent.typed_value;
 
     if (tv.val.castTag(.function)) |func_payload| {
+        const is_global = o.dg.functionIsGlobal(tv);
         const fwd_decl_writer = o.dg.fwd_decl.writer();
-        try o.dg.renderFunctionSignature(fwd_decl_writer);
+        if (is_global) {
+            try fwd_decl_writer.writeAll("ZIG_EXTERN_C ");
+        }
+        try o.dg.renderFunctionSignature(fwd_decl_writer, is_global);
         try fwd_decl_writer.writeAll(";\n");
 
         const func: *Module.Fn = func_payload.data;
         const instructions = func.body.instructions;
         const writer = o.code.writer();
         try writer.writeAll("\n");
-        try o.dg.renderFunctionSignature(writer);
+        try o.dg.renderFunctionSignature(writer, is_global);
         if (instructions.len == 0) {
             try writer.writeAll(" {}\n");
             return;
@@ -363,7 +368,8 @@ pub fn genDecl(o: *Object) !void {
         try writer.writeAll("}\n");
     } else if (tv.val.tag() == .extern_fn) {
         const writer = o.code.writer();
-        try o.dg.renderFunctionSignature(writer);
+        try writer.writeAll("ZIG_EXTERN_C ");
+        try o.dg.renderFunctionSignature(writer, true);
         try writer.writeAll(";\n");
     } else {
         const writer = o.code.writer();
@@ -381,20 +387,20 @@ pub fn genDecl(o: *Object) !void {
     }
 }
 
-pub fn genHeader(comp: *Compilation, dg: *DeclGen) error{ AnalysisFail, OutOfMemory }!void {
+pub fn genHeader(dg: *DeclGen) error{ AnalysisFail, OutOfMemory }!void {
     const tracy = trace(@src());
     defer tracy.end();
 
-    switch (decl.typed_value.most_recent.typed_value.ty.zigTypeTag()) {
+    const tv = dg.decl.typed_value.most_recent.typed_value;
+    const writer = dg.fwd_decl.writer();
+
+    switch (tv.ty.zigTypeTag()) {
         .Fn => {
-            dg.renderFunctionSignature() catch |err| switch (err) {
-                error.AnalysisFail => {
-                    try dg.module.failed_decls.put(dg.module.gpa, decl, dg.error_msg.?);
-                    dg.error_msg = null;
-                    return error.AnalysisFail;
-                },
-                else => |e| return e,
-            };
+            const is_global = dg.functionIsGlobal(tv);
+            if (is_global) {
+                try writer.writeAll("ZIG_EXTERN_C ");
+            }
+            try dg.renderFunctionSignature(writer, is_global);
             try dg.fwd_decl.appendSlice(";\n");
         },
         else => {},
diff --git a/src/link/C.zig b/src/link/C.zig
index 0bca77f25f..32d859f43e 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -130,8 +130,10 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
-    const module = self.base.options.module orelse
-        return error.LinkingWithoutZigSourceUnimplemented;
+    const module = self.base.options.module.?;
+
+    // This code path happens exclusively with -ofmt=c. The flush logic for
+    // emit-h is in `flushEmitH` below.
 
     // We collect a list of buffers to write, and write them all at once with pwritev 😎
     var all_buffers = std.ArrayList(std.os.iovec_const).init(comp.gpa);
@@ -187,6 +189,46 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     try file.pwritevAll(all_buffers.items, 0);
 }
 
+pub fn flushEmitH(module: *Module) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const emit_h_loc = module.emit_h orelse return;
+
+    // We collect a list of buffers to write, and write them all at once with pwritev 😎
+    var all_buffers = std.ArrayList(std.os.iovec_const).init(module.gpa);
+    defer all_buffers.deinit();
+
+    try all_buffers.ensureCapacity(module.decl_table.count() + 1);
+
+    var file_size: u64 = zig_h.len;
+    all_buffers.appendAssumeCapacity(.{
+        .iov_base = zig_h,
+        .iov_len = zig_h.len,
+    });
+
+    for (module.decl_table.items()) |kv| {
+        const emit_h = kv.value.getEmitH(module);
+        const buf = emit_h.fwd_decl.items;
+        all_buffers.appendAssumeCapacity(.{
+            .iov_base = buf.ptr,
+            .iov_len = buf.len,
+        });
+        file_size += buf.len;
+    }
+
+    const directory = emit_h_loc.directory orelse module.comp.local_cache_directory;
+    const file = try directory.handle.createFile(emit_h_loc.basename, .{
+        // We set the end position explicitly below; by not truncating the file, we possibly
+        // make it easier on the file system by doing 1 reallocation instead of two.
+        .truncate = false,
+    });
+    defer file.close();
+
+    try file.setEndPos(file_size);
+    try file.pwritevAll(all_buffers.items, 0);
+}
+
 pub fn updateDeclExports(
     self: *C,
     module: *Module,
diff --git a/src/link/C/zig.h b/src/link/C/zig.h
index fed799d348..640dfb2345 100644
--- a/src/link/C/zig.h
+++ b/src/link/C/zig.h
@@ -23,11 +23,17 @@
 #endif
 
 #if __STDC_VERSION__ >= 199901L
-#define zig_restrict restrict
+#define ZIG_RESTRICT restrict
 #elif defined(__GNUC__)
-#define zig_restrict __restrict
+#define ZIG_RESTRICT __restrict
 #else
-#define zig_restrict
+#define ZIG_RESTRICT
+#endif
+
+#ifdef __cplusplus
+#define ZIG_EXTERN_C extern "C"
+#else
+#define ZIG_EXTERN_C
 #endif
 
 #if defined(_MSC_VER)
@@ -48,5 +54,5 @@
 #include <stddef.h>
 #define int128_t __int128
 #define uint128_t unsigned __int128
-void *memcpy (void *zig_restrict, const void *zig_restrict, size_t);
+ZIG_EXTERN_C void *memcpy (void *ZIG_RESTRICT, const void *ZIG_RESTRICT, size_t);
 
diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig
index e66dabc147..9163ac1662 100644
--- a/test/stage2/cbe.zig
+++ b/test/stage2/cbe.zig
@@ -180,7 +180,7 @@ pub fn addCases(ctx: *TestContext) !void {
         \\    unreachable;
         \\}
     ,
-        \\zig_noreturn void _start(void);
+        \\ZIG_EXTERN_C zig_noreturn void _start(void);
         \\
         \\zig_noreturn void _start(void) {
         \\    zig_breakpoint();
@@ -191,37 +191,37 @@ pub fn addCases(ctx: *TestContext) !void {
     ctx.h("simple header", linux_x64,
         \\export fn start() void{}
     ,
-        \\void start(void);
+        \\ZIG_EXTERN_C void start(void);
         \\
     );
     ctx.h("header with single param function", linux_x64,
         \\export fn start(a: u8) void{}
     ,
-        \\void start(uint8_t arg0);
+        \\ZIG_EXTERN_C void start(uint8_t a0);
         \\
     );
     ctx.h("header with multiple param function", linux_x64,
         \\export fn start(a: u8, b: u8, c: u8) void{}
     ,
-        \\void start(uint8_t arg0, uint8_t arg1, uint8_t arg2);
+        \\ZIG_EXTERN_C void start(uint8_t a0, uint8_t a1, uint8_t a2);
         \\
     );
     ctx.h("header with u32 param function", linux_x64,
         \\export fn start(a: u32) void{}
     ,
-        \\void start(uint32_t arg0);
+        \\ZIG_EXTERN_C void start(uint32_t a0);
         \\
     );
     ctx.h("header with usize param function", linux_x64,
         \\export fn start(a: usize) void{}
     ,
-        \\void start(uintptr_t arg0);
+        \\ZIG_EXTERN_C void start(uintptr_t a0);
         \\
     );
     ctx.h("header with bool param function", linux_x64,
         \\export fn start(a: bool) void{}
     ,
-        \\void start(bool arg0);
+        \\ZIG_EXTERN_C void start(bool a0);
         \\
     );
     ctx.h("header with noreturn function", linux_x64,
@@ -229,7 +229,7 @@ pub fn addCases(ctx: *TestContext) !void {
         \\    unreachable;
         \\}
     ,
-        \\zig_noreturn void start(void);
+        \\ZIG_EXTERN_C zig_noreturn void start(void);
         \\
     );
     ctx.h("header with multiple functions", linux_x64,
@@ -237,15 +237,15 @@ pub fn addCases(ctx: *TestContext) !void {
         \\export fn b() void{}
         \\export fn c() void{}
     ,
-        \\void a(void);
-        \\void b(void);
-        \\void c(void);
+        \\ZIG_EXTERN_C void a(void);
+        \\ZIG_EXTERN_C void b(void);
+        \\ZIG_EXTERN_C void c(void);
         \\
     );
     ctx.h("header with multiple includes", linux_x64,
         \\export fn start(a: u32, b: usize) void{}
     ,
-        \\void start(uint32_t arg0, uintptr_t arg1);
+        \\ZIG_EXTERN_C void start(uint32_t a0, uintptr_t a1);
         \\
     );
 }
-- 
cgit v1.2.3


From 3e39d0c44fbf0cfb56ef0c00fc8ec7c0d5e1c1ec Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 5 Jan 2021 17:41:22 -0700
Subject: minor fixups from moving identifiers and files around

---
 CMakeLists.txt      | 2 +-
 src/Compilation.zig | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2ed25b93d2..ebb4f74ffa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -559,7 +559,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/link/MachO.zig"
     "${CMAKE_SOURCE_DIR}/src/link/MachO/Trie.zig"
     "${CMAKE_SOURCE_DIR}/src/link/Wasm.zig"
-    "${CMAKE_SOURCE_DIR}/src/link/cbe.h"
+    "${CMAKE_SOURCE_DIR}/src/link/C/zig.h"
     "${CMAKE_SOURCE_DIR}/src/link/msdos-stub.bin"
     "${CMAKE_SOURCE_DIR}/src/liveness.zig"
     "${CMAKE_SOURCE_DIR}/src/llvm_backend.zig"
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 552c550149..bf8fd73e93 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -3123,7 +3123,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         });
         break :blk try directory.join(arena, &[_][]const u8{bin_basename});
     } else "";
-    if (comp.emit_h != null) {
+    if (mod.emit_h != null) {
         log.warn("-femit-h is not available in the stage1 backend; no .h file will be produced", .{});
     }
     const emit_h_path = try stage1LocPath(arena, mod.emit_h, directory);
-- 
cgit v1.2.3


From ab5f7b51566ad2c8e635e81b8515a97206daf8ab Mon Sep 17 00:00:00 2001
From: g-w1 <jacoblevgw@gmail.com>
Date: Sat, 21 Nov 2020 21:12:33 -0500
Subject: stage2: add compile log statement

---
 src/Compilation.zig  | 12 ++++++++++++
 src/Module.zig       | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 src/astgen.zig       | 12 ++++++++++++
 src/zir.zig          | 23 ++++++++++++++++++++---
 src/zir_sema.zig     | 22 ++++++++++++++++++++++
 test/stage2/test.zig | 18 ++++++++++++++++++
 6 files changed, 135 insertions(+), 3 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index bf8fd73e93..4efb845a82 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1351,6 +1351,9 @@ pub fn totalErrorCount(self: *Compilation) usize {
             module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
+        for (module.compile_log_decls.items()) |entry| {
+            total += entry.value.items.len;
+        }
     }
 
     // The "no entry point found" error only counts if there are no other errors.
@@ -1407,6 +1410,15 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
             });
             try AllErrors.addPlain(&arena, &errors, msg);
         }
+        for (module.compile_log_decls.items()) |entry| {
+            const decl = entry.key;
+            const path = decl.scope.subFilePath();
+            const source = try decl.scope.getSource(module);
+            for (entry.value.items) |src_loc| {
+                const err_msg = ErrorMsg{ .byte_offset = src_loc, .msg = "found compile log statement" };
+                try AllErrors.add(&arena, &errors, path, source, err_msg);
+            }
+        }
     }
 
     if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) {
diff --git a/src/Module.zig b/src/Module.zig
index 6395a7f3a5..c53efb7a2d 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -61,6 +61,8 @@ failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
 /// emit-h failing for that Decl. This table is also how we tell if a Decl has
 /// failed emit-h or succeeded.
 emit_h_failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
+/// A Decl can have multiple compileLogs, but only one error, so we map a Decl to a the src locs of all the compileLogs
+compile_log_decls: std.AutoArrayHashMapUnmanaged(*Decl, ArrayListUnmanaged(usize)) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator.
 failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{},
@@ -936,6 +938,11 @@ pub fn deinit(self: *Module) void {
     }
     self.failed_exports.deinit(gpa);
 
+    for (self.compile_log_decls.items()) |*entry| {
+        entry.value.deinit(gpa);
+    }
+    self.compile_log_decls.deinit(gpa);
+
     for (self.decl_exports.items()) |entry| {
         const export_list = entry.value;
         gpa.free(export_list);
@@ -1881,6 +1888,9 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     if (self.emit_h_failed_decls.remove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
+    if (self.compile_log_decls.remove(decl)) |*entry| {
+        entry.value.deinit(self.gpa);
+    }
     self.deleteDeclExports(decl);
     self.comp.bin_file.freeDecl(decl);
 
@@ -1971,6 +1981,9 @@ fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     if (self.emit_h_failed_decls.remove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
+    if (self.compile_log_decls.remove(decl)) |*entry| {
+        entry.value.deinit(self.gpa);
+    }
     decl.analysis = .outdated;
 }
 
@@ -3151,6 +3164,44 @@ pub fn failNode(
     return self.fail(scope, src, format, args);
 }
 
+fn addCompileLog(self: *Module, decl: *Decl, src: usize) error{OutOfMemory}!void {
+    const entry = try self.compile_log_decls.getOrPutValue(self.gpa, decl, .{});
+    try entry.value.append(self.gpa, src);
+}
+
+pub fn failCompileLog(
+    self: *Module,
+    scope: *Scope,
+    src: usize,
+) InnerError!void {
+    switch (scope.tag) {
+        .decl => {
+            const decl = scope.cast(Scope.DeclAnalysis).?.decl;
+            try self.addCompileLog(decl, src);
+        },
+        .block => {
+            const block = scope.cast(Scope.Block).?;
+            try self.addCompileLog(block.decl, src);
+        },
+        .gen_zir => {
+            const gen_zir = scope.cast(Scope.GenZIR).?;
+            try self.addCompileLog(gen_zir.decl, src);
+        },
+        .local_val => {
+            const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir;
+            try self.addCompileLog(gen_zir.decl, src);
+        },
+        .local_ptr => {
+            const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir;
+            try self.addCompileLog(gen_zir.decl, src);
+        },
+        .zir_module,
+        .file,
+        .container,
+        => unreachable,
+    }
+}
+
 fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Compilation.ErrorMsg) InnerError {
     {
         errdefer err_msg.destroy(self.gpa);
diff --git a/src/astgen.zig b/src/astgen.zig
index 8275a05d77..7f5d596dda 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -2346,6 +2346,16 @@ fn typeOf(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.BuiltinCal
         items[param_i] = try expr(mod, scope, .none, param);
     return rlWrap(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.TypeOfPeer, .{ .items = items }, .{}));
 }
+fn compileLog(mod: *Module, scope: *Scope, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
+    const tree = scope.tree();
+    const arena = scope.arena();
+    const src = tree.token_locs[call.builtin_token].start;
+    const params = call.params();
+    var targets = try arena.alloc(*zir.Inst, params.len);
+    for (params) |param, param_i|
+        targets[param_i] = try expr(mod, scope, .none, param);
+    return addZIRInst(mod, scope, src, zir.Inst.CompileLog, .{ .to_log = targets }, .{});
+}
 
 fn builtinCall(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.BuiltinCall) InnerError!*zir.Inst {
     const tree = scope.tree();
@@ -2377,6 +2387,8 @@ fn builtinCall(mod: *Module, scope: *Scope, rl: ResultLoc, call: *ast.Node.Built
         return compileError(mod, scope, call);
     } else if (mem.eql(u8, builtin_name, "@setEvalBranchQuota")) {
         return setEvalBranchQuota(mod, scope, call);
+    } else if (mem.eql(u8, builtin_name, "@compileLog")) {
+        return compileLog(mod, scope, call);
     } else {
         return mod.failTok(scope, call.builtin_token, "invalid builtin function: '{s}'", .{builtin_name});
     }
diff --git a/src/zir.zig b/src/zir.zig
index 8019d0e030..63850d67db 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -127,9 +127,8 @@ pub const Inst = struct {
         coerce_to_ptr_elem,
         /// Emit an error message and fail compilation.
         compileerror,
-        /// Changes the maximum number of backwards branches that compile-time
-        /// code execution can use before giving up and making a compile error.
-        set_eval_branch_quota,
+        /// Log compile time variables and emit an error message.
+        compilelog,
         /// Conditional branch. Splits control flow based on a boolean condition value.
         condbr,
         /// Special case, has no textual representation.
@@ -223,6 +222,9 @@ pub const Inst = struct {
         @"return",
         /// Same as `return` but there is no operand; the operand is implicitly the void value.
         returnvoid,
+        /// Changes the maximum number of backwards branches that compile-time
+        /// code execution can use before giving up and making a compile error.
+        set_eval_branch_quota,
         /// Integer shift-left. Zeroes are shifted in from the right hand side.
         shl,
         /// Integer shift-right. Arithmetic or logical depending on the signedness of the integer type.
@@ -407,6 +409,7 @@ pub const Inst = struct {
                 .declval => DeclVal,
                 .declval_in_module => DeclValInModule,
                 .coerce_result_block_ptr => CoerceResultBlockPtr,
+                .compilelog => CompileLog,
                 .loop => Loop,
                 .@"const" => Const,
                 .str => Str,
@@ -540,6 +543,7 @@ pub const Inst = struct {
                 .typeof_peer,
                 .resolve_inferred_alloc,
                 .set_eval_branch_quota,
+                .compilelog,
                 => false,
 
                 .@"break",
@@ -723,6 +727,19 @@ pub const Inst = struct {
         kw_args: struct {},
     };
 
+    pub const CompileLog = struct {
+        pub const base_tag = Tag.compilelog;
+        base: Inst,
+
+        positionals: struct {
+            to_log: []*Inst,
+        },
+        kw_args: struct {
+            /// If we have seen it already so don't make another error
+            seen: bool = false,
+        },
+    };
+
     pub const Const = struct {
         pub const base_tag = Tag.@"const";
         base: Inst,
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index dbe10c4bde..596cd5b24e 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -57,6 +57,7 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
         .coerce_result_ptr => return analyzeInstCoerceResultPtr(mod, scope, old_inst.castTag(.coerce_result_ptr).?),
         .coerce_to_ptr_elem => return analyzeInstCoerceToPtrElem(mod, scope, old_inst.castTag(.coerce_to_ptr_elem).?),
         .compileerror => return analyzeInstCompileError(mod, scope, old_inst.castTag(.compileerror).?),
+        .compilelog => return analyzeInstCompileLog(mod, scope, old_inst.castTag(.compilelog).?),
         .@"const" => return analyzeInstConst(mod, scope, old_inst.castTag(.@"const").?),
         .dbg_stmt => return analyzeInstDbgStmt(mod, scope, old_inst.castTag(.dbg_stmt).?),
         .declref => return analyzeInstDeclRef(mod, scope, old_inst.castTag(.declref).?),
@@ -630,6 +631,27 @@ fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) In
     return mod.fail(scope, inst.base.src, "{s}", .{msg});
 }
 
+fn analyzeInstCompileLog(mod: *Module, scope: *Scope, inst: *zir.Inst.CompileLog) InnerError!*Inst {
+    std.debug.print("| ", .{});
+    for (inst.positionals.to_log) |item, i| {
+        const to_log = try resolveInst(mod, scope, item);
+        if (to_log.value()) |val| {
+            std.debug.print("{}", .{val});
+        } else {
+            std.debug.print("(runtime value)", .{});
+        }
+        if (i != inst.positionals.to_log.len - 1) std.debug.print(", ", .{});
+    }
+    std.debug.print("\n", .{});
+    if (!inst.kw_args.seen) {
+
+        // so that we do not give multiple compile errors if it gets evaled twice
+        inst.kw_args.seen = true;
+        try mod.failCompileLog(scope, inst.base.src);
+    }
+    return mod.constVoid(scope, inst.base.src);
+}
+
 fn analyzeInstArg(mod: *Module, scope: *Scope, inst: *zir.Inst.Arg) InnerError!*Inst {
     const tracy = trace(@src());
     defer tracy.end();
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index 6e25dc283b..829ab421ea 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -1243,6 +1243,24 @@ pub fn addCases(ctx: *TestContext) !void {
             \\}
         , &[_][]const u8{":3:9: error: redefinition of 'testing'"});
     }
+    ctx.compileError("compileLog", linux_x64,
+        \\export fn _start() noreturn {
+        \\  const b = true;
+        \\  var f: u32 = 1;
+        \\  @compileLog(b, 20, f, x);
+        \\  @compileLog(1000);
+        \\  var bruh: usize = true;
+        \\  unreachable;
+        \\}
+        \\fn x() void {}
+    , &[_][]const u8{
+        ":4:3: error: found compile log statement",
+        ":5:3: error: found compile log statement",
+        ":6:21: error: expected usize, found bool",
+    });
+    // TODO if this is here it invalidates the compile error checker:
+    // "| true, 20, (runtime value), (function)"
+    // "| 1000"
 
     {
         var case = ctx.obj("extern variable has no type", linux_x64);
-- 
cgit v1.2.3


From b1cfa923bee5210fd78c7508d1af92dde3361c8c Mon Sep 17 00:00:00 2001
From: Timon Kruiper <timonkruiper@gmail.com>
Date: Wed, 6 Jan 2021 01:27:06 +0100
Subject: stage2: rename and move files related to LLVM backend

---
 CMakeLists.txt                |   4 +-
 src/Compilation.zig           |   2 +-
 src/codegen/llvm.zig          | 722 ++++++++++++++++++++++++++++++++++++++++++
 src/codegen/llvm/bindings.zig | 571 +++++++++++++++++++++++++++++++++
 src/link.zig                  |   2 +-
 src/link/Coff.zig             |   2 +-
 src/link/Elf.zig              |   2 +-
 src/llvm_backend.zig          | 722 ------------------------------------------
 src/llvm_bindings.zig         | 574 ---------------------------------
 src/main.zig                  |   6 +-
 src/mingw.zig                 |   2 +-
 src/target.zig                |   2 +-
 test/stage2/llvm.zig          |  43 +++
 test/stage2/llvm_backend.zig  |  43 ---
 test/stage2/test.zig          |   2 +-
 15 files changed, 1348 insertions(+), 1351 deletions(-)
 create mode 100644 src/codegen/llvm.zig
 create mode 100644 src/codegen/llvm/bindings.zig
 delete mode 100644 src/llvm_backend.zig
 delete mode 100644 src/llvm_bindings.zig
 create mode 100644 test/stage2/llvm.zig
 delete mode 100644 test/stage2/llvm_backend.zig

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2ed25b93d2..eb4b465ef3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -541,6 +541,8 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/codegen/aarch64.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/arm.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/c.zig"
+    "${CMAKE_SOURCE_DIR}/src/codegen/llvm.zig"
+    "${CMAKE_SOURCE_DIR}/src/codegen/llvm/bindings.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/riscv64.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/spu-mk2.zig"
     "${CMAKE_SOURCE_DIR}/src/codegen/wasm.zig"
@@ -562,8 +564,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/link/cbe.h"
     "${CMAKE_SOURCE_DIR}/src/link/msdos-stub.bin"
     "${CMAKE_SOURCE_DIR}/src/liveness.zig"
-    "${CMAKE_SOURCE_DIR}/src/llvm_backend.zig"
-    "${CMAKE_SOURCE_DIR}/src/llvm_bindings.zig"
     "${CMAKE_SOURCE_DIR}/src/main.zig"
     "${CMAKE_SOURCE_DIR}/src/mingw.zig"
     "${CMAKE_SOURCE_DIR}/src/musl.zig"
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 9912520437..77f9f79bae 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2116,7 +2116,7 @@ pub fn addCCArgs(
         try argv.append("-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS");
     }
 
-    const llvm_triple = try @import("llvm_backend.zig").targetTriple(arena, target);
+    const llvm_triple = try @import("codegen/llvm.zig").targetTriple(arena, target);
     try argv.appendSlice(&[_][]const u8{ "-target", llvm_triple });
 
     switch (ext) {
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
new file mode 100644
index 0000000000..df97f7e0ef
--- /dev/null
+++ b/src/codegen/llvm.zig
@@ -0,0 +1,722 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+const Compilation = @import("../Compilation.zig");
+const llvm = @import("llvm/bindings.zig");
+const link = @import("../link.zig");
+const log = std.log.scoped(.codegen);
+
+const Module = @import("../Module.zig");
+const TypedValue = @import("../TypedValue.zig");
+const ir = @import("../ir.zig");
+const Inst = ir.Inst;
+
+const Value = @import("../value.zig").Value;
+const Type = @import("../type.zig").Type;
+
+pub fn targetTriple(allocator: *Allocator, target: std.Target) ![:0]u8 {
+    const llvm_arch = switch (target.cpu.arch) {
+        .arm => "arm",
+        .armeb => "armeb",
+        .aarch64 => "aarch64",
+        .aarch64_be => "aarch64_be",
+        .aarch64_32 => "aarch64_32",
+        .arc => "arc",
+        .avr => "avr",
+        .bpfel => "bpfel",
+        .bpfeb => "bpfeb",
+        .hexagon => "hexagon",
+        .mips => "mips",
+        .mipsel => "mipsel",
+        .mips64 => "mips64",
+        .mips64el => "mips64el",
+        .msp430 => "msp430",
+        .powerpc => "powerpc",
+        .powerpc64 => "powerpc64",
+        .powerpc64le => "powerpc64le",
+        .r600 => "r600",
+        .amdgcn => "amdgcn",
+        .riscv32 => "riscv32",
+        .riscv64 => "riscv64",
+        .sparc => "sparc",
+        .sparcv9 => "sparcv9",
+        .sparcel => "sparcel",
+        .s390x => "s390x",
+        .tce => "tce",
+        .tcele => "tcele",
+        .thumb => "thumb",
+        .thumbeb => "thumbeb",
+        .i386 => "i386",
+        .x86_64 => "x86_64",
+        .xcore => "xcore",
+        .nvptx => "nvptx",
+        .nvptx64 => "nvptx64",
+        .le32 => "le32",
+        .le64 => "le64",
+        .amdil => "amdil",
+        .amdil64 => "amdil64",
+        .hsail => "hsail",
+        .hsail64 => "hsail64",
+        .spir => "spir",
+        .spir64 => "spir64",
+        .kalimba => "kalimba",
+        .shave => "shave",
+        .lanai => "lanai",
+        .wasm32 => "wasm32",
+        .wasm64 => "wasm64",
+        .renderscript32 => "renderscript32",
+        .renderscript64 => "renderscript64",
+        .ve => "ve",
+        .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII,
+    };
+    // TODO Add a sub-arch for some architectures depending on CPU features.
+
+    const llvm_os = switch (target.os.tag) {
+        .freestanding => "unknown",
+        .ananas => "ananas",
+        .cloudabi => "cloudabi",
+        .dragonfly => "dragonfly",
+        .freebsd => "freebsd",
+        .fuchsia => "fuchsia",
+        .ios => "ios",
+        .kfreebsd => "kfreebsd",
+        .linux => "linux",
+        .lv2 => "lv2",
+        .macos => "macosx",
+        .netbsd => "netbsd",
+        .openbsd => "openbsd",
+        .solaris => "solaris",
+        .windows => "windows",
+        .haiku => "haiku",
+        .minix => "minix",
+        .rtems => "rtems",
+        .nacl => "nacl",
+        .cnk => "cnk",
+        .aix => "aix",
+        .cuda => "cuda",
+        .nvcl => "nvcl",
+        .amdhsa => "amdhsa",
+        .ps4 => "ps4",
+        .elfiamcu => "elfiamcu",
+        .tvos => "tvos",
+        .watchos => "watchos",
+        .mesa3d => "mesa3d",
+        .contiki => "contiki",
+        .amdpal => "amdpal",
+        .hermit => "hermit",
+        .hurd => "hurd",
+        .wasi => "wasi",
+        .emscripten => "emscripten",
+        .uefi => "windows",
+        .other => "unknown",
+    };
+
+    const llvm_abi = switch (target.abi) {
+        .none => "unknown",
+        .gnu => "gnu",
+        .gnuabin32 => "gnuabin32",
+        .gnuabi64 => "gnuabi64",
+        .gnueabi => "gnueabi",
+        .gnueabihf => "gnueabihf",
+        .gnux32 => "gnux32",
+        .code16 => "code16",
+        .eabi => "eabi",
+        .eabihf => "eabihf",
+        .android => "android",
+        .musl => "musl",
+        .musleabi => "musleabi",
+        .musleabihf => "musleabihf",
+        .msvc => "msvc",
+        .itanium => "itanium",
+        .cygnus => "cygnus",
+        .coreclr => "coreclr",
+        .simulator => "simulator",
+        .macabi => "macabi",
+    };
+
+    return std.fmt.allocPrintZ(allocator, "{s}-unknown-{s}-{s}", .{ llvm_arch, llvm_os, llvm_abi });
+}
+
+pub const LLVMIRModule = struct {
+    module: *Module,
+    llvm_module: *const llvm.Module,
+    context: *const llvm.Context,
+    target_machine: *const llvm.TargetMachine,
+    builder: *const llvm.Builder,
+
+    object_path: []const u8,
+
+    gpa: *Allocator,
+    err_msg: ?*Compilation.ErrorMsg = null,
+
+    // TODO: The fields below should really move into a different struct,
+    //       because they are only valid when generating a function
+
+    /// This stores the LLVM values used in a function, such that they can be
+    /// referred to in other instructions. This table is cleared before every function is generated.
+    func_inst_table: std.AutoHashMapUnmanaged(*Inst, *const llvm.Value) = .{},
+
+    /// These fields are used to refer to the LLVM value of the function paramaters in an Arg instruction.
+    args: []*const llvm.Value = &[_]*const llvm.Value{},
+    arg_index: usize = 0,
+
+    entry_block: *const llvm.BasicBlock = undefined,
+    /// This fields stores the last alloca instruction, such that we can append more alloca instructions
+    /// to the top of the function.
+    latest_alloca_inst: ?*const llvm.Value = null,
+
+    pub fn create(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*LLVMIRModule {
+        const self = try allocator.create(LLVMIRModule);
+        errdefer allocator.destroy(self);
+
+        const gpa = options.module.?.gpa;
+
+        const obj_basename = try std.zig.binNameAlloc(gpa, .{
+            .root_name = options.root_name,
+            .target = options.target,
+            .output_mode = .Obj,
+        });
+        defer gpa.free(obj_basename);
+
+        const o_directory = options.module.?.zig_cache_artifact_directory;
+        const object_path = try o_directory.join(gpa, &[_][]const u8{obj_basename});
+        errdefer gpa.free(object_path);
+
+        const context = llvm.Context.create();
+        errdefer context.dispose();
+
+        initializeLLVMTargets();
+
+        const root_nameZ = try gpa.dupeZ(u8, options.root_name);
+        defer gpa.free(root_nameZ);
+        const llvm_module = llvm.Module.createWithName(root_nameZ.ptr, context);
+        errdefer llvm_module.dispose();
+
+        const llvm_target_triple = try targetTriple(gpa, options.target);
+        defer gpa.free(llvm_target_triple);
+
+        var error_message: [*:0]const u8 = undefined;
+        var target: *const llvm.Target = undefined;
+        if (llvm.Target.getFromTriple(llvm_target_triple.ptr, &target, &error_message)) {
+            defer llvm.disposeMessage(error_message);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.print(
+                \\Zig is expecting LLVM to understand this target: '{s}'
+                \\However LLVM responded with: "{s}"
+                \\Zig is unable to continue. This is a bug in Zig:
+                \\https://github.com/ziglang/zig/issues/438
+                \\
+            ,
+                .{
+                    llvm_target_triple,
+                    error_message,
+                },
+            );
+            return error.InvalidLLVMTriple;
+        }
+
+        const opt_level: llvm.CodeGenOptLevel = if (options.optimize_mode == .Debug) .None else .Aggressive;
+        const target_machine = llvm.TargetMachine.create(
+            target,
+            llvm_target_triple.ptr,
+            "",
+            "",
+            opt_level,
+            .Static,
+            .Default,
+        );
+        errdefer target_machine.dispose();
+
+        const builder = context.createBuilder();
+        errdefer builder.dispose();
+
+        self.* = .{
+            .module = options.module.?,
+            .llvm_module = llvm_module,
+            .context = context,
+            .target_machine = target_machine,
+            .builder = builder,
+            .object_path = object_path,
+            .gpa = gpa,
+        };
+        return self;
+    }
+
+    pub fn deinit(self: *LLVMIRModule, allocator: *Allocator) void {
+        self.builder.dispose();
+        self.target_machine.dispose();
+        self.llvm_module.dispose();
+        self.context.dispose();
+
+        self.func_inst_table.deinit(self.gpa);
+        self.gpa.free(self.object_path);
+
+        allocator.destroy(self);
+    }
+
+    fn initializeLLVMTargets() void {
+        llvm.initializeAllTargets();
+        llvm.initializeAllTargetInfos();
+        llvm.initializeAllTargetMCs();
+        llvm.initializeAllAsmPrinters();
+        llvm.initializeAllAsmParsers();
+    }
+
+    pub fn flushModule(self: *LLVMIRModule, comp: *Compilation) !void {
+        if (comp.verbose_llvm_ir) {
+            const dump = self.llvm_module.printToString();
+            defer llvm.disposeMessage(dump);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.writeAll(std.mem.spanZ(dump));
+        }
+
+        {
+            var error_message: [*:0]const u8 = undefined;
+            // verifyModule always allocs the error_message even if there is no error
+            defer llvm.disposeMessage(error_message);
+
+            if (self.llvm_module.verify(.ReturnStatus, &error_message)) {
+                const stderr = std.io.getStdErr().outStream();
+                try stderr.print("broken LLVM module found: {s}\nThis is a bug in the Zig compiler.", .{error_message});
+                return error.BrokenLLVMModule;
+            }
+        }
+
+        const object_pathZ = try self.gpa.dupeZ(u8, self.object_path);
+        defer self.gpa.free(object_pathZ);
+
+        var error_message: [*:0]const u8 = undefined;
+        if (self.target_machine.emitToFile(
+            self.llvm_module,
+            object_pathZ.ptr,
+            .ObjectFile,
+            &error_message,
+        )) {
+            defer llvm.disposeMessage(error_message);
+
+            const stderr = std.io.getStdErr().outStream();
+            try stderr.print("LLVM failed to emit file: {s}\n", .{error_message});
+            return error.FailedToEmit;
+        }
+    }
+
+    pub fn updateDecl(self: *LLVMIRModule, module: *Module, decl: *Module.Decl) !void {
+        self.gen(module, decl) catch |err| switch (err) {
+            error.CodegenFail => {
+                decl.analysis = .codegen_failure;
+                try module.failed_decls.put(module.gpa, decl, self.err_msg.?);
+                self.err_msg = null;
+                return;
+            },
+            else => |e| return e,
+        };
+    }
+
+    fn gen(self: *LLVMIRModule, module: *Module, decl: *Module.Decl) !void {
+        const typed_value = decl.typed_value.most_recent.typed_value;
+        const src = decl.src();
+
+        log.debug("gen: {s} type: {}, value: {}", .{ decl.name, typed_value.ty, typed_value.val });
+
+        if (typed_value.val.castTag(.function)) |func_payload| {
+            const func = func_payload.data;
+
+            const llvm_func = try self.resolveLLVMFunction(func.owner_decl, src);
+
+            // This gets the LLVM values from the function and stores them in `self.args`.
+            const fn_param_len = func.owner_decl.typed_value.most_recent.typed_value.ty.fnParamLen();
+            var args = try self.gpa.alloc(*const llvm.Value, fn_param_len);
+            defer self.gpa.free(args);
+
+            for (args) |*arg, i| {
+                arg.* = llvm.getParam(llvm_func, @intCast(c_uint, i));
+            }
+            self.args = args;
+            self.arg_index = 0;
+
+            // Make sure no other LLVM values from other functions can be referenced
+            self.func_inst_table.clearRetainingCapacity();
+
+            // We remove all the basic blocks of a function to support incremental
+            // compilation!
+            // TODO: remove all basic blocks if functions can have more than one
+            if (llvm_func.getFirstBasicBlock()) |bb| {
+                bb.deleteBasicBlock();
+            }
+
+            self.entry_block = self.context.appendBasicBlock(llvm_func, "Entry");
+            self.builder.positionBuilderAtEnd(self.entry_block);
+            self.latest_alloca_inst = null;
+
+            const instructions = func.body.instructions;
+            for (instructions) |inst| {
+                const opt_llvm_val: ?*const llvm.Value = switch (inst.tag) {
+                    .add => try self.genAdd(inst.castTag(.add).?),
+                    .alloc => try self.genAlloc(inst.castTag(.alloc).?),
+                    .arg => try self.genArg(inst.castTag(.arg).?),
+                    .bitcast => try self.genBitCast(inst.castTag(.bitcast).?),
+                    .breakpoint => try self.genBreakpoint(inst.castTag(.breakpoint).?),
+                    .call => try self.genCall(inst.castTag(.call).?),
+                    .intcast => try self.genIntCast(inst.castTag(.intcast).?),
+                    .load => try self.genLoad(inst.castTag(.load).?),
+                    .not => try self.genNot(inst.castTag(.not).?),
+                    .ret => try self.genRet(inst.castTag(.ret).?),
+                    .retvoid => self.genRetVoid(inst.castTag(.retvoid).?),
+                    .store => try self.genStore(inst.castTag(.store).?),
+                    .sub => try self.genSub(inst.castTag(.sub).?),
+                    .unreach => self.genUnreach(inst.castTag(.unreach).?),
+                    .dbg_stmt => blk: {
+                        // TODO: implement debug info
+                        break :blk null;
+                    },
+                    else => |tag| return self.fail(src, "TODO implement LLVM codegen for Zir instruction: {}", .{tag}),
+                };
+                if (opt_llvm_val) |llvm_val| try self.func_inst_table.putNoClobber(self.gpa, inst, llvm_val);
+            }
+        } else if (typed_value.val.castTag(.extern_fn)) |extern_fn| {
+            _ = try self.resolveLLVMFunction(extern_fn.data, src);
+        } else {
+            _ = try self.resolveGlobalDecl(decl, src);
+        }
+    }
+
+    fn genCall(self: *LLVMIRModule, inst: *Inst.Call) !?*const llvm.Value {
+        if (inst.func.value()) |func_value| {
+            const fn_decl = if (func_value.castTag(.extern_fn)) |extern_fn|
+                extern_fn.data
+            else if (func_value.castTag(.function)) |func_payload|
+                func_payload.data.owner_decl
+            else
+                unreachable;
+
+            const zig_fn_type = fn_decl.typed_value.most_recent.typed_value.ty;
+            const llvm_fn = try self.resolveLLVMFunction(fn_decl, inst.base.src);
+
+            const num_args = inst.args.len;
+
+            const llvm_param_vals = try self.gpa.alloc(*const llvm.Value, num_args);
+            defer self.gpa.free(llvm_param_vals);
+
+            for (inst.args) |arg, i| {
+                llvm_param_vals[i] = try self.resolveInst(arg);
+            }
+
+            // TODO: LLVMBuildCall2 handles opaque function pointers, according to llvm docs
+            //       Do we need that?
+            const call = self.builder.buildCall(
+                llvm_fn,
+                if (num_args == 0) null else llvm_param_vals.ptr,
+                @intCast(c_uint, num_args),
+                "",
+            );
+
+            const return_type = zig_fn_type.fnReturnType();
+            if (return_type.tag() == .noreturn) {
+                _ = self.builder.buildUnreachable();
+            }
+
+            // No need to store the LLVM value if the return type is void or noreturn
+            if (!return_type.hasCodeGenBits()) return null;
+
+            return call;
+        } else {
+            return self.fail(inst.base.src, "TODO implement calling runtime known function pointer LLVM backend", .{});
+        }
+    }
+
+    fn genRetVoid(self: *LLVMIRModule, inst: *Inst.NoOp) ?*const llvm.Value {
+        _ = self.builder.buildRetVoid();
+        return null;
+    }
+
+    fn genRet(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
+        _ = self.builder.buildRet(try self.resolveInst(inst.operand));
+        return null;
+    }
+
+    fn genNot(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
+        return self.builder.buildNot(try self.resolveInst(inst.operand), "");
+    }
+
+    fn genUnreach(self: *LLVMIRModule, inst: *Inst.NoOp) ?*const llvm.Value {
+        _ = self.builder.buildUnreachable();
+        return null;
+    }
+
+    fn genAdd(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
+        const lhs = try self.resolveInst(inst.lhs);
+        const rhs = try self.resolveInst(inst.rhs);
+
+        if (!inst.base.ty.isInt())
+            return self.fail(inst.base.src, "TODO implement 'genAdd' for type {}", .{inst.base.ty});
+
+        return if (inst.base.ty.isSignedInt())
+            self.builder.buildNSWAdd(lhs, rhs, "")
+        else
+            self.builder.buildNUWAdd(lhs, rhs, "");
+    }
+
+    fn genSub(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
+        const lhs = try self.resolveInst(inst.lhs);
+        const rhs = try self.resolveInst(inst.rhs);
+
+        if (!inst.base.ty.isInt())
+            return self.fail(inst.base.src, "TODO implement 'genSub' for type {}", .{inst.base.ty});
+
+        return if (inst.base.ty.isSignedInt())
+            self.builder.buildNSWSub(lhs, rhs, "")
+        else
+            self.builder.buildNUWSub(lhs, rhs, "");
+    }
+
+    fn genIntCast(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
+        const val = try self.resolveInst(inst.operand);
+
+        const signed = inst.base.ty.isSignedInt();
+        // TODO: Should we use intcast here or just a simple bitcast?
+        //       LLVM does truncation vs bitcast (+signed extension) in the intcast depending on the sizes
+        return self.builder.buildIntCast2(val, try self.getLLVMType(inst.base.ty, inst.base.src), signed, "");
+    }
+
+    fn genBitCast(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
+        const val = try self.resolveInst(inst.operand);
+        const dest_type = try self.getLLVMType(inst.base.ty, inst.base.src);
+
+        return self.builder.buildBitCast(val, dest_type, "");
+    }
+
+    fn genArg(self: *LLVMIRModule, inst: *Inst.Arg) !?*const llvm.Value {
+        const arg_val = self.args[self.arg_index];
+        self.arg_index += 1;
+
+        const ptr_val = self.buildAlloca(try self.getLLVMType(inst.base.ty, inst.base.src));
+        _ = self.builder.buildStore(arg_val, ptr_val);
+        return self.builder.buildLoad(ptr_val, "");
+    }
+
+    fn genAlloc(self: *LLVMIRModule, inst: *Inst.NoOp) !?*const llvm.Value {
+        // buildAlloca expects the pointee type, not the pointer type, so assert that
+        // a Payload.PointerSimple is passed to the alloc instruction.
+        const pointee_type = inst.base.ty.castPointer().?.data;
+
+        // TODO: figure out a way to get the name of the var decl.
+        // TODO: set alignment and volatile
+        return self.buildAlloca(try self.getLLVMType(pointee_type, inst.base.src));
+    }
+
+    /// Use this instead of builder.buildAlloca, because this function makes sure to
+    /// put the alloca instruction at the top of the function!
+    fn buildAlloca(self: *LLVMIRModule, t: *const llvm.Type) *const llvm.Value {
+        if (self.latest_alloca_inst) |latest_alloc| {
+            // builder.positionBuilder adds it before the instruction,
+            // but we want to put it after the last alloca instruction.
+            self.builder.positionBuilder(self.entry_block, latest_alloc.getNextInstruction().?);
+        } else {
+            // There might have been other instructions emitted before the
+            // first alloca has been generated. However the alloca should still
+            // be first in the function.
+            if (self.entry_block.getFirstInstruction()) |first_inst| {
+                self.builder.positionBuilder(self.entry_block, first_inst);
+            }
+        }
+        defer self.builder.positionBuilderAtEnd(self.entry_block);
+
+        const val = self.builder.buildAlloca(t, "");
+        self.latest_alloca_inst = val;
+        return val;
+    }
+
+    fn genStore(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
+        const val = try self.resolveInst(inst.rhs);
+        const ptr = try self.resolveInst(inst.lhs);
+        _ = self.builder.buildStore(val, ptr);
+        return null;
+    }
+
+    fn genLoad(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
+        const ptr_val = try self.resolveInst(inst.operand);
+        return self.builder.buildLoad(ptr_val, "");
+    }
+
+    fn genBreakpoint(self: *LLVMIRModule, inst: *Inst.NoOp) !?*const llvm.Value {
+        const llvn_fn = self.getIntrinsic("llvm.debugtrap");
+        _ = self.builder.buildCall(llvn_fn, null, 0, "");
+        return null;
+    }
+
+    fn getIntrinsic(self: *LLVMIRModule, name: []const u8) *const llvm.Value {
+        const id = llvm.lookupIntrinsicID(name.ptr, name.len);
+        assert(id != 0);
+        // TODO: add support for overload intrinsics by passing the prefix of the intrinsic
+        //       to `lookupIntrinsicID` and then passing the correct types to
+        //       `getIntrinsicDeclaration`
+        return self.llvm_module.getIntrinsicDeclaration(id, null, 0);
+    }
+
+    fn resolveInst(self: *LLVMIRModule, inst: *ir.Inst) !*const llvm.Value {
+        if (inst.value()) |val| {
+            return self.genTypedValue(inst.src, .{ .ty = inst.ty, .val = val });
+        }
+        if (self.func_inst_table.get(inst)) |value| return value;
+
+        return self.fail(inst.src, "TODO implement global llvm values (or the value is not in the func_inst_table table)", .{});
+    }
+
+    fn genTypedValue(self: *LLVMIRModule, src: usize, tv: TypedValue) error{ OutOfMemory, CodegenFail }!*const llvm.Value {
+        const llvm_type = try self.getLLVMType(tv.ty, src);
+
+        if (tv.val.isUndef())
+            return llvm_type.getUndef();
+
+        switch (tv.ty.zigTypeTag()) {
+            .Bool => return if (tv.val.toBool()) llvm_type.constAllOnes() else llvm_type.constNull(),
+            .Int => {
+                var bigint_space: Value.BigIntSpace = undefined;
+                const bigint = tv.val.toBigInt(&bigint_space);
+
+                if (bigint.eqZero()) return llvm_type.constNull();
+
+                if (bigint.limbs.len != 1) {
+                    return self.fail(src, "TODO implement bigger bigint", .{});
+                }
+                const llvm_int = llvm_type.constInt(bigint.limbs[0], false);
+                if (!bigint.positive) {
+                    return llvm.constNeg(llvm_int);
+                }
+                return llvm_int;
+            },
+            .Pointer => switch (tv.val.tag()) {
+                .decl_ref => {
+                    const decl = tv.val.castTag(.decl_ref).?.data;
+                    const val = try self.resolveGlobalDecl(decl, src);
+
+                    const usize_type = try self.getLLVMType(Type.initTag(.usize), src);
+
+                    // TODO: second index should be the index into the memory!
+                    var indices: [2]*const llvm.Value = .{
+                        usize_type.constNull(),
+                        usize_type.constNull(),
+                    };
+
+                    // TODO: consider using buildInBoundsGEP2 for opaque pointers
+                    return self.builder.buildInBoundsGEP(val, &indices, 2, "");
+                },
+                else => return self.fail(src, "TODO implement const of pointer type '{}'", .{tv.ty}),
+            },
+            .Array => {
+                if (tv.val.castTag(.bytes)) |payload| {
+                    const zero_sentinel = if (tv.ty.sentinel()) |sentinel| blk: {
+                        if (sentinel.tag() == .zero) break :blk true;
+                        return self.fail(src, "TODO handle other sentinel values", .{});
+                    } else false;
+
+                    return self.context.constString(payload.data.ptr, @intCast(c_uint, payload.data.len), !zero_sentinel);
+                } else {
+                    return self.fail(src, "TODO handle more array values", .{});
+                }
+            },
+            else => return self.fail(src, "TODO implement const of type '{}'", .{tv.ty}),
+        }
+    }
+
+    fn getLLVMType(self: *LLVMIRModule, t: Type, src: usize) error{ OutOfMemory, CodegenFail }!*const llvm.Type {
+        switch (t.zigTypeTag()) {
+            .Void => return self.context.voidType(),
+            .NoReturn => return self.context.voidType(),
+            .Int => {
+                const info = t.intInfo(self.module.getTarget());
+                return self.context.intType(info.bits);
+            },
+            .Bool => return self.context.intType(1),
+            .Pointer => {
+                if (t.isSlice()) {
+                    return self.fail(src, "TODO: LLVM backend: implement slices", .{});
+                } else {
+                    const elem_type = try self.getLLVMType(t.elemType(), src);
+                    return elem_type.pointerType(0);
+                }
+            },
+            .Array => {
+                const elem_type = try self.getLLVMType(t.elemType(), src);
+                return elem_type.arrayType(@intCast(c_uint, t.abiSize(self.module.getTarget())));
+            },
+            else => return self.fail(src, "TODO implement getLLVMType for type '{}'", .{t}),
+        }
+    }
+
+    fn resolveGlobalDecl(self: *LLVMIRModule, decl: *Module.Decl, src: usize) error{ OutOfMemory, CodegenFail }!*const llvm.Value {
+        // TODO: do we want to store this in our own datastructure?
+        if (self.llvm_module.getNamedGlobal(decl.name)) |val| return val;
+
+        const typed_value = decl.typed_value.most_recent.typed_value;
+
+        // TODO: remove this redundant `getLLVMType`, it is also called in `genTypedValue`.
+        const llvm_type = try self.getLLVMType(typed_value.ty, src);
+        const val = try self.genTypedValue(src, typed_value);
+        const global = self.llvm_module.addGlobal(llvm_type, decl.name);
+        llvm.setInitializer(global, val);
+
+        // TODO ask the Decl if it is const
+        // https://github.com/ziglang/zig/issues/7582
+
+        return global;
+    }
+
+    /// If the llvm function does not exist, create it
+    fn resolveLLVMFunction(self: *LLVMIRModule, func: *Module.Decl, src: usize) !*const llvm.Value {
+        // TODO: do we want to store this in our own datastructure?
+        if (self.llvm_module.getNamedFunction(func.name)) |llvm_fn| return llvm_fn;
+
+        const zig_fn_type = func.typed_value.most_recent.typed_value.ty;
+        const return_type = zig_fn_type.fnReturnType();
+
+        const fn_param_len = zig_fn_type.fnParamLen();
+
+        const fn_param_types = try self.gpa.alloc(Type, fn_param_len);
+        defer self.gpa.free(fn_param_types);
+        zig_fn_type.fnParamTypes(fn_param_types);
+
+        const llvm_param = try self.gpa.alloc(*const llvm.Type, fn_param_len);
+        defer self.gpa.free(llvm_param);
+
+        for (fn_param_types) |fn_param, i| {
+            llvm_param[i] = try self.getLLVMType(fn_param, src);
+        }
+
+        const fn_type = llvm.Type.functionType(
+            try self.getLLVMType(return_type, src),
+            if (fn_param_len == 0) null else llvm_param.ptr,
+            @intCast(c_uint, fn_param_len),
+            false,
+        );
+        const llvm_fn = self.llvm_module.addFunction(func.name, fn_type);
+
+        if (return_type.tag() == .noreturn) {
+            self.addFnAttr(llvm_fn, "noreturn");
+        }
+
+        return llvm_fn;
+    }
+
+    // Helper functions
+    fn addAttr(self: LLVMIRModule, val: *const llvm.Value, index: llvm.AttributeIndex, name: []const u8) void {
+        const kind_id = llvm.getEnumAttributeKindForName(name.ptr, name.len);
+        assert(kind_id != 0);
+        const llvm_attr = self.context.createEnumAttribute(kind_id, 0);
+        val.addAttributeAtIndex(index, llvm_attr);
+    }
+
+    fn addFnAttr(self: *LLVMIRModule, val: *const llvm.Value, attr_name: []const u8) void {
+        // TODO: improve this API, `addAttr(-1, attr_name)`
+        self.addAttr(val, std.math.maxInt(llvm.AttributeIndex), attr_name);
+    }
+
+    pub fn fail(self: *LLVMIRModule, src: usize, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
+        @setCold(true);
+        assert(self.err_msg == null);
+        self.err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
+        return error.CodegenFail;
+    }
+};
diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig
new file mode 100644
index 0000000000..cd928877b1
--- /dev/null
+++ b/src/codegen/llvm/bindings.zig
@@ -0,0 +1,571 @@
+//! We do this instead of @cImport because the self-hosted compiler is easier
+//! to bootstrap if it does not depend on translate-c.
+
+const LLVMBool = bool;
+pub const AttributeIndex = c_uint;
+
+/// Make sure to use the *InContext functions instead of the global ones.
+pub const Context = opaque {
+    pub const create = LLVMContextCreate;
+    extern fn LLVMContextCreate() *const Context;
+
+    pub const dispose = LLVMContextDispose;
+    extern fn LLVMContextDispose(C: *const Context) void;
+
+    pub const createEnumAttribute = LLVMCreateEnumAttribute;
+    extern fn LLVMCreateEnumAttribute(*const Context, KindID: c_uint, Val: u64) *const Attribute;
+
+    pub const intType = LLVMIntTypeInContext;
+    extern fn LLVMIntTypeInContext(C: *const Context, NumBits: c_uint) *const Type;
+
+    pub const voidType = LLVMVoidTypeInContext;
+    extern fn LLVMVoidTypeInContext(C: *const Context) *const Type;
+
+    pub const constString = LLVMConstStringInContext;
+    extern fn LLVMConstStringInContext(C: *const Context, Str: [*]const u8, Length: c_uint, DontNullTerminate: LLVMBool) *const Value;
+
+    pub const appendBasicBlock = LLVMAppendBasicBlockInContext;
+    extern fn LLVMAppendBasicBlockInContext(C: *const Context, Fn: *const Value, Name: [*:0]const u8) *const BasicBlock;
+
+    pub const createBuilder = LLVMCreateBuilderInContext;
+    extern fn LLVMCreateBuilderInContext(C: *const Context) *const Builder;
+};
+
+pub const Value = opaque {
+    pub const addAttributeAtIndex = LLVMAddAttributeAtIndex;
+    extern fn LLVMAddAttributeAtIndex(*const Value, Idx: AttributeIndex, A: *const Attribute) void;
+
+    pub const getFirstBasicBlock = LLVMGetFirstBasicBlock;
+    extern fn LLVMGetFirstBasicBlock(Fn: *const Value) ?*const BasicBlock;
+
+    pub const getNextInstruction = LLVMGetNextInstruction;
+    extern fn LLVMGetNextInstruction(Inst: *const Value) ?*const Value;
+};
+
+pub const Type = opaque {
+    pub const functionType = LLVMFunctionType;
+    extern fn LLVMFunctionType(ReturnType: *const Type, ParamTypes: ?[*]*const Type, ParamCount: c_uint, IsVarArg: LLVMBool) *const Type;
+
+    pub const constNull = LLVMConstNull;
+    extern fn LLVMConstNull(Ty: *const Type) *const Value;
+
+    pub const constAllOnes = LLVMConstAllOnes;
+    extern fn LLVMConstAllOnes(Ty: *const Type) *const Value;
+
+    pub const constInt = LLVMConstInt;
+    extern fn LLVMConstInt(IntTy: *const Type, N: c_ulonglong, SignExtend: LLVMBool) *const Value;
+
+    pub const constArray = LLVMConstArray;
+    extern fn LLVMConstArray(ElementTy: *const Type, ConstantVals: ?[*]*const Value, Length: c_uint) *const Value;
+
+    pub const getUndef = LLVMGetUndef;
+    extern fn LLVMGetUndef(Ty: *const Type) *const Value;
+
+    pub const pointerType = LLVMPointerType;
+    extern fn LLVMPointerType(ElementType: *const Type, AddressSpace: c_uint) *const Type;
+
+    pub const arrayType = LLVMArrayType;
+    extern fn LLVMArrayType(ElementType: *const Type, ElementCount: c_uint) *const Type;
+};
+
+pub const Module = opaque {
+    pub const createWithName = LLVMModuleCreateWithNameInContext;
+    extern fn LLVMModuleCreateWithNameInContext(ModuleID: [*:0]const u8, C: *const Context) *const Module;
+
+    pub const dispose = LLVMDisposeModule;
+    extern fn LLVMDisposeModule(*const Module) void;
+
+    pub const verify = LLVMVerifyModule;
+    extern fn LLVMVerifyModule(*const Module, Action: VerifierFailureAction, OutMessage: *[*:0]const u8) LLVMBool;
+
+    pub const addFunction = LLVMAddFunction;
+    extern fn LLVMAddFunction(*const Module, Name: [*:0]const u8, FunctionTy: *const Type) *const Value;
+
+    pub const getNamedFunction = LLVMGetNamedFunction;
+    extern fn LLVMGetNamedFunction(*const Module, Name: [*:0]const u8) ?*const Value;
+
+    pub const getIntrinsicDeclaration = LLVMGetIntrinsicDeclaration;
+    extern fn LLVMGetIntrinsicDeclaration(Mod: *const Module, ID: c_uint, ParamTypes: ?[*]*const Type, ParamCount: usize) *const Value;
+
+    pub const printToString = LLVMPrintModuleToString;
+    extern fn LLVMPrintModuleToString(*const Module) [*:0]const u8;
+
+    pub const addGlobal = LLVMAddGlobal;
+    extern fn LLVMAddGlobal(M: *const Module, Ty: *const Type, Name: [*:0]const u8) *const Value;
+
+    pub const getNamedGlobal = LLVMGetNamedGlobal;
+    extern fn LLVMGetNamedGlobal(M: *const Module, Name: [*:0]const u8) ?*const Value;
+};
+
+pub const lookupIntrinsicID = LLVMLookupIntrinsicID;
+extern fn LLVMLookupIntrinsicID(Name: [*]const u8, NameLen: usize) c_uint;
+
+pub const disposeMessage = LLVMDisposeMessage;
+extern fn LLVMDisposeMessage(Message: [*:0]const u8) void;
+
+pub const VerifierFailureAction = extern enum {
+    AbortProcess,
+    PrintMessage,
+    ReturnStatus,
+};
+
+pub const constNeg = LLVMConstNeg;
+extern fn LLVMConstNeg(ConstantVal: *const Value) *const Value;
+
+pub const setInitializer = LLVMSetInitializer;
+extern fn LLVMSetInitializer(GlobalVar: *const Value, ConstantVal: *const Value) void;
+
+pub const getParam = LLVMGetParam;
+extern fn LLVMGetParam(Fn: *const Value, Index: c_uint) *const Value;
+
+pub const getEnumAttributeKindForName = LLVMGetEnumAttributeKindForName;
+extern fn LLVMGetEnumAttributeKindForName(Name: [*]const u8, SLen: usize) c_uint;
+
+pub const Attribute = opaque {};
+
+pub const Builder = opaque {
+    pub const dispose = LLVMDisposeBuilder;
+    extern fn LLVMDisposeBuilder(Builder: *const Builder) void;
+
+    pub const positionBuilder = LLVMPositionBuilder;
+    extern fn LLVMPositionBuilder(Builder: *const Builder, Block: *const BasicBlock, Instr: *const Value) void;
+
+    pub const positionBuilderAtEnd = LLVMPositionBuilderAtEnd;
+    extern fn LLVMPositionBuilderAtEnd(Builder: *const Builder, Block: *const BasicBlock) void;
+
+    pub const getInsertBlock = LLVMGetInsertBlock;
+    extern fn LLVMGetInsertBlock(Builder: *const Builder) *const BasicBlock;
+
+    pub const buildCall = LLVMBuildCall;
+    extern fn LLVMBuildCall(*const Builder, Fn: *const Value, Args: ?[*]*const Value, NumArgs: c_uint, Name: [*:0]const u8) *const Value;
+
+    pub const buildCall2 = LLVMBuildCall2;
+    extern fn LLVMBuildCall2(*const Builder, *const Type, Fn: *const Value, Args: [*]*const Value, NumArgs: c_uint, Name: [*:0]const u8) *const Value;
+
+    pub const buildRetVoid = LLVMBuildRetVoid;
+    extern fn LLVMBuildRetVoid(*const Builder) *const Value;
+
+    pub const buildRet = LLVMBuildRet;
+    extern fn LLVMBuildRet(*const Builder, V: *const Value) *const Value;
+
+    pub const buildUnreachable = LLVMBuildUnreachable;
+    extern fn LLVMBuildUnreachable(*const Builder) *const Value;
+
+    pub const buildAlloca = LLVMBuildAlloca;
+    extern fn LLVMBuildAlloca(*const Builder, Ty: *const Type, Name: [*:0]const u8) *const Value;
+
+    pub const buildStore = LLVMBuildStore;
+    extern fn LLVMBuildStore(*const Builder, Val: *const Value, Ptr: *const Value) *const Value;
+
+    pub const buildLoad = LLVMBuildLoad;
+    extern fn LLVMBuildLoad(*const Builder, PointerVal: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildNot = LLVMBuildNot;
+    extern fn LLVMBuildNot(*const Builder, V: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildNSWAdd = LLVMBuildNSWAdd;
+    extern fn LLVMBuildNSWAdd(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildNUWAdd = LLVMBuildNUWAdd;
+    extern fn LLVMBuildNUWAdd(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildNSWSub = LLVMBuildNSWSub;
+    extern fn LLVMBuildNSWSub(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildNUWSub = LLVMBuildNUWSub;
+    extern fn LLVMBuildNUWSub(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
+
+    pub const buildIntCast2 = LLVMBuildIntCast2;
+    extern fn LLVMBuildIntCast2(*const Builder, Val: *const Value, DestTy: *const Type, IsSigned: LLVMBool, Name: [*:0]const u8) *const Value;
+
+    pub const buildBitCast = LLVMBuildBitCast;
+    extern fn LLVMBuildBitCast(*const Builder, Val: *const Value, DestTy: *const Type, Name: [*:0]const u8) *const Value;
+
+    pub const buildInBoundsGEP = LLVMBuildInBoundsGEP;
+    extern fn LLVMBuildInBoundsGEP(B: *const Builder, Pointer: *const Value, Indices: [*]*const Value, NumIndices: c_uint, Name: [*:0]const u8) *const Value;
+};
+
+pub const BasicBlock = opaque {
+    pub const deleteBasicBlock = LLVMDeleteBasicBlock;
+    extern fn LLVMDeleteBasicBlock(BB: *const BasicBlock) void;
+
+    pub const getFirstInstruction = LLVMGetFirstInstruction;
+    extern fn LLVMGetFirstInstruction(BB: *const BasicBlock) ?*const Value;
+};
+
+pub const TargetMachine = opaque {
+    pub const create = LLVMCreateTargetMachine;
+    extern fn LLVMCreateTargetMachine(
+        T: *const Target,
+        Triple: [*:0]const u8,
+        CPU: [*:0]const u8,
+        Features: [*:0]const u8,
+        Level: CodeGenOptLevel,
+        Reloc: RelocMode,
+        CodeModel: CodeMode,
+    ) *const TargetMachine;
+
+    pub const dispose = LLVMDisposeTargetMachine;
+    extern fn LLVMDisposeTargetMachine(T: *const TargetMachine) void;
+
+    pub const emitToFile = LLVMTargetMachineEmitToFile;
+    extern fn LLVMTargetMachineEmitToFile(*const TargetMachine, M: *const Module, Filename: [*:0]const u8, codegen: CodeGenFileType, ErrorMessage: *[*:0]const u8) LLVMBool;
+};
+
+pub const CodeMode = extern enum {
+    Default,
+    JITDefault,
+    Tiny,
+    Small,
+    Kernel,
+    Medium,
+    Large,
+};
+
+pub const CodeGenOptLevel = extern enum {
+    None,
+    Less,
+    Default,
+    Aggressive,
+};
+
+pub const RelocMode = extern enum {
+    Default,
+    Static,
+    PIC,
+    DynamicNoPic,
+    ROPI,
+    RWPI,
+    ROPI_RWPI,
+};
+
+pub const CodeGenFileType = extern enum {
+    AssemblyFile,
+    ObjectFile,
+};
+
+pub const Target = opaque {
+    pub const getFromTriple = LLVMGetTargetFromTriple;
+    extern fn LLVMGetTargetFromTriple(Triple: [*:0]const u8, T: **const Target, ErrorMessage: *[*:0]const u8) LLVMBool;
+};
+
+extern fn LLVMInitializeAArch64TargetInfo() void;
+extern fn LLVMInitializeAMDGPUTargetInfo() void;
+extern fn LLVMInitializeARMTargetInfo() void;
+extern fn LLVMInitializeAVRTargetInfo() void;
+extern fn LLVMInitializeBPFTargetInfo() void;
+extern fn LLVMInitializeHexagonTargetInfo() void;
+extern fn LLVMInitializeLanaiTargetInfo() void;
+extern fn LLVMInitializeMipsTargetInfo() void;
+extern fn LLVMInitializeMSP430TargetInfo() void;
+extern fn LLVMInitializeNVPTXTargetInfo() void;
+extern fn LLVMInitializePowerPCTargetInfo() void;
+extern fn LLVMInitializeRISCVTargetInfo() void;
+extern fn LLVMInitializeSparcTargetInfo() void;
+extern fn LLVMInitializeSystemZTargetInfo() void;
+extern fn LLVMInitializeWebAssemblyTargetInfo() void;
+extern fn LLVMInitializeX86TargetInfo() void;
+extern fn LLVMInitializeXCoreTargetInfo() void;
+extern fn LLVMInitializeAArch64Target() void;
+extern fn LLVMInitializeAMDGPUTarget() void;
+extern fn LLVMInitializeARMTarget() void;
+extern fn LLVMInitializeAVRTarget() void;
+extern fn LLVMInitializeBPFTarget() void;
+extern fn LLVMInitializeHexagonTarget() void;
+extern fn LLVMInitializeLanaiTarget() void;
+extern fn LLVMInitializeMipsTarget() void;
+extern fn LLVMInitializeMSP430Target() void;
+extern fn LLVMInitializeNVPTXTarget() void;
+extern fn LLVMInitializePowerPCTarget() void;
+extern fn LLVMInitializeRISCVTarget() void;
+extern fn LLVMInitializeSparcTarget() void;
+extern fn LLVMInitializeSystemZTarget() void;
+extern fn LLVMInitializeWebAssemblyTarget() void;
+extern fn LLVMInitializeX86Target() void;
+extern fn LLVMInitializeXCoreTarget() void;
+extern fn LLVMInitializeAArch64TargetMC() void;
+extern fn LLVMInitializeAMDGPUTargetMC() void;
+extern fn LLVMInitializeARMTargetMC() void;
+extern fn LLVMInitializeAVRTargetMC() void;
+extern fn LLVMInitializeBPFTargetMC() void;
+extern fn LLVMInitializeHexagonTargetMC() void;
+extern fn LLVMInitializeLanaiTargetMC() void;
+extern fn LLVMInitializeMipsTargetMC() void;
+extern fn LLVMInitializeMSP430TargetMC() void;
+extern fn LLVMInitializeNVPTXTargetMC() void;
+extern fn LLVMInitializePowerPCTargetMC() void;
+extern fn LLVMInitializeRISCVTargetMC() void;
+extern fn LLVMInitializeSparcTargetMC() void;
+extern fn LLVMInitializeSystemZTargetMC() void;
+extern fn LLVMInitializeWebAssemblyTargetMC() void;
+extern fn LLVMInitializeX86TargetMC() void;
+extern fn LLVMInitializeXCoreTargetMC() void;
+extern fn LLVMInitializeAArch64AsmPrinter() void;
+extern fn LLVMInitializeAMDGPUAsmPrinter() void;
+extern fn LLVMInitializeARMAsmPrinter() void;
+extern fn LLVMInitializeAVRAsmPrinter() void;
+extern fn LLVMInitializeBPFAsmPrinter() void;
+extern fn LLVMInitializeHexagonAsmPrinter() void;
+extern fn LLVMInitializeLanaiAsmPrinter() void;
+extern fn LLVMInitializeMipsAsmPrinter() void;
+extern fn LLVMInitializeMSP430AsmPrinter() void;
+extern fn LLVMInitializeNVPTXAsmPrinter() void;
+extern fn LLVMInitializePowerPCAsmPrinter() void;
+extern fn LLVMInitializeRISCVAsmPrinter() void;
+extern fn LLVMInitializeSparcAsmPrinter() void;
+extern fn LLVMInitializeSystemZAsmPrinter() void;
+extern fn LLVMInitializeWebAssemblyAsmPrinter() void;
+extern fn LLVMInitializeX86AsmPrinter() void;
+extern fn LLVMInitializeXCoreAsmPrinter() void;
+extern fn LLVMInitializeAArch64AsmParser() void;
+extern fn LLVMInitializeAMDGPUAsmParser() void;
+extern fn LLVMInitializeARMAsmParser() void;
+extern fn LLVMInitializeAVRAsmParser() void;
+extern fn LLVMInitializeBPFAsmParser() void;
+extern fn LLVMInitializeHexagonAsmParser() void;
+extern fn LLVMInitializeLanaiAsmParser() void;
+extern fn LLVMInitializeMipsAsmParser() void;
+extern fn LLVMInitializeMSP430AsmParser() void;
+extern fn LLVMInitializePowerPCAsmParser() void;
+extern fn LLVMInitializeRISCVAsmParser() void;
+extern fn LLVMInitializeSparcAsmParser() void;
+extern fn LLVMInitializeSystemZAsmParser() void;
+extern fn LLVMInitializeWebAssemblyAsmParser() void;
+extern fn LLVMInitializeX86AsmParser() void;
+
+pub const initializeAllTargetInfos = LLVMInitializeAllTargetInfos;
+fn LLVMInitializeAllTargetInfos() callconv(.C) void {
+    LLVMInitializeAArch64TargetInfo();
+    LLVMInitializeAMDGPUTargetInfo();
+    LLVMInitializeARMTargetInfo();
+    LLVMInitializeAVRTargetInfo();
+    LLVMInitializeBPFTargetInfo();
+    LLVMInitializeHexagonTargetInfo();
+    LLVMInitializeLanaiTargetInfo();
+    LLVMInitializeMipsTargetInfo();
+    LLVMInitializeMSP430TargetInfo();
+    LLVMInitializeNVPTXTargetInfo();
+    LLVMInitializePowerPCTargetInfo();
+    LLVMInitializeRISCVTargetInfo();
+    LLVMInitializeSparcTargetInfo();
+    LLVMInitializeSystemZTargetInfo();
+    LLVMInitializeWebAssemblyTargetInfo();
+    LLVMInitializeX86TargetInfo();
+    LLVMInitializeXCoreTargetInfo();
+}
+pub const initializeAllTargets = LLVMInitializeAllTargets;
+fn LLVMInitializeAllTargets() callconv(.C) void {
+    LLVMInitializeAArch64Target();
+    LLVMInitializeAMDGPUTarget();
+    LLVMInitializeARMTarget();
+    LLVMInitializeAVRTarget();
+    LLVMInitializeBPFTarget();
+    LLVMInitializeHexagonTarget();
+    LLVMInitializeLanaiTarget();
+    LLVMInitializeMipsTarget();
+    LLVMInitializeMSP430Target();
+    LLVMInitializeNVPTXTarget();
+    LLVMInitializePowerPCTarget();
+    LLVMInitializeRISCVTarget();
+    LLVMInitializeSparcTarget();
+    LLVMInitializeSystemZTarget();
+    LLVMInitializeWebAssemblyTarget();
+    LLVMInitializeX86Target();
+    LLVMInitializeXCoreTarget();
+}
+pub const initializeAllTargetMCs = LLVMInitializeAllTargetMCs;
+fn LLVMInitializeAllTargetMCs() callconv(.C) void {
+    LLVMInitializeAArch64TargetMC();
+    LLVMInitializeAMDGPUTargetMC();
+    LLVMInitializeARMTargetMC();
+    LLVMInitializeAVRTargetMC();
+    LLVMInitializeBPFTargetMC();
+    LLVMInitializeHexagonTargetMC();
+    LLVMInitializeLanaiTargetMC();
+    LLVMInitializeMipsTargetMC();
+    LLVMInitializeMSP430TargetMC();
+    LLVMInitializeNVPTXTargetMC();
+    LLVMInitializePowerPCTargetMC();
+    LLVMInitializeRISCVTargetMC();
+    LLVMInitializeSparcTargetMC();
+    LLVMInitializeSystemZTargetMC();
+    LLVMInitializeWebAssemblyTargetMC();
+    LLVMInitializeX86TargetMC();
+    LLVMInitializeXCoreTargetMC();
+}
+pub const initializeAllAsmPrinters = LLVMInitializeAllAsmPrinters;
+fn LLVMInitializeAllAsmPrinters() callconv(.C) void {
+    LLVMInitializeAArch64AsmPrinter();
+    LLVMInitializeAMDGPUAsmPrinter();
+    LLVMInitializeARMAsmPrinter();
+    LLVMInitializeAVRAsmPrinter();
+    LLVMInitializeBPFAsmPrinter();
+    LLVMInitializeHexagonAsmPrinter();
+    LLVMInitializeLanaiAsmPrinter();
+    LLVMInitializeMipsAsmPrinter();
+    LLVMInitializeMSP430AsmPrinter();
+    LLVMInitializeNVPTXAsmPrinter();
+    LLVMInitializePowerPCAsmPrinter();
+    LLVMInitializeRISCVAsmPrinter();
+    LLVMInitializeSparcAsmPrinter();
+    LLVMInitializeSystemZAsmPrinter();
+    LLVMInitializeWebAssemblyAsmPrinter();
+    LLVMInitializeX86AsmPrinter();
+    LLVMInitializeXCoreAsmPrinter();
+}
+pub const initializeAllAsmParsers = LLVMInitializeAllAsmParsers;
+fn LLVMInitializeAllAsmParsers() callconv(.C) void {
+    LLVMInitializeAArch64AsmParser();
+    LLVMInitializeAMDGPUAsmParser();
+    LLVMInitializeARMAsmParser();
+    LLVMInitializeAVRAsmParser();
+    LLVMInitializeBPFAsmParser();
+    LLVMInitializeHexagonAsmParser();
+    LLVMInitializeLanaiAsmParser();
+    LLVMInitializeMipsAsmParser();
+    LLVMInitializeMSP430AsmParser();
+    LLVMInitializePowerPCAsmParser();
+    LLVMInitializeRISCVAsmParser();
+    LLVMInitializeSparcAsmParser();
+    LLVMInitializeSystemZAsmParser();
+    LLVMInitializeWebAssemblyAsmParser();
+    LLVMInitializeX86AsmParser();
+}
+
+extern fn ZigLLDLinkCOFF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
+extern fn ZigLLDLinkELF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
+extern fn ZigLLDLinkMachO(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
+extern fn ZigLLDLinkWasm(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
+
+pub const LinkCOFF = ZigLLDLinkCOFF;
+pub const LinkELF = ZigLLDLinkELF;
+pub const LinkMachO = ZigLLDLinkMachO;
+pub const LinkWasm = ZigLLDLinkWasm;
+
+pub const ObjectFormatType = extern enum(c_int) {
+    Unknown,
+    COFF,
+    ELF,
+    MachO,
+    Wasm,
+    XCOFF,
+};
+
+pub const GetHostCPUName = LLVMGetHostCPUName;
+extern fn LLVMGetHostCPUName() ?[*:0]u8;
+
+pub const GetNativeFeatures = ZigLLVMGetNativeFeatures;
+extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8;
+
+pub const WriteArchive = ZigLLVMWriteArchive;
+extern fn ZigLLVMWriteArchive(
+    archive_name: [*:0]const u8,
+    file_names_ptr: [*]const [*:0]const u8,
+    file_names_len: usize,
+    os_type: OSType,
+) bool;
+
+pub const OSType = extern enum(c_int) {
+    UnknownOS = 0,
+    Ananas = 1,
+    CloudABI = 2,
+    Darwin = 3,
+    DragonFly = 4,
+    FreeBSD = 5,
+    Fuchsia = 6,
+    IOS = 7,
+    KFreeBSD = 8,
+    Linux = 9,
+    Lv2 = 10,
+    MacOSX = 11,
+    NetBSD = 12,
+    OpenBSD = 13,
+    Solaris = 14,
+    Win32 = 15,
+    Haiku = 16,
+    Minix = 17,
+    RTEMS = 18,
+    NaCl = 19,
+    CNK = 20,
+    AIX = 21,
+    CUDA = 22,
+    NVCL = 23,
+    AMDHSA = 24,
+    PS4 = 25,
+    ELFIAMCU = 26,
+    TvOS = 27,
+    WatchOS = 28,
+    Mesa3D = 29,
+    Contiki = 30,
+    AMDPAL = 31,
+    HermitCore = 32,
+    Hurd = 33,
+    WASI = 34,
+    Emscripten = 35,
+};
+
+pub const ArchType = extern enum(c_int) {
+    UnknownArch = 0,
+    arm = 1,
+    armeb = 2,
+    aarch64 = 3,
+    aarch64_be = 4,
+    aarch64_32 = 5,
+    arc = 6,
+    avr = 7,
+    bpfel = 8,
+    bpfeb = 9,
+    hexagon = 10,
+    mips = 11,
+    mipsel = 12,
+    mips64 = 13,
+    mips64el = 14,
+    msp430 = 15,
+    ppc = 16,
+    ppc64 = 17,
+    ppc64le = 18,
+    r600 = 19,
+    amdgcn = 20,
+    riscv32 = 21,
+    riscv64 = 22,
+    sparc = 23,
+    sparcv9 = 24,
+    sparcel = 25,
+    systemz = 26,
+    tce = 27,
+    tcele = 28,
+    thumb = 29,
+    thumbeb = 30,
+    x86 = 31,
+    x86_64 = 32,
+    xcore = 33,
+    nvptx = 34,
+    nvptx64 = 35,
+    le32 = 36,
+    le64 = 37,
+    amdil = 38,
+    amdil64 = 39,
+    hsail = 40,
+    hsail64 = 41,
+    spir = 42,
+    spir64 = 43,
+    kalimba = 44,
+    shave = 45,
+    lanai = 46,
+    wasm32 = 47,
+    wasm64 = 48,
+    renderscript32 = 49,
+    renderscript64 = 50,
+    ve = 51,
+};
+
+pub const ParseCommandLineOptions = ZigLLVMParseCommandLineOptions;
+extern fn ZigLLVMParseCommandLineOptions(argc: usize, argv: [*]const [*:0]const u8) void;
+
+pub const WriteImportLibrary = ZigLLVMWriteImportLibrary;
+extern fn ZigLLVMWriteImportLibrary(
+    def_path: [*:0]const u8,
+    arch: ArchType,
+    output_lib_path: [*c]const u8,
+    kill_at: bool,
+) bool;
diff --git a/src/link.zig b/src/link.zig
index 3dbfb3b922..7893e69cd5 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -567,7 +567,7 @@ pub const File = struct {
             std.debug.print("\n", .{});
         }
 
-        const llvm = @import("llvm_bindings.zig");
+        const llvm = @import("codegen/llvm/bindings.zig");
         const os_type = @import("target.zig").osToLLVM(base.options.target.os.tag);
         const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type);
         if (bad) return error.UnableToWriteArchive;
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 117bdef4a7..f7d646356c 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -16,7 +16,7 @@ const link = @import("../link.zig");
 const build_options = @import("build_options");
 const Cache = @import("../Cache.zig");
 const mingw = @import("../mingw.zig");
-const llvm_backend = @import("../llvm_backend.zig");
+const llvm_backend = @import("../codegen/llvm.zig");
 
 const allocation_padding = 4 / 3;
 const minimum_text_block_size = 64 * allocation_padding;
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index b6b8fd7750..f3073824a5 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -24,7 +24,7 @@ const build_options = @import("build_options");
 const target_util = @import("../target.zig");
 const glibc = @import("../glibc.zig");
 const Cache = @import("../Cache.zig");
-const llvm_backend = @import("../llvm_backend.zig");
+const llvm_backend = @import("../codegen/llvm.zig");
 
 const default_entry_addr = 0x8000000;
 
diff --git a/src/llvm_backend.zig b/src/llvm_backend.zig
deleted file mode 100644
index 2bac4523c9..0000000000
--- a/src/llvm_backend.zig
+++ /dev/null
@@ -1,722 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const Allocator = std.mem.Allocator;
-const Compilation = @import("Compilation.zig");
-const llvm = @import("llvm_bindings.zig");
-const link = @import("link.zig");
-const log = std.log.scoped(.codegen);
-
-const Module = @import("Module.zig");
-const TypedValue = @import("TypedValue.zig");
-const ir = @import("ir.zig");
-const Inst = ir.Inst;
-
-const Value = @import("value.zig").Value;
-const Type = @import("type.zig").Type;
-
-pub fn targetTriple(allocator: *Allocator, target: std.Target) ![:0]u8 {
-    const llvm_arch = switch (target.cpu.arch) {
-        .arm => "arm",
-        .armeb => "armeb",
-        .aarch64 => "aarch64",
-        .aarch64_be => "aarch64_be",
-        .aarch64_32 => "aarch64_32",
-        .arc => "arc",
-        .avr => "avr",
-        .bpfel => "bpfel",
-        .bpfeb => "bpfeb",
-        .hexagon => "hexagon",
-        .mips => "mips",
-        .mipsel => "mipsel",
-        .mips64 => "mips64",
-        .mips64el => "mips64el",
-        .msp430 => "msp430",
-        .powerpc => "powerpc",
-        .powerpc64 => "powerpc64",
-        .powerpc64le => "powerpc64le",
-        .r600 => "r600",
-        .amdgcn => "amdgcn",
-        .riscv32 => "riscv32",
-        .riscv64 => "riscv64",
-        .sparc => "sparc",
-        .sparcv9 => "sparcv9",
-        .sparcel => "sparcel",
-        .s390x => "s390x",
-        .tce => "tce",
-        .tcele => "tcele",
-        .thumb => "thumb",
-        .thumbeb => "thumbeb",
-        .i386 => "i386",
-        .x86_64 => "x86_64",
-        .xcore => "xcore",
-        .nvptx => "nvptx",
-        .nvptx64 => "nvptx64",
-        .le32 => "le32",
-        .le64 => "le64",
-        .amdil => "amdil",
-        .amdil64 => "amdil64",
-        .hsail => "hsail",
-        .hsail64 => "hsail64",
-        .spir => "spir",
-        .spir64 => "spir64",
-        .kalimba => "kalimba",
-        .shave => "shave",
-        .lanai => "lanai",
-        .wasm32 => "wasm32",
-        .wasm64 => "wasm64",
-        .renderscript32 => "renderscript32",
-        .renderscript64 => "renderscript64",
-        .ve => "ve",
-        .spu_2 => return error.LLVMBackendDoesNotSupportSPUMarkII,
-    };
-    // TODO Add a sub-arch for some architectures depending on CPU features.
-
-    const llvm_os = switch (target.os.tag) {
-        .freestanding => "unknown",
-        .ananas => "ananas",
-        .cloudabi => "cloudabi",
-        .dragonfly => "dragonfly",
-        .freebsd => "freebsd",
-        .fuchsia => "fuchsia",
-        .ios => "ios",
-        .kfreebsd => "kfreebsd",
-        .linux => "linux",
-        .lv2 => "lv2",
-        .macos => "macosx",
-        .netbsd => "netbsd",
-        .openbsd => "openbsd",
-        .solaris => "solaris",
-        .windows => "windows",
-        .haiku => "haiku",
-        .minix => "minix",
-        .rtems => "rtems",
-        .nacl => "nacl",
-        .cnk => "cnk",
-        .aix => "aix",
-        .cuda => "cuda",
-        .nvcl => "nvcl",
-        .amdhsa => "amdhsa",
-        .ps4 => "ps4",
-        .elfiamcu => "elfiamcu",
-        .tvos => "tvos",
-        .watchos => "watchos",
-        .mesa3d => "mesa3d",
-        .contiki => "contiki",
-        .amdpal => "amdpal",
-        .hermit => "hermit",
-        .hurd => "hurd",
-        .wasi => "wasi",
-        .emscripten => "emscripten",
-        .uefi => "windows",
-        .other => "unknown",
-    };
-
-    const llvm_abi = switch (target.abi) {
-        .none => "unknown",
-        .gnu => "gnu",
-        .gnuabin32 => "gnuabin32",
-        .gnuabi64 => "gnuabi64",
-        .gnueabi => "gnueabi",
-        .gnueabihf => "gnueabihf",
-        .gnux32 => "gnux32",
-        .code16 => "code16",
-        .eabi => "eabi",
-        .eabihf => "eabihf",
-        .android => "android",
-        .musl => "musl",
-        .musleabi => "musleabi",
-        .musleabihf => "musleabihf",
-        .msvc => "msvc",
-        .itanium => "itanium",
-        .cygnus => "cygnus",
-        .coreclr => "coreclr",
-        .simulator => "simulator",
-        .macabi => "macabi",
-    };
-
-    return std.fmt.allocPrintZ(allocator, "{s}-unknown-{s}-{s}", .{ llvm_arch, llvm_os, llvm_abi });
-}
-
-pub const LLVMIRModule = struct {
-    module: *Module,
-    llvm_module: *const llvm.Module,
-    context: *const llvm.Context,
-    target_machine: *const llvm.TargetMachine,
-    builder: *const llvm.Builder,
-
-    object_path: []const u8,
-
-    gpa: *Allocator,
-    err_msg: ?*Compilation.ErrorMsg = null,
-
-    // TODO: The fields below should really move into a different struct,
-    //       because they are only valid when generating a function
-
-    /// This stores the LLVM values used in a function, such that they can be
-    /// referred to in other instructions. This table is cleared before every function is generated.
-    func_inst_table: std.AutoHashMapUnmanaged(*Inst, *const llvm.Value) = .{},
-
-    /// These fields are used to refer to the LLVM value of the function paramaters in an Arg instruction.
-    args: []*const llvm.Value = &[_]*const llvm.Value{},
-    arg_index: usize = 0,
-
-    entry_block: *const llvm.BasicBlock = undefined,
-    /// This fields stores the last alloca instruction, such that we can append more alloca instructions
-    /// to the top of the function.
-    latest_alloca_inst: ?*const llvm.Value = null,
-
-    pub fn create(allocator: *Allocator, sub_path: []const u8, options: link.Options) !*LLVMIRModule {
-        const self = try allocator.create(LLVMIRModule);
-        errdefer allocator.destroy(self);
-
-        const gpa = options.module.?.gpa;
-
-        const obj_basename = try std.zig.binNameAlloc(gpa, .{
-            .root_name = options.root_name,
-            .target = options.target,
-            .output_mode = .Obj,
-        });
-        defer gpa.free(obj_basename);
-
-        const o_directory = options.module.?.zig_cache_artifact_directory;
-        const object_path = try o_directory.join(gpa, &[_][]const u8{obj_basename});
-        errdefer gpa.free(object_path);
-
-        const context = llvm.Context.create();
-        errdefer context.dispose();
-
-        initializeLLVMTargets();
-
-        const root_nameZ = try gpa.dupeZ(u8, options.root_name);
-        defer gpa.free(root_nameZ);
-        const llvm_module = llvm.Module.createWithName(root_nameZ.ptr, context);
-        errdefer llvm_module.dispose();
-
-        const llvm_target_triple = try targetTriple(gpa, options.target);
-        defer gpa.free(llvm_target_triple);
-
-        var error_message: [*:0]const u8 = undefined;
-        var target: *const llvm.Target = undefined;
-        if (llvm.Target.getFromTriple(llvm_target_triple.ptr, &target, &error_message)) {
-            defer llvm.disposeMessage(error_message);
-
-            const stderr = std.io.getStdErr().outStream();
-            try stderr.print(
-                \\Zig is expecting LLVM to understand this target: '{s}'
-                \\However LLVM responded with: "{s}"
-                \\Zig is unable to continue. This is a bug in Zig:
-                \\https://github.com/ziglang/zig/issues/438
-                \\
-            ,
-                .{
-                    llvm_target_triple,
-                    error_message,
-                },
-            );
-            return error.InvalidLLVMTriple;
-        }
-
-        const opt_level: llvm.CodeGenOptLevel = if (options.optimize_mode == .Debug) .None else .Aggressive;
-        const target_machine = llvm.TargetMachine.create(
-            target,
-            llvm_target_triple.ptr,
-            "",
-            "",
-            opt_level,
-            .Static,
-            .Default,
-        );
-        errdefer target_machine.dispose();
-
-        const builder = context.createBuilder();
-        errdefer builder.dispose();
-
-        self.* = .{
-            .module = options.module.?,
-            .llvm_module = llvm_module,
-            .context = context,
-            .target_machine = target_machine,
-            .builder = builder,
-            .object_path = object_path,
-            .gpa = gpa,
-        };
-        return self;
-    }
-
-    pub fn deinit(self: *LLVMIRModule, allocator: *Allocator) void {
-        self.builder.dispose();
-        self.target_machine.dispose();
-        self.llvm_module.dispose();
-        self.context.dispose();
-
-        self.func_inst_table.deinit(self.gpa);
-        self.gpa.free(self.object_path);
-
-        allocator.destroy(self);
-    }
-
-    fn initializeLLVMTargets() void {
-        llvm.initializeAllTargets();
-        llvm.initializeAllTargetInfos();
-        llvm.initializeAllTargetMCs();
-        llvm.initializeAllAsmPrinters();
-        llvm.initializeAllAsmParsers();
-    }
-
-    pub fn flushModule(self: *LLVMIRModule, comp: *Compilation) !void {
-        if (comp.verbose_llvm_ir) {
-            const dump = self.llvm_module.printToString();
-            defer llvm.disposeMessage(dump);
-
-            const stderr = std.io.getStdErr().outStream();
-            try stderr.writeAll(std.mem.spanZ(dump));
-        }
-
-        {
-            var error_message: [*:0]const u8 = undefined;
-            // verifyModule always allocs the error_message even if there is no error
-            defer llvm.disposeMessage(error_message);
-
-            if (self.llvm_module.verify(.ReturnStatus, &error_message)) {
-                const stderr = std.io.getStdErr().outStream();
-                try stderr.print("broken LLVM module found: {s}\nThis is a bug in the Zig compiler.", .{error_message});
-                return error.BrokenLLVMModule;
-            }
-        }
-
-        const object_pathZ = try self.gpa.dupeZ(u8, self.object_path);
-        defer self.gpa.free(object_pathZ);
-
-        var error_message: [*:0]const u8 = undefined;
-        if (self.target_machine.emitToFile(
-            self.llvm_module,
-            object_pathZ.ptr,
-            .ObjectFile,
-            &error_message,
-        )) {
-            defer llvm.disposeMessage(error_message);
-
-            const stderr = std.io.getStdErr().outStream();
-            try stderr.print("LLVM failed to emit file: {s}\n", .{error_message});
-            return error.FailedToEmit;
-        }
-    }
-
-    pub fn updateDecl(self: *LLVMIRModule, module: *Module, decl: *Module.Decl) !void {
-        self.gen(module, decl) catch |err| switch (err) {
-            error.CodegenFail => {
-                decl.analysis = .codegen_failure;
-                try module.failed_decls.put(module.gpa, decl, self.err_msg.?);
-                self.err_msg = null;
-                return;
-            },
-            else => |e| return e,
-        };
-    }
-
-    fn gen(self: *LLVMIRModule, module: *Module, decl: *Module.Decl) !void {
-        const typed_value = decl.typed_value.most_recent.typed_value;
-        const src = decl.src();
-
-        log.debug("gen: {s} type: {}, value: {}", .{ decl.name, typed_value.ty, typed_value.val });
-
-        if (typed_value.val.castTag(.function)) |func_payload| {
-            const func = func_payload.data;
-
-            const llvm_func = try self.resolveLLVMFunction(func.owner_decl, src);
-
-            // This gets the LLVM values from the function and stores them in `self.args`.
-            const fn_param_len = func.owner_decl.typed_value.most_recent.typed_value.ty.fnParamLen();
-            var args = try self.gpa.alloc(*const llvm.Value, fn_param_len);
-            defer self.gpa.free(args);
-
-            for (args) |*arg, i| {
-                arg.* = llvm.getParam(llvm_func, @intCast(c_uint, i));
-            }
-            self.args = args;
-            self.arg_index = 0;
-
-            // Make sure no other LLVM values from other functions can be referenced
-            self.func_inst_table.clearRetainingCapacity();
-
-            // We remove all the basic blocks of a function to support incremental
-            // compilation!
-            // TODO: remove all basic blocks if functions can have more than one
-            if (llvm_func.getFirstBasicBlock()) |bb| {
-                bb.deleteBasicBlock();
-            }
-
-            self.entry_block = self.context.appendBasicBlock(llvm_func, "Entry");
-            self.builder.positionBuilderAtEnd(self.entry_block);
-            self.latest_alloca_inst = null;
-
-            const instructions = func.body.instructions;
-            for (instructions) |inst| {
-                const opt_llvm_val: ?*const llvm.Value = switch (inst.tag) {
-                    .add => try self.genAdd(inst.castTag(.add).?),
-                    .alloc => try self.genAlloc(inst.castTag(.alloc).?),
-                    .arg => try self.genArg(inst.castTag(.arg).?),
-                    .bitcast => try self.genBitCast(inst.castTag(.bitcast).?),
-                    .breakpoint => try self.genBreakpoint(inst.castTag(.breakpoint).?),
-                    .call => try self.genCall(inst.castTag(.call).?),
-                    .intcast => try self.genIntCast(inst.castTag(.intcast).?),
-                    .load => try self.genLoad(inst.castTag(.load).?),
-                    .not => try self.genNot(inst.castTag(.not).?),
-                    .ret => try self.genRet(inst.castTag(.ret).?),
-                    .retvoid => self.genRetVoid(inst.castTag(.retvoid).?),
-                    .store => try self.genStore(inst.castTag(.store).?),
-                    .sub => try self.genSub(inst.castTag(.sub).?),
-                    .unreach => self.genUnreach(inst.castTag(.unreach).?),
-                    .dbg_stmt => blk: {
-                        // TODO: implement debug info
-                        break :blk null;
-                    },
-                    else => |tag| return self.fail(src, "TODO implement LLVM codegen for Zir instruction: {}", .{tag}),
-                };
-                if (opt_llvm_val) |llvm_val| try self.func_inst_table.putNoClobber(self.gpa, inst, llvm_val);
-            }
-        } else if (typed_value.val.castTag(.extern_fn)) |extern_fn| {
-            _ = try self.resolveLLVMFunction(extern_fn.data, src);
-        } else {
-            _ = try self.resolveGlobalDecl(decl, src);
-        }
-    }
-
-    fn genCall(self: *LLVMIRModule, inst: *Inst.Call) !?*const llvm.Value {
-        if (inst.func.value()) |func_value| {
-            const fn_decl = if (func_value.castTag(.extern_fn)) |extern_fn|
-                extern_fn.data
-            else if (func_value.castTag(.function)) |func_payload|
-                func_payload.data.owner_decl
-            else
-                unreachable;
-
-            const zig_fn_type = fn_decl.typed_value.most_recent.typed_value.ty;
-            const llvm_fn = try self.resolveLLVMFunction(fn_decl, inst.base.src);
-
-            const num_args = inst.args.len;
-
-            const llvm_param_vals = try self.gpa.alloc(*const llvm.Value, num_args);
-            defer self.gpa.free(llvm_param_vals);
-
-            for (inst.args) |arg, i| {
-                llvm_param_vals[i] = try self.resolveInst(arg);
-            }
-
-            // TODO: LLVMBuildCall2 handles opaque function pointers, according to llvm docs
-            //       Do we need that?
-            const call = self.builder.buildCall(
-                llvm_fn,
-                if (num_args == 0) null else llvm_param_vals.ptr,
-                @intCast(c_uint, num_args),
-                "",
-            );
-
-            const return_type = zig_fn_type.fnReturnType();
-            if (return_type.tag() == .noreturn) {
-                _ = self.builder.buildUnreachable();
-            }
-
-            // No need to store the LLVM value if the return type is void or noreturn
-            if (!return_type.hasCodeGenBits()) return null;
-
-            return call;
-        } else {
-            return self.fail(inst.base.src, "TODO implement calling runtime known function pointer LLVM backend", .{});
-        }
-    }
-
-    fn genRetVoid(self: *LLVMIRModule, inst: *Inst.NoOp) ?*const llvm.Value {
-        _ = self.builder.buildRetVoid();
-        return null;
-    }
-
-    fn genRet(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
-        _ = self.builder.buildRet(try self.resolveInst(inst.operand));
-        return null;
-    }
-
-    fn genNot(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
-        return self.builder.buildNot(try self.resolveInst(inst.operand), "");
-    }
-
-    fn genUnreach(self: *LLVMIRModule, inst: *Inst.NoOp) ?*const llvm.Value {
-        _ = self.builder.buildUnreachable();
-        return null;
-    }
-
-    fn genAdd(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
-        const lhs = try self.resolveInst(inst.lhs);
-        const rhs = try self.resolveInst(inst.rhs);
-
-        if (!inst.base.ty.isInt())
-            return self.fail(inst.base.src, "TODO implement 'genAdd' for type {}", .{inst.base.ty});
-
-        return if (inst.base.ty.isSignedInt())
-            self.builder.buildNSWAdd(lhs, rhs, "")
-        else
-            self.builder.buildNUWAdd(lhs, rhs, "");
-    }
-
-    fn genSub(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
-        const lhs = try self.resolveInst(inst.lhs);
-        const rhs = try self.resolveInst(inst.rhs);
-
-        if (!inst.base.ty.isInt())
-            return self.fail(inst.base.src, "TODO implement 'genSub' for type {}", .{inst.base.ty});
-
-        return if (inst.base.ty.isSignedInt())
-            self.builder.buildNSWSub(lhs, rhs, "")
-        else
-            self.builder.buildNUWSub(lhs, rhs, "");
-    }
-
-    fn genIntCast(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
-        const val = try self.resolveInst(inst.operand);
-
-        const signed = inst.base.ty.isSignedInt();
-        // TODO: Should we use intcast here or just a simple bitcast?
-        //       LLVM does truncation vs bitcast (+signed extension) in the intcast depending on the sizes
-        return self.builder.buildIntCast2(val, try self.getLLVMType(inst.base.ty, inst.base.src), signed, "");
-    }
-
-    fn genBitCast(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
-        const val = try self.resolveInst(inst.operand);
-        const dest_type = try self.getLLVMType(inst.base.ty, inst.base.src);
-
-        return self.builder.buildBitCast(val, dest_type, "");
-    }
-
-    fn genArg(self: *LLVMIRModule, inst: *Inst.Arg) !?*const llvm.Value {
-        const arg_val = self.args[self.arg_index];
-        self.arg_index += 1;
-
-        const ptr_val = self.buildAlloca(try self.getLLVMType(inst.base.ty, inst.base.src));
-        _ = self.builder.buildStore(arg_val, ptr_val);
-        return self.builder.buildLoad(ptr_val, "");
-    }
-
-    fn genAlloc(self: *LLVMIRModule, inst: *Inst.NoOp) !?*const llvm.Value {
-        // buildAlloca expects the pointee type, not the pointer type, so assert that
-        // a Payload.PointerSimple is passed to the alloc instruction.
-        const pointee_type = inst.base.ty.castPointer().?.data;
-
-        // TODO: figure out a way to get the name of the var decl.
-        // TODO: set alignment and volatile
-        return self.buildAlloca(try self.getLLVMType(pointee_type, inst.base.src));
-    }
-
-    /// Use this instead of builder.buildAlloca, because this function makes sure to
-    /// put the alloca instruction at the top of the function!
-    fn buildAlloca(self: *LLVMIRModule, t: *const llvm.Type) *const llvm.Value {
-        if (self.latest_alloca_inst) |latest_alloc| {
-            // builder.positionBuilder adds it before the instruction,
-            // but we want to put it after the last alloca instruction.
-            self.builder.positionBuilder(self.entry_block, latest_alloc.getNextInstruction().?);
-        } else {
-            // There might have been other instructions emitted before the
-            // first alloca has been generated. However the alloca should still
-            // be first in the function.
-            if (self.entry_block.getFirstInstruction()) |first_inst| {
-                self.builder.positionBuilder(self.entry_block, first_inst);
-            }
-        }
-        defer self.builder.positionBuilderAtEnd(self.entry_block);
-
-        const val = self.builder.buildAlloca(t, "");
-        self.latest_alloca_inst = val;
-        return val;
-    }
-
-    fn genStore(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value {
-        const val = try self.resolveInst(inst.rhs);
-        const ptr = try self.resolveInst(inst.lhs);
-        _ = self.builder.buildStore(val, ptr);
-        return null;
-    }
-
-    fn genLoad(self: *LLVMIRModule, inst: *Inst.UnOp) !?*const llvm.Value {
-        const ptr_val = try self.resolveInst(inst.operand);
-        return self.builder.buildLoad(ptr_val, "");
-    }
-
-    fn genBreakpoint(self: *LLVMIRModule, inst: *Inst.NoOp) !?*const llvm.Value {
-        const llvn_fn = self.getIntrinsic("llvm.debugtrap");
-        _ = self.builder.buildCall(llvn_fn, null, 0, "");
-        return null;
-    }
-
-    fn getIntrinsic(self: *LLVMIRModule, name: []const u8) *const llvm.Value {
-        const id = llvm.lookupIntrinsicID(name.ptr, name.len);
-        assert(id != 0);
-        // TODO: add support for overload intrinsics by passing the prefix of the intrinsic
-        //       to `lookupIntrinsicID` and then passing the correct types to
-        //       `getIntrinsicDeclaration`
-        return self.llvm_module.getIntrinsicDeclaration(id, null, 0);
-    }
-
-    fn resolveInst(self: *LLVMIRModule, inst: *ir.Inst) !*const llvm.Value {
-        if (inst.value()) |val| {
-            return self.genTypedValue(inst.src, .{ .ty = inst.ty, .val = val });
-        }
-        if (self.func_inst_table.get(inst)) |value| return value;
-
-        return self.fail(inst.src, "TODO implement global llvm values (or the value is not in the func_inst_table table)", .{});
-    }
-
-    fn genTypedValue(self: *LLVMIRModule, src: usize, tv: TypedValue) error{ OutOfMemory, CodegenFail }!*const llvm.Value {
-        const llvm_type = try self.getLLVMType(tv.ty, src);
-
-        if (tv.val.isUndef())
-            return llvm_type.getUndef();
-
-        switch (tv.ty.zigTypeTag()) {
-            .Bool => return if (tv.val.toBool()) llvm_type.constAllOnes() else llvm_type.constNull(),
-            .Int => {
-                var bigint_space: Value.BigIntSpace = undefined;
-                const bigint = tv.val.toBigInt(&bigint_space);
-
-                if (bigint.eqZero()) return llvm_type.constNull();
-
-                if (bigint.limbs.len != 1) {
-                    return self.fail(src, "TODO implement bigger bigint", .{});
-                }
-                const llvm_int = llvm_type.constInt(bigint.limbs[0], false);
-                if (!bigint.positive) {
-                    return llvm.constNeg(llvm_int);
-                }
-                return llvm_int;
-            },
-            .Pointer => switch (tv.val.tag()) {
-                .decl_ref => {
-                    const decl = tv.val.castTag(.decl_ref).?.data;
-                    const val = try self.resolveGlobalDecl(decl, src);
-
-                    const usize_type = try self.getLLVMType(Type.initTag(.usize), src);
-
-                    // TODO: second index should be the index into the memory!
-                    var indices: [2]*const llvm.Value = .{
-                        usize_type.constNull(),
-                        usize_type.constNull(),
-                    };
-
-                    // TODO: consider using buildInBoundsGEP2 for opaque pointers
-                    return self.builder.buildInBoundsGEP(val, &indices, 2, "");
-                },
-                else => return self.fail(src, "TODO implement const of pointer type '{}'", .{tv.ty}),
-            },
-            .Array => {
-                if (tv.val.castTag(.bytes)) |payload| {
-                    const zero_sentinel = if (tv.ty.sentinel()) |sentinel| blk: {
-                        if (sentinel.tag() == .zero) break :blk true;
-                        return self.fail(src, "TODO handle other sentinel values", .{});
-                    } else false;
-
-                    return self.context.constString(payload.data.ptr, @intCast(c_uint, payload.data.len), !zero_sentinel);
-                } else {
-                    return self.fail(src, "TODO handle more array values", .{});
-                }
-            },
-            else => return self.fail(src, "TODO implement const of type '{}'", .{tv.ty}),
-        }
-    }
-
-    fn getLLVMType(self: *LLVMIRModule, t: Type, src: usize) error{ OutOfMemory, CodegenFail }!*const llvm.Type {
-        switch (t.zigTypeTag()) {
-            .Void => return self.context.voidType(),
-            .NoReturn => return self.context.voidType(),
-            .Int => {
-                const info = t.intInfo(self.module.getTarget());
-                return self.context.intType(info.bits);
-            },
-            .Bool => return self.context.intType(1),
-            .Pointer => {
-                if (t.isSlice()) {
-                    return self.fail(src, "TODO: LLVM backend: implement slices", .{});
-                } else {
-                    const elem_type = try self.getLLVMType(t.elemType(), src);
-                    return elem_type.pointerType(0);
-                }
-            },
-            .Array => {
-                const elem_type = try self.getLLVMType(t.elemType(), src);
-                return elem_type.arrayType(@intCast(c_uint, t.abiSize(self.module.getTarget())));
-            },
-            else => return self.fail(src, "TODO implement getLLVMType for type '{}'", .{t}),
-        }
-    }
-
-    fn resolveGlobalDecl(self: *LLVMIRModule, decl: *Module.Decl, src: usize) error{ OutOfMemory, CodegenFail }!*const llvm.Value {
-        // TODO: do we want to store this in our own datastructure?
-        if (self.llvm_module.getNamedGlobal(decl.name)) |val| return val;
-
-        const typed_value = decl.typed_value.most_recent.typed_value;
-
-        // TODO: remove this redundant `getLLVMType`, it is also called in `genTypedValue`.
-        const llvm_type = try self.getLLVMType(typed_value.ty, src);
-        const val = try self.genTypedValue(src, typed_value);
-        const global = self.llvm_module.addGlobal(llvm_type, decl.name);
-        llvm.setInitializer(global, val);
-
-        // TODO ask the Decl if it is const
-        // https://github.com/ziglang/zig/issues/7582
-
-        return global;
-    }
-
-    /// If the llvm function does not exist, create it
-    fn resolveLLVMFunction(self: *LLVMIRModule, func: *Module.Decl, src: usize) !*const llvm.Value {
-        // TODO: do we want to store this in our own datastructure?
-        if (self.llvm_module.getNamedFunction(func.name)) |llvm_fn| return llvm_fn;
-
-        const zig_fn_type = func.typed_value.most_recent.typed_value.ty;
-        const return_type = zig_fn_type.fnReturnType();
-
-        const fn_param_len = zig_fn_type.fnParamLen();
-
-        const fn_param_types = try self.gpa.alloc(Type, fn_param_len);
-        defer self.gpa.free(fn_param_types);
-        zig_fn_type.fnParamTypes(fn_param_types);
-
-        const llvm_param = try self.gpa.alloc(*const llvm.Type, fn_param_len);
-        defer self.gpa.free(llvm_param);
-
-        for (fn_param_types) |fn_param, i| {
-            llvm_param[i] = try self.getLLVMType(fn_param, src);
-        }
-
-        const fn_type = llvm.Type.functionType(
-            try self.getLLVMType(return_type, src),
-            if (fn_param_len == 0) null else llvm_param.ptr,
-            @intCast(c_uint, fn_param_len),
-            false,
-        );
-        const llvm_fn = self.llvm_module.addFunction(func.name, fn_type);
-
-        if (return_type.tag() == .noreturn) {
-            self.addFnAttr(llvm_fn, "noreturn");
-        }
-
-        return llvm_fn;
-    }
-
-    // Helper functions
-    fn addAttr(self: LLVMIRModule, val: *const llvm.Value, index: llvm.AttributeIndex, name: []const u8) void {
-        const kind_id = llvm.getEnumAttributeKindForName(name.ptr, name.len);
-        assert(kind_id != 0);
-        const llvm_attr = self.context.createEnumAttribute(kind_id, 0);
-        val.addAttributeAtIndex(index, llvm_attr);
-    }
-
-    fn addFnAttr(self: *LLVMIRModule, val: *const llvm.Value, attr_name: []const u8) void {
-        // TODO: improve this API, `addAttr(-1, attr_name)`
-        self.addAttr(val, std.math.maxInt(llvm.AttributeIndex), attr_name);
-    }
-
-    pub fn fail(self: *LLVMIRModule, src: usize, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
-        @setCold(true);
-        assert(self.err_msg == null);
-        self.err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
-        return error.CodegenFail;
-    }
-};
diff --git a/src/llvm_bindings.zig b/src/llvm_bindings.zig
deleted file mode 100644
index 1af64e85f6..0000000000
--- a/src/llvm_bindings.zig
+++ /dev/null
@@ -1,574 +0,0 @@
-//! We do this instead of @cImport because the self-hosted compiler is easier
-//! to bootstrap if it does not depend on translate-c.
-
-const std = @import("std");
-const assert = std.debug.assert;
-
-const LLVMBool = bool;
-pub const AttributeIndex = c_uint;
-
-/// Make sure to use the *InContext functions instead of the global ones.
-pub const Context = opaque {
-    pub const create = LLVMContextCreate;
-    extern fn LLVMContextCreate() *const Context;
-
-    pub const dispose = LLVMContextDispose;
-    extern fn LLVMContextDispose(C: *const Context) void;
-
-    pub const createEnumAttribute = LLVMCreateEnumAttribute;
-    extern fn LLVMCreateEnumAttribute(*const Context, KindID: c_uint, Val: u64) *const Attribute;
-
-    pub const intType = LLVMIntTypeInContext;
-    extern fn LLVMIntTypeInContext(C: *const Context, NumBits: c_uint) *const Type;
-
-    pub const voidType = LLVMVoidTypeInContext;
-    extern fn LLVMVoidTypeInContext(C: *const Context) *const Type;
-
-    pub const constString = LLVMConstStringInContext;
-    extern fn LLVMConstStringInContext(C: *const Context, Str: [*]const u8, Length: c_uint, DontNullTerminate: LLVMBool) *const Value;
-
-    pub const appendBasicBlock = LLVMAppendBasicBlockInContext;
-    extern fn LLVMAppendBasicBlockInContext(C: *const Context, Fn: *const Value, Name: [*:0]const u8) *const BasicBlock;
-
-    pub const createBuilder = LLVMCreateBuilderInContext;
-    extern fn LLVMCreateBuilderInContext(C: *const Context) *const Builder;
-};
-
-pub const Value = opaque {
-    pub const addAttributeAtIndex = LLVMAddAttributeAtIndex;
-    extern fn LLVMAddAttributeAtIndex(*const Value, Idx: AttributeIndex, A: *const Attribute) void;
-
-    pub const getFirstBasicBlock = LLVMGetFirstBasicBlock;
-    extern fn LLVMGetFirstBasicBlock(Fn: *const Value) ?*const BasicBlock;
-
-    pub const getNextInstruction = LLVMGetNextInstruction;
-    extern fn LLVMGetNextInstruction(Inst: *const Value) ?*const Value;
-};
-
-pub const Type = opaque {
-    pub const functionType = LLVMFunctionType;
-    extern fn LLVMFunctionType(ReturnType: *const Type, ParamTypes: ?[*]*const Type, ParamCount: c_uint, IsVarArg: LLVMBool) *const Type;
-
-    pub const constNull = LLVMConstNull;
-    extern fn LLVMConstNull(Ty: *const Type) *const Value;
-
-    pub const constAllOnes = LLVMConstAllOnes;
-    extern fn LLVMConstAllOnes(Ty: *const Type) *const Value;
-
-    pub const constInt = LLVMConstInt;
-    extern fn LLVMConstInt(IntTy: *const Type, N: c_ulonglong, SignExtend: LLVMBool) *const Value;
-
-    pub const constArray = LLVMConstArray;
-    extern fn LLVMConstArray(ElementTy: *const Type, ConstantVals: ?[*]*const Value, Length: c_uint) *const Value;
-
-    pub const getUndef = LLVMGetUndef;
-    extern fn LLVMGetUndef(Ty: *const Type) *const Value;
-
-    pub const pointerType = LLVMPointerType;
-    extern fn LLVMPointerType(ElementType: *const Type, AddressSpace: c_uint) *const Type;
-
-    pub const arrayType = LLVMArrayType;
-    extern fn LLVMArrayType(ElementType: *const Type, ElementCount: c_uint) *const Type;
-};
-
-pub const Module = opaque {
-    pub const createWithName = LLVMModuleCreateWithNameInContext;
-    extern fn LLVMModuleCreateWithNameInContext(ModuleID: [*:0]const u8, C: *const Context) *const Module;
-
-    pub const dispose = LLVMDisposeModule;
-    extern fn LLVMDisposeModule(*const Module) void;
-
-    pub const verify = LLVMVerifyModule;
-    extern fn LLVMVerifyModule(*const Module, Action: VerifierFailureAction, OutMessage: *[*:0]const u8) LLVMBool;
-
-    pub const addFunction = LLVMAddFunction;
-    extern fn LLVMAddFunction(*const Module, Name: [*:0]const u8, FunctionTy: *const Type) *const Value;
-
-    pub const getNamedFunction = LLVMGetNamedFunction;
-    extern fn LLVMGetNamedFunction(*const Module, Name: [*:0]const u8) ?*const Value;
-
-    pub const getIntrinsicDeclaration = LLVMGetIntrinsicDeclaration;
-    extern fn LLVMGetIntrinsicDeclaration(Mod: *const Module, ID: c_uint, ParamTypes: ?[*]*const Type, ParamCount: usize) *const Value;
-
-    pub const printToString = LLVMPrintModuleToString;
-    extern fn LLVMPrintModuleToString(*const Module) [*:0]const u8;
-
-    pub const addGlobal = LLVMAddGlobal;
-    extern fn LLVMAddGlobal(M: *const Module, Ty: *const Type, Name: [*:0]const u8) *const Value;
-
-    pub const getNamedGlobal = LLVMGetNamedGlobal;
-    extern fn LLVMGetNamedGlobal(M: *const Module, Name: [*:0]const u8) ?*const Value;
-};
-
-pub const lookupIntrinsicID = LLVMLookupIntrinsicID;
-extern fn LLVMLookupIntrinsicID(Name: [*]const u8, NameLen: usize) c_uint;
-
-pub const disposeMessage = LLVMDisposeMessage;
-extern fn LLVMDisposeMessage(Message: [*:0]const u8) void;
-
-pub const VerifierFailureAction = extern enum {
-    AbortProcess,
-    PrintMessage,
-    ReturnStatus,
-};
-
-pub const constNeg = LLVMConstNeg;
-extern fn LLVMConstNeg(ConstantVal: *const Value) *const Value;
-
-pub const setInitializer = LLVMSetInitializer;
-extern fn LLVMSetInitializer(GlobalVar: *const Value, ConstantVal: *const Value) void;
-
-pub const getParam = LLVMGetParam;
-extern fn LLVMGetParam(Fn: *const Value, Index: c_uint) *const Value;
-
-pub const getEnumAttributeKindForName = LLVMGetEnumAttributeKindForName;
-extern fn LLVMGetEnumAttributeKindForName(Name: [*]const u8, SLen: usize) c_uint;
-
-pub const Attribute = opaque {};
-
-pub const Builder = opaque {
-    pub const dispose = LLVMDisposeBuilder;
-    extern fn LLVMDisposeBuilder(Builder: *const Builder) void;
-
-    pub const positionBuilder = LLVMPositionBuilder;
-    extern fn LLVMPositionBuilder(Builder: *const Builder, Block: *const BasicBlock, Instr: *const Value) void;
-
-    pub const positionBuilderAtEnd = LLVMPositionBuilderAtEnd;
-    extern fn LLVMPositionBuilderAtEnd(Builder: *const Builder, Block: *const BasicBlock) void;
-
-    pub const getInsertBlock = LLVMGetInsertBlock;
-    extern fn LLVMGetInsertBlock(Builder: *const Builder) *const BasicBlock;
-
-    pub const buildCall = LLVMBuildCall;
-    extern fn LLVMBuildCall(*const Builder, Fn: *const Value, Args: ?[*]*const Value, NumArgs: c_uint, Name: [*:0]const u8) *const Value;
-
-    pub const buildCall2 = LLVMBuildCall2;
-    extern fn LLVMBuildCall2(*const Builder, *const Type, Fn: *const Value, Args: [*]*const Value, NumArgs: c_uint, Name: [*:0]const u8) *const Value;
-
-    pub const buildRetVoid = LLVMBuildRetVoid;
-    extern fn LLVMBuildRetVoid(*const Builder) *const Value;
-
-    pub const buildRet = LLVMBuildRet;
-    extern fn LLVMBuildRet(*const Builder, V: *const Value) *const Value;
-
-    pub const buildUnreachable = LLVMBuildUnreachable;
-    extern fn LLVMBuildUnreachable(*const Builder) *const Value;
-
-    pub const buildAlloca = LLVMBuildAlloca;
-    extern fn LLVMBuildAlloca(*const Builder, Ty: *const Type, Name: [*:0]const u8) *const Value;
-
-    pub const buildStore = LLVMBuildStore;
-    extern fn LLVMBuildStore(*const Builder, Val: *const Value, Ptr: *const Value) *const Value;
-
-    pub const buildLoad = LLVMBuildLoad;
-    extern fn LLVMBuildLoad(*const Builder, PointerVal: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildNot = LLVMBuildNot;
-    extern fn LLVMBuildNot(*const Builder, V: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildNSWAdd = LLVMBuildNSWAdd;
-    extern fn LLVMBuildNSWAdd(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildNUWAdd = LLVMBuildNUWAdd;
-    extern fn LLVMBuildNUWAdd(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildNSWSub = LLVMBuildNSWSub;
-    extern fn LLVMBuildNSWSub(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildNUWSub = LLVMBuildNUWSub;
-    extern fn LLVMBuildNUWSub(*const Builder, LHS: *const Value, RHS: *const Value, Name: [*:0]const u8) *const Value;
-
-    pub const buildIntCast2 = LLVMBuildIntCast2;
-    extern fn LLVMBuildIntCast2(*const Builder, Val: *const Value, DestTy: *const Type, IsSigned: LLVMBool, Name: [*:0]const u8) *const Value;
-
-    pub const buildBitCast = LLVMBuildBitCast;
-    extern fn LLVMBuildBitCast(*const Builder, Val: *const Value, DestTy: *const Type, Name: [*:0]const u8) *const Value;
-
-    pub const buildInBoundsGEP = LLVMBuildInBoundsGEP;
-    extern fn LLVMBuildInBoundsGEP(B: *const Builder, Pointer: *const Value, Indices: [*]*const Value, NumIndices: c_uint, Name: [*:0]const u8) *const Value;
-};
-
-pub const BasicBlock = opaque {
-    pub const deleteBasicBlock = LLVMDeleteBasicBlock;
-    extern fn LLVMDeleteBasicBlock(BB: *const BasicBlock) void;
-
-    pub const getFirstInstruction = LLVMGetFirstInstruction;
-    extern fn LLVMGetFirstInstruction(BB: *const BasicBlock) ?*const Value;
-};
-
-pub const TargetMachine = opaque {
-    pub const create = LLVMCreateTargetMachine;
-    extern fn LLVMCreateTargetMachine(
-        T: *const Target,
-        Triple: [*:0]const u8,
-        CPU: [*:0]const u8,
-        Features: [*:0]const u8,
-        Level: CodeGenOptLevel,
-        Reloc: RelocMode,
-        CodeModel: CodeMode,
-    ) *const TargetMachine;
-
-    pub const dispose = LLVMDisposeTargetMachine;
-    extern fn LLVMDisposeTargetMachine(T: *const TargetMachine) void;
-
-    pub const emitToFile = LLVMTargetMachineEmitToFile;
-    extern fn LLVMTargetMachineEmitToFile(*const TargetMachine, M: *const Module, Filename: [*:0]const u8, codegen: CodeGenFileType, ErrorMessage: *[*:0]const u8) LLVMBool;
-};
-
-pub const CodeMode = extern enum {
-    Default,
-    JITDefault,
-    Tiny,
-    Small,
-    Kernel,
-    Medium,
-    Large,
-};
-
-pub const CodeGenOptLevel = extern enum {
-    None,
-    Less,
-    Default,
-    Aggressive,
-};
-
-pub const RelocMode = extern enum {
-    Default,
-    Static,
-    PIC,
-    DynamicNoPic,
-    ROPI,
-    RWPI,
-    ROPI_RWPI,
-};
-
-pub const CodeGenFileType = extern enum {
-    AssemblyFile,
-    ObjectFile,
-};
-
-pub const Target = opaque {
-    pub const getFromTriple = LLVMGetTargetFromTriple;
-    extern fn LLVMGetTargetFromTriple(Triple: [*:0]const u8, T: **const Target, ErrorMessage: *[*:0]const u8) LLVMBool;
-};
-
-extern fn LLVMInitializeAArch64TargetInfo() void;
-extern fn LLVMInitializeAMDGPUTargetInfo() void;
-extern fn LLVMInitializeARMTargetInfo() void;
-extern fn LLVMInitializeAVRTargetInfo() void;
-extern fn LLVMInitializeBPFTargetInfo() void;
-extern fn LLVMInitializeHexagonTargetInfo() void;
-extern fn LLVMInitializeLanaiTargetInfo() void;
-extern fn LLVMInitializeMipsTargetInfo() void;
-extern fn LLVMInitializeMSP430TargetInfo() void;
-extern fn LLVMInitializeNVPTXTargetInfo() void;
-extern fn LLVMInitializePowerPCTargetInfo() void;
-extern fn LLVMInitializeRISCVTargetInfo() void;
-extern fn LLVMInitializeSparcTargetInfo() void;
-extern fn LLVMInitializeSystemZTargetInfo() void;
-extern fn LLVMInitializeWebAssemblyTargetInfo() void;
-extern fn LLVMInitializeX86TargetInfo() void;
-extern fn LLVMInitializeXCoreTargetInfo() void;
-extern fn LLVMInitializeAArch64Target() void;
-extern fn LLVMInitializeAMDGPUTarget() void;
-extern fn LLVMInitializeARMTarget() void;
-extern fn LLVMInitializeAVRTarget() void;
-extern fn LLVMInitializeBPFTarget() void;
-extern fn LLVMInitializeHexagonTarget() void;
-extern fn LLVMInitializeLanaiTarget() void;
-extern fn LLVMInitializeMipsTarget() void;
-extern fn LLVMInitializeMSP430Target() void;
-extern fn LLVMInitializeNVPTXTarget() void;
-extern fn LLVMInitializePowerPCTarget() void;
-extern fn LLVMInitializeRISCVTarget() void;
-extern fn LLVMInitializeSparcTarget() void;
-extern fn LLVMInitializeSystemZTarget() void;
-extern fn LLVMInitializeWebAssemblyTarget() void;
-extern fn LLVMInitializeX86Target() void;
-extern fn LLVMInitializeXCoreTarget() void;
-extern fn LLVMInitializeAArch64TargetMC() void;
-extern fn LLVMInitializeAMDGPUTargetMC() void;
-extern fn LLVMInitializeARMTargetMC() void;
-extern fn LLVMInitializeAVRTargetMC() void;
-extern fn LLVMInitializeBPFTargetMC() void;
-extern fn LLVMInitializeHexagonTargetMC() void;
-extern fn LLVMInitializeLanaiTargetMC() void;
-extern fn LLVMInitializeMipsTargetMC() void;
-extern fn LLVMInitializeMSP430TargetMC() void;
-extern fn LLVMInitializeNVPTXTargetMC() void;
-extern fn LLVMInitializePowerPCTargetMC() void;
-extern fn LLVMInitializeRISCVTargetMC() void;
-extern fn LLVMInitializeSparcTargetMC() void;
-extern fn LLVMInitializeSystemZTargetMC() void;
-extern fn LLVMInitializeWebAssemblyTargetMC() void;
-extern fn LLVMInitializeX86TargetMC() void;
-extern fn LLVMInitializeXCoreTargetMC() void;
-extern fn LLVMInitializeAArch64AsmPrinter() void;
-extern fn LLVMInitializeAMDGPUAsmPrinter() void;
-extern fn LLVMInitializeARMAsmPrinter() void;
-extern fn LLVMInitializeAVRAsmPrinter() void;
-extern fn LLVMInitializeBPFAsmPrinter() void;
-extern fn LLVMInitializeHexagonAsmPrinter() void;
-extern fn LLVMInitializeLanaiAsmPrinter() void;
-extern fn LLVMInitializeMipsAsmPrinter() void;
-extern fn LLVMInitializeMSP430AsmPrinter() void;
-extern fn LLVMInitializeNVPTXAsmPrinter() void;
-extern fn LLVMInitializePowerPCAsmPrinter() void;
-extern fn LLVMInitializeRISCVAsmPrinter() void;
-extern fn LLVMInitializeSparcAsmPrinter() void;
-extern fn LLVMInitializeSystemZAsmPrinter() void;
-extern fn LLVMInitializeWebAssemblyAsmPrinter() void;
-extern fn LLVMInitializeX86AsmPrinter() void;
-extern fn LLVMInitializeXCoreAsmPrinter() void;
-extern fn LLVMInitializeAArch64AsmParser() void;
-extern fn LLVMInitializeAMDGPUAsmParser() void;
-extern fn LLVMInitializeARMAsmParser() void;
-extern fn LLVMInitializeAVRAsmParser() void;
-extern fn LLVMInitializeBPFAsmParser() void;
-extern fn LLVMInitializeHexagonAsmParser() void;
-extern fn LLVMInitializeLanaiAsmParser() void;
-extern fn LLVMInitializeMipsAsmParser() void;
-extern fn LLVMInitializeMSP430AsmParser() void;
-extern fn LLVMInitializePowerPCAsmParser() void;
-extern fn LLVMInitializeRISCVAsmParser() void;
-extern fn LLVMInitializeSparcAsmParser() void;
-extern fn LLVMInitializeSystemZAsmParser() void;
-extern fn LLVMInitializeWebAssemblyAsmParser() void;
-extern fn LLVMInitializeX86AsmParser() void;
-
-pub const initializeAllTargetInfos = LLVMInitializeAllTargetInfos;
-fn LLVMInitializeAllTargetInfos() callconv(.C) void {
-    LLVMInitializeAArch64TargetInfo();
-    LLVMInitializeAMDGPUTargetInfo();
-    LLVMInitializeARMTargetInfo();
-    LLVMInitializeAVRTargetInfo();
-    LLVMInitializeBPFTargetInfo();
-    LLVMInitializeHexagonTargetInfo();
-    LLVMInitializeLanaiTargetInfo();
-    LLVMInitializeMipsTargetInfo();
-    LLVMInitializeMSP430TargetInfo();
-    LLVMInitializeNVPTXTargetInfo();
-    LLVMInitializePowerPCTargetInfo();
-    LLVMInitializeRISCVTargetInfo();
-    LLVMInitializeSparcTargetInfo();
-    LLVMInitializeSystemZTargetInfo();
-    LLVMInitializeWebAssemblyTargetInfo();
-    LLVMInitializeX86TargetInfo();
-    LLVMInitializeXCoreTargetInfo();
-}
-pub const initializeAllTargets = LLVMInitializeAllTargets;
-fn LLVMInitializeAllTargets() callconv(.C) void {
-    LLVMInitializeAArch64Target();
-    LLVMInitializeAMDGPUTarget();
-    LLVMInitializeARMTarget();
-    LLVMInitializeAVRTarget();
-    LLVMInitializeBPFTarget();
-    LLVMInitializeHexagonTarget();
-    LLVMInitializeLanaiTarget();
-    LLVMInitializeMipsTarget();
-    LLVMInitializeMSP430Target();
-    LLVMInitializeNVPTXTarget();
-    LLVMInitializePowerPCTarget();
-    LLVMInitializeRISCVTarget();
-    LLVMInitializeSparcTarget();
-    LLVMInitializeSystemZTarget();
-    LLVMInitializeWebAssemblyTarget();
-    LLVMInitializeX86Target();
-    LLVMInitializeXCoreTarget();
-}
-pub const initializeAllTargetMCs = LLVMInitializeAllTargetMCs;
-fn LLVMInitializeAllTargetMCs() callconv(.C) void {
-    LLVMInitializeAArch64TargetMC();
-    LLVMInitializeAMDGPUTargetMC();
-    LLVMInitializeARMTargetMC();
-    LLVMInitializeAVRTargetMC();
-    LLVMInitializeBPFTargetMC();
-    LLVMInitializeHexagonTargetMC();
-    LLVMInitializeLanaiTargetMC();
-    LLVMInitializeMipsTargetMC();
-    LLVMInitializeMSP430TargetMC();
-    LLVMInitializeNVPTXTargetMC();
-    LLVMInitializePowerPCTargetMC();
-    LLVMInitializeRISCVTargetMC();
-    LLVMInitializeSparcTargetMC();
-    LLVMInitializeSystemZTargetMC();
-    LLVMInitializeWebAssemblyTargetMC();
-    LLVMInitializeX86TargetMC();
-    LLVMInitializeXCoreTargetMC();
-}
-pub const initializeAllAsmPrinters = LLVMInitializeAllAsmPrinters;
-fn LLVMInitializeAllAsmPrinters() callconv(.C) void {
-    LLVMInitializeAArch64AsmPrinter();
-    LLVMInitializeAMDGPUAsmPrinter();
-    LLVMInitializeARMAsmPrinter();
-    LLVMInitializeAVRAsmPrinter();
-    LLVMInitializeBPFAsmPrinter();
-    LLVMInitializeHexagonAsmPrinter();
-    LLVMInitializeLanaiAsmPrinter();
-    LLVMInitializeMipsAsmPrinter();
-    LLVMInitializeMSP430AsmPrinter();
-    LLVMInitializeNVPTXAsmPrinter();
-    LLVMInitializePowerPCAsmPrinter();
-    LLVMInitializeRISCVAsmPrinter();
-    LLVMInitializeSparcAsmPrinter();
-    LLVMInitializeSystemZAsmPrinter();
-    LLVMInitializeWebAssemblyAsmPrinter();
-    LLVMInitializeX86AsmPrinter();
-    LLVMInitializeXCoreAsmPrinter();
-}
-pub const initializeAllAsmParsers = LLVMInitializeAllAsmParsers;
-fn LLVMInitializeAllAsmParsers() callconv(.C) void {
-    LLVMInitializeAArch64AsmParser();
-    LLVMInitializeAMDGPUAsmParser();
-    LLVMInitializeARMAsmParser();
-    LLVMInitializeAVRAsmParser();
-    LLVMInitializeBPFAsmParser();
-    LLVMInitializeHexagonAsmParser();
-    LLVMInitializeLanaiAsmParser();
-    LLVMInitializeMipsAsmParser();
-    LLVMInitializeMSP430AsmParser();
-    LLVMInitializePowerPCAsmParser();
-    LLVMInitializeRISCVAsmParser();
-    LLVMInitializeSparcAsmParser();
-    LLVMInitializeSystemZAsmParser();
-    LLVMInitializeWebAssemblyAsmParser();
-    LLVMInitializeX86AsmParser();
-}
-
-extern fn ZigLLDLinkCOFF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
-extern fn ZigLLDLinkELF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
-extern fn ZigLLDLinkMachO(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
-extern fn ZigLLDLinkWasm(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int;
-
-pub const LinkCOFF = ZigLLDLinkCOFF;
-pub const LinkELF = ZigLLDLinkELF;
-pub const LinkMachO = ZigLLDLinkMachO;
-pub const LinkWasm = ZigLLDLinkWasm;
-
-pub const ObjectFormatType = extern enum(c_int) {
-    Unknown,
-    COFF,
-    ELF,
-    MachO,
-    Wasm,
-    XCOFF,
-};
-
-pub const GetHostCPUName = LLVMGetHostCPUName;
-extern fn LLVMGetHostCPUName() ?[*:0]u8;
-
-pub const GetNativeFeatures = ZigLLVMGetNativeFeatures;
-extern fn ZigLLVMGetNativeFeatures() ?[*:0]u8;
-
-pub const WriteArchive = ZigLLVMWriteArchive;
-extern fn ZigLLVMWriteArchive(
-    archive_name: [*:0]const u8,
-    file_names_ptr: [*]const [*:0]const u8,
-    file_names_len: usize,
-    os_type: OSType,
-) bool;
-
-pub const OSType = extern enum(c_int) {
-    UnknownOS = 0,
-    Ananas = 1,
-    CloudABI = 2,
-    Darwin = 3,
-    DragonFly = 4,
-    FreeBSD = 5,
-    Fuchsia = 6,
-    IOS = 7,
-    KFreeBSD = 8,
-    Linux = 9,
-    Lv2 = 10,
-    MacOSX = 11,
-    NetBSD = 12,
-    OpenBSD = 13,
-    Solaris = 14,
-    Win32 = 15,
-    Haiku = 16,
-    Minix = 17,
-    RTEMS = 18,
-    NaCl = 19,
-    CNK = 20,
-    AIX = 21,
-    CUDA = 22,
-    NVCL = 23,
-    AMDHSA = 24,
-    PS4 = 25,
-    ELFIAMCU = 26,
-    TvOS = 27,
-    WatchOS = 28,
-    Mesa3D = 29,
-    Contiki = 30,
-    AMDPAL = 31,
-    HermitCore = 32,
-    Hurd = 33,
-    WASI = 34,
-    Emscripten = 35,
-};
-
-pub const ArchType = extern enum(c_int) {
-    UnknownArch = 0,
-    arm = 1,
-    armeb = 2,
-    aarch64 = 3,
-    aarch64_be = 4,
-    aarch64_32 = 5,
-    arc = 6,
-    avr = 7,
-    bpfel = 8,
-    bpfeb = 9,
-    hexagon = 10,
-    mips = 11,
-    mipsel = 12,
-    mips64 = 13,
-    mips64el = 14,
-    msp430 = 15,
-    ppc = 16,
-    ppc64 = 17,
-    ppc64le = 18,
-    r600 = 19,
-    amdgcn = 20,
-    riscv32 = 21,
-    riscv64 = 22,
-    sparc = 23,
-    sparcv9 = 24,
-    sparcel = 25,
-    systemz = 26,
-    tce = 27,
-    tcele = 28,
-    thumb = 29,
-    thumbeb = 30,
-    x86 = 31,
-    x86_64 = 32,
-    xcore = 33,
-    nvptx = 34,
-    nvptx64 = 35,
-    le32 = 36,
-    le64 = 37,
-    amdil = 38,
-    amdil64 = 39,
-    hsail = 40,
-    hsail64 = 41,
-    spir = 42,
-    spir64 = 43,
-    kalimba = 44,
-    shave = 45,
-    lanai = 46,
-    wasm32 = 47,
-    wasm64 = 48,
-    renderscript32 = 49,
-    renderscript64 = 50,
-    ve = 51,
-};
-
-pub const ParseCommandLineOptions = ZigLLVMParseCommandLineOptions;
-extern fn ZigLLVMParseCommandLineOptions(argc: usize, argv: [*]const [*:0]const u8) void;
-
-pub const WriteImportLibrary = ZigLLVMWriteImportLibrary;
-extern fn ZigLLVMWriteImportLibrary(
-    def_path: [*:0]const u8,
-    arch: ArchType,
-    output_lib_path: [*c]const u8,
-    kill_at: bool,
-) bool;
diff --git a/src/main.zig b/src/main.zig
index f026b6b4b4..ac31437f85 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1703,7 +1703,7 @@ fn buildOutputType(
     if (build_options.have_llvm and emit_asm != .no) {
         // LLVM has no way to set this non-globally.
         const argv = [_][*:0]const u8{ "zig (LLVM option parsing)", "--x86-asm-syntax=intel" };
-        @import("llvm_bindings.zig").ParseCommandLineOptions(argv.len, &argv);
+        @import("codegen/llvm/bindings.zig").ParseCommandLineOptions(argv.len, &argv);
     }
 
     gimmeMoreOfThoseSweetSweetFileDescriptors();
@@ -2890,7 +2890,7 @@ pub fn punt_to_lld(arena: *Allocator, args: []const []const u8) error{OutOfMemor
         argv[i] = try arena.dupeZ(u8, arg); // TODO If there was an argsAllocZ we could avoid this allocation.
     }
     const exit_code = rc: {
-        const llvm = @import("llvm_bindings.zig");
+        const llvm = @import("codegen/llvm/bindings.zig");
         const argc = @intCast(c_int, argv.len);
         if (mem.eql(u8, args[1], "ld.lld")) {
             break :rc llvm.LinkELF(argc, argv.ptr, true);
@@ -3275,7 +3275,7 @@ fn detectNativeTargetInfo(gpa: *Allocator, cross_target: std.zig.CrossTarget) !s
         if (!build_options.have_llvm)
             fatal("CPU features detection is not yet available for {s} without LLVM extensions", .{@tagName(arch)});
 
-        const llvm = @import("llvm_bindings.zig");
+        const llvm = @import("codegen/llvm/bindings.zig");
         const llvm_cpu_name = llvm.GetHostCPUName();
         const llvm_cpu_features = llvm.GetNativeFeatures();
         info.target.cpu = try detectNativeCpuWithLLVM(arch, llvm_cpu_name, llvm_cpu_features);
diff --git a/src/mingw.zig b/src/mingw.zig
index d55cc28b2b..0a7669f995 100644
--- a/src/mingw.zig
+++ b/src/mingw.zig
@@ -405,7 +405,7 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
     });
     errdefer comp.gpa.free(lib_final_path);
 
-    const llvm = @import("llvm_bindings.zig");
+    const llvm = @import("codegen/llvm/bindings.zig");
     const arch_type = @import("target.zig").archToLLVM(target.cpu.arch);
     const def_final_path_z = try arena.dupeZ(u8, def_final_path);
     const lib_final_path_z = try arena.dupeZ(u8, lib_final_path);
diff --git a/src/target.zig b/src/target.zig
index d33e8d06b0..daac577c7b 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -1,5 +1,5 @@
 const std = @import("std");
-const llvm = @import("llvm_bindings.zig");
+const llvm = @import("codegen/llvm/bindings.zig");
 
 pub const ArchOsAbi = struct {
     arch: std.Target.Cpu.Arch,
diff --git a/test/stage2/llvm.zig b/test/stage2/llvm.zig
new file mode 100644
index 0000000000..ebb5fc390f
--- /dev/null
+++ b/test/stage2/llvm.zig
@@ -0,0 +1,43 @@
+const std = @import("std");
+const TestContext = @import("../../src/test.zig").TestContext;
+const build_options = @import("build_options");
+
+// These tests should work with all platforms, but we're using linux_x64 for
+// now for consistency. Will be expanded eventually.
+const linux_x64 = std.zig.CrossTarget{
+    .cpu_arch = .x86_64,
+    .os_tag = .linux,
+};
+
+pub fn addCases(ctx: *TestContext) !void {
+    {
+        var case = ctx.exeUsingLlvmBackend("simple addition and subtraction", linux_x64);
+
+        case.addCompareOutput(
+            \\fn add(a: i32, b: i32) i32 {
+            \\    return a + b;
+            \\}
+            \\
+            \\export fn main() c_int {
+            \\    var a: i32 = -5;
+            \\    const x = add(a, 7);
+            \\    var y = add(2, 0);
+            \\    y -= x;
+            \\    return y;
+            \\}
+        , "");
+    }
+
+    {
+        var case = ctx.exeUsingLlvmBackend("hello world", linux_x64);
+
+        case.addCompareOutput(
+            \\extern fn puts(s: [*:0]const u8) c_int;
+            \\
+            \\export fn main() c_int {
+            \\    _ = puts("hello world!");
+            \\    return 0;
+            \\}
+        , "hello world!" ++ std.cstr.line_sep);
+    }
+}
diff --git a/test/stage2/llvm_backend.zig b/test/stage2/llvm_backend.zig
deleted file mode 100644
index ebb5fc390f..0000000000
--- a/test/stage2/llvm_backend.zig
+++ /dev/null
@@ -1,43 +0,0 @@
-const std = @import("std");
-const TestContext = @import("../../src/test.zig").TestContext;
-const build_options = @import("build_options");
-
-// These tests should work with all platforms, but we're using linux_x64 for
-// now for consistency. Will be expanded eventually.
-const linux_x64 = std.zig.CrossTarget{
-    .cpu_arch = .x86_64,
-    .os_tag = .linux,
-};
-
-pub fn addCases(ctx: *TestContext) !void {
-    {
-        var case = ctx.exeUsingLlvmBackend("simple addition and subtraction", linux_x64);
-
-        case.addCompareOutput(
-            \\fn add(a: i32, b: i32) i32 {
-            \\    return a + b;
-            \\}
-            \\
-            \\export fn main() c_int {
-            \\    var a: i32 = -5;
-            \\    const x = add(a, 7);
-            \\    var y = add(2, 0);
-            \\    y -= x;
-            \\    return y;
-            \\}
-        , "");
-    }
-
-    {
-        var case = ctx.exeUsingLlvmBackend("hello world", linux_x64);
-
-        case.addCompareOutput(
-            \\extern fn puts(s: [*:0]const u8) c_int;
-            \\
-            \\export fn main() c_int {
-            \\    _ = puts("hello world!");
-            \\    return 0;
-            \\}
-        , "hello world!" ++ std.cstr.line_sep);
-    }
-}
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index 6e25dc283b..5c21ccd669 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -31,7 +31,7 @@ pub fn addCases(ctx: *TestContext) !void {
     try @import("spu-ii.zig").addCases(ctx);
     try @import("arm.zig").addCases(ctx);
     try @import("aarch64.zig").addCases(ctx);
-    try @import("llvm_backend.zig").addCases(ctx);
+    try @import("llvm.zig").addCases(ctx);
 
     {
         var case = ctx.exe("hello world with updates", linux_x64);
-- 
cgit v1.2.3


From efe94a9a1289db274be326d6d66a19caeee86d33 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 6 Jan 2021 16:47:09 -0700
Subject: stage2: C backend: support unused Decls

---
 src/Compilation.zig |  4 ++++
 src/link/C.zig      | 54 +++++++++++++++++++++++++++++++----------------------
 test/stage2/cbe.zig | 10 ++++++++++
 3 files changed, 46 insertions(+), 22 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 4efb845a82..8e2da5aec4 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -527,6 +527,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             if (options.root_pkg == null)
                 break :blk false;
 
+            // If we are outputting .c code we must use Zig backend.
+            if (ofmt == .c)
+                break :blk false;
+
             // If we are the stage1 compiler, we depend on the stage1 c++ llvm backend
             // to compile zig code.
             if (build_options.is_stage1)
diff --git a/src/link/C.zig b/src/link/C.zig
index 32d859f43e..0ebfe288fc 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -151,36 +151,46 @@ pub fn flushModule(self: *C, comp: *Compilation) !void {
     var fn_count: usize = 0;
 
     // Forward decls and non-functions first.
+    // TODO: performance investigation: would keeping a list of Decls that we should
+    // generate, rather than querying here, be faster?
     for (module.decl_table.items()) |kv| {
         const decl = kv.value;
-        const decl_tv = decl.typed_value.most_recent.typed_value;
-        const buf = buf: {
-            if (decl_tv.val.castTag(.function)) |_| {
-                fn_count += 1;
-                break :buf decl.fn_link.c.fwd_decl.items;
-            } else {
-                break :buf decl.link.c.code.items;
-            }
-        };
-        all_buffers.appendAssumeCapacity(.{
-            .iov_base = buf.ptr,
-            .iov_len = buf.len,
-        });
-        file_size += buf.len;
+        switch (decl.typed_value) {
+            .most_recent => |tvm| {
+                const buf = buf: {
+                    if (tvm.typed_value.val.castTag(.function)) |_| {
+                        fn_count += 1;
+                        break :buf decl.fn_link.c.fwd_decl.items;
+                    } else {
+                        break :buf decl.link.c.code.items;
+                    }
+                };
+                all_buffers.appendAssumeCapacity(.{
+                    .iov_base = buf.ptr,
+                    .iov_len = buf.len,
+                });
+                file_size += buf.len;
+            },
+            .never_succeeded => continue,
+        }
     }
 
     // Now the function bodies.
     try all_buffers.ensureCapacity(all_buffers.items.len + fn_count);
     for (module.decl_table.items()) |kv| {
         const decl = kv.value;
-        const decl_tv = decl.typed_value.most_recent.typed_value;
-        if (decl_tv.val.castTag(.function)) |_| {
-            const buf = decl.link.c.code.items;
-            all_buffers.appendAssumeCapacity(.{
-                .iov_base = buf.ptr,
-                .iov_len = buf.len,
-            });
-            file_size += buf.len;
+        switch (decl.typed_value) {
+            .most_recent => |tvm| {
+                if (tvm.typed_value.val.castTag(.function)) |_| {
+                    const buf = decl.link.c.code.items;
+                    all_buffers.appendAssumeCapacity(.{
+                        .iov_base = buf.ptr,
+                        .iov_len = buf.len,
+                    });
+                    file_size += buf.len;
+                }
+            },
+            .never_succeeded => continue,
         }
     }
 
diff --git a/test/stage2/cbe.zig b/test/stage2/cbe.zig
index 9163ac1662..6d4e2062bf 100644
--- a/test/stage2/cbe.zig
+++ b/test/stage2/cbe.zig
@@ -29,6 +29,16 @@ pub fn addCases(ctx: *TestContext) !void {
             \\    return 0;
             \\}
         , "yo" ++ std.cstr.line_sep);
+
+        // Add an unused Decl
+        case.addCompareOutput(
+            \\extern fn puts(s: [*:0]const u8) c_int;
+            \\export fn main() c_int {
+            \\    _ = puts("yo!");
+            \\    return 0;
+            \\}
+            \\fn unused() void {}
+        , "yo!" ++ std.cstr.line_sep);
     }
 
     {
-- 
cgit v1.2.3


From 5ee04315272ba19e3d329ad21d38e9e425e21ab5 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Wed, 6 Jan 2021 17:40:25 -0700
Subject: stage2: update to new ArrayListHashMap API

---
 src/Compilation.zig | 2 +-
 src/Module.zig      | 8 ++++----
 src/link/C.zig      | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index e2213954ef..95c9435aa7 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1564,7 +1564,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                 };
 
                 fwd_decl.* = dg.fwd_decl.moveToUnmanaged();
-                fwd_decl.shrink(module.gpa, fwd_decl.items.len);
+                fwd_decl.shrinkAndFree(module.gpa, fwd_decl.items.len);
             },
         },
         .analyze_decl => |decl| {
diff --git a/src/Module.zig b/src/Module.zig
index 0856b7a606..603c97ffee 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -1885,10 +1885,10 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     if (self.failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.emit_h_failed_decls.remove(decl)) |entry| {
+    if (self.emit_h_failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.compile_log_decls.remove(decl)) |*entry| {
+    if (self.compile_log_decls.swapRemove(decl)) |*entry| {
         entry.value.deinit(self.gpa);
     }
     self.deleteDeclExports(decl);
@@ -1978,10 +1978,10 @@ fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     if (self.failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.emit_h_failed_decls.remove(decl)) |entry| {
+    if (self.emit_h_failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.compile_log_decls.remove(decl)) |*entry| {
+    if (self.compile_log_decls.swapRemove(decl)) |*entry| {
         entry.value.deinit(self.gpa);
     }
     decl.analysis = .outdated;
diff --git a/src/link/C.zig b/src/link/C.zig
index 0ebfe288fc..a60d0efd8e 100644
--- a/src/link/C.zig
+++ b/src/link/C.zig
@@ -112,8 +112,8 @@ pub fn updateDecl(self: *C, module: *Module, decl: *Module.Decl) !void {
     code.* = object.code.moveToUnmanaged();
 
     // Free excess allocated memory for this Decl.
-    fwd_decl.shrink(module.gpa, fwd_decl.items.len);
-    code.shrink(module.gpa, code.items.len);
+    fwd_decl.shrinkAndFree(module.gpa, fwd_decl.items.len);
+    code.shrinkAndFree(module.gpa, code.items.len);
 }
 
 pub fn updateDeclLineNumber(self: *C, module: *Module, decl: *Module.Decl) !void {
-- 
cgit v1.2.3


From a9b505fa7774e2e8451bedfa7bea27d7227572e7 Mon Sep 17 00:00:00 2001
From: Jay Petacat <jay@jayschwa.net>
Date: Tue, 5 Jan 2021 20:57:18 -0500
Subject: Reduce use of deprecated IO types

Related: #4917
---
 doc/docgen.zig                      |  8 ++++----
 lib/std/Progress.zig                |  2 +-
 lib/std/atomic/queue.zig            |  8 ++++----
 lib/std/build.zig                   | 10 +++++-----
 lib/std/build/run.zig               |  4 ++--
 lib/std/child_process.zig           |  2 +-
 lib/std/coff.zig                    |  8 ++++----
 lib/std/crypto/benchmark.zig        |  2 +-
 lib/std/debug.zig                   | 36 +++++++++++++++++------------------
 lib/std/dwarf.zig                   | 10 +++++-----
 lib/std/heap/logging_allocator.zig  |  2 +-
 lib/std/io/buffered_atomic_file.zig |  2 +-
 lib/std/io/fixed_buffer_stream.zig  |  2 +-
 lib/std/io/test.zig                 |  6 +++---
 lib/std/json.zig                    | 16 ++++++++--------
 lib/std/json/write_stream.zig       |  2 +-
 lib/std/net.zig                     |  4 ++--
 lib/std/net/test.zig                |  2 +-
 lib/std/os.zig                      |  2 +-
 lib/std/os/test.zig                 |  6 +++---
 lib/std/special/build_runner.zig    |  4 ++--
 lib/std/unicode/throughput_test.zig |  2 +-
 lib/std/zig/cross_target.zig        | 14 +++++++-------
 lib/std/zig/parser_test.zig         |  6 +++---
 lib/std/zig/perf_test.zig           |  2 +-
 lib/std/zig/render.zig              |  4 ++--
 src/Cache.zig                       |  2 +-
 src/Compilation.zig                 |  8 ++++----
 src/DepTokenizer.zig                |  2 +-
 src/Module.zig                      |  2 +-
 src/codegen/llvm.zig                |  8 ++++----
 src/libc_installation.zig           |  6 +++---
 src/main.zig                        | 38 ++++++++++++++++++-------------------
 src/print_env.zig                   | 10 +++++-----
 src/print_targets.zig               | 10 +++++-----
 src/test.zig                        |  2 +-
 src/translate_c.zig                 |  2 +-
 src/zir.zig                         |  4 ++--
 test/compare_output.zig             | 30 ++++++++++++++---------------
 test/stage1/behavior/bugs/5487.zig  |  6 +++---
 test/tests.zig                      |  8 ++++----
 tools/update_clang_options.zig      |  8 ++++----
 tools/update_glibc.zig              |  6 +++---
 43 files changed, 159 insertions(+), 159 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/doc/docgen.zig b/doc/docgen.zig
index 7e0bedbb6c..90e3e32201 100644
--- a/doc/docgen.zig
+++ b/doc/docgen.zig
@@ -40,9 +40,9 @@ pub fn main() !void {
     var out_file = try fs.cwd().createFile(out_file_name, .{});
     defer out_file.close();
 
-    const input_file_bytes = try in_file.inStream().readAllAlloc(allocator, max_doc_file_size);
+    const input_file_bytes = try in_file.reader().readAllAlloc(allocator, max_doc_file_size);
 
-    var buffered_out_stream = io.bufferedOutStream(out_file.writer());
+    var buffered_writer = io.bufferedWriter(out_file.writer());
 
     var tokenizer = Tokenizer.init(in_file_name, input_file_bytes);
     var toc = try genToc(allocator, &tokenizer);
@@ -50,8 +50,8 @@ pub fn main() !void {
     try fs.cwd().makePath(tmp_dir_name);
     defer fs.cwd().deleteTree(tmp_dir_name) catch {};
 
-    try genHtml(allocator, &tokenizer, &toc, buffered_out_stream.writer(), zig_exe);
-    try buffered_out_stream.flush();
+    try genHtml(allocator, &tokenizer, &toc, buffered_writer.writer(), zig_exe);
+    try buffered_writer.flush();
 }
 
 const Token = struct {
diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig
index 6226e34248..ca9fb8ea1f 100644
--- a/lib/std/Progress.zig
+++ b/lib/std/Progress.zig
@@ -271,7 +271,7 @@ fn refreshWithHeldLock(self: *Progress) void {
 pub fn log(self: *Progress, comptime format: []const u8, args: anytype) void {
     const file = self.terminal orelse return;
     self.refresh();
-    file.outStream().print(format, args) catch {
+    file.writer().print(format, args) catch {
         self.terminal = null;
         return;
     };
diff --git a/lib/std/atomic/queue.zig b/lib/std/atomic/queue.zig
index 19d04d041a..fa3711cd9f 100644
--- a/lib/std/atomic/queue.zig
+++ b/lib/std/atomic/queue.zig
@@ -122,7 +122,7 @@ pub fn Queue(comptime T: type) type {
 
         /// Dumps the contents of the queue to `stderr`.
         pub fn dump(self: *Self) void {
-            self.dumpToStream(std.io.getStdErr().outStream()) catch return;
+            self.dumpToStream(std.io.getStdErr().writer()) catch return;
         }
 
         /// Dumps the contents of the queue to `stream`.
@@ -351,7 +351,7 @@ test "std.atomic.Queue dump" {
 
     // Test empty stream
     fbs.reset();
-    try queue.dumpToStream(fbs.outStream());
+    try queue.dumpToStream(fbs.writer());
     expect(mem.eql(u8, buffer[0..fbs.pos],
         \\head: (null)
         \\tail: (null)
@@ -367,7 +367,7 @@ test "std.atomic.Queue dump" {
     queue.put(&node_0);
 
     fbs.reset();
-    try queue.dumpToStream(fbs.outStream());
+    try queue.dumpToStream(fbs.writer());
 
     var expected = try std.fmt.bufPrint(expected_buffer[0..],
         \\head: 0x{x}=1
@@ -387,7 +387,7 @@ test "std.atomic.Queue dump" {
     queue.put(&node_1);
 
     fbs.reset();
-    try queue.dumpToStream(fbs.outStream());
+    try queue.dumpToStream(fbs.writer());
 
     expected = try std.fmt.bufPrint(expected_buffer[0..],
         \\head: 0x{x}=1
diff --git a/lib/std/build.zig b/lib/std/build.zig
index 0d17b4753a..cb4cb229e3 100644
--- a/lib/std/build.zig
+++ b/lib/std/build.zig
@@ -1042,7 +1042,7 @@ pub const Builder = struct {
 
         try child.spawn();
 
-        const stdout = try child.stdout.?.inStream().readAllAlloc(self.allocator, max_output_size);
+        const stdout = try child.stdout.?.reader().readAllAlloc(self.allocator, max_output_size);
         errdefer self.allocator.free(stdout);
 
         const term = try child.wait();
@@ -1849,7 +1849,7 @@ pub const LibExeObjStep = struct {
     }
 
     pub fn addBuildOption(self: *LibExeObjStep, comptime T: type, name: []const u8, value: T) void {
-        const out = self.build_options_contents.outStream();
+        const out = self.build_options_contents.writer();
         switch (T) {
             []const []const u8 => {
                 out.print("pub const {z}: []const []const u8 = &[_][]const u8{{\n", .{name}) catch unreachable;
@@ -2295,16 +2295,16 @@ pub const LibExeObjStep = struct {
             } else {
                 var mcpu_buffer = std.ArrayList(u8).init(builder.allocator);
 
-                try mcpu_buffer.outStream().print("-mcpu={s}", .{cross.cpu.model.name});
+                try mcpu_buffer.writer().print("-mcpu={s}", .{cross.cpu.model.name});
 
                 for (all_features) |feature, i_usize| {
                     const i = @intCast(std.Target.Cpu.Feature.Set.Index, i_usize);
                     const in_cpu_set = populated_cpu_features.isEnabled(i);
                     const in_actual_set = cross.cpu.features.isEnabled(i);
                     if (in_cpu_set and !in_actual_set) {
-                        try mcpu_buffer.outStream().print("-{s}", .{feature.name});
+                        try mcpu_buffer.writer().print("-{s}", .{feature.name});
                     } else if (!in_cpu_set and in_actual_set) {
-                        try mcpu_buffer.outStream().print("+{s}", .{feature.name});
+                        try mcpu_buffer.writer().print("+{s}", .{feature.name});
                     }
                 }
 
diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig
index 36a4a1a843..8f8fa2eba0 100644
--- a/lib/std/build/run.zig
+++ b/lib/std/build/run.zig
@@ -200,7 +200,7 @@ pub const RunStep = struct {
 
         switch (self.stdout_action) {
             .expect_exact, .expect_matches => {
-                stdout = child.stdout.?.inStream().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable;
+                stdout = child.stdout.?.reader().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable;
             },
             .inherit, .ignore => {},
         }
@@ -210,7 +210,7 @@ pub const RunStep = struct {
 
         switch (self.stderr_action) {
             .expect_exact, .expect_matches => {
-                stderr = child.stderr.?.inStream().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable;
+                stderr = child.stderr.?.reader().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable;
             },
             .inherit, .ignore => {},
         }
diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig
index 4360cc7d73..d37dd9fdf5 100644
--- a/lib/std/child_process.zig
+++ b/lib/std/child_process.zig
@@ -922,7 +922,7 @@ fn writeIntFd(fd: i32, value: ErrInt) !void {
         .capable_io_mode = .blocking,
         .intended_io_mode = .blocking,
     };
-    file.outStream().writeIntNative(u64, @intCast(u64, value)) catch return error.SystemResources;
+    file.writer().writeIntNative(u64, @intCast(u64, value)) catch return error.SystemResources;
 }
 
 fn readIntFd(fd: i32) !ErrInt {
diff --git a/lib/std/coff.zig b/lib/std/coff.zig
index fdc2ec5a82..85000bf8cb 100644
--- a/lib/std/coff.zig
+++ b/lib/std/coff.zig
@@ -127,7 +127,7 @@ pub const Coff = struct {
     pub fn loadHeader(self: *Coff) !void {
         const pe_pointer_offset = 0x3C;
 
-        const in = self.in_file.inStream();
+        const in = self.in_file.reader();
 
         var magic: [2]u8 = undefined;
         try in.readNoEof(magic[0..]);
@@ -163,7 +163,7 @@ pub const Coff = struct {
     }
 
     fn loadOptionalHeader(self: *Coff) !void {
-        const in = self.in_file.inStream();
+        const in = self.in_file.reader();
         self.pe_header.magic = try in.readIntLittle(u16);
         // For now we're only interested in finding the reference to the .pdb,
         // so we'll skip most of this header, which size is different in 32
@@ -206,7 +206,7 @@ pub const Coff = struct {
         const debug_dir = &self.pe_header.data_directory[DEBUG_DIRECTORY];
         const file_offset = debug_dir.virtual_address - header.virtual_address + header.pointer_to_raw_data;
 
-        const in = self.in_file.inStream();
+        const in = self.in_file.reader();
         try self.in_file.seekTo(file_offset);
 
         // Find the correct DebugDirectoryEntry, and where its data is stored.
@@ -257,7 +257,7 @@ pub const Coff = struct {
 
         try self.sections.ensureCapacity(self.coff_header.number_of_sections);
 
-        const in = self.in_file.inStream();
+        const in = self.in_file.reader();
 
         var name: [8]u8 = undefined;
 
diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig
index 5b3b837d13..00336aef87 100644
--- a/lib/std/crypto/benchmark.zig
+++ b/lib/std/crypto/benchmark.zig
@@ -314,7 +314,7 @@ fn mode(comptime x: comptime_int) comptime_int {
 }
 
 pub fn main() !void {
-    const stdout = std.io.getStdOut().outStream();
+    const stdout = std.io.getStdOut().writer();
 
     var buffer: [1024]u8 = undefined;
     var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]);
diff --git a/lib/std/debug.zig b/lib/std/debug.zig
index 073f68da5d..b19f96892f 100644
--- a/lib/std/debug.zig
+++ b/lib/std/debug.zig
@@ -517,15 +517,15 @@ fn populateModule(di: *ModuleDebugInfo, mod: *Module) !void {
 
     const modi = di.pdb.getStreamById(mod.mod_info.ModuleSymStream) orelse return error.MissingDebugInfo;
 
-    const signature = try modi.inStream().readIntLittle(u32);
+    const signature = try modi.reader().readIntLittle(u32);
     if (signature != 4)
         return error.InvalidDebugInfo;
 
     mod.symbols = try allocator.alloc(u8, mod.mod_info.SymByteSize - 4);
-    try modi.inStream().readNoEof(mod.symbols);
+    try modi.reader().readNoEof(mod.symbols);
 
     mod.subsect_info = try allocator.alloc(u8, mod.mod_info.C13ByteSize);
-    try modi.inStream().readNoEof(mod.subsect_info);
+    try modi.reader().readNoEof(mod.subsect_info);
 
     var sect_offset: usize = 0;
     var skip_len: usize = undefined;
@@ -704,11 +704,11 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
         try di.pdb.openFile(di.coff, path);
 
         var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo;
-        const version = try pdb_stream.inStream().readIntLittle(u32);
-        const signature = try pdb_stream.inStream().readIntLittle(u32);
-        const age = try pdb_stream.inStream().readIntLittle(u32);
+        const version = try pdb_stream.reader().readIntLittle(u32);
+        const signature = try pdb_stream.reader().readIntLittle(u32);
+        const age = try pdb_stream.reader().readIntLittle(u32);
         var guid: [16]u8 = undefined;
-        try pdb_stream.inStream().readNoEof(&guid);
+        try pdb_stream.reader().readNoEof(&guid);
         if (version != 20000404) // VC70, only value observed by LLVM team
             return error.UnknownPDBVersion;
         if (!mem.eql(u8, &di.coff.guid, &guid) or di.coff.age != age)
@@ -716,9 +716,9 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
         // We validated the executable and pdb match.
 
         const string_table_index = str_tab_index: {
-            const name_bytes_len = try pdb_stream.inStream().readIntLittle(u32);
+            const name_bytes_len = try pdb_stream.reader().readIntLittle(u32);
             const name_bytes = try allocator.alloc(u8, name_bytes_len);
-            try pdb_stream.inStream().readNoEof(name_bytes);
+            try pdb_stream.reader().readNoEof(name_bytes);
 
             const HashTableHeader = packed struct {
                 Size: u32,
@@ -728,17 +728,17 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
                     return cap * 2 / 3 + 1;
                 }
             };
-            const hash_tbl_hdr = try pdb_stream.inStream().readStruct(HashTableHeader);
+            const hash_tbl_hdr = try pdb_stream.reader().readStruct(HashTableHeader);
             if (hash_tbl_hdr.Capacity == 0)
                 return error.InvalidDebugInfo;
 
             if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity))
                 return error.InvalidDebugInfo;
 
-            const present = try readSparseBitVector(&pdb_stream.inStream(), allocator);
+            const present = try readSparseBitVector(&pdb_stream.reader(), allocator);
             if (present.len != hash_tbl_hdr.Size)
                 return error.InvalidDebugInfo;
-            const deleted = try readSparseBitVector(&pdb_stream.inStream(), allocator);
+            const deleted = try readSparseBitVector(&pdb_stream.reader(), allocator);
 
             const Bucket = struct {
                 first: u32,
@@ -746,8 +746,8 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
             };
             const bucket_list = try allocator.alloc(Bucket, present.len);
             for (present) |_| {
-                const name_offset = try pdb_stream.inStream().readIntLittle(u32);
-                const name_index = try pdb_stream.inStream().readIntLittle(u32);
+                const name_offset = try pdb_stream.reader().readIntLittle(u32);
+                const name_index = try pdb_stream.reader().readIntLittle(u32);
                 const name = mem.spanZ(std.meta.assumeSentinel(name_bytes.ptr + name_offset, 0));
                 if (mem.eql(u8, name, "/names")) {
                     break :str_tab_index name_index;
@@ -762,7 +762,7 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
         const dbi = di.pdb.dbi;
 
         // Dbi Header
-        const dbi_stream_header = try dbi.inStream().readStruct(pdb.DbiStreamHeader);
+        const dbi_stream_header = try dbi.reader().readStruct(pdb.DbiStreamHeader);
         if (dbi_stream_header.VersionHeader != 19990903) // V70, only value observed by LLVM team
             return error.UnknownPDBVersion;
         if (dbi_stream_header.Age != age)
@@ -776,7 +776,7 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
         // Module Info Substream
         var mod_info_offset: usize = 0;
         while (mod_info_offset != mod_info_size) {
-            const mod_info = try dbi.inStream().readStruct(pdb.ModInfo);
+            const mod_info = try dbi.reader().readStruct(pdb.ModInfo);
             var this_record_len: usize = @sizeOf(pdb.ModInfo);
 
             const module_name = try dbi.readNullTermString(allocator);
@@ -814,14 +814,14 @@ fn readCoffDebugInfo(allocator: *mem.Allocator, coff_file: File) !ModuleDebugInf
         var sect_contribs = ArrayList(pdb.SectionContribEntry).init(allocator);
         var sect_cont_offset: usize = 0;
         if (section_contrib_size != 0) {
-            const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.inStream().readIntLittle(u32));
+            const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.reader().readIntLittle(u32));
             if (ver != pdb.SectionContrSubstreamVersion.Ver60)
                 return error.InvalidDebugInfo;
             sect_cont_offset += @sizeOf(u32);
         }
         while (sect_cont_offset != section_contrib_size) {
             const entry = try sect_contribs.addOne();
-            entry.* = try dbi.inStream().readStruct(pdb.SectionContribEntry);
+            entry.* = try dbi.reader().readStruct(pdb.SectionContribEntry);
             sect_cont_offset += @sizeOf(pdb.SectionContribEntry);
 
             if (sect_cont_offset > section_contrib_size)
diff --git a/lib/std/dwarf.zig b/lib/std/dwarf.zig
index 2e732cbc75..6769a139da 100644
--- a/lib/std/dwarf.zig
+++ b/lib/std/dwarf.zig
@@ -408,7 +408,7 @@ pub const DwarfInfo = struct {
 
     fn scanAllFunctions(di: *DwarfInfo) !void {
         var stream = io.fixedBufferStream(di.debug_info);
-        const in = &stream.inStream();
+        const in = &stream.reader();
         const seekable = &stream.seekableStream();
         var this_unit_offset: u64 = 0;
 
@@ -512,7 +512,7 @@ pub const DwarfInfo = struct {
 
     fn scanAllCompileUnits(di: *DwarfInfo) !void {
         var stream = io.fixedBufferStream(di.debug_info);
-        const in = &stream.inStream();
+        const in = &stream.reader();
         const seekable = &stream.seekableStream();
         var this_unit_offset: u64 = 0;
 
@@ -585,7 +585,7 @@ pub const DwarfInfo = struct {
             if (di.debug_ranges) |debug_ranges| {
                 if (compile_unit.die.getAttrSecOffset(AT_ranges)) |ranges_offset| {
                     var stream = io.fixedBufferStream(debug_ranges);
-                    const in = &stream.inStream();
+                    const in = &stream.reader();
                     const seekable = &stream.seekableStream();
 
                     // All the addresses in the list are relative to the value
@@ -640,7 +640,7 @@ pub const DwarfInfo = struct {
 
     fn parseAbbrevTable(di: *DwarfInfo, offset: u64) !AbbrevTable {
         var stream = io.fixedBufferStream(di.debug_abbrev);
-        const in = &stream.inStream();
+        const in = &stream.reader();
         const seekable = &stream.seekableStream();
 
         try seekable.seekTo(offset);
@@ -691,7 +691,7 @@ pub const DwarfInfo = struct {
 
     pub fn getLineNumberInfo(di: *DwarfInfo, compile_unit: CompileUnit, target_address: usize) !debug.LineInfo {
         var stream = io.fixedBufferStream(di.debug_line);
-        const in = &stream.inStream();
+        const in = &stream.reader();
         const seekable = &stream.seekableStream();
 
         const compile_unit_cwd = try compile_unit.die.getAttrString(di, AT_comp_dir);
diff --git a/lib/std/heap/logging_allocator.zig b/lib/std/heap/logging_allocator.zig
index 47a584bb1d..7920138e9b 100644
--- a/lib/std/heap/logging_allocator.zig
+++ b/lib/std/heap/logging_allocator.zig
@@ -89,7 +89,7 @@ test "LoggingAllocator" {
 
     var allocator_buf: [10]u8 = undefined;
     var fixedBufferAllocator = std.mem.validationWrap(std.heap.FixedBufferAllocator.init(&allocator_buf));
-    const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.outStream()).allocator;
+    const allocator = &loggingAllocator(&fixedBufferAllocator.allocator, fbs.writer()).allocator;
 
     var a = try allocator.alloc(u8, 10);
     a = allocator.shrink(a, 5);
diff --git a/lib/std/io/buffered_atomic_file.zig b/lib/std/io/buffered_atomic_file.zig
index 9d65e9d193..66c349d318 100644
--- a/lib/std/io/buffered_atomic_file.zig
+++ b/lib/std/io/buffered_atomic_file.zig
@@ -38,7 +38,7 @@ pub const BufferedAtomicFile = struct {
         self.atomic_file = try dir.atomicFile(dest_path, atomic_file_options);
         errdefer self.atomic_file.deinit();
 
-        self.file_stream = self.atomic_file.file.outStream();
+        self.file_stream = self.atomic_file.file.writer();
         self.buffered_stream = .{ .unbuffered_writer = self.file_stream };
         return self;
     }
diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig
index b36949e538..1711b6da1b 100644
--- a/lib/std/io/fixed_buffer_stream.zig
+++ b/lib/std/io/fixed_buffer_stream.zig
@@ -45,7 +45,7 @@ pub fn FixedBufferStream(comptime Buffer: type) type {
             return .{ .context = self };
         }
 
-        /// Deprecated: use `inStream`
+        /// Deprecated: use `reader`
         pub fn inStream(self: *Self) InStream {
             return .{ .context = self };
         }
diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig
index 85357ae58a..9fdef0de1d 100644
--- a/lib/std/io/test.zig
+++ b/lib/std/io/test.zig
@@ -30,8 +30,8 @@ test "write a file, read it, then delete it" {
         var file = try tmp.dir.createFile(tmp_file_name, .{});
         defer file.close();
 
-        var buf_stream = io.bufferedOutStream(file.outStream());
-        const st = buf_stream.outStream();
+        var buf_stream = io.bufferedWriter(file.writer());
+        const st = buf_stream.writer();
         try st.print("begin", .{});
         try st.writeAll(data[0..]);
         try st.print("end", .{});
@@ -72,7 +72,7 @@ test "BitStreams with File Stream" {
         var file = try tmp.dir.createFile(tmp_file_name, .{});
         defer file.close();
 
-        var bit_stream = io.bitOutStream(builtin.endian, file.outStream());
+        var bit_stream = io.bitWriter(builtin.endian, file.writer());
 
         try bit_stream.writeBits(@as(u2, 1), 1);
         try bit_stream.writeBits(@as(u5, 2), 2);
diff --git a/lib/std/json.zig b/lib/std/json.zig
index 87808a7350..5cbdb4319e 100644
--- a/lib/std/json.zig
+++ b/lib/std/json.zig
@@ -1323,31 +1323,31 @@ test "Value.jsonStringify" {
     {
         var buffer: [10]u8 = undefined;
         var fbs = std.io.fixedBufferStream(&buffer);
-        try @as(Value, .Null).jsonStringify(.{}, fbs.outStream());
+        try @as(Value, .Null).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "null");
     }
     {
         var buffer: [10]u8 = undefined;
         var fbs = std.io.fixedBufferStream(&buffer);
-        try (Value{ .Bool = true }).jsonStringify(.{}, fbs.outStream());
+        try (Value{ .Bool = true }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "true");
     }
     {
         var buffer: [10]u8 = undefined;
         var fbs = std.io.fixedBufferStream(&buffer);
-        try (Value{ .Integer = 42 }).jsonStringify(.{}, fbs.outStream());
+        try (Value{ .Integer = 42 }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "42");
     }
     {
         var buffer: [10]u8 = undefined;
         var fbs = std.io.fixedBufferStream(&buffer);
-        try (Value{ .Float = 42 }).jsonStringify(.{}, fbs.outStream());
+        try (Value{ .Float = 42 }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
     }
     {
         var buffer: [10]u8 = undefined;
         var fbs = std.io.fixedBufferStream(&buffer);
-        try (Value{ .String = "weeee" }).jsonStringify(.{}, fbs.outStream());
+        try (Value{ .String = "weeee" }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
     }
     {
@@ -1360,7 +1360,7 @@ test "Value.jsonStringify" {
         };
         try (Value{
             .Array = Array.fromOwnedSlice(undefined, &vals),
-        }).jsonStringify(.{}, fbs.outStream());
+        }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
     }
     {
@@ -1369,7 +1369,7 @@ test "Value.jsonStringify" {
         var obj = ObjectMap.init(testing.allocator);
         defer obj.deinit();
         try obj.putNoClobber("a", .{ .String = "b" });
-        try (Value{ .Object = obj }).jsonStringify(.{}, fbs.outStream());
+        try (Value{ .Object = obj }).jsonStringify(.{}, fbs.writer());
         testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
     }
 }
@@ -2223,7 +2223,7 @@ test "write json then parse it" {
     var out_buffer: [1000]u8 = undefined;
 
     var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
-    const out_stream = fixed_buffer_stream.outStream();
+    const out_stream = fixed_buffer_stream.writer();
     var jw = writeStream(out_stream, 4);
 
     try jw.beginObject();
diff --git a/lib/std/json/write_stream.zig b/lib/std/json/write_stream.zig
index 9322ca5429..b4a8aed84c 100644
--- a/lib/std/json/write_stream.zig
+++ b/lib/std/json/write_stream.zig
@@ -238,7 +238,7 @@ pub fn writeStream(
 test "json write stream" {
     var out_buf: [1024]u8 = undefined;
     var slice_stream = std.io.fixedBufferStream(&out_buf);
-    const out = slice_stream.outStream();
+    const out = slice_stream.writer();
 
     var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
     defer arena_allocator.deinit();
diff --git a/lib/std/net.zig b/lib/std/net.zig
index da35bd88b0..037df76907 100644
--- a/lib/std/net.zig
+++ b/lib/std/net.zig
@@ -1106,7 +1106,7 @@ fn linuxLookupNameFromHosts(
     };
     defer file.close();
 
-    const stream = std.io.bufferedInStream(file.inStream()).inStream();
+    const stream = std.io.bufferedReader(file.reader()).reader();
     var line_buf: [512]u8 = undefined;
     while (stream.readUntilDelimiterOrEof(&line_buf, '\n') catch |err| switch (err) {
         error.StreamTooLong => blk: {
@@ -1304,7 +1304,7 @@ fn getResolvConf(allocator: *mem.Allocator, rc: *ResolvConf) !void {
     };
     defer file.close();
 
-    const stream = std.io.bufferedInStream(file.inStream()).inStream();
+    const stream = std.io.bufferedReader(file.reader()).reader();
     var line_buf: [512]u8 = undefined;
     while (stream.readUntilDelimiterOrEof(&line_buf, '\n') catch |err| switch (err) {
         error.StreamTooLong => blk: {
diff --git a/lib/std/net/test.zig b/lib/std/net/test.zig
index 110c6ec9c0..74ae1ddf4f 100644
--- a/lib/std/net/test.zig
+++ b/lib/std/net/test.zig
@@ -249,6 +249,6 @@ fn testServer(server: *net.StreamServer) anyerror!void {
 
     var client = try server.accept();
 
-    const stream = client.file.outStream();
+    const stream = client.file.writer();
     try stream.print("hello from server\n", .{});
 }
diff --git a/lib/std/os.zig b/lib/std/os.zig
index e9bf6379cc..42bdf486d7 100644
--- a/lib/std/os.zig
+++ b/lib/std/os.zig
@@ -191,7 +191,7 @@ fn getRandomBytesDevURandom(buf: []u8) !void {
         .capable_io_mode = .blocking,
         .intended_io_mode = .blocking,
     };
-    const stream = file.inStream();
+    const stream = file.reader();
     stream.readNoEof(buf) catch return error.Unexpected;
 }
 
diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig
index 65dee90e62..0904a8585f 100644
--- a/lib/std/os/test.zig
+++ b/lib/std/os/test.zig
@@ -475,7 +475,7 @@ test "mmap" {
         const file = try tmp.dir.createFile(test_out_file, .{});
         defer file.close();
 
-        const stream = file.outStream();
+        const stream = file.writer();
 
         var i: u32 = 0;
         while (i < alloc_size / @sizeOf(u32)) : (i += 1) {
@@ -499,7 +499,7 @@ test "mmap" {
         defer os.munmap(data);
 
         var mem_stream = io.fixedBufferStream(data);
-        const stream = mem_stream.inStream();
+        const stream = mem_stream.reader();
 
         var i: u32 = 0;
         while (i < alloc_size / @sizeOf(u32)) : (i += 1) {
@@ -523,7 +523,7 @@ test "mmap" {
         defer os.munmap(data);
 
         var mem_stream = io.fixedBufferStream(data);
-        const stream = mem_stream.inStream();
+        const stream = mem_stream.reader();
 
         var i: u32 = alloc_size / 2 / @sizeOf(u32);
         while (i < alloc_size / @sizeOf(u32)) : (i += 1) {
diff --git a/lib/std/special/build_runner.zig b/lib/std/special/build_runner.zig
index f2fa2dc3b8..0b7baf0fc1 100644
--- a/lib/std/special/build_runner.zig
+++ b/lib/std/special/build_runner.zig
@@ -57,8 +57,8 @@ pub fn main() !void {
 
     var targets = ArrayList([]const u8).init(allocator);
 
-    const stderr_stream = io.getStdErr().outStream();
-    const stdout_stream = io.getStdOut().outStream();
+    const stderr_stream = io.getStdErr().writer();
+    const stdout_stream = io.getStdOut().writer();
 
     while (nextArg(args, &arg_idx)) |arg| {
         if (mem.startsWith(u8, arg, "-D")) {
diff --git a/lib/std/unicode/throughput_test.zig b/lib/std/unicode/throughput_test.zig
index 2676a30cb2..8f9f9d9cb7 100644
--- a/lib/std/unicode/throughput_test.zig
+++ b/lib/std/unicode/throughput_test.zig
@@ -45,7 +45,7 @@ fn benchmarkCodepointCount(buf: []const u8) !ResultCount {
 }
 
 pub fn main() !void {
-    const stdout = std.io.getStdOut().outStream();
+    const stdout = std.io.getStdOut().writer();
 
     const args = try std.process.argsAlloc(std.heap.page_allocator);
 
diff --git a/lib/std/zig/cross_target.zig b/lib/std/zig/cross_target.zig
index 060dd83caf..f0849f9a03 100644
--- a/lib/std/zig/cross_target.zig
+++ b/lib/std/zig/cross_target.zig
@@ -519,29 +519,29 @@ pub const CrossTarget = struct {
         var result = std.ArrayList(u8).init(allocator);
         defer result.deinit();
 
-        try result.outStream().print("{s}-{s}", .{ arch_name, os_name });
+        try result.writer().print("{s}-{s}", .{ arch_name, os_name });
 
         // The zig target syntax does not allow specifying a max os version with no min, so
         // if either are present, we need the min.
         if (self.os_version_min != null or self.os_version_max != null) {
             switch (self.getOsVersionMin()) {
                 .none => {},
-                .semver => |v| try result.outStream().print(".{}", .{v}),
-                .windows => |v| try result.outStream().print("{s}", .{v}),
+                .semver => |v| try result.writer().print(".{}", .{v}),
+                .windows => |v| try result.writer().print("{s}", .{v}),
             }
         }
         if (self.os_version_max) |max| {
             switch (max) {
                 .none => {},
-                .semver => |v| try result.outStream().print("...{}", .{v}),
-                .windows => |v| try result.outStream().print("..{s}", .{v}),
+                .semver => |v| try result.writer().print("...{}", .{v}),
+                .windows => |v| try result.writer().print("..{s}", .{v}),
             }
         }
 
         if (self.glibc_version) |v| {
-            try result.outStream().print("-{s}.{}", .{ @tagName(self.getAbi()), v });
+            try result.writer().print("-{s}.{}", .{ @tagName(self.getAbi()), v });
         } else if (self.abi) |abi| {
-            try result.outStream().print("-{s}", .{@tagName(abi)});
+            try result.writer().print("-{s}", .{@tagName(abi)});
         }
 
         return result.toOwnedSlice();
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index 63d04f9a80..d7cc1208a2 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -3734,7 +3734,7 @@ const maxInt = std.math.maxInt;
 var fixed_buffer_mem: [100 * 1024]u8 = undefined;
 
 fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 {
-    const stderr = io.getStdErr().outStream();
+    const stderr = io.getStdErr().writer();
 
     const tree = try std.zig.parse(allocator, source);
     defer tree.deinit();
@@ -3767,8 +3767,8 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
     var buffer = std.ArrayList(u8).init(allocator);
     errdefer buffer.deinit();
 
-    const outStream = buffer.outStream();
-    anything_changed.* = try std.zig.render(allocator, outStream, tree);
+    const writer = buffer.writer();
+    anything_changed.* = try std.zig.render(allocator, writer, tree);
     return buffer.toOwnedSlice();
 }
 fn testTransform(source: []const u8, expected_source: []const u8) !void {
diff --git a/lib/std/zig/perf_test.zig b/lib/std/zig/perf_test.zig
index e1c58c1469..b111170902 100644
--- a/lib/std/zig/perf_test.zig
+++ b/lib/std/zig/perf_test.zig
@@ -29,7 +29,7 @@ pub fn main() !void {
     const mb_per_sec = bytes_per_sec / (1024 * 1024);
 
     var stdout_file = std.io.getStdOut();
-    const stdout = stdout_file.outStream();
+    const stdout = stdout_file.writer();
     try stdout.print("{:.3} MiB/s, {} KiB used \n", .{ mb_per_sec, memory_used / 1024 });
 }
 
diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig
index 6931b19250..b882d71cfd 100644
--- a/lib/std/zig/render.zig
+++ b/lib/std/zig/render.zig
@@ -790,7 +790,7 @@ fn renderExpression(
                         const section_exprs = row_exprs[0..section_end];
 
                         // Null stream for counting the printed length of each expression
-                        var line_find_stream = std.io.findByteOutStream('\n', std.io.null_out_stream);
+                        var line_find_stream = std.io.findByteOutStream('\n', std.io.null_writer);
                         var counting_stream = std.io.countingOutStream(line_find_stream.writer());
                         var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, counting_stream.writer());
 
@@ -954,7 +954,7 @@ fn renderExpression(
             const expr_outputs_one_line = blk: {
                 // render field expressions until a LF is found
                 for (field_inits) |field_init| {
-                    var find_stream = std.io.findByteOutStream('\n', std.io.null_out_stream);
+                    var find_stream = std.io.findByteOutStream('\n', std.io.null_writer);
                     var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, find_stream.writer());
 
                     try renderExpression(allocator, &auto_indenting_stream, tree, field_init, Space.None);
diff --git a/src/Cache.zig b/src/Cache.zig
index 9f8beaabc7..f5ffb34dbe 100644
--- a/src/Cache.zig
+++ b/src/Cache.zig
@@ -285,7 +285,7 @@ pub const Manifest = struct {
             };
         }
 
-        const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, manifest_file_size_max);
+        const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
         defer self.cache.gpa.free(file_contents);
 
         const input_file_count = self.files.items.len;
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 95c9435aa7..a81926bf19 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1820,7 +1820,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{});
         defer out_zig_file.close();
 
-        var bos = std.io.bufferedOutStream(out_zig_file.writer());
+        var bos = std.io.bufferedWriter(out_zig_file.writer());
         _ = try std.zig.render(comp.gpa, bos.writer(), tree);
         try bos.flush();
 
@@ -2750,7 +2750,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
 
     switch (target.os.getVersionRange()) {
         .none => try buffer.appendSlice(" .none = {} }\n"),
-        .semver => |semver| try buffer.outStream().print(
+        .semver => |semver| try buffer.writer().print(
             \\ .semver = .{{
             \\        .min = .{{
             \\            .major = {},
@@ -2773,7 +2773,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
             semver.max.minor,
             semver.max.patch,
         }),
-        .linux => |linux| try buffer.outStream().print(
+        .linux => |linux| try buffer.writer().print(
             \\ .linux = .{{
             \\        .range = .{{
             \\            .min = .{{
@@ -2807,7 +2807,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
             linux.glibc.minor,
             linux.glibc.patch,
         }),
-        .windows => |windows| try buffer.outStream().print(
+        .windows => |windows| try buffer.writer().print(
             \\ .windows = .{{
             \\        .min = {s},
             \\        .max = {s},
diff --git a/src/DepTokenizer.zig b/src/DepTokenizer.zig
index 0fe1310cd8..b246a22bd0 100644
--- a/src/DepTokenizer.zig
+++ b/src/DepTokenizer.zig
@@ -910,7 +910,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
             },
             else => {
                 try buffer.appendSlice("ERROR: ");
-                try token.printError(buffer.outStream());
+                try token.printError(buffer.writer());
                 break;
             },
         }
diff --git a/src/Module.zig b/src/Module.zig
index 603c97ffee..9707e79e2a 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -1649,7 +1649,7 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
                 var msg = std.ArrayList(u8).init(self.gpa);
                 defer msg.deinit();
 
-                try parse_err.render(tree.token_ids, msg.outStream());
+                try parse_err.render(tree.token_ids, msg.writer());
                 const err_msg = try self.gpa.create(Compilation.ErrorMsg);
                 err_msg.* = .{
                     .msg = msg.toOwnedSlice(),
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
index df97f7e0ef..9b35c42c6e 100644
--- a/src/codegen/llvm.zig
+++ b/src/codegen/llvm.zig
@@ -200,7 +200,7 @@ pub const LLVMIRModule = struct {
         if (llvm.Target.getFromTriple(llvm_target_triple.ptr, &target, &error_message)) {
             defer llvm.disposeMessage(error_message);
 
-            const stderr = std.io.getStdErr().outStream();
+            const stderr = std.io.getStdErr().writer();
             try stderr.print(
                 \\Zig is expecting LLVM to understand this target: '{s}'
                 \\However LLVM responded with: "{s}"
@@ -268,7 +268,7 @@ pub const LLVMIRModule = struct {
             const dump = self.llvm_module.printToString();
             defer llvm.disposeMessage(dump);
 
-            const stderr = std.io.getStdErr().outStream();
+            const stderr = std.io.getStdErr().writer();
             try stderr.writeAll(std.mem.spanZ(dump));
         }
 
@@ -278,7 +278,7 @@ pub const LLVMIRModule = struct {
             defer llvm.disposeMessage(error_message);
 
             if (self.llvm_module.verify(.ReturnStatus, &error_message)) {
-                const stderr = std.io.getStdErr().outStream();
+                const stderr = std.io.getStdErr().writer();
                 try stderr.print("broken LLVM module found: {s}\nThis is a bug in the Zig compiler.", .{error_message});
                 return error.BrokenLLVMModule;
             }
@@ -296,7 +296,7 @@ pub const LLVMIRModule = struct {
         )) {
             defer llvm.disposeMessage(error_message);
 
-            const stderr = std.io.getStdErr().outStream();
+            const stderr = std.io.getStdErr().writer();
             try stderr.print("LLVM failed to emit file: {s}\n", .{error_message});
             return error.FailedToEmit;
         }
diff --git a/src/libc_installation.zig b/src/libc_installation.zig
index bc317869e8..05c85578ea 100644
--- a/src/libc_installation.zig
+++ b/src/libc_installation.zig
@@ -338,7 +338,7 @@ pub const LibCInstallation = struct {
 
         for (searches) |search| {
             result_buf.shrinkAndFree(0);
-            try result_buf.outStream().print("{s}\\Include\\{s}\\ucrt", .{ search.path, search.version });
+            try result_buf.writer().print("{s}\\Include\\{s}\\ucrt", .{ search.path, search.version });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
                 error.FileNotFound,
@@ -384,7 +384,7 @@ pub const LibCInstallation = struct {
 
         for (searches) |search| {
             result_buf.shrinkAndFree(0);
-            try result_buf.outStream().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ search.path, search.version, arch_sub_dir });
+            try result_buf.writer().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ search.path, search.version, arch_sub_dir });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
                 error.FileNotFound,
@@ -438,7 +438,7 @@ pub const LibCInstallation = struct {
 
         for (searches) |search| {
             result_buf.shrinkAndFree(0);
-            const stream = result_buf.outStream();
+            const stream = result_buf.writer();
             try stream.print("{s}\\Lib\\{s}\\um\\{s}", .{ search.path, search.version, arch_sub_dir });
 
             var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) {
diff --git a/src/main.zig b/src/main.zig
index cd89f269d0..7829901acc 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -196,7 +196,7 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
         return cmdInit(gpa, arena, cmd_args, .Lib);
     } else if (mem.eql(u8, cmd, "targets")) {
         const info = try detectNativeTargetInfo(arena, .{});
-        const stdout = io.getStdOut().outStream();
+        const stdout = io.getStdOut().writer();
         return @import("print_targets.zig").cmdTargets(arena, cmd_args, stdout, info.target);
     } else if (mem.eql(u8, cmd, "version")) {
         try std.io.getStdOut().writeAll(build_options.version ++ "\n");
@@ -1944,8 +1944,8 @@ fn buildOutputType(
         }
     }
 
-    const stdin = std.io.getStdIn().inStream();
-    const stderr = std.io.getStdErr().outStream();
+    const stdin = std.io.getStdIn().reader();
+    const stderr = std.io.getStdErr().writer();
     var repl_buf: [1024]u8 = undefined;
 
     while (watch) {
@@ -2114,9 +2114,9 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
         var zig_file = try o_dir.createFile(translated_zig_basename, .{});
         defer zig_file.close();
 
-        var bos = io.bufferedOutStream(zig_file.writer());
-        _ = try std.zig.render(comp.gpa, bos.writer(), tree);
-        try bos.flush();
+        var bw = io.bufferedWriter(zig_file.writer());
+        _ = try std.zig.render(comp.gpa, bw.writer(), tree);
+        try bw.flush();
 
         man.writeManifest() catch |err| warn("failed to write cache manifest: {s}", .{@errorName(err)});
 
@@ -2187,9 +2187,9 @@ pub fn cmdLibC(gpa: *Allocator, args: []const []const u8) !void {
         };
         defer libc.deinit(gpa);
 
-        var bos = io.bufferedOutStream(io.getStdOut().writer());
-        try libc.render(bos.writer());
-        try bos.flush();
+        var bw = io.bufferedWriter(io.getStdOut().writer());
+        try libc.render(bw.writer());
+        try bw.flush();
     }
 }
 
@@ -2570,7 +2570,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
             const arg = args[i];
             if (mem.startsWith(u8, arg, "-")) {
                 if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
-                    const stdout = io.getStdOut().outStream();
+                    const stdout = io.getStdOut().writer();
                     try stdout.writeAll(usage_fmt);
                     return cleanExit();
                 } else if (mem.eql(u8, arg, "--color")) {
@@ -2600,7 +2600,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
             fatal("cannot use --stdin with positional arguments", .{});
         }
 
-        const stdin = io.getStdIn().inStream();
+        const stdin = io.getStdIn().reader();
 
         const source_code = try stdin.readAllAlloc(gpa, max_src_size);
         defer gpa.free(source_code);
@@ -2617,14 +2617,14 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
             process.exit(1);
         }
         if (check_flag) {
-            const anything_changed = try std.zig.render(gpa, io.null_out_stream, tree);
+            const anything_changed = try std.zig.render(gpa, io.null_writer, tree);
             const code = if (anything_changed) @as(u8, 1) else @as(u8, 0);
             process.exit(code);
         }
 
-        var bos = io.bufferedOutStream(io.getStdOut().writer());
-        _ = try std.zig.render(gpa, bos.writer(), tree);
-        try bos.flush();
+        var bw = io.bufferedWriter(io.getStdOut().writer());
+        _ = try std.zig.render(gpa, bw.writer(), tree);
+        try bw.flush();
         return;
     }
 
@@ -2774,7 +2774,7 @@ fn fmtPathFile(
     }
 
     if (check_mode) {
-        const anything_changed = try std.zig.render(fmt.gpa, io.null_out_stream, tree);
+        const anything_changed = try std.zig.render(fmt.gpa, io.null_writer, tree);
         if (anything_changed) {
             const stdout = io.getStdOut().writer();
             try stdout.print("{s}\n", .{file_path});
@@ -2823,11 +2823,11 @@ fn printErrMsgToFile(
 
     var text_buf = std.ArrayList(u8).init(gpa);
     defer text_buf.deinit();
-    const out_stream = text_buf.outStream();
-    try parse_error.render(tree.token_ids, out_stream);
+    const writer = text_buf.writer();
+    try parse_error.render(tree.token_ids, writer);
     const text = text_buf.items;
 
-    const stream = file.outStream();
+    const stream = file.writer();
     try stream.print("{s}:{d}:{d}: error: {s}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text });
 
     if (!color_on) return;
diff --git a/src/print_env.zig b/src/print_env.zig
index bcf4a983ab..d62e1f62fd 100644
--- a/src/print_env.zig
+++ b/src/print_env.zig
@@ -20,10 +20,10 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: std.fs.File.Wri
     const global_cache_dir = try introspect.resolveGlobalCacheDir(gpa);
     defer gpa.free(global_cache_dir);
 
-    var bos = std.io.bufferedOutStream(stdout);
-    const bos_stream = bos.outStream();
+    var bw = std.io.bufferedWriter(stdout);
+    const w = bw.writer();
 
-    var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream);
+    var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);
     try jws.beginObject();
 
     try jws.objectField("zig_exe");
@@ -42,6 +42,6 @@ pub fn cmdEnv(gpa: *Allocator, args: []const []const u8, stdout: std.fs.File.Wri
     try jws.emitString(build_options.version);
 
     try jws.endObject();
-    try bos_stream.writeByte('\n');
-    try bos.flush();
+    try w.writeByte('\n');
+    try bw.flush();
 }
diff --git a/src/print_targets.zig b/src/print_targets.zig
index cf55eee516..e24a2294a1 100644
--- a/src/print_targets.zig
+++ b/src/print_targets.zig
@@ -26,9 +26,9 @@ pub fn cmdTargets(
     const glibc_abi = try glibc.loadMetaData(allocator, zig_lib_directory.handle);
     defer glibc_abi.destroy(allocator);
 
-    var bos = io.bufferedOutStream(stdout);
-    const bos_stream = bos.outStream();
-    var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream);
+    var bw = io.bufferedWriter(stdout);
+    const w = bw.writer();
+    var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);
 
     try jws.beginObject();
 
@@ -156,6 +156,6 @@ pub fn cmdTargets(
 
     try jws.endObject();
 
-    try bos_stream.writeByte('\n');
-    return bos.flush();
+    try w.writeByte('\n');
+    return bw.flush();
 }
diff --git a/src/test.zig b/src/test.zig
index 682fb5078f..2553e57ff4 100644
--- a/src/test.zig
+++ b/src/test.zig
@@ -738,7 +738,7 @@ pub const TestContext = struct {
                     write_node.activate();
                     var out_zir = std.ArrayList(u8).init(allocator);
                     defer out_zir.deinit();
-                    try new_zir_module.writeToStream(allocator, out_zir.outStream());
+                    try new_zir_module.writeToStream(allocator, out_zir.writer());
                     write_node.end();
 
                     var test_node = update_node.start("assert", 0);
diff --git a/src/translate_c.zig b/src/translate_c.zig
index 9369c6d4b8..8df7963267 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -5268,7 +5268,7 @@ fn appendTokenFmt(c: *Context, token_id: Token.Id, comptime format: []const u8,
     try c.token_locs.ensureCapacity(c.gpa, c.token_locs.items.len + 1);
 
     const start_index = c.source_buffer.items.len;
-    try c.source_buffer.outStream().print(format ++ " ", args);
+    try c.source_buffer.writer().print(format ++ " ", args);
 
     c.token_ids.appendAssumeCapacity(token_id);
     c.token_locs.appendAssumeCapacity(.{
diff --git a/src/zir.zig b/src/zir.zig
index 63850d67db..b3d004f1dc 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1116,7 +1116,7 @@ pub const Module = struct {
 
     /// This is a debugging utility for rendering the tree to stderr.
     pub fn dump(self: Module) void {
-        self.writeToStream(std.heap.page_allocator, std.io.getStdErr().outStream()) catch {};
+        self.writeToStream(std.heap.page_allocator, std.io.getStdErr().writer()) catch {};
     }
 
     const DeclAndIndex = struct {
@@ -3254,7 +3254,7 @@ pub fn dumpZir(allocator: *Allocator, kind: []const u8, decl_name: [*:0]const u8
 
     try write.inst_table.ensureCapacity(@intCast(u32, instructions.len));
 
-    const stderr = std.io.getStdErr().outStream();
+    const stderr = std.io.getStdErr().writer();
     try stderr.print("{s} {s} {{ // unanalyzed\n", .{ kind, decl_name });
 
     for (instructions) |inst| {
diff --git a/test/compare_output.zig b/test/compare_output.zig
index 9dc80f202d..a6e9835ea1 100644
--- a/test/compare_output.zig
+++ b/test/compare_output.zig
@@ -22,7 +22,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\
             \\pub fn main() void {
             \\    privateFunction();
-            \\    const stdout = getStdOut().outStream();
+            \\    const stdout = getStdOut().writer();
             \\    stdout.print("OK 2\n", .{}) catch unreachable;
             \\}
             \\
@@ -37,7 +37,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\// purposefully conflicting function with main.zig
             \\// but it's private so it should be OK
             \\fn privateFunction() void {
-            \\    const stdout = getStdOut().outStream();
+            \\    const stdout = getStdOut().writer();
             \\    stdout.print("OK 1\n", .{}) catch unreachable;
             \\}
             \\
@@ -63,7 +63,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         tc.addSourceFile("foo.zig",
             \\usingnamespace @import("std").io;
             \\pub fn foo_function() void {
-            \\    const stdout = getStdOut().outStream();
+            \\    const stdout = getStdOut().writer();
             \\    stdout.print("OK\n", .{}) catch unreachable;
             \\}
         );
@@ -74,7 +74,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\
             \\pub fn bar_function() void {
             \\    if (foo_function()) {
-            \\        const stdout = getStdOut().outStream();
+            \\        const stdout = getStdOut().writer();
             \\        stdout.print("OK\n", .{}) catch unreachable;
             \\    }
             \\}
@@ -106,7 +106,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\pub const a_text = "OK\n";
             \\
             \\pub fn ok() void {
-            \\    const stdout = io.getStdOut().outStream();
+            \\    const stdout = io.getStdOut().writer();
             \\    stdout.print(b_text, .{}) catch unreachable;
             \\}
         );
@@ -124,7 +124,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\const io = @import("std").io;
         \\
         \\pub fn main() void {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("Hello, world!\n{d:4} {x:3} {c}\n", .{@as(u32, 12), @as(u16, 0x12), @as(u8, 'a')}) catch unreachable;
         \\}
     , "Hello, world!\n  12  12 a\n");
@@ -267,7 +267,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\    var x_local : i32 = print_ok(x);
         \\}
         \\fn print_ok(val: @TypeOf(x)) @TypeOf(foo) {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("OK\n", .{}) catch unreachable;
         \\    return 0;
         \\}
@@ -349,7 +349,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\pub fn main() void {
         \\    const bar = Bar {.field2 = 13,};
         \\    const foo = Foo {.field1 = bar,};
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    if (!foo.method()) {
         \\        stdout.print("BAD\n", .{}) catch unreachable;
         \\    }
@@ -363,7 +363,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
     cases.add("defer with only fallthrough",
         \\const io = @import("std").io;
         \\pub fn main() void {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("before\n", .{}) catch unreachable;
         \\    defer stdout.print("defer1\n", .{}) catch unreachable;
         \\    defer stdout.print("defer2\n", .{}) catch unreachable;
@@ -376,7 +376,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\const io = @import("std").io;
         \\const os = @import("std").os;
         \\pub fn main() void {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("before\n", .{}) catch unreachable;
         \\    defer stdout.print("defer1\n", .{}) catch unreachable;
         \\    defer stdout.print("defer2\n", .{}) catch unreachable;
@@ -393,7 +393,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\    do_test() catch return;
         \\}
         \\fn do_test() !void {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("before\n", .{}) catch unreachable;
         \\    defer stdout.print("defer1\n", .{}) catch unreachable;
         \\    errdefer stdout.print("deferErr\n", .{}) catch unreachable;
@@ -412,7 +412,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\    do_test() catch return;
         \\}
         \\fn do_test() !void {
-        \\    const stdout = io.getStdOut().outStream();
+        \\    const stdout = io.getStdOut().writer();
         \\    stdout.print("before\n", .{}) catch unreachable;
         \\    defer stdout.print("defer1\n", .{}) catch unreachable;
         \\    errdefer stdout.print("deferErr\n", .{}) catch unreachable;
@@ -429,7 +429,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\const io = @import("std").io;
             \\
             \\pub fn main() void {
-            \\    const stdout = io.getStdOut().outStream();
+            \\    const stdout = io.getStdOut().writer();
             \\    stdout.print(foo_txt, .{}) catch unreachable;
             \\}
         , "1234\nabcd\n");
@@ -448,7 +448,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\
             \\pub fn main() !void {
             \\    var args_it = std.process.args();
-            \\    const stdout = io.getStdOut().outStream();
+            \\    const stdout = io.getStdOut().writer();
             \\    var index: usize = 0;
             \\    _ = args_it.skip();
             \\    while (args_it.next(allocator)) |arg_or_err| : (index += 1) {
@@ -487,7 +487,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\
             \\pub fn main() !void {
             \\    var args_it = std.process.args();
-            \\    const stdout = io.getStdOut().outStream();
+            \\    const stdout = io.getStdOut().writer();
             \\    var index: usize = 0;
             \\    _ = args_it.skip();
             \\    while (args_it.next(allocator)) |arg_or_err| : (index += 1) {
diff --git a/test/stage1/behavior/bugs/5487.zig b/test/stage1/behavior/bugs/5487.zig
index 02fa677a44..bf530a8a02 100644
--- a/test/stage1/behavior/bugs/5487.zig
+++ b/test/stage1/behavior/bugs/5487.zig
@@ -3,10 +3,10 @@ const io = @import("std").io;
 pub fn write(_: void, bytes: []const u8) !usize {
     return 0;
 }
-pub fn outStream() io.OutStream(void, @typeInfo(@typeInfo(@TypeOf(write)).Fn.return_type.?).ErrorUnion.error_set, write) {
-    return io.OutStream(void, @typeInfo(@typeInfo(@TypeOf(write)).Fn.return_type.?).ErrorUnion.error_set, write){ .context = {} };
+pub fn writer() io.Writer(void, @typeInfo(@typeInfo(@TypeOf(write)).Fn.return_type.?).ErrorUnion.error_set, write) {
+    return io.Writer(void, @typeInfo(@typeInfo(@TypeOf(write)).Fn.return_type.?).ErrorUnion.error_set, write){ .context = {} };
 }
 
 test "crash" {
-    _ = io.multiOutStream(.{outStream()});
+    _ = io.multiWriter(.{writer()});
 }
diff --git a/test/tests.zig b/test/tests.zig
index 5ee381e5c2..ec6d9e1df8 100644
--- a/test/tests.zig
+++ b/test/tests.zig
@@ -652,9 +652,9 @@ pub const StackTracesContext = struct {
             }
             child.spawn() catch |err| debug.panic("Unable to spawn {s}: {s}\n", .{ full_exe_path, @errorName(err) });
 
-            const stdout = child.stdout.?.inStream().readAllAlloc(b.allocator, max_stdout_size) catch unreachable;
+            const stdout = child.stdout.?.reader().readAllAlloc(b.allocator, max_stdout_size) catch unreachable;
             defer b.allocator.free(stdout);
-            const stderrFull = child.stderr.?.inStream().readAllAlloc(b.allocator, max_stdout_size) catch unreachable;
+            const stderrFull = child.stderr.?.reader().readAllAlloc(b.allocator, max_stdout_size) catch unreachable;
             defer b.allocator.free(stderrFull);
             var stderr = stderrFull;
 
@@ -875,8 +875,8 @@ pub const CompileErrorContext = struct {
             var stdout_buf = ArrayList(u8).init(b.allocator);
             var stderr_buf = ArrayList(u8).init(b.allocator);
 
-            child.stdout.?.inStream().readAllArrayList(&stdout_buf, max_stdout_size) catch unreachable;
-            child.stderr.?.inStream().readAllArrayList(&stderr_buf, max_stdout_size) catch unreachable;
+            child.stdout.?.reader().readAllArrayList(&stdout_buf, max_stdout_size) catch unreachable;
+            child.stderr.?.reader().readAllArrayList(&stderr_buf, max_stdout_size) catch unreachable;
 
             const term = child.wait() catch |err| {
                 debug.panic("Unable to spawn {s}: {s}\n", .{ zig_args.items[0], @errorName(err) });
diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig
index fd64bb062a..ad380e0139 100644
--- a/tools/update_clang_options.zig
+++ b/tools/update_clang_options.zig
@@ -388,8 +388,8 @@ pub fn main() anyerror!void {
     // "W" and "Wl,". So we sort this list in order of descending priority.
     std.sort.sort(*json.ObjectMap, all_objects.items, {}, objectLessThan);
 
-    var stdout_bos = std.io.bufferedOutStream(std.io.getStdOut().outStream());
-    const stdout = stdout_bos.outStream();
+    var buffered_stdout = std.io.bufferedWriter(std.io.getStdOut().writer());
+    const stdout = buffered_stdout.writer();
     try stdout.writeAll(
         \\// This file is generated by tools/update_clang_options.zig.
         \\// zig fmt: off
@@ -469,7 +469,7 @@ pub fn main() anyerror!void {
         \\
     );
 
-    try stdout_bos.flush();
+    try buffered_stdout.flush();
 }
 
 // TODO we should be able to import clang_options.zig but currently this is problematic because it will
@@ -611,7 +611,7 @@ fn objectLessThan(context: void, a: *json.ObjectMap, b: *json.ObjectMap) bool {
 }
 
 fn usageAndExit(file: fs.File, arg0: []const u8, code: u8) noreturn {
-    file.outStream().print(
+    file.writer().print(
         \\Usage: {} /path/to/llvm-tblgen /path/to/git/llvm/llvm-project
         \\Alternative Usage: zig run /path/to/git/zig/tools/update_clang_options.zig -- /path/to/llvm-tblgen /path/to/git/llvm/llvm-project
         \\
diff --git a/tools/update_glibc.zig b/tools/update_glibc.zig
index 03b746f0ab..1e23bf0ff8 100644
--- a/tools/update_glibc.zig
+++ b/tools/update_glibc.zig
@@ -239,7 +239,7 @@ pub fn main() !void {
         const vers_txt_path = try fs.path.join(allocator, &[_][]const u8{ glibc_out_dir, "vers.txt" });
         const vers_txt_file = try fs.cwd().createFile(vers_txt_path, .{});
         defer vers_txt_file.close();
-        var buffered = std.io.bufferedOutStream(vers_txt_file.writer());
+        var buffered = std.io.bufferedWriter(vers_txt_file.writer());
         const vers_txt = buffered.writer();
         for (global_ver_list) |name, i| {
             _ = global_ver_set.put(name, i) catch unreachable;
@@ -251,7 +251,7 @@ pub fn main() !void {
         const fns_txt_path = try fs.path.join(allocator, &[_][]const u8{ glibc_out_dir, "fns.txt" });
         const fns_txt_file = try fs.cwd().createFile(fns_txt_path, .{});
         defer fns_txt_file.close();
-        var buffered = std.io.bufferedOutStream(fns_txt_file.writer());
+        var buffered = std.io.bufferedWriter(fns_txt_file.writer());
         const fns_txt = buffered.writer();
         for (global_fn_list) |name, i| {
             const entry = global_fn_set.getEntry(name).?;
@@ -282,7 +282,7 @@ pub fn main() !void {
         const abilist_txt_path = try fs.path.join(allocator, &[_][]const u8{ glibc_out_dir, "abi.txt" });
         const abilist_txt_file = try fs.cwd().createFile(abilist_txt_path, .{});
         defer abilist_txt_file.close();
-        var buffered = std.io.bufferedOutStream(abilist_txt_file.writer());
+        var buffered = std.io.bufferedWriter(abilist_txt_file.writer());
         const abilist_txt = buffered.writer();
 
         // first iterate over the abi lists
-- 
cgit v1.2.3


From 31802c6c68a98bdbe34766d3cfdaf65b782851da Mon Sep 17 00:00:00 2001
From: Jonathan Marler <johnnymarler@gmail.com>
Date: Sun, 3 Jan 2021 13:49:51 -0700
Subject: remove z/Z format specifiers

Zig's format system is flexible enough to add custom formatters.  This PR removes the new z/Z format specifiers that were added for printing Zig identifiers and replaces them with custom formatters.
---
 lib/std/build.zig   | 32 +++++++++----------
 lib/std/fmt.zig     | 89 +++++++++++++++++------------------------------------
 lib/std/fs/wasi.zig |  2 +-
 lib/std/zig.zig     |  2 ++
 lib/std/zig/fmt.zig | 71 ++++++++++++++++++++++++++++++++++++++++++
 src/Compilation.zig | 44 +++++++++++++-------------
 src/codegen/c.zig   |  4 +--
 src/translate_c.zig |  7 +++--
 src/value.zig       |  4 +--
 src/zir.zig         |  8 ++---
 10 files changed, 153 insertions(+), 110 deletions(-)
 create mode 100644 lib/std/zig/fmt.zig

(limited to 'src/Compilation.zig')

diff --git a/lib/std/build.zig b/lib/std/build.zig
index cb4cb229e3..5fd64cad0b 100644
--- a/lib/std/build.zig
+++ b/lib/std/build.zig
@@ -1852,25 +1852,25 @@ pub const LibExeObjStep = struct {
         const out = self.build_options_contents.writer();
         switch (T) {
             []const []const u8 => {
-                out.print("pub const {z}: []const []const u8 = &[_][]const u8{{\n", .{name}) catch unreachable;
+                out.print("pub const {}: []const []const u8 = &[_][]const u8{{\n", .{std.zig.fmtId(name)}) catch unreachable;
                 for (value) |slice| {
-                    out.print("    \"{Z}\",\n", .{slice}) catch unreachable;
+                    out.print("    \"{}\",\n", .{std.zig.fmtEscapes(slice)}) catch unreachable;
                 }
                 out.writeAll("};\n") catch unreachable;
                 return;
             },
             [:0]const u8 => {
-                out.print("pub const {z}: [:0]const u8 = \"{Z}\";\n", .{ name, value }) catch unreachable;
+                out.print("pub const {}: [:0]const u8 = \"{}\";\n", .{ std.zig.fmtId(name), std.zig.fmtEscapes(value) }) catch unreachable;
                 return;
             },
             []const u8 => {
-                out.print("pub const {z}: []const u8 = \"{Z}\";\n", .{ name, value }) catch unreachable;
+                out.print("pub const {}: []const u8 = \"{}\";\n", .{ std.zig.fmtId(name), std.zig.fmtEscapes(value) }) catch unreachable;
                 return;
             },
             ?[]const u8 => {
-                out.print("pub const {z}: ?[]const u8 = ", .{name}) catch unreachable;
+                out.print("pub const {}: ?[]const u8 = ", .{std.zig.fmtId(name)}) catch unreachable;
                 if (value) |payload| {
-                    out.print("\"{Z}\";\n", .{payload}) catch unreachable;
+                    out.print("\"{}\";\n", .{std.zig.fmtEscapes(payload)}) catch unreachable;
                 } else {
                     out.writeAll("null;\n") catch unreachable;
                 }
@@ -1878,14 +1878,14 @@ pub const LibExeObjStep = struct {
             },
             std.builtin.Version => {
                 out.print(
-                    \\pub const {z}: @import("builtin").Version = .{{
+                    \\pub const {}: @import("builtin").Version = .{{
                     \\    .major = {d},
                     \\    .minor = {d},
                     \\    .patch = {d},
                     \\}};
                     \\
                 , .{
-                    name,
+                    std.zig.fmtId(name),
 
                     value.major,
                     value.minor,
@@ -1894,23 +1894,23 @@ pub const LibExeObjStep = struct {
             },
             std.SemanticVersion => {
                 out.print(
-                    \\pub const {z}: @import("std").SemanticVersion = .{{
+                    \\pub const {}: @import("std").SemanticVersion = .{{
                     \\    .major = {d},
                     \\    .minor = {d},
                     \\    .patch = {d},
                     \\
                 , .{
-                    name,
+                    std.zig.fmtId(name),
 
                     value.major,
                     value.minor,
                     value.patch,
                 }) catch unreachable;
                 if (value.pre) |some| {
-                    out.print("    .pre = \"{Z}\",\n", .{some}) catch unreachable;
+                    out.print("    .pre = \"{}\",\n", .{std.zig.fmtEscapes(some)}) catch unreachable;
                 }
                 if (value.build) |some| {
-                    out.print("    .build = \"{Z}\",\n", .{some}) catch unreachable;
+                    out.print("    .build = \"{}\",\n", .{std.zig.fmtEscapes(some)}) catch unreachable;
                 }
                 out.writeAll("};\n") catch unreachable;
                 return;
@@ -1919,15 +1919,15 @@ pub const LibExeObjStep = struct {
         }
         switch (@typeInfo(T)) {
             .Enum => |enum_info| {
-                out.print("pub const {z} = enum {{\n", .{@typeName(T)}) catch unreachable;
+                out.print("pub const {} = enum {{\n", .{std.zig.fmtId(@typeName(T))}) catch unreachable;
                 inline for (enum_info.fields) |field| {
-                    out.print("    {z},\n", .{field.name}) catch unreachable;
+                    out.print("    {},\n", .{std.zig.fmtId(field.name)}) catch unreachable;
                 }
                 out.writeAll("};\n") catch unreachable;
             },
             else => {},
         }
-        out.print("pub const {z}: {s} = {};\n", .{ name, @typeName(T), value }) catch unreachable;
+        out.print("pub const {}: {s} = {};\n", .{ std.zig.fmtId(name), @typeName(T), value }) catch unreachable;
     }
 
     /// The value is the path in the cache dir.
@@ -2157,7 +2157,7 @@ pub const LibExeObjStep = struct {
             // Render build artifact options at the last minute, now that the path is known.
             for (self.build_options_artifact_args.items) |item| {
                 const out = self.build_options_contents.writer();
-                out.print("pub const {s}: []const u8 = \"{Z}\";\n", .{ item.name, item.artifact.getOutputPath() }) catch unreachable;
+                out.print("pub const {s}: []const u8 = \"{}\";\n", .{ item.name, std.zig.fmtEscapes(item.artifact.getOutputPath()) }) catch unreachable;
             }
 
             const build_options_file = try fs.path.join(
diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig
index bc9ac92283..d8b81eb264 100644
--- a/lib/std/fmt.zig
+++ b/lib/std/fmt.zig
@@ -715,9 +715,9 @@ pub fn formatText(
         }
         return;
     } else if (comptime std.mem.eql(u8, fmt, "z")) {
-        return formatZigIdentifier(bytes, options, writer);
+        @compileError("specifier 'z' has been deprecated, wrap your argument in std.zig.fmtId instead");
     } else if (comptime std.mem.eql(u8, fmt, "Z")) {
-        return formatZigEscapes(bytes, options, writer);
+        @compileError("specifier 'Z' has been deprecated, wrap your argument in std.zig.fmtEscapes instead");
     } else {
         @compileError("Unknown format string: '" ++ fmt ++ "'");
     }
@@ -782,52 +782,6 @@ pub fn formatBuf(
     }
 }
 
-/// Print the string as a Zig identifier escaping it with @"" syntax if needed.
-pub fn formatZigIdentifier(
-    bytes: []const u8,
-    options: FormatOptions,
-    writer: anytype,
-) !void {
-    if (isValidZigIdentifier(bytes)) {
-        return writer.writeAll(bytes);
-    }
-    try writer.writeAll("@\"");
-    try formatZigEscapes(bytes, options, writer);
-    try writer.writeByte('"');
-}
-
-fn isValidZigIdentifier(bytes: []const u8) bool {
-    for (bytes) |c, i| {
-        switch (c) {
-            '_', 'a'...'z', 'A'...'Z' => {},
-            '0'...'9' => if (i == 0) return false,
-            else => return false,
-        }
-    }
-    return std.zig.Token.getKeyword(bytes) == null;
-}
-
-pub fn formatZigEscapes(
-    bytes: []const u8,
-    options: FormatOptions,
-    writer: anytype,
-) !void {
-    for (bytes) |byte| switch (byte) {
-        '\n' => try writer.writeAll("\\n"),
-        '\r' => try writer.writeAll("\\r"),
-        '\t' => try writer.writeAll("\\t"),
-        '\\' => try writer.writeAll("\\\\"),
-        '"' => try writer.writeAll("\\\""),
-        '\'' => try writer.writeAll("\\'"),
-        ' ', '!', '#'...'&', '('...'[', ']'...'~' => try writer.writeByte(byte),
-        // Use hex escapes for rest any unprintable characters.
-        else => {
-            try writer.writeAll("\\x");
-            try formatInt(byte, 16, false, .{ .width = 2, .fill = '0' }, writer);
-        },
-    };
-}
-
 /// Print a float in scientific notation to the specified precision. Null uses full precision.
 /// It should be the case that every full precision, printed value can be re-parsed back to the
 /// same type unambiguously.
@@ -1173,6 +1127,32 @@ pub const ParseIntError = error{
     InvalidCharacter,
 };
 
+/// Creates a Formatter type from a format function. Wrapping data in Formatter(func) causes
+/// the data to be formatted using the given function `func`.  `func` must be of the following
+/// form:
+///
+///     fn formatExample(
+///         data: T,
+///         comptime fmt: []const u8,
+///         options: std.fmt.FormatOptions,
+///         writer: anytype,
+///     ) !void;
+///
+pub fn Formatter(comptime format_fn: anytype) type {
+    const Data = @typeInfo(@TypeOf(format_fn)).Fn.args[0].arg_type.?;
+    return struct {
+        data: Data,
+        pub fn format(
+            self: @This(),
+            comptime fmt: []const u8,
+            options: std.fmt.FormatOptions,
+            writer: anytype,
+        ) @TypeOf(writer).Error!void {
+            try format_fn(self.data, fmt, options, writer);
+        }
+    };
+}
+
 /// Parses the string `buf` as signed or unsigned representation in the
 /// specified radix of an integral value of type `T`.
 ///
@@ -1608,17 +1588,6 @@ test "escape non-printable" {
     try testFmt("ab\\xFFc", "{E}", .{"ab\xffc"});
 }
 
-test "escape invalid identifiers" {
-    try testFmt("@\"while\"", "{z}", .{"while"});
-    try testFmt("hello", "{z}", .{"hello"});
-    try testFmt("@\"11\\\"23\"", "{z}", .{"11\"23"});
-    try testFmt("@\"11\\x0f23\"", "{z}", .{"11\x0F23"});
-    try testFmt("\\x0f", "{Z}", .{0x0f});
-    try testFmt(
-        \\" \\ hi \x07 \x11 \" derp \'"
-    , "\"{Z}\"", .{" \\ hi \x07 \x11 \" derp '"});
-}
-
 test "pointer" {
     {
         const value = @intToPtr(*align(1) i32, 0xdeadbeef);
@@ -1898,7 +1867,7 @@ test "bytes.hex" {
     try testFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", .{bytes_with_zeros});
 }
 
-fn testFmt(expected: []const u8, comptime template: []const u8, args: anytype) !void {
+pub fn testFmt(expected: []const u8, comptime template: []const u8, args: anytype) !void {
     var buf: [100]u8 = undefined;
     const result = try bufPrint(buf[0..], template, args);
     if (mem.eql(u8, result, expected)) return;
diff --git a/lib/std/fs/wasi.zig b/lib/std/fs/wasi.zig
index 900adc5e2d..cf5431f994 100644
--- a/lib/std/fs/wasi.zig
+++ b/lib/std/fs/wasi.zig
@@ -38,7 +38,7 @@ pub const PreopenType = union(PreopenTypeTag) {
     pub fn format(self: Self, comptime fmt: []const u8, options: std.fmt.FormatOptions, out_stream: anytype) !void {
         try out_stream.print("PreopenType{{ ", .{});
         switch (self) {
-            PreopenType.Dir => |path| try out_stream.print(".Dir = '{z}'", .{path}),
+            PreopenType.Dir => |path| try out_stream.print(".Dir = '{}'", .{std.zig.fmtId(path)}),
         }
         return out_stream.print(" }}", .{});
     }
diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 5d95031e02..c39eb6b05f 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -8,6 +8,8 @@ const tokenizer = @import("zig/tokenizer.zig");
 
 pub const Token = tokenizer.Token;
 pub const Tokenizer = tokenizer.Tokenizer;
+pub const fmtId = @import("zig/fmt.zig").fmtId;
+pub const fmtEscapes = @import("zig/fmt.zig").fmtEscapes;
 pub const parse = @import("zig/parse.zig").parse;
 pub const parseStringLiteral = @import("zig/string_literal.zig").parse;
 pub const render = @import("zig/render.zig").render;
diff --git a/lib/std/zig/fmt.zig b/lib/std/zig/fmt.zig
new file mode 100644
index 0000000000..d852f2b49d
--- /dev/null
+++ b/lib/std/zig/fmt.zig
@@ -0,0 +1,71 @@
+const std = @import("std");
+const mem = std.mem;
+
+/// Print the string as a Zig identifier escaping it with @"" syntax if needed.
+pub fn formatId(
+    bytes: []const u8,
+    comptime fmt: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    if (isValidId(bytes)) {
+        return writer.writeAll(bytes);
+    }
+    try writer.writeAll("@\"");
+    try formatEscapes(bytes, fmt, options, writer);
+    try writer.writeByte('"');
+}
+
+/// Return a Formatter for a Zig identifier
+pub fn fmtId(bytes: []const u8) std.fmt.Formatter(formatId) {
+    return .{ .data = bytes };
+}
+
+pub fn isValidId(bytes: []const u8) bool {
+    for (bytes) |c, i| {
+        switch (c) {
+            '_', 'a'...'z', 'A'...'Z' => {},
+            '0'...'9' => if (i == 0) return false,
+            else => return false,
+        }
+    }
+    return std.zig.Token.getKeyword(bytes) == null;
+}
+
+pub fn formatEscapes(
+    bytes: []const u8,
+    comptime fmt: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    for (bytes) |byte| switch (byte) {
+        '\n' => try writer.writeAll("\\n"),
+        '\r' => try writer.writeAll("\\r"),
+        '\t' => try writer.writeAll("\\t"),
+        '\\' => try writer.writeAll("\\\\"),
+        '"' => try writer.writeAll("\\\""),
+        '\'' => try writer.writeAll("\\'"),
+        ' ', '!', '#'...'&', '('...'[', ']'...'~' => try writer.writeByte(byte),
+        // Use hex escapes for rest any unprintable characters.
+        else => {
+            try writer.writeAll("\\x");
+            try std.fmt.formatInt(byte, 16, false, .{ .width = 2, .fill = '0' }, writer);
+        },
+    };
+}
+
+/// Return a Formatter for Zig Escapes
+pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(formatEscapes) {
+    return .{ .data = bytes };
+}
+
+test "escape invalid identifiers" {
+    try std.fmt.testFmt("@\"while\"", "{}", .{fmtId("while")});
+    try std.fmt.testFmt("hello", "{}", .{fmtId("hello")});
+    try std.fmt.testFmt("@\"11\\\"23\"", "{}", .{fmtId("11\"23")});
+    try std.fmt.testFmt("@\"11\\x0f23\"", "{}", .{fmtId("11\x0F23")});
+    try std.fmt.testFmt("\\x0f", "{}", .{fmtEscapes("\x0f")});
+    try std.fmt.testFmt(
+        \\" \\ hi \x07 \x11 \" derp \'"
+    , "\"{}\"", .{fmtEscapes(" \\ hi \x07 \x11 \" derp '")});
+}
diff --git a/src/Compilation.zig b/src/Compilation.zig
index a81926bf19..75b9ac7520 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2703,27 +2703,27 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const arch = Target.current.cpu.arch;
         \\/// Deprecated
         \\pub const endian = Target.current.cpu.arch.endian();
-        \\pub const output_mode = OutputMode.{z};
-        \\pub const link_mode = LinkMode.{z};
+        \\pub const output_mode = OutputMode.{};
+        \\pub const link_mode = LinkMode.{};
         \\pub const is_test = {};
         \\pub const single_threaded = {};
-        \\pub const abi = Abi.{z};
+        \\pub const abi = Abi.{};
         \\pub const cpu: Cpu = Cpu{{
-        \\    .arch = .{z},
-        \\    .model = &Target.{z}.cpu.{z},
-        \\    .features = Target.{z}.featureSet(&[_]Target.{z}.Feature{{
+        \\    .arch = .{},
+        \\    .model = &Target.{}.cpu.{},
+        \\    .features = Target.{}.featureSet(&[_]Target.{}.Feature{{
         \\
     , .{
-        @tagName(comp.bin_file.options.output_mode),
-        @tagName(comp.bin_file.options.link_mode),
+        std.zig.fmtId(@tagName(comp.bin_file.options.output_mode)),
+        std.zig.fmtId(@tagName(comp.bin_file.options.link_mode)),
         comp.bin_file.options.is_test,
         comp.bin_file.options.single_threaded,
-        @tagName(target.abi),
-        @tagName(target.cpu.arch),
-        generic_arch_name,
-        target.cpu.model.name,
-        generic_arch_name,
-        generic_arch_name,
+        std.zig.fmtId(@tagName(target.abi)),
+        std.zig.fmtId(@tagName(target.cpu.arch)),
+        std.zig.fmtId(generic_arch_name),
+        std.zig.fmtId(target.cpu.model.name),
+        std.zig.fmtId(generic_arch_name),
+        std.zig.fmtId(generic_arch_name),
     });
 
     for (target.cpu.arch.allFeaturesList()) |feature, index_usize| {
@@ -2742,10 +2742,10 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\    }}),
         \\}};
         \\pub const os = Os{{
-        \\    .tag = .{z},
+        \\    .tag = .{},
         \\    .version_range = .{{
     ,
-        .{@tagName(target.os.tag)},
+        .{std.zig.fmtId(@tagName(target.os.tag))},
     );
 
     switch (target.os.getVersionRange()) {
@@ -2828,8 +2828,8 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         (comp.bin_file.options.skip_linker_dependencies and comp.bin_file.options.parent_compilation_link_libc);
 
     try buffer.writer().print(
-        \\pub const object_format = ObjectFormat.{z};
-        \\pub const mode = Mode.{z};
+        \\pub const object_format = ObjectFormat.{};
+        \\pub const mode = Mode.{};
         \\pub const link_libc = {};
         \\pub const link_libcpp = {};
         \\pub const have_error_return_tracing = {};
@@ -2837,11 +2837,11 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const position_independent_code = {};
         \\pub const position_independent_executable = {};
         \\pub const strip_debug_info = {};
-        \\pub const code_model = CodeModel.{z};
+        \\pub const code_model = CodeModel.{};
         \\
     , .{
-        @tagName(comp.bin_file.options.object_format),
-        @tagName(comp.bin_file.options.optimize_mode),
+        std.zig.fmtId(@tagName(comp.bin_file.options.object_format)),
+        std.zig.fmtId(@tagName(comp.bin_file.options.optimize_mode)),
         link_libc,
         comp.bin_file.options.link_libcpp,
         comp.bin_file.options.error_return_tracing,
@@ -2849,7 +2849,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         comp.bin_file.options.pic,
         comp.bin_file.options.pie,
         comp.bin_file.options.strip,
-        @tagName(comp.bin_file.options.machine_code_model),
+        std.zig.fmtId(@tagName(comp.bin_file.options.machine_code_model)),
     });
 
     if (comp.bin_file.options.is_test) {
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 3de79e2a3d..8c85f482fd 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -169,8 +169,8 @@ pub const DeclGen = struct {
                     .undef, .empty_struct_value, .empty_array => try writer.writeAll("{}"),
                     .bytes => {
                         const bytes = val.castTag(.bytes).?.data;
-                        // TODO: make our own C string escape instead of using {Z}
-                        try writer.print("\"{Z}\"", .{bytes});
+                        // TODO: make our own C string escape instead of using std.zig.fmtEscapes
+                        try writer.print("\"{}\"", .{std.zig.fmtEscapes(bytes)});
                     },
                     else => {
                         // Fall back to generic implementation.
diff --git a/src/translate_c.zig b/src/translate_c.zig
index 8df7963267..baca8c12de 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -2031,7 +2031,7 @@ fn transStringLiteral(
             const bytes_ptr = stmt.getString_bytes_begin_size(&len);
             const str = bytes_ptr[0..len];
 
-            const token = try appendTokenFmt(rp.c, .StringLiteral, "\"{Z}\"", .{str});
+            const token = try appendTokenFmt(rp.c, .StringLiteral, "\"{}\"", .{std.zig.fmtEscapes(str)});
             const node = try rp.c.arena.create(ast.Node.OneToken);
             node.* = .{
                 .base = .{ .tag = .StringLiteral },
@@ -2944,7 +2944,8 @@ fn transCharLiteral(
                 if (val > 255)
                     break :blk try transCreateNodeInt(rp.c, val);
             }
-            const token = try appendTokenFmt(rp.c, .CharLiteral, "'{Z}'", .{@intCast(u8, val)});
+            const val_array = [_]u8 { @intCast(u8, val) };
+            const token = try appendTokenFmt(rp.c, .CharLiteral, "'{}'", .{std.zig.fmtEscapes(&val_array)});
             const node = try rp.c.arena.create(ast.Node.OneToken);
             node.* = .{
                 .base = .{ .tag = .CharLiteral },
@@ -5315,7 +5316,7 @@ fn isZigPrimitiveType(name: []const u8) bool {
 }
 
 fn appendIdentifier(c: *Context, name: []const u8) !ast.TokenIndex {
-    return appendTokenFmt(c, .Identifier, "{z}", .{name});
+    return appendTokenFmt(c, .Identifier, "{}", .{std.zig.fmtId(name)});
 }
 
 fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node {
diff --git a/src/value.zig b/src/value.zig
index 11c385b446..036e3b71bf 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -491,8 +491,8 @@ pub const Value = extern union {
                 val = elem_ptr.array_ptr;
             },
             .empty_array => return out_stream.writeAll(".{}"),
-            .enum_literal => return out_stream.print(".{z}", .{self.castTag(.enum_literal).?.data}),
-            .bytes => return out_stream.print("\"{Z}\"", .{self.castTag(.bytes).?.data}),
+            .enum_literal => return out_stream.print(".{}", .{std.zig.fmtId(self.castTag(.enum_literal).?.data)}),
+            .bytes => return out_stream.print("\"{}\"", .{std.zig.fmtEscapes(self.castTag(.bytes).?.data)}),
             .repeated => {
                 try out_stream.writeAll("(repeated) ");
                 val = val.castTag(.repeated).?.data;
diff --git a/src/zir.zig b/src/zir.zig
index b3d004f1dc..246e9dda1f 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -1308,17 +1308,17 @@ const Writer = struct {
                 try stream.writeByte('}');
             },
             bool => return stream.writeByte("01"[@boolToInt(param)]),
-            []u8, []const u8 => return stream.print("\"{Z}\"", .{param}),
+            []u8, []const u8 => return stream.print("\"{}\"", .{std.zig.fmtEscapes(param)}),
             BigIntConst, usize => return stream.print("{}", .{param}),
             TypedValue => return stream.print("TypedValue{{ .ty = {}, .val = {}}}", .{ param.ty, param.val }),
             *IrModule.Decl => return stream.print("Decl({s})", .{param.name}),
             *Inst.Block => {
                 const name = self.block_table.get(param).?;
-                return stream.print("\"{Z}\"", .{name});
+                return stream.print("\"{}\"", .{std.zig.fmtEscapes(name)});
             },
             *Inst.Loop => {
                 const name = self.loop_table.get(param).?;
-                return stream.print("\"{Z}\"", .{name});
+                return stream.print("\"{}\"", .{std.zig.fmtEscapes(name)});
             },
             [][]const u8 => {
                 try stream.writeByte('[');
@@ -1326,7 +1326,7 @@ const Writer = struct {
                     if (i != 0) {
                         try stream.writeAll(", ");
                     }
-                    try stream.print("\"{Z}\"", .{str});
+                    try stream.print("\"{}\"", .{std.zig.fmtEscapes(str)});
                 }
                 try stream.writeByte(']');
             },
-- 
cgit v1.2.3


From a0ad2dee6a0b3bb7fd04032f6113206f7b4e73eb Mon Sep 17 00:00:00 2001
From: Jay Petacat <jay@jayschwa.net>
Date: Mon, 4 Jan 2021 23:48:34 -0500
Subject: builtin: Add zig_version

This will enable code to perform version checks and make it easier to
support multiple versions of Zig.

Within the SemVer implementation, an intermediate value needed to be
coerced to a slice to workaround a comptime bug.

Closes #6466
---
 lib/std/SemanticVersion.zig | 11 ++++++++++-
 src/Compilation.zig         |  6 ++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/SemanticVersion.zig b/lib/std/SemanticVersion.zig
index dc9e6d8572..76767de8b5 100644
--- a/lib/std/SemanticVersion.zig
+++ b/lib/std/SemanticVersion.zig
@@ -102,7 +102,7 @@ pub fn parse(text: []const u8) !Version {
     if (extra_index == null) return ver;
 
     // Slice optional pre-release or build metadata components.
-    const extra = text[extra_index.?..text.len];
+    const extra: []const u8 = text[extra_index.?..text.len];
     if (extra[0] == '-') {
         const build_index = std.mem.indexOfScalar(u8, extra, '+');
         ver.pre = extra[1..(build_index orelse extra.len)];
@@ -293,3 +293,12 @@ fn testFmt(expected: []const u8, comptime template: []const u8, args: anytype) !
     std.debug.warn("\n======================================\n", .{});
     return error.TestFailed;
 }
+
+test "zig_version" {
+    // An approximate Zig build that predates this test.
+    const older_version = .{ .major = 0, .minor = 8, .patch = 0, .pre = "dev.874" };
+
+    // Simulated compatibility check using Zig version.
+    const compatible = comptime @import("builtin").zig_version.order(older_version) == .gt;
+    if (!compatible) @compileError("zig_version test failed");
+}
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 75b9ac7520..b017dde8c8 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2703,6 +2703,11 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\pub const arch = Target.current.cpu.arch;
         \\/// Deprecated
         \\pub const endian = Target.current.cpu.arch.endian();
+        \\
+        \\/// Zig version. When writing code that supports multiple versions of Zig, prefer
+        \\/// feature detection (i.e. with `@hasDecl` or `@hasField`) over version checks.
+        \\pub const zig_version = try @import("std").SemanticVersion.parse("{s}");
+        \\
         \\pub const output_mode = OutputMode.{};
         \\pub const link_mode = LinkMode.{};
         \\pub const is_test = {};
@@ -2714,6 +2719,7 @@ pub fn generateBuiltinZigSource(comp: *Compilation, allocator: *Allocator) ![]u8
         \\    .features = Target.{}.featureSet(&[_]Target.{}.Feature{{
         \\
     , .{
+        build_options.version,
         std.zig.fmtId(@tagName(comp.bin_file.options.output_mode)),
         std.zig.fmtId(@tagName(comp.bin_file.options.link_mode)),
         comp.bin_file.options.is_test,
-- 
cgit v1.2.3


From 8932c2d7456fc86b9e92c7976cedcce798caef1a Mon Sep 17 00:00:00 2001
From: Lee Cannon <leecannon@leecannon.xyz>
Date: Sun, 20 Dec 2020 21:41:00 +0000
Subject: Added support for no red zone

---
 lib/std/build.zig        |  4 ++++
 src/Compilation.zig      | 12 +++++++++++-
 src/glibc.zig            |  1 +
 src/libcxx.zig           |  2 ++
 src/libunwind.zig        |  1 +
 src/link.zig             |  1 +
 src/main.zig             |  8 ++++++++
 src/musl.zig             |  1 +
 src/stage1.zig           |  1 +
 src/stage1/all_types.hpp |  1 +
 src/stage1/codegen.cpp   |  4 ++++
 src/stage1/stage1.cpp    |  1 +
 src/stage1/stage1.h      |  1 +
 13 files changed, 37 insertions(+), 1 deletion(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/build.zig b/lib/std/build.zig
index 5fd64cad0b..869671c785 100644
--- a/lib/std/build.zig
+++ b/lib/std/build.zig
@@ -1262,6 +1262,7 @@ pub const LibExeObjStep = struct {
     disable_stack_probing: bool,
     disable_sanitize_c: bool,
     sanitize_thread: bool,
+    no_red_zone: bool = false,
     rdynamic: bool,
     c_std: Builder.CStd,
     override_lib_dir: ?[]const u8,
@@ -2260,6 +2261,9 @@ pub const LibExeObjStep = struct {
         if (self.disable_stack_probing) {
             try zig_args.append("-fno-stack-check");
         }
+        if (self.no_red_zone) {
+            try zig_args.append("-fno-red-zone");
+        }
         if (self.disable_sanitize_c) {
             try zig_args.append("-fno-sanitize-c");
         }
diff --git a/src/Compilation.zig b/src/Compilation.zig
index b017dde8c8..1588544010 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -392,6 +392,7 @@ pub const InitOptions = struct {
     want_pie: ?bool = null,
     want_sanitize_c: ?bool = null,
     want_stack_check: ?bool = null,
+    no_red_zone: bool = false,
     want_valgrind: ?bool = null,
     want_tsan: ?bool = null,
     want_compiler_rt: ?bool = null,
@@ -773,6 +774,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(pie);
         cache.hash.add(tsan);
         cache.hash.add(stack_check);
+        cache.hash.add(options.no_red_zone);
         cache.hash.add(link_mode);
         cache.hash.add(options.function_sections);
         cache.hash.add(strip);
@@ -982,6 +984,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .valgrind = valgrind,
             .tsan = tsan,
             .stack_check = stack_check,
+            .no_red_zone = options.no_red_zone,
             .single_threaded = single_threaded,
             .verbose_link = options.verbose_link,
             .machine_code_model = options.machine_code_model,
@@ -2255,7 +2258,11 @@ pub fn addCCArgs(
             } else if (!comp.sanitize_c and comp.bin_file.options.tsan) {
                 try argv.append("-fsanitize=thread");
             }
-
+            
+            if (comp.bin_file.options.no_red_zone) {
+                try argv.append("-mno-red-zone");
+            }
+            
             switch (comp.bin_file.options.optimize_mode) {
                 .Debug => {
                     // windows c runtime requires -D_DEBUG if using debug libraries
@@ -2960,6 +2967,7 @@ fn buildOutputFromZig(
         .function_sections = true,
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
@@ -3198,6 +3206,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .tsan_enabled = comp.bin_file.options.tsan,
         .function_sections = comp.bin_file.options.function_sections,
         .enable_stack_probing = comp.bin_file.options.stack_check,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .enable_time_report = comp.time_report,
         .enable_stack_report = comp.stack_report,
         .test_is_evented = comp.test_evented_io,
@@ -3342,6 +3351,7 @@ pub fn build_crt_file(
         .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/glibc.zig b/src/glibc.zig
index e7b7b1b1cf..28ed31f7bb 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -934,6 +934,7 @@ fn buildSharedLib(
         .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .emit_h = null,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index e79a0106e9..dc9fdc15e7 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -167,6 +167,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
@@ -284,6 +285,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index 7710b1aa77..cfc0cb00aa 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -108,6 +108,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
+        .no_red_zone = comp.bin_file.options.no_red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/link.zig b/src/link.zig
index ffb69adee9..04cfce99ab 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -77,6 +77,7 @@ pub const Options = struct {
     valgrind: bool,
     tsan: bool,
     stack_check: bool,
+    no_red_zone: bool,
     single_threaded: bool,
     verbose_link: bool,
     dll_export_fns: bool,
diff --git a/src/main.zig b/src/main.zig
index b50f89e5c2..a7a23c2b2d 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -282,6 +282,8 @@ const usage_build_generic =
     \\  -fno-PIE                  Force-disable Position Independent Executable
     \\  -fstack-check             Enable stack probing in unsafe builds
     \\  -fno-stack-check          Disable stack probing in safe builds
+    \\  -fred-zone                Enable the "red-zone"
+    \\  -fno-red-zone             Disable the "red-zone"
     \\  -fsanitize-c              Enable C undefined behavior detection in unsafe builds
     \\  -fno-sanitize-c           Disable C undefined behavior detection in safe builds
     \\  -fvalgrind                Include valgrind client requests in release builds
@@ -505,6 +507,7 @@ fn buildOutputType(
     var want_pie: ?bool = null;
     var want_sanitize_c: ?bool = null;
     var want_stack_check: ?bool = null;
+    var no_red_zone: bool = false;
     var want_valgrind: ?bool = null;
     var want_tsan: ?bool = null;
     var want_compiler_rt: ?bool = null;
@@ -843,6 +846,10 @@ fn buildOutputType(
                         want_stack_check = true;
                     } else if (mem.eql(u8, arg, "-fno-stack-check")) {
                         want_stack_check = false;
+                    } else if (mem.eql(u8, arg, "-fred-zone")) {
+                        no_red_zone = false;
+                    } else if (mem.eql(u8, arg, "-fno-red-zone")) {
+                        no_red_zone = true;
                     } else if (mem.eql(u8, arg, "-fsanitize-c")) {
                         want_sanitize_c = true;
                     } else if (mem.eql(u8, arg, "-fno-sanitize-c")) {
@@ -1760,6 +1767,7 @@ fn buildOutputType(
         .want_pie = want_pie,
         .want_sanitize_c = want_sanitize_c,
         .want_stack_check = want_stack_check,
+        .no_red_zone = no_red_zone,
         .want_valgrind = want_valgrind,
         .want_tsan = want_tsan,
         .want_compiler_rt = want_compiler_rt,
diff --git a/src/musl.zig b/src/musl.zig
index a865e78623..d9060307d0 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -206,6 +206,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .optimize_mode = comp.compilerRtOptMode(),
                 .want_sanitize_c = false,
                 .want_stack_check = false,
+                .no_red_zone = comp.bin_file.options.no_red_zone,
                 .want_valgrind = false,
                 .want_tsan = false,
                 .emit_h = null,
diff --git a/src/stage1.zig b/src/stage1.zig
index cf3a252ce8..721ab93fcd 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -119,6 +119,7 @@ pub const Module = extern struct {
     tsan_enabled: bool,
     function_sections: bool,
     enable_stack_probing: bool,
+    no_red_zone: bool,
     enable_time_report: bool,
     enable_stack_report: bool,
     test_is_evented: bool,
diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index 24b54ad794..af8305d4b0 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -2195,6 +2195,7 @@ struct CodeGen {
     bool link_mode_dynamic;
     bool dll_export_fns;
     bool have_stack_probing;
+    bool no_red_zone;
     bool function_sections;
     bool test_is_evented;
     bool valgrind_enabled;
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index 386bc43086..fbc13f2638 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -513,6 +513,10 @@ static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) {
     } else {
         maybe_import_dll(g, llvm_fn, linkage);
     }
+    
+    if (g->no_red_zone) {
+        addLLVMFnAttr(llvm_fn, "noredzone");
+    }
 
     if (fn->alignstack_value != 0) {
         addLLVMFnAttrInt(llvm_fn, "alignstack", fn->alignstack_value);
diff --git a/src/stage1/stage1.cpp b/src/stage1/stage1.cpp
index 1195a83931..7cf34c9a47 100644
--- a/src/stage1/stage1.cpp
+++ b/src/stage1/stage1.cpp
@@ -91,6 +91,7 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) {
     g->have_pic = stage1->pic;
     g->have_pie = stage1->pie;
     g->have_stack_probing = stage1->enable_stack_probing;
+    g->no_red_zone = stage1->no_red_zone;
     g->is_single_threaded = stage1->is_single_threaded;
     g->valgrind_enabled = stage1->valgrind_enabled;
     g->tsan_enabled = stage1->tsan_enabled;
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index dbf6491699..66e0effbf6 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -188,6 +188,7 @@ struct ZigStage1 {
     bool tsan_enabled;
     bool function_sections;
     bool enable_stack_probing;
+    bool no_red_zone;
     bool enable_time_report;
     bool enable_stack_report;
     bool test_is_evented;
-- 
cgit v1.2.3


From 5b2a79848ced20db80f3f4ce46b3ef7f4a051d53 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 11 Jan 2021 22:01:16 -0700
Subject: stage2: cleanups regarding red zone CLI flags

 * CLI: change to -mred-zone and -mno-red-zone to match gcc/clang.
 * build.zig: remove the double negative and make it an optional bool.
   This follows precedent from other flags, allowing the compiler CLI to
   be the decider of what is default instead of duplicating the default
   value into the build system code.
 * Compilation: make it an optional `want_red_zone` instead of a
   `no_red_zone` bool. The default is decided by a call to
   `target_util.hasRedZone`.
 * When creating a Clang command line, put -mred-zone on the command
   line if we are forcing it to be enabled.
 * Update update_clang_options.zig with respect to the recent {s}/{} format changes.
 * `zig cc` integration with red zone preference.
---
 lib/std/build.zig              | 11 ++++++--
 src/Compilation.zig            | 21 ++++++++------
 src/clang_options_data.zig     | 18 ++++++++++--
 src/glibc.zig                  |  2 +-
 src/libcxx.zig                 |  4 +--
 src/libunwind.zig              |  2 +-
 src/link.zig                   |  2 +-
 src/main.zig                   | 20 +++++++------
 src/musl.zig                   |  2 +-
 src/stage1.zig                 |  2 +-
 src/stage1/all_types.hpp       |  2 +-
 src/stage1/codegen.cpp         |  4 +--
 src/stage1/stage1.cpp          |  2 +-
 src/stage1/stage1.h            |  2 +-
 src/target.zig                 | 18 ++++++++++++
 tools/update_clang_options.zig | 64 ++++++++++++++++++++++++------------------
 16 files changed, 114 insertions(+), 62 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/build.zig b/lib/std/build.zig
index 869671c785..381488d800 100644
--- a/lib/std/build.zig
+++ b/lib/std/build.zig
@@ -1262,7 +1262,6 @@ pub const LibExeObjStep = struct {
     disable_stack_probing: bool,
     disable_sanitize_c: bool,
     sanitize_thread: bool,
-    no_red_zone: bool = false,
     rdynamic: bool,
     c_std: Builder.CStd,
     override_lib_dir: ?[]const u8,
@@ -1333,6 +1332,8 @@ pub const LibExeObjStep = struct {
     /// Position Independent Executable
     pie: ?bool = null,
 
+    red_zone: ?bool = null,
+
     subsystem: ?builtin.SubSystem = null,
 
     /// Overrides the default stack size
@@ -2261,8 +2262,12 @@ pub const LibExeObjStep = struct {
         if (self.disable_stack_probing) {
             try zig_args.append("-fno-stack-check");
         }
-        if (self.no_red_zone) {
-            try zig_args.append("-fno-red-zone");
+        if (self.red_zone) |red_zone| {
+            if (red_zone) {
+                try zig_args.append("-mred-zone");
+            } else {
+                try zig_args.append("-mno-red-zone");
+            }
         }
         if (self.disable_sanitize_c) {
             try zig_args.append("-fno-sanitize-c");
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 1588544010..0efad3362d 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -392,7 +392,7 @@ pub const InitOptions = struct {
     want_pie: ?bool = null,
     want_sanitize_c: ?bool = null,
     want_stack_check: ?bool = null,
-    no_red_zone: bool = false,
+    want_red_zone: ?bool = null,
     want_valgrind: ?bool = null,
     want_tsan: ?bool = null,
     want_compiler_rt: ?bool = null,
@@ -744,6 +744,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         } else null;
 
         const strip = options.strip or !target_util.hasDebugInfo(options.target);
+        const red_zone = options.want_red_zone orelse target_util.hasRedZone(options.target);
 
         // We put everything into the cache hash that *cannot be modified during an incremental update*.
         // For example, one cannot change the target between updates, but one can change source files,
@@ -774,7 +775,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(pie);
         cache.hash.add(tsan);
         cache.hash.add(stack_check);
-        cache.hash.add(options.no_red_zone);
+        cache.hash.add(red_zone);
         cache.hash.add(link_mode);
         cache.hash.add(options.function_sections);
         cache.hash.add(strip);
@@ -984,7 +985,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .valgrind = valgrind,
             .tsan = tsan,
             .stack_check = stack_check,
-            .no_red_zone = options.no_red_zone,
+            .red_zone = red_zone,
             .single_threaded = single_threaded,
             .verbose_link = options.verbose_link,
             .machine_code_model = options.machine_code_model,
@@ -2258,11 +2259,13 @@ pub fn addCCArgs(
             } else if (!comp.sanitize_c and comp.bin_file.options.tsan) {
                 try argv.append("-fsanitize=thread");
             }
-            
-            if (comp.bin_file.options.no_red_zone) {
+
+            if (comp.bin_file.options.red_zone) {
+                try argv.append("-mred-zone");
+            } else if (target_util.hasRedZone(target)) {
                 try argv.append("-mno-red-zone");
             }
-            
+
             switch (comp.bin_file.options.optimize_mode) {
                 .Debug => {
                     // windows c runtime requires -D_DEBUG if using debug libraries
@@ -2967,7 +2970,7 @@ fn buildOutputFromZig(
         .function_sections = true,
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
@@ -3206,7 +3209,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .tsan_enabled = comp.bin_file.options.tsan,
         .function_sections = comp.bin_file.options.function_sections,
         .enable_stack_probing = comp.bin_file.options.stack_check,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .red_zone = comp.bin_file.options.red_zone,
         .enable_time_report = comp.time_report,
         .enable_stack_report = comp.stack_report,
         .test_is_evented = comp.test_evented_io,
@@ -3351,7 +3354,7 @@ pub fn build_crt_file(
         .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/clang_options_data.zig b/src/clang_options_data.zig
index 52dc8c0280..c901c6045f 100644
--- a/src/clang_options_data.zig
+++ b/src/clang_options_data.zig
@@ -3803,7 +3803,14 @@ flagpd1("mno-qpx"),
 flagpd1("mno-rdpid"),
 flagpd1("mno-rdrnd"),
 flagpd1("mno-rdseed"),
-flagpd1("mno-red-zone"),
+.{
+    .name = "mno-red-zone",
+    .syntax = .flag,
+    .zig_equivalent = .no_red_zone,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 flagpd1("mno-reference-types"),
 flagpd1("mno-relax"),
 flagpd1("mno-relax-all"),
@@ -3901,7 +3908,14 @@ flagpd1("mrdseed"),
 flagpd1("mreassociate"),
 flagpd1("mrecip"),
 flagpd1("mrecord-mcount"),
-flagpd1("mred-zone"),
+.{
+    .name = "mred-zone",
+    .syntax = .flag,
+    .zig_equivalent = .red_zone,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 flagpd1("mreference-types"),
 sepd1("mregparm"),
 flagpd1("mrelax"),
diff --git a/src/glibc.zig b/src/glibc.zig
index 28ed31f7bb..b154bd530d 100644
--- a/src/glibc.zig
+++ b/src/glibc.zig
@@ -934,7 +934,7 @@ fn buildSharedLib(
         .optimize_mode = comp.compilerRtOptMode(),
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .emit_h = null,
diff --git a/src/libcxx.zig b/src/libcxx.zig
index dc9fdc15e7..11989998e9 100644
--- a/src/libcxx.zig
+++ b/src/libcxx.zig
@@ -167,7 +167,7 @@ pub fn buildLibCXX(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
@@ -285,7 +285,7 @@ pub fn buildLibCXXABI(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = comp.bin_file.options.tsan,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/libunwind.zig b/src/libunwind.zig
index cfc0cb00aa..c2d0475d6f 100644
--- a/src/libunwind.zig
+++ b/src/libunwind.zig
@@ -108,7 +108,7 @@ pub fn buildStaticLib(comp: *Compilation) !void {
         .link_mode = link_mode,
         .want_sanitize_c = false,
         .want_stack_check = false,
-        .no_red_zone = comp.bin_file.options.no_red_zone,
+        .want_red_zone = comp.bin_file.options.red_zone,
         .want_valgrind = false,
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
diff --git a/src/link.zig b/src/link.zig
index 04cfce99ab..6914131bea 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -77,7 +77,7 @@ pub const Options = struct {
     valgrind: bool,
     tsan: bool,
     stack_check: bool,
-    no_red_zone: bool,
+    red_zone: bool,
     single_threaded: bool,
     verbose_link: bool,
     dll_export_fns: bool,
diff --git a/src/main.zig b/src/main.zig
index a7a23c2b2d..867aa348b1 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -267,6 +267,8 @@ const usage_build_generic =
     \\  -mcmodel=[default|tiny|   Limit range of code and data virtual addresses
     \\            small|kernel|
     \\            medium|large]
+    \\  -mred-zone                Force-enable the "red-zone"
+    \\  -mno-red-zone             Force-disable the "red-zone"
     \\  --name [name]             Override root name (not a file path)
     \\  -O [mode]                 Choose what to optimize for
     \\    Debug                   (default) Optimizations off, safety on
@@ -282,8 +284,6 @@ const usage_build_generic =
     \\  -fno-PIE                  Force-disable Position Independent Executable
     \\  -fstack-check             Enable stack probing in unsafe builds
     \\  -fno-stack-check          Disable stack probing in safe builds
-    \\  -fred-zone                Enable the "red-zone"
-    \\  -fno-red-zone             Disable the "red-zone"
     \\  -fsanitize-c              Enable C undefined behavior detection in unsafe builds
     \\  -fno-sanitize-c           Disable C undefined behavior detection in safe builds
     \\  -fvalgrind                Include valgrind client requests in release builds
@@ -507,7 +507,7 @@ fn buildOutputType(
     var want_pie: ?bool = null;
     var want_sanitize_c: ?bool = null;
     var want_stack_check: ?bool = null;
-    var no_red_zone: bool = false;
+    var want_red_zone: ?bool = null;
     var want_valgrind: ?bool = null;
     var want_tsan: ?bool = null;
     var want_compiler_rt: ?bool = null;
@@ -846,10 +846,10 @@ fn buildOutputType(
                         want_stack_check = true;
                     } else if (mem.eql(u8, arg, "-fno-stack-check")) {
                         want_stack_check = false;
-                    } else if (mem.eql(u8, arg, "-fred-zone")) {
-                        no_red_zone = false;
-                    } else if (mem.eql(u8, arg, "-fno-red-zone")) {
-                        no_red_zone = true;
+                    } else if (mem.eql(u8, arg, "-mred-zone")) {
+                        want_red_zone = true;
+                    } else if (mem.eql(u8, arg, "-mno-red-zone")) {
+                        want_red_zone = false;
                     } else if (mem.eql(u8, arg, "-fsanitize-c")) {
                         want_sanitize_c = true;
                     } else if (mem.eql(u8, arg, "-fno-sanitize-c")) {
@@ -1075,6 +1075,8 @@ fn buildOutputType(
                     .no_pic => want_pic = false,
                     .pie => want_pie = true,
                     .no_pie => want_pie = false,
+                    .red_zone => want_red_zone = true,
+                    .no_red_zone => want_red_zone = false,
                     .nostdlib => ensure_libc_on_non_freestanding = false,
                     .nostdlib_cpp => ensure_libcpp_on_non_freestanding = false,
                     .shared => {
@@ -1767,7 +1769,7 @@ fn buildOutputType(
         .want_pie = want_pie,
         .want_sanitize_c = want_sanitize_c,
         .want_stack_check = want_stack_check,
-        .no_red_zone = no_red_zone,
+        .want_red_zone = want_red_zone,
         .want_valgrind = want_valgrind,
         .want_tsan = want_tsan,
         .want_compiler_rt = want_compiler_rt,
@@ -2977,6 +2979,8 @@ pub const ClangArgIterator = struct {
         framework_dir,
         framework,
         nostdlibinc,
+        red_zone,
+        no_red_zone,
     };
 
     const Args = struct {
diff --git a/src/musl.zig b/src/musl.zig
index d9060307d0..0cb6983e28 100644
--- a/src/musl.zig
+++ b/src/musl.zig
@@ -206,7 +206,7 @@ pub fn buildCRTFile(comp: *Compilation, crt_file: CRTFile) !void {
                 .optimize_mode = comp.compilerRtOptMode(),
                 .want_sanitize_c = false,
                 .want_stack_check = false,
-                .no_red_zone = comp.bin_file.options.no_red_zone,
+                .want_red_zone = comp.bin_file.options.red_zone,
                 .want_valgrind = false,
                 .want_tsan = false,
                 .emit_h = null,
diff --git a/src/stage1.zig b/src/stage1.zig
index 721ab93fcd..44fd8e109e 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -119,7 +119,7 @@ pub const Module = extern struct {
     tsan_enabled: bool,
     function_sections: bool,
     enable_stack_probing: bool,
-    no_red_zone: bool,
+    red_zone: bool,
     enable_time_report: bool,
     enable_stack_report: bool,
     test_is_evented: bool,
diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index af8305d4b0..c235bb362d 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -2195,7 +2195,7 @@ struct CodeGen {
     bool link_mode_dynamic;
     bool dll_export_fns;
     bool have_stack_probing;
-    bool no_red_zone;
+    bool red_zone;
     bool function_sections;
     bool test_is_evented;
     bool valgrind_enabled;
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index fbc13f2638..10943483f4 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -513,8 +513,8 @@ static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) {
     } else {
         maybe_import_dll(g, llvm_fn, linkage);
     }
-    
-    if (g->no_red_zone) {
+
+    if (!g->red_zone) {
         addLLVMFnAttr(llvm_fn, "noredzone");
     }
 
diff --git a/src/stage1/stage1.cpp b/src/stage1/stage1.cpp
index 7cf34c9a47..d680121577 100644
--- a/src/stage1/stage1.cpp
+++ b/src/stage1/stage1.cpp
@@ -91,7 +91,7 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) {
     g->have_pic = stage1->pic;
     g->have_pie = stage1->pie;
     g->have_stack_probing = stage1->enable_stack_probing;
-    g->no_red_zone = stage1->no_red_zone;
+    g->red_zone = stage1->red_zone;
     g->is_single_threaded = stage1->is_single_threaded;
     g->valgrind_enabled = stage1->valgrind_enabled;
     g->tsan_enabled = stage1->tsan_enabled;
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index 66e0effbf6..6db3621cf5 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -188,7 +188,7 @@ struct ZigStage1 {
     bool tsan_enabled;
     bool function_sections;
     bool enable_stack_probing;
-    bool no_red_zone;
+    bool red_zone;
     bool enable_time_report;
     bool enable_stack_report;
     bool test_is_evented;
diff --git a/src/target.zig b/src/target.zig
index daac577c7b..c3df682ce0 100644
--- a/src/target.zig
+++ b/src/target.zig
@@ -349,3 +349,21 @@ pub fn defaultCompilerRtOptimizeMode(target: std.Target) std.builtin.Mode {
         return .ReleaseFast;
     }
 }
+
+pub fn hasRedZone(target: std.Target) bool {
+    return switch (target.cpu.arch) {
+        .x86_64,
+        .i386,
+        .wasm32,
+        .wasm64,
+        .powerpc,
+        .powerpc64,
+        .powerpc64le,
+        .aarch64,
+        .aarch64_be,
+        .aarch64_32,
+        => true,
+
+        else => false,
+    };
+}
diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig
index ad380e0139..189a2ed7fa 100644
--- a/tools/update_clang_options.zig
+++ b/tools/update_clang_options.zig
@@ -248,6 +248,14 @@ const known_options = [_]KnownOpt{
         .name = "mtune",
         .ident = "mcpu",
     },
+    .{
+        .name = "mred-zone",
+        .ident = "red_zone",
+    },
+    .{
+        .name = "mno-red-zone",
+        .ident = "no_red_zone",
+    },
     .{
         .name = "MD",
         .ident = "dep_file",
@@ -340,9 +348,9 @@ pub fn main() anyerror!void {
     const child_args = [_][]const u8{
         llvm_tblgen_exe,
         "--dump-json",
-        try std.fmt.allocPrint(allocator, "{}/clang/include/clang/Driver/Options.td", .{llvm_src_root}),
-        try std.fmt.allocPrint(allocator, "-I={}/llvm/include", .{llvm_src_root}),
-        try std.fmt.allocPrint(allocator, "-I={}/clang/include/clang/Driver", .{llvm_src_root}),
+        try std.fmt.allocPrint(allocator, "{s}/clang/include/clang/Driver/Options.td", .{llvm_src_root}),
+        try std.fmt.allocPrint(allocator, "-I={s}/llvm/include", .{llvm_src_root}),
+        try std.fmt.allocPrint(allocator, "-I={s}/clang/include/clang/Driver", .{llvm_src_root}),
     };
 
     const child_result = try std.ChildProcess.exec(.{
@@ -351,11 +359,11 @@ pub fn main() anyerror!void {
         .max_output_bytes = 100 * 1024 * 1024,
     });
 
-    std.debug.warn("{}\n", .{child_result.stderr});
+    std.debug.warn("{s}\n", .{child_result.stderr});
 
     const json_text = switch (child_result.term) {
         .Exited => |code| if (code == 0) child_result.stdout else {
-            std.debug.warn("llvm-tblgen exited with code {}\n", .{code});
+            std.debug.warn("llvm-tblgen exited with code {d}\n", .{code});
             std.process.exit(1);
         },
         else => {
@@ -412,7 +420,7 @@ pub fn main() anyerror!void {
             } else if (std.mem.eql(u8, prefix, "/")) {
                 pslash = true;
             } else {
-                std.debug.warn("{} has unrecognized prefix '{}'\n", .{ name, prefix });
+                std.debug.warn("{s} has unrecognized prefix '{s}'\n", .{ name, prefix });
                 std.process.exit(1);
             }
         }
@@ -422,7 +430,7 @@ pub fn main() anyerror!void {
             // `-MT foo` is ambiguous because there is also an -MT flag
             // The canonical way to specify the flag is with `/MT` and so we make this
             // the only way.
-            try stdout.print("flagpsl(\"{}\"),\n", .{name});
+            try stdout.print("flagpsl(\"{s}\"),\n", .{name});
         } else if (knownOption(name)) |ident| {
 
             // Workaround the fact that in 'Options.td'  -Ofast is listed as 'joined'
@@ -430,34 +438,34 @@ pub fn main() anyerror!void {
 
             try stdout.print(
                 \\.{{
-                \\    .name = "{}",
-                \\    .syntax = {},
-                \\    .zig_equivalent = .{},
-                \\    .pd1 = {},
-                \\    .pd2 = {},
-                \\    .psl = {},
+                \\    .name = "{s}",
+                \\    .syntax = {s},
+                \\    .zig_equivalent = .{s},
+                \\    .pd1 = {s},
+                \\    .pd2 = {s},
+                \\    .psl = {s},
                 \\}},
                 \\
             , .{ name, final_syntax, ident, pd1, pd2, pslash });
         } else if (pd1 and !pd2 and !pslash and syntax == .flag) {
-            try stdout.print("flagpd1(\"{}\"),\n", .{name});
+            try stdout.print("flagpd1(\"{s}\"),\n", .{name});
         } else if (!pd1 and !pd2 and pslash and syntax == .flag) {
-            try stdout.print("flagpsl(\"{}\"),\n", .{name});
+            try stdout.print("flagpsl(\"{s}\"),\n", .{name});
         } else if (pd1 and !pd2 and !pslash and syntax == .joined) {
-            try stdout.print("joinpd1(\"{}\"),\n", .{name});
+            try stdout.print("joinpd1(\"{s}\"),\n", .{name});
         } else if (pd1 and !pd2 and !pslash and syntax == .joined_or_separate) {
-            try stdout.print("jspd1(\"{}\"),\n", .{name});
+            try stdout.print("jspd1(\"{s}\"),\n", .{name});
         } else if (pd1 and !pd2 and !pslash and syntax == .separate) {
-            try stdout.print("sepd1(\"{}\"),\n", .{name});
+            try stdout.print("sepd1(\"{s}\"),\n", .{name});
         } else {
             try stdout.print(
                 \\.{{
-                \\    .name = "{}",
-                \\    .syntax = {},
+                \\    .name = "{s}",
+                \\    .syntax = {s},
                 \\    .zig_equivalent = .other,
-                \\    .pd1 = {},
-                \\    .pd2 = {},
-                \\    .psl = {},
+                \\    .pd1 = {s},
+                \\    .pd2 = {s},
+                \\    .psl = {s},
                 \\}},
                 \\
             , .{ name, syntax, pd1, pd2, pslash });
@@ -506,8 +514,8 @@ const Syntax = union(enum) {
         out_stream: anytype,
     ) !void {
         switch (self) {
-            .multi_arg => |n| return out_stream.print(".{{.{}={}}}", .{ @tagName(self), n }),
-            else => return out_stream.print(".{}", .{@tagName(self)}),
+            .multi_arg => |n| return out_stream.print(".{{.{s}={}}}", .{ @tagName(self), n }),
+            else => return out_stream.print(".{s}", .{@tagName(self)}),
         }
     }
 };
@@ -559,9 +567,9 @@ fn objSyntax(obj: *json.ObjectMap) Syntax {
         return .flag;
     }
     const key = obj.get("!name").?.String;
-    std.debug.warn("{} (key {}) has unrecognized superclasses:\n", .{ name, key });
+    std.debug.warn("{s} (key {s}) has unrecognized superclasses:\n", .{ name, key });
     for (obj.get("!superclasses").?.Array.items) |superclass_json| {
-        std.debug.warn(" {}\n", .{superclass_json.String});
+        std.debug.warn(" {s}\n", .{superclass_json.String});
     }
     std.process.exit(1);
 }
@@ -612,7 +620,7 @@ fn objectLessThan(context: void, a: *json.ObjectMap, b: *json.ObjectMap) bool {
 
 fn usageAndExit(file: fs.File, arg0: []const u8, code: u8) noreturn {
     file.writer().print(
-        \\Usage: {} /path/to/llvm-tblgen /path/to/git/llvm/llvm-project
+        \\Usage: {s} /path/to/llvm-tblgen /path/to/git/llvm/llvm-project
         \\Alternative Usage: zig run /path/to/git/zig/tools/update_clang_options.zig -- /path/to/llvm-tblgen /path/to/git/llvm/llvm-project
         \\
         \\Prints to stdout Zig code which you can use to replace the file src/clang_options_data.zig.
-- 
cgit v1.2.3


From a9667b5a859a589056f23df2b74b91fede0bbbfa Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 14 Jan 2021 20:41:37 -0700
Subject: organize std lib concurrency primitives and add RwLock

 * move concurrency primitives that always operate on kernel threads to
   the std.Thread namespace
 * remove std.SpinLock. Nobody should use this in a non-freestanding
   environment; the other primitives are always preferable. In
   freestanding, it will be necessary to put custom spin logic in there,
   so there are no use cases for a std lib version.
 * move some std lib files to the top level fields convention
 * add std.Thread.spinLoopHint
 * add std.Thread.Condition
 * add std.Thread.Semaphore
 * new implementation of std.Thread.Mutex for Windows and non-pthreads Linux
 * add std.Thread.RwLock

Implementations provided by @kprotty
---
 CMakeLists.txt                             |  11 +-
 lib/std/Progress.zig                       |   2 +-
 lib/std/ResetEvent.zig                     | 297 ---------------
 lib/std/SpinLock.zig                       |  86 -----
 lib/std/StaticResetEvent.zig               | 396 --------------------
 lib/std/Thread.zig                         | 558 +++++++++++++++++++++++++++++
 lib/std/Thread/AutoResetEvent.zig          | 228 ++++++++++++
 lib/std/Thread/Condition.zig               | 182 ++++++++++
 lib/std/Thread/Mutex.zig                   | 303 ++++++++++++++++
 lib/std/Thread/ResetEvent.zig              | 297 +++++++++++++++
 lib/std/Thread/RwLock.zig                  | 308 ++++++++++++++++
 lib/std/Thread/Semaphore.zig               |  39 ++
 lib/std/Thread/StaticResetEvent.zig        | 396 ++++++++++++++++++++
 lib/std/atomic/queue.zig                   |   4 +-
 lib/std/auto_reset_event.zig               | 226 ------------
 lib/std/c.zig                              |   7 +
 lib/std/c/darwin.zig                       |   6 +-
 lib/std/c/emscripten.zig                   |   3 +
 lib/std/c/freebsd.zig                      |   3 +
 lib/std/c/fuchsia.zig                      |   3 +
 lib/std/c/haiku.zig                        |   9 +
 lib/std/c/hermit.zig                       |   3 +
 lib/std/c/linux.zig                        |  26 ++
 lib/std/c/netbsd.zig                       |  16 +
 lib/std/c/openbsd.zig                      |   3 +
 lib/std/debug.zig                          |   8 +-
 lib/std/event/lock.zig                     |   2 +-
 lib/std/event/loop.zig                     |   6 +-
 lib/std/event/wait_group.zig               |   2 +-
 lib/std/fs/test.zig                        |   4 +-
 lib/std/heap/general_purpose_allocator.zig |  10 +-
 lib/std/mutex.zig                          | 379 --------------------
 lib/std/once.zig                           |   2 +-
 lib/std/os/windows/bits.zig                |   5 +
 lib/std/os/windows/kernel32.zig            |  13 +
 lib/std/std.zig                            |  13 +-
 lib/std/thread.zig                         | 526 ---------------------------
 src/Compilation.zig                        |   2 +-
 src/ThreadPool.zig                         |   4 +-
 src/WaitGroup.zig                          |   4 +-
 40 files changed, 2438 insertions(+), 1954 deletions(-)
 delete mode 100644 lib/std/ResetEvent.zig
 delete mode 100644 lib/std/SpinLock.zig
 delete mode 100644 lib/std/StaticResetEvent.zig
 create mode 100644 lib/std/Thread.zig
 create mode 100644 lib/std/Thread/AutoResetEvent.zig
 create mode 100644 lib/std/Thread/Condition.zig
 create mode 100644 lib/std/Thread/Mutex.zig
 create mode 100644 lib/std/Thread/ResetEvent.zig
 create mode 100644 lib/std/Thread/RwLock.zig
 create mode 100644 lib/std/Thread/Semaphore.zig
 create mode 100644 lib/std/Thread/StaticResetEvent.zig
 delete mode 100644 lib/std/auto_reset_event.zig
 delete mode 100644 lib/std/mutex.zig
 delete mode 100644 lib/std/thread.zig

(limited to 'src/Compilation.zig')

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0dfec5a953..a0c3ae84fa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -334,7 +334,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/atomic/int.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/atomic/queue.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/auto_reset_event.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/builtin.zig"
@@ -409,7 +408,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/meta.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/meta/trailer_flags.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/meta/trait.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/mutex.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/bits.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/bits/linux.zig"
@@ -426,8 +424,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/ntstatus.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/windows/win32error.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/Progress.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/ResetEvent.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/StaticResetEvent.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/pdb.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/process.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/rand.zig"
@@ -494,7 +490,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt/udivmodti4.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt/udivti3.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/special/compiler_rt/umodti3.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/SpinLock.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/start.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/std.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/target.zig"
@@ -513,7 +508,11 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/target/systemz.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/target/wasm.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/target/x86.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/thread.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Thread.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Thread/AutoResetEvent.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Thread/Mutex.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Thread/ResetEvent.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/Thread/StaticResetEvent.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/time.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/unicode.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/zig.zig"
diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig
index ca9fb8ea1f..b75ad106a4 100644
--- a/lib/std/Progress.zig
+++ b/lib/std/Progress.zig
@@ -50,7 +50,7 @@ done: bool = true,
 /// Protects the `refresh` function, as well as `node.recently_updated_child`.
 /// Without this, callsites would call `Node.end` and then free `Node` memory
 /// while it was still being accessed by the `refresh` function.
-update_lock: std.Mutex = .{},
+update_lock: std.Thread.Mutex = .{},
 
 /// Keeps track of how many columns in the terminal have been output, so that
 /// we can move the cursor back later.
diff --git a/lib/std/ResetEvent.zig b/lib/std/ResetEvent.zig
deleted file mode 100644
index 4443fdcdfb..0000000000
--- a/lib/std/ResetEvent.zig
+++ /dev/null
@@ -1,297 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-
-//! A thread-safe resource which supports blocking until signaled.
-//! This API is for kernel threads, not evented I/O.
-//! This API requires being initialized at runtime, and initialization
-//! can fail. Once initialized, the core operations cannot fail.
-//! If you need an abstraction that cannot fail to be initialized, see
-//! `std.StaticResetEvent`. However if you can handle initialization failure,
-//! it is preferred to use `ResetEvent`.
-
-const ResetEvent = @This();
-const std = @import("std.zig");
-const builtin = std.builtin;
-const testing = std.testing;
-const assert = std.debug.assert;
-const c = std.c;
-const os = std.os;
-const time = std.time;
-
-impl: Impl,
-
-pub const Impl = if (builtin.single_threaded)
-    std.StaticResetEvent.DebugEvent
-else if (std.Target.current.isDarwin())
-    DarwinEvent
-else if (std.Thread.use_pthreads)
-    PosixEvent
-else
-    std.StaticResetEvent.AtomicEvent;
-
-pub const InitError = error{SystemResources};
-
-/// After `init`, it is legal to call any other function.
-pub fn init(ev: *ResetEvent) InitError!void {
-    return ev.impl.init();
-}
-
-/// This function is not thread-safe.
-/// After `deinit`, the only legal function to call is `init`.
-pub fn deinit(ev: *ResetEvent) void {
-    return ev.impl.deinit();
-}
-
-/// Sets the event if not already set and wakes up all the threads waiting on
-/// the event. It is safe to call `set` multiple times before calling `wait`.
-/// However it is illegal to call `set` after `wait` is called until the event
-/// is `reset`. This function is thread-safe.
-pub fn set(ev: *ResetEvent) void {
-    return ev.impl.set();
-}
-
-/// Resets the event to its original, unset state.
-/// This function is *not* thread-safe. It is equivalent to calling
-/// `deinit` followed by `init` but without the possibility of failure.
-pub fn reset(ev: *ResetEvent) void {
-    return ev.impl.reset();
-}
-
-/// Wait for the event to be set by blocking the current thread.
-/// Thread-safe. No spurious wakeups.
-/// Upon return from `wait`, the only functions available to be called
-/// in `ResetEvent` are `reset` and `deinit`.
-pub fn wait(ev: *ResetEvent) void {
-    return ev.impl.wait();
-}
-
-pub const TimedWaitResult = enum { event_set, timed_out };
-
-/// Wait for the event to be set by blocking the current thread.
-/// A timeout in nanoseconds can be provided as a hint for how
-/// long the thread should block on the unset event before returning
-/// `TimedWaitResult.timed_out`.
-/// Thread-safe. No precision of timing is guaranteed.
-/// Upon return from `wait`, the only functions available to be called
-/// in `ResetEvent` are `reset` and `deinit`.
-pub fn timedWait(ev: *ResetEvent, timeout_ns: u64) TimedWaitResult {
-    return ev.impl.timedWait(timeout_ns);
-}
-
-/// Apple has decided to not support POSIX semaphores, so we go with a
-/// different approach using Grand Central Dispatch. This API is exposed
-/// by libSystem so it is guaranteed to be available on all Darwin platforms.
-pub const DarwinEvent = struct {
-    sem: c.dispatch_semaphore_t = undefined,
-
-    pub fn init(ev: *DarwinEvent) !void {
-        ev.* = .{
-            .sem = c.dispatch_semaphore_create(0) orelse return error.SystemResources,
-        };
-    }
-
-    pub fn deinit(ev: *DarwinEvent) void {
-        c.dispatch_release(ev.sem);
-        ev.* = undefined;
-    }
-
-    pub fn set(ev: *DarwinEvent) void {
-        // Empirically this returns the numerical value of the semaphore.
-        _ = c.dispatch_semaphore_signal(ev.sem);
-    }
-
-    pub fn wait(ev: *DarwinEvent) void {
-        assert(c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_FOREVER) == 0);
-    }
-
-    pub fn timedWait(ev: *DarwinEvent, timeout_ns: u64) TimedWaitResult {
-        const t = c.dispatch_time(c.DISPATCH_TIME_NOW, @intCast(i64, timeout_ns));
-        if (c.dispatch_semaphore_wait(ev.sem, t) != 0) {
-            return .timed_out;
-        } else {
-            return .event_set;
-        }
-    }
-
-    pub fn reset(ev: *DarwinEvent) void {
-        // Keep calling until the semaphore goes back down to 0.
-        while (c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_NOW) == 0) {}
-    }
-};
-
-/// POSIX semaphores must be initialized at runtime because they are allowed to
-/// be implemented as file descriptors, in which case initialization would require
-/// a syscall to open the fd.
-pub const PosixEvent = struct {
-    sem: c.sem_t = undefined,
-
-    pub fn init(ev: *PosixEvent) !void {
-        switch (c.getErrno(c.sem_init(&ev.sem, 0, 0))) {
-            0 => return,
-            else => return error.SystemResources,
-        }
-    }
-
-    pub fn deinit(ev: *PosixEvent) void {
-        assert(c.sem_destroy(&ev.sem) == 0);
-        ev.* = undefined;
-    }
-
-    pub fn set(ev: *PosixEvent) void {
-        assert(c.sem_post(&ev.sem) == 0);
-    }
-
-    pub fn wait(ev: *PosixEvent) void {
-        while (true) {
-            switch (c.getErrno(c.sem_wait(&ev.sem))) {
-                0 => return,
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                else => unreachable,
-            }
-        }
-    }
-
-    pub fn timedWait(ev: *PosixEvent, timeout_ns: u64) TimedWaitResult {
-        var ts: os.timespec = undefined;
-        var timeout_abs = timeout_ns;
-        os.clock_gettime(os.CLOCK_REALTIME, &ts) catch return .timed_out;
-        timeout_abs += @intCast(u64, ts.tv_sec) * time.ns_per_s;
-        timeout_abs += @intCast(u64, ts.tv_nsec);
-        ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), @divFloor(timeout_abs, time.ns_per_s));
-        ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), @mod(timeout_abs, time.ns_per_s));
-        while (true) {
-            switch (c.getErrno(c.sem_timedwait(&ev.sem, &ts))) {
-                0 => return .event_set,
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                c.ETIMEDOUT => return .timed_out,
-                else => unreachable,
-            }
-        }
-    }
-
-    pub fn reset(ev: *PosixEvent) void {
-        while (true) {
-            switch (c.getErrno(c.sem_trywait(&ev.sem))) {
-                0 => continue, // Need to make it go to zero.
-                c.EINTR => continue,
-                c.EINVAL => unreachable,
-                c.EAGAIN => return, // The semaphore currently has the value zero.
-                else => unreachable,
-            }
-        }
-    }
-};
-
-test "basic usage" {
-    var event: ResetEvent = undefined;
-    try event.init();
-    defer event.deinit();
-
-    // test event setting
-    event.set();
-
-    // test event resetting
-    event.reset();
-
-    // test event waiting (non-blocking)
-    event.set();
-    event.wait();
-    event.reset();
-
-    event.set();
-    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
-
-    // test cross-thread signaling
-    if (builtin.single_threaded)
-        return;
-
-    const Context = struct {
-        const Self = @This();
-
-        value: u128,
-        in: ResetEvent,
-        out: ResetEvent,
-
-        fn init(self: *Self) !void {
-            self.* = .{
-                .value = 0,
-                .in = undefined,
-                .out = undefined,
-            };
-            try self.in.init();
-            try self.out.init();
-        }
-
-        fn deinit(self: *Self) void {
-            self.in.deinit();
-            self.out.deinit();
-            self.* = undefined;
-        }
-
-        fn sender(self: *Self) void {
-            // update value and signal input
-            testing.expect(self.value == 0);
-            self.value = 1;
-            self.in.set();
-
-            // wait for receiver to update value and signal output
-            self.out.wait();
-            testing.expect(self.value == 2);
-
-            // update value and signal final input
-            self.value = 3;
-            self.in.set();
-        }
-
-        fn receiver(self: *Self) void {
-            // wait for sender to update value and signal input
-            self.in.wait();
-            assert(self.value == 1);
-
-            // update value and signal output
-            self.in.reset();
-            self.value = 2;
-            self.out.set();
-
-            // wait for sender to update value and signal final input
-            self.in.wait();
-            assert(self.value == 3);
-        }
-
-        fn sleeper(self: *Self) void {
-            self.in.set();
-            time.sleep(time.ns_per_ms * 2);
-            self.value = 5;
-            self.out.set();
-        }
-
-        fn timedWaiter(self: *Self) !void {
-            self.in.wait();
-            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
-            try self.out.timedWait(time.ns_per_ms * 100);
-            testing.expect(self.value == 5);
-        }
-    };
-
-    var context: Context = undefined;
-    try context.init();
-    defer context.deinit();
-    const receiver = try std.Thread.spawn(&context, Context.receiver);
-    defer receiver.wait();
-    context.sender();
-
-    if (false) {
-        // I have now observed this fail on macOS, Windows, and Linux.
-        // https://github.com/ziglang/zig/issues/7009
-        var timed = Context.init();
-        defer timed.deinit();
-        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
-        defer sleeper.wait();
-        try timed.timedWaiter();
-    }
-}
diff --git a/lib/std/SpinLock.zig b/lib/std/SpinLock.zig
deleted file mode 100644
index a16cfa930b..0000000000
--- a/lib/std/SpinLock.zig
+++ /dev/null
@@ -1,86 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-//! A mutually exclusive lock that grinds the CPU rather than interacting with
-//! the operating system. It does however yield to the OS scheduler while
-//! spinning, when targeting an OS that supports it.
-//! This struct can be initialized directly and statically initialized. The
-//! default state is unlocked.
-
-state: State = State.Unlocked,
-
-const std = @import("std.zig");
-const builtin = @import("builtin");
-const SpinLock = @This();
-
-const State = enum(u8) {
-    Unlocked,
-    Locked,
-};
-
-pub const Held = struct {
-    spinlock: *SpinLock,
-
-    pub fn release(self: Held) void {
-        @atomicStore(State, &self.spinlock.state, .Unlocked, .Release);
-    }
-};
-
-pub fn tryAcquire(self: *SpinLock) ?Held {
-    return switch (@atomicRmw(State, &self.state, .Xchg, .Locked, .Acquire)) {
-        .Unlocked => Held{ .spinlock = self },
-        .Locked => null,
-    };
-}
-
-pub fn acquire(self: *SpinLock) Held {
-    while (true) {
-        return self.tryAcquire() orelse {
-            yield();
-            continue;
-        };
-    }
-}
-
-pub fn yield() void {
-    // On native windows, SwitchToThread is too expensive,
-    // and yielding for 380-410 iterations was found to be
-    // a nice sweet spot. Posix systems on the other hand,
-    // especially linux, perform better by yielding the thread.
-    switch (builtin.os.tag) {
-        .windows => loopHint(400),
-        else => std.os.sched_yield() catch loopHint(1),
-    }
-}
-
-/// Hint to the cpu that execution is spinning
-/// for the given amount of iterations.
-pub fn loopHint(iterations: usize) void {
-    var i = iterations;
-    while (i != 0) : (i -= 1) {
-        switch (builtin.arch) {
-            // these instructions use a memory clobber as they
-            // flush the pipeline of any speculated reads/writes.
-            .i386, .x86_64 => asm volatile ("pause"
-                :
-                :
-                : "memory"
-            ),
-            .arm, .aarch64 => asm volatile ("yield"
-                :
-                :
-                : "memory"
-            ),
-            else => std.os.sched_yield() catch {},
-        }
-    }
-}
-
-test "basic usage" {
-    var lock: SpinLock = .{};
-
-    const held = lock.acquire();
-    defer held.release();
-}
diff --git a/lib/std/StaticResetEvent.zig b/lib/std/StaticResetEvent.zig
deleted file mode 100644
index 4e551a565e..0000000000
--- a/lib/std/StaticResetEvent.zig
+++ /dev/null
@@ -1,396 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-
-//! A thread-safe resource which supports blocking until signaled.
-//! This API is for kernel threads, not evented I/O.
-//! This API is statically initializable. It cannot fail to be initialized
-//! and it requires no deinitialization. The downside is that it may not
-//! integrate as cleanly into other synchronization APIs, or, in a worst case,
-//! may be forced to fall back on spin locking. As a rule of thumb, prefer
-//! to use `std.ResetEvent` when possible, and use `StaticResetEvent` when
-//! the logic needs stronger API guarantees.
-
-const std = @import("std.zig");
-const StaticResetEvent = @This();
-const SpinLock = std.SpinLock;
-const assert = std.debug.assert;
-const os = std.os;
-const time = std.time;
-const linux = std.os.linux;
-const windows = std.os.windows;
-const testing = std.testing;
-
-impl: Impl = .{},
-
-pub const Impl = if (std.builtin.single_threaded)
-    DebugEvent
-else
-    AtomicEvent;
-
-/// Sets the event if not already set and wakes up all the threads waiting on
-/// the event. It is safe to call `set` multiple times before calling `wait`.
-/// However it is illegal to call `set` after `wait` is called until the event
-/// is `reset`. This function is thread-safe.
-pub fn set(ev: *StaticResetEvent) void {
-    return ev.impl.set();
-}
-
-/// Wait for the event to be set by blocking the current thread.
-/// Thread-safe. No spurious wakeups.
-/// Upon return from `wait`, the only function available to be called
-/// in `StaticResetEvent` is `reset`.
-pub fn wait(ev: *StaticResetEvent) void {
-    return ev.impl.wait();
-}
-
-/// Resets the event to its original, unset state.
-/// This function is *not* thread-safe. It is equivalent to calling
-/// `deinit` followed by `init` but without the possibility of failure.
-pub fn reset(ev: *StaticResetEvent) void {
-    return ev.impl.reset();
-}
-
-pub const TimedWaitResult = std.ResetEvent.TimedWaitResult;
-
-/// Wait for the event to be set by blocking the current thread.
-/// A timeout in nanoseconds can be provided as a hint for how
-/// long the thread should block on the unset event before returning
-/// `TimedWaitResult.timed_out`.
-/// Thread-safe. No precision of timing is guaranteed.
-/// Upon return from `timedWait`, the only function available to be called
-/// in `StaticResetEvent` is `reset`.
-pub fn timedWait(ev: *StaticResetEvent, timeout_ns: u64) TimedWaitResult {
-    return ev.impl.timedWait(timeout_ns);
-}
-
-/// For single-threaded builds, we use this to detect deadlocks.
-/// In unsafe modes this ends up being no-ops.
-pub const DebugEvent = struct {
-    state: State = State.unset,
-
-    const State = enum {
-        unset,
-        set,
-        waited,
-    };
-
-    /// This function is provided so that this type can be re-used inside
-    /// `std.ResetEvent`.
-    pub fn init(ev: *DebugEvent) void {
-        ev.* = .{};
-    }
-
-    /// This function is provided so that this type can be re-used inside
-    /// `std.ResetEvent`.
-    pub fn deinit(ev: *DebugEvent) void {
-        ev.* = undefined;
-    }
-
-    pub fn set(ev: *DebugEvent) void {
-        switch (ev.state) {
-            .unset => ev.state = .set,
-            .set => {},
-            .waited => unreachable, // Not allowed to call `set` until `reset`.
-        }
-    }
-
-    pub fn wait(ev: *DebugEvent) void {
-        switch (ev.state) {
-            .unset => unreachable, // Deadlock detected.
-            .set => return,
-            .waited => unreachable, // Not allowed to call `wait` until `reset`.
-        }
-    }
-
-    pub fn timedWait(ev: *DebugEvent, timeout: u64) TimedWaitResult {
-        switch (ev.state) {
-            .unset => return .timed_out,
-            .set => return .event_set,
-            .waited => unreachable, // Not allowed to call `wait` until `reset`.
-        }
-    }
-
-    pub fn reset(ev: *DebugEvent) void {
-        ev.state = .unset;
-    }
-};
-
-pub const AtomicEvent = struct {
-    waiters: u32 = 0,
-
-    const WAKE = 1 << 0;
-    const WAIT = 1 << 1;
-
-    /// This function is provided so that this type can be re-used inside
-    /// `std.ResetEvent`.
-    pub fn init(ev: *AtomicEvent) void {
-        ev.* = .{};
-    }
-
-    /// This function is provided so that this type can be re-used inside
-    /// `std.ResetEvent`.
-    pub fn deinit(ev: *AtomicEvent) void {
-        ev.* = undefined;
-    }
-
-    pub fn set(ev: *AtomicEvent) void {
-        const waiters = @atomicRmw(u32, &ev.waiters, .Xchg, WAKE, .Release);
-        if (waiters >= WAIT) {
-            return Futex.wake(&ev.waiters, waiters >> 1);
-        }
-    }
-
-    pub fn wait(ev: *AtomicEvent) void {
-        switch (ev.timedWait(null)) {
-            .timed_out => unreachable,
-            .event_set => return,
-        }
-    }
-
-    pub fn timedWait(ev: *AtomicEvent, timeout: ?u64) TimedWaitResult {
-        var waiters = @atomicLoad(u32, &ev.waiters, .Acquire);
-        while (waiters != WAKE) {
-            waiters = @cmpxchgWeak(u32, &ev.waiters, waiters, waiters + WAIT, .Acquire, .Acquire) orelse {
-                if (Futex.wait(&ev.waiters, timeout)) |_| {
-                    return .event_set;
-                } else |_| {
-                    return .timed_out;
-                }
-            };
-        }
-        return .event_set;
-    }
-
-    pub fn reset(ev: *AtomicEvent) void {
-        @atomicStore(u32, &ev.waiters, 0, .Monotonic);
-    }
-
-    pub const Futex = switch (std.Target.current.os.tag) {
-        .windows => WindowsFutex,
-        .linux => LinuxFutex,
-        else => SpinFutex,
-    };
-
-    pub const SpinFutex = struct {
-        fn wake(waiters: *u32, wake_count: u32) void {}
-
-        fn wait(waiters: *u32, timeout: ?u64) !void {
-            var timer: time.Timer = undefined;
-            if (timeout != null)
-                timer = time.Timer.start() catch return error.TimedOut;
-
-            while (@atomicLoad(u32, waiters, .Acquire) != WAKE) {
-                SpinLock.yield();
-                if (timeout) |timeout_ns| {
-                    if (timer.read() >= timeout_ns)
-                        return error.TimedOut;
-                }
-            }
-        }
-    };
-
-    pub const LinuxFutex = struct {
-        fn wake(waiters: *u32, wake_count: u32) void {
-            const waiting = std.math.maxInt(i32); // wake_count
-            const ptr = @ptrCast(*const i32, waiters);
-            const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, waiting);
-            assert(linux.getErrno(rc) == 0);
-        }
-
-        fn wait(waiters: *u32, timeout: ?u64) !void {
-            var ts: linux.timespec = undefined;
-            var ts_ptr: ?*linux.timespec = null;
-            if (timeout) |timeout_ns| {
-                ts_ptr = &ts;
-                ts.tv_sec = @intCast(isize, timeout_ns / time.ns_per_s);
-                ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);
-            }
-
-            while (true) {
-                const waiting = @atomicLoad(u32, waiters, .Acquire);
-                if (waiting == WAKE)
-                    return;
-                const expected = @intCast(i32, waiting);
-                const ptr = @ptrCast(*const i32, waiters);
-                const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);
-                switch (linux.getErrno(rc)) {
-                    0 => continue,
-                    os.ETIMEDOUT => return error.TimedOut,
-                    os.EINTR => continue,
-                    os.EAGAIN => return,
-                    else => unreachable,
-                }
-            }
-        }
-    };
-
-    pub const WindowsFutex = struct {
-        pub fn wake(waiters: *u32, wake_count: u32) void {
-            const handle = getEventHandle() orelse return SpinFutex.wake(waiters, wake_count);
-            const key = @ptrCast(*const c_void, waiters);
-
-            var waiting = wake_count;
-            while (waiting != 0) : (waiting -= 1) {
-                const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
-                assert(rc == .SUCCESS);
-            }
-        }
-
-        pub fn wait(waiters: *u32, timeout: ?u64) !void {
-            const handle = getEventHandle() orelse return SpinFutex.wait(waiters, timeout);
-            const key = @ptrCast(*const c_void, waiters);
-
-            // NT uses timeouts in units of 100ns with negative value being relative
-            var timeout_ptr: ?*windows.LARGE_INTEGER = null;
-            var timeout_value: windows.LARGE_INTEGER = undefined;
-            if (timeout) |timeout_ns| {
-                timeout_ptr = &timeout_value;
-                timeout_value = -@intCast(windows.LARGE_INTEGER, timeout_ns / 100);
-            }
-
-            // NtWaitForKeyedEvent doesnt have spurious wake-ups
-            var rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
-            switch (rc) {
-                .TIMEOUT => {
-                    // update the wait count to signal that we're not waiting anymore.
-                    // if the .set() thread already observed that we are, perform a
-                    // matching NtWaitForKeyedEvent so that the .set() thread doesn't
-                    // deadlock trying to run NtReleaseKeyedEvent above.
-                    var waiting = @atomicLoad(u32, waiters, .Monotonic);
-                    while (true) {
-                        if (waiting == WAKE) {
-                            rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
-                            assert(rc == .WAIT_0);
-                            break;
-                        } else {
-                            waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - WAIT, .Acquire, .Monotonic) orelse break;
-                            continue;
-                        }
-                    }
-                    return error.TimedOut;
-                },
-                .WAIT_0 => {},
-                else => unreachable,
-            }
-        }
-
-        var event_handle: usize = EMPTY;
-        const EMPTY = ~@as(usize, 0);
-        const LOADING = EMPTY - 1;
-
-        pub fn getEventHandle() ?windows.HANDLE {
-            var handle = @atomicLoad(usize, &event_handle, .Monotonic);
-            while (true) {
-                switch (handle) {
-                    EMPTY => handle = @cmpxchgWeak(usize, &event_handle, EMPTY, LOADING, .Acquire, .Monotonic) orelse {
-                        const handle_ptr = @ptrCast(*windows.HANDLE, &handle);
-                        const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
-                        if (windows.ntdll.NtCreateKeyedEvent(handle_ptr, access_mask, null, 0) != .SUCCESS)
-                            handle = 0;
-                        @atomicStore(usize, &event_handle, handle, .Monotonic);
-                        return @intToPtr(?windows.HANDLE, handle);
-                    },
-                    LOADING => {
-                        SpinLock.yield();
-                        handle = @atomicLoad(usize, &event_handle, .Monotonic);
-                    },
-                    else => {
-                        return @intToPtr(?windows.HANDLE, handle);
-                    },
-                }
-            }
-        }
-    };
-};
-
-test "basic usage" {
-    var event = StaticResetEvent{};
-
-    // test event setting
-    event.set();
-
-    // test event resetting
-    event.reset();
-
-    // test event waiting (non-blocking)
-    event.set();
-    event.wait();
-    event.reset();
-
-    event.set();
-    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
-
-    // test cross-thread signaling
-    if (std.builtin.single_threaded)
-        return;
-
-    const Context = struct {
-        const Self = @This();
-
-        value: u128 = 0,
-        in: StaticResetEvent = .{},
-        out: StaticResetEvent = .{},
-
-        fn sender(self: *Self) void {
-            // update value and signal input
-            testing.expect(self.value == 0);
-            self.value = 1;
-            self.in.set();
-
-            // wait for receiver to update value and signal output
-            self.out.wait();
-            testing.expect(self.value == 2);
-
-            // update value and signal final input
-            self.value = 3;
-            self.in.set();
-        }
-
-        fn receiver(self: *Self) void {
-            // wait for sender to update value and signal input
-            self.in.wait();
-            assert(self.value == 1);
-
-            // update value and signal output
-            self.in.reset();
-            self.value = 2;
-            self.out.set();
-
-            // wait for sender to update value and signal final input
-            self.in.wait();
-            assert(self.value == 3);
-        }
-
-        fn sleeper(self: *Self) void {
-            self.in.set();
-            time.sleep(time.ns_per_ms * 2);
-            self.value = 5;
-            self.out.set();
-        }
-
-        fn timedWaiter(self: *Self) !void {
-            self.in.wait();
-            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
-            try self.out.timedWait(time.ns_per_ms * 100);
-            testing.expect(self.value == 5);
-        }
-    };
-
-    var context = Context{};
-    const receiver = try std.Thread.spawn(&context, Context.receiver);
-    defer receiver.wait();
-    context.sender();
-
-    if (false) {
-        // I have now observed this fail on macOS, Windows, and Linux.
-        // https://github.com/ziglang/zig/issues/7009
-        var timed = Context.init();
-        defer timed.deinit();
-        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
-        defer sleeper.wait();
-        try timed.timedWaiter();
-    }
-}
diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig
new file mode 100644
index 0000000000..f878f43539
--- /dev/null
+++ b/lib/std/Thread.zig
@@ -0,0 +1,558 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! This struct represents a kernel thread, and acts as a namespace for concurrency
+//! primitives that operate on kernel threads. For concurrency primitives that support
+//! both evented I/O and async I/O, see the respective names in the top level std namespace.
+
+data: Data,
+
+pub const AutoResetEvent = @import("Thread/AutoResetEvent.zig");
+pub const ResetEvent = @import("Thread/ResetEvent.zig");
+pub const StaticResetEvent = @import("Thread/StaticResetEvent.zig");
+pub const Mutex = @import("Thread/Mutex.zig");
+pub const Semaphore = @import("Thread/Semaphore.zig");
+pub const Condition = @import("Thread/Condition.zig");
+
+pub const use_pthreads = std.Target.current.os.tag != .windows and builtin.link_libc;
+
+const Thread = @This();
+const std = @import("std.zig");
+const builtin = std.builtin;
+const os = std.os;
+const mem = std.mem;
+const windows = std.os.windows;
+const c = std.c;
+const assert = std.debug.assert;
+
+const bad_startfn_ret = "expected return type of startFn to be 'u8', 'noreturn', 'void', or '!void'";
+
+/// Represents a kernel thread handle.
+/// May be an integer or a pointer depending on the platform.
+/// On Linux and POSIX, this is the same as Id.
+pub const Handle = if (use_pthreads)
+    c.pthread_t
+else switch (std.Target.current.os.tag) {
+    .linux => i32,
+    .windows => windows.HANDLE,
+    else => void,
+};
+
+/// Represents a unique ID per thread.
+/// May be an integer or pointer depending on the platform.
+/// On Linux and POSIX, this is the same as Handle.
+pub const Id = switch (std.Target.current.os.tag) {
+    .windows => windows.DWORD,
+    else => Handle,
+};
+
+pub const Data = if (use_pthreads)
+    struct {
+        handle: Thread.Handle,
+        memory: []u8,
+    }
+else switch (std.Target.current.os.tag) {
+    .linux => struct {
+        handle: Thread.Handle,
+        memory: []align(mem.page_size) u8,
+    },
+    .windows => struct {
+        handle: Thread.Handle,
+        alloc_start: *c_void,
+        heap_handle: windows.HANDLE,
+    },
+    else => struct {},
+};
+
+/// Signals the processor that it is inside a busy-wait spin-loop ("spin lock").
+pub fn spinLoopHint() void {
+    switch (std.Target.current.cpu.arch) {
+        .i386, .x86_64 => asm volatile ("pause"
+            :
+            :
+            : "memory"
+        ),
+        .arm, .aarch64 => asm volatile ("yield"
+            :
+            :
+            : "memory"
+        ),
+        else => {},
+    }
+}
+
+/// Returns the ID of the calling thread.
+/// Makes a syscall every time the function is called.
+/// On Linux and POSIX, this Id is the same as a Handle.
+pub fn getCurrentId() Id {
+    if (use_pthreads) {
+        return c.pthread_self();
+    } else
+        return switch (std.Target.current.os.tag) {
+            .linux => os.linux.gettid(),
+            .windows => windows.kernel32.GetCurrentThreadId(),
+            else => @compileError("Unsupported OS"),
+        };
+}
+
+/// Returns the handle of this thread.
+/// On Linux and POSIX, this is the same as Id.
+/// On Linux, it is possible that the thread spawned with `spawn`
+/// finishes executing entirely before the clone syscall completes. In this
+/// case, this function will return 0 rather than the no-longer-existing thread's
+/// pid.
+pub fn handle(self: Thread) Handle {
+    return self.data.handle;
+}
+
+pub fn wait(self: *Thread) void {
+    if (use_pthreads) {
+        const err = c.pthread_join(self.data.handle, null);
+        switch (err) {
+            0 => {},
+            os.EINVAL => unreachable,
+            os.ESRCH => unreachable,
+            os.EDEADLK => unreachable,
+            else => unreachable,
+        }
+        std.heap.c_allocator.free(self.data.memory);
+        std.heap.c_allocator.destroy(self);
+    } else switch (std.Target.current.os.tag) {
+        .linux => {
+            while (true) {
+                const pid_value = @atomicLoad(i32, &self.data.handle, .SeqCst);
+                if (pid_value == 0) break;
+                const rc = os.linux.futex_wait(&self.data.handle, os.linux.FUTEX_WAIT, pid_value, null);
+                switch (os.linux.getErrno(rc)) {
+                    0 => continue,
+                    os.EINTR => continue,
+                    os.EAGAIN => continue,
+                    else => unreachable,
+                }
+            }
+            os.munmap(self.data.memory);
+        },
+        .windows => {
+            windows.WaitForSingleObjectEx(self.data.handle, windows.INFINITE, false) catch unreachable;
+            windows.CloseHandle(self.data.handle);
+            windows.HeapFree(self.data.heap_handle, 0, self.data.alloc_start);
+        },
+        else => @compileError("Unsupported OS"),
+    }
+}
+
+pub const SpawnError = error{
+    /// A system-imposed limit on the number of threads was encountered.
+    /// There are a number of limits that may trigger this error:
+    /// *  the  RLIMIT_NPROC soft resource limit (set via setrlimit(2)),
+    ///    which limits the number of processes and threads for  a  real
+    ///    user ID, was reached;
+    /// *  the kernel's system-wide limit on the number of processes and
+    ///    threads,  /proc/sys/kernel/threads-max,  was   reached   (see
+    ///    proc(5));
+    /// *  the  maximum  number  of  PIDs, /proc/sys/kernel/pid_max, was
+    ///    reached (see proc(5)); or
+    /// *  the PID limit (pids.max) imposed by the cgroup "process  num‐
+    ///    ber" (PIDs) controller was reached.
+    ThreadQuotaExceeded,
+
+    /// The kernel cannot allocate sufficient memory to allocate a task structure
+    /// for the child, or to copy those parts of the caller's context that need to
+    /// be copied.
+    SystemResources,
+
+    /// Not enough userland memory to spawn the thread.
+    OutOfMemory,
+
+    /// `mlockall` is enabled, and the memory needed to spawn the thread
+    /// would exceed the limit.
+    LockedMemoryLimitExceeded,
+
+    Unexpected,
+};
+
+/// caller must call wait on the returned thread
+/// fn startFn(@TypeOf(context)) T
+/// where T is u8, noreturn, void, or !void
+/// caller must call wait on the returned thread
+pub fn spawn(context: anytype, comptime startFn: anytype) SpawnError!*Thread {
+    if (builtin.single_threaded) @compileError("cannot spawn thread when building in single-threaded mode");
+    // TODO compile-time call graph analysis to determine stack upper bound
+    // https://github.com/ziglang/zig/issues/157
+    const default_stack_size = 16 * 1024 * 1024;
+
+    const Context = @TypeOf(context);
+    comptime assert(@typeInfo(@TypeOf(startFn)).Fn.args[0].arg_type.? == Context);
+
+    if (std.Target.current.os.tag == .windows) {
+        const WinThread = struct {
+            const OuterContext = struct {
+                thread: Thread,
+                inner: Context,
+            };
+            fn threadMain(raw_arg: windows.LPVOID) callconv(.C) windows.DWORD {
+                const arg = if (@sizeOf(Context) == 0) {} else @ptrCast(*Context, @alignCast(@alignOf(Context), raw_arg)).*;
+
+                switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
+                    .NoReturn => {
+                        startFn(arg);
+                    },
+                    .Void => {
+                        startFn(arg);
+                        return 0;
+                    },
+                    .Int => |info| {
+                        if (info.bits != 8) {
+                            @compileError(bad_startfn_ret);
+                        }
+                        return startFn(arg);
+                    },
+                    .ErrorUnion => |info| {
+                        if (info.payload != void) {
+                            @compileError(bad_startfn_ret);
+                        }
+                        startFn(arg) catch |err| {
+                            std.debug.warn("error: {s}\n", .{@errorName(err)});
+                            if (@errorReturnTrace()) |trace| {
+                                std.debug.dumpStackTrace(trace.*);
+                            }
+                        };
+                        return 0;
+                    },
+                    else => @compileError(bad_startfn_ret),
+                }
+            }
+        };
+
+        const heap_handle = windows.kernel32.GetProcessHeap() orelse return error.OutOfMemory;
+        const byte_count = @alignOf(WinThread.OuterContext) + @sizeOf(WinThread.OuterContext);
+        const bytes_ptr = windows.kernel32.HeapAlloc(heap_handle, 0, byte_count) orelse return error.OutOfMemory;
+        errdefer assert(windows.kernel32.HeapFree(heap_handle, 0, bytes_ptr) != 0);
+        const bytes = @ptrCast([*]u8, bytes_ptr)[0..byte_count];
+        const outer_context = std.heap.FixedBufferAllocator.init(bytes).allocator.create(WinThread.OuterContext) catch unreachable;
+        outer_context.* = WinThread.OuterContext{
+            .thread = Thread{
+                .data = Thread.Data{
+                    .heap_handle = heap_handle,
+                    .alloc_start = bytes_ptr,
+                    .handle = undefined,
+                },
+            },
+            .inner = context,
+        };
+
+        const parameter = if (@sizeOf(Context) == 0) null else @ptrCast(*c_void, &outer_context.inner);
+        outer_context.thread.data.handle = windows.kernel32.CreateThread(null, default_stack_size, WinThread.threadMain, parameter, 0, null) orelse {
+            switch (windows.kernel32.GetLastError()) {
+                else => |err| return windows.unexpectedError(err),
+            }
+        };
+        return &outer_context.thread;
+    }
+
+    const MainFuncs = struct {
+        fn linuxThreadMain(ctx_addr: usize) callconv(.C) u8 {
+            const arg = if (@sizeOf(Context) == 0) {} else @intToPtr(*const Context, ctx_addr).*;
+
+            switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
+                .NoReturn => {
+                    startFn(arg);
+                },
+                .Void => {
+                    startFn(arg);
+                    return 0;
+                },
+                .Int => |info| {
+                    if (info.bits != 8) {
+                        @compileError(bad_startfn_ret);
+                    }
+                    return startFn(arg);
+                },
+                .ErrorUnion => |info| {
+                    if (info.payload != void) {
+                        @compileError(bad_startfn_ret);
+                    }
+                    startFn(arg) catch |err| {
+                        std.debug.warn("error: {s}\n", .{@errorName(err)});
+                        if (@errorReturnTrace()) |trace| {
+                            std.debug.dumpStackTrace(trace.*);
+                        }
+                    };
+                    return 0;
+                },
+                else => @compileError(bad_startfn_ret),
+            }
+        }
+        fn posixThreadMain(ctx: ?*c_void) callconv(.C) ?*c_void {
+            const arg = if (@sizeOf(Context) == 0) {} else @ptrCast(*Context, @alignCast(@alignOf(Context), ctx)).*;
+
+            switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
+                .NoReturn => {
+                    startFn(arg);
+                },
+                .Void => {
+                    startFn(arg);
+                    return null;
+                },
+                .Int => |info| {
+                    if (info.bits != 8) {
+                        @compileError(bad_startfn_ret);
+                    }
+                    // pthreads don't support exit status, ignore value
+                    _ = startFn(arg);
+                    return null;
+                },
+                .ErrorUnion => |info| {
+                    if (info.payload != void) {
+                        @compileError(bad_startfn_ret);
+                    }
+                    startFn(arg) catch |err| {
+                        std.debug.warn("error: {s}\n", .{@errorName(err)});
+                        if (@errorReturnTrace()) |trace| {
+                            std.debug.dumpStackTrace(trace.*);
+                        }
+                    };
+                    return null;
+                },
+                else => @compileError(bad_startfn_ret),
+            }
+        }
+    };
+
+    if (Thread.use_pthreads) {
+        var attr: c.pthread_attr_t = undefined;
+        if (c.pthread_attr_init(&attr) != 0) return error.SystemResources;
+        defer assert(c.pthread_attr_destroy(&attr) == 0);
+
+        const thread_obj = try std.heap.c_allocator.create(Thread);
+        errdefer std.heap.c_allocator.destroy(thread_obj);
+        if (@sizeOf(Context) > 0) {
+            thread_obj.data.memory = try std.heap.c_allocator.allocAdvanced(
+                u8,
+                @alignOf(Context),
+                @sizeOf(Context),
+                .at_least,
+            );
+            errdefer std.heap.c_allocator.free(thread_obj.data.memory);
+            mem.copy(u8, thread_obj.data.memory, mem.asBytes(&context));
+        } else {
+            thread_obj.data.memory = @as([*]u8, undefined)[0..0];
+        }
+
+        // Use the same set of parameters used by the libc-less impl.
+        assert(c.pthread_attr_setstacksize(&attr, default_stack_size) == 0);
+        assert(c.pthread_attr_setguardsize(&attr, mem.page_size) == 0);
+
+        const err = c.pthread_create(
+            &thread_obj.data.handle,
+            &attr,
+            MainFuncs.posixThreadMain,
+            thread_obj.data.memory.ptr,
+        );
+        switch (err) {
+            0 => return thread_obj,
+            os.EAGAIN => return error.SystemResources,
+            os.EPERM => unreachable,
+            os.EINVAL => unreachable,
+            else => return os.unexpectedErrno(@intCast(usize, err)),
+        }
+
+        return thread_obj;
+    }
+
+    var guard_end_offset: usize = undefined;
+    var stack_end_offset: usize = undefined;
+    var thread_start_offset: usize = undefined;
+    var context_start_offset: usize = undefined;
+    var tls_start_offset: usize = undefined;
+    const mmap_len = blk: {
+        var l: usize = mem.page_size;
+        // Allocate a guard page right after the end of the stack region
+        guard_end_offset = l;
+        // The stack itself, which grows downwards.
+        l = mem.alignForward(l + default_stack_size, mem.page_size);
+        stack_end_offset = l;
+        // Above the stack, so that it can be in the same mmap call, put the Thread object.
+        l = mem.alignForward(l, @alignOf(Thread));
+        thread_start_offset = l;
+        l += @sizeOf(Thread);
+        // Next, the Context object.
+        if (@sizeOf(Context) != 0) {
+            l = mem.alignForward(l, @alignOf(Context));
+            context_start_offset = l;
+            l += @sizeOf(Context);
+        }
+        // Finally, the Thread Local Storage, if any.
+        l = mem.alignForward(l, os.linux.tls.tls_image.alloc_align);
+        tls_start_offset = l;
+        l += os.linux.tls.tls_image.alloc_size;
+        // Round the size to the page size.
+        break :blk mem.alignForward(l, mem.page_size);
+    };
+
+    const mmap_slice = mem: {
+        // Map the whole stack with no rw permissions to avoid
+        // committing the whole region right away
+        const mmap_slice = os.mmap(
+            null,
+            mmap_len,
+            os.PROT_NONE,
+            os.MAP_PRIVATE | os.MAP_ANONYMOUS,
+            -1,
+            0,
+        ) catch |err| switch (err) {
+            error.MemoryMappingNotSupported => unreachable,
+            error.AccessDenied => unreachable,
+            error.PermissionDenied => unreachable,
+            else => |e| return e,
+        };
+        errdefer os.munmap(mmap_slice);
+
+        // Map everything but the guard page as rw
+        os.mprotect(
+            mmap_slice[guard_end_offset..],
+            os.PROT_READ | os.PROT_WRITE,
+        ) catch |err| switch (err) {
+            error.AccessDenied => unreachable,
+            else => |e| return e,
+        };
+
+        break :mem mmap_slice;
+    };
+
+    const mmap_addr = @ptrToInt(mmap_slice.ptr);
+
+    const thread_ptr = @alignCast(@alignOf(Thread), @intToPtr(*Thread, mmap_addr + thread_start_offset));
+    thread_ptr.data.memory = mmap_slice;
+
+    var arg: usize = undefined;
+    if (@sizeOf(Context) != 0) {
+        arg = mmap_addr + context_start_offset;
+        const context_ptr = @alignCast(@alignOf(Context), @intToPtr(*Context, arg));
+        context_ptr.* = context;
+    }
+
+    if (std.Target.current.os.tag == .linux) {
+        const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
+            os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
+            os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
+            os.CLONE_DETACHED | os.CLONE_SETTLS;
+        // This structure is only needed when targeting i386
+        var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
+
+        const tls_area = mmap_slice[tls_start_offset..];
+        const tp_value = os.linux.tls.prepareTLS(tls_area);
+
+        const newtls = blk: {
+            if (std.Target.current.cpu.arch == .i386) {
+                user_desc = os.linux.user_desc{
+                    .entry_number = os.linux.tls.tls_image.gdt_entry_number,
+                    .base_addr = tp_value,
+                    .limit = 0xfffff,
+                    .seg_32bit = 1,
+                    .contents = 0, // Data
+                    .read_exec_only = 0,
+                    .limit_in_pages = 1,
+                    .seg_not_present = 0,
+                    .useable = 1,
+                };
+                break :blk @ptrToInt(&user_desc);
+            } else {
+                break :blk tp_value;
+            }
+        };
+
+        const rc = os.linux.clone(
+            MainFuncs.linuxThreadMain,
+            mmap_addr + stack_end_offset,
+            flags,
+            arg,
+            &thread_ptr.data.handle,
+            newtls,
+            &thread_ptr.data.handle,
+        );
+        switch (os.errno(rc)) {
+            0 => return thread_ptr,
+            os.EAGAIN => return error.ThreadQuotaExceeded,
+            os.EINVAL => unreachable,
+            os.ENOMEM => return error.SystemResources,
+            os.ENOSPC => unreachable,
+            os.EPERM => unreachable,
+            os.EUSERS => unreachable,
+            else => |err| return os.unexpectedErrno(err),
+        }
+    } else {
+        @compileError("Unsupported OS");
+    }
+}
+
+pub const CpuCountError = error{
+    PermissionDenied,
+    SystemResources,
+    Unexpected,
+};
+
+pub fn cpuCount() CpuCountError!usize {
+    if (std.Target.current.os.tag == .linux) {
+        const cpu_set = try os.sched_getaffinity(0);
+        return @as(usize, os.CPU_COUNT(cpu_set)); // TODO should not need this usize cast
+    }
+    if (std.Target.current.os.tag == .windows) {
+        return os.windows.peb().NumberOfProcessors;
+    }
+    if (std.Target.current.os.tag == .openbsd) {
+        var count: c_int = undefined;
+        var count_size: usize = @sizeOf(c_int);
+        const mib = [_]c_int{ os.CTL_HW, os.HW_NCPUONLINE };
+        os.sysctl(&mib, &count, &count_size, null, 0) catch |err| switch (err) {
+            error.NameTooLong, error.UnknownName => unreachable,
+            else => |e| return e,
+        };
+        return @intCast(usize, count);
+    }
+    var count: c_int = undefined;
+    var count_len: usize = @sizeOf(c_int);
+    const name = if (comptime std.Target.current.isDarwin()) "hw.logicalcpu" else "hw.ncpu";
+    os.sysctlbynameZ(name, &count, &count_len, null, 0) catch |err| switch (err) {
+        error.NameTooLong, error.UnknownName => unreachable,
+        else => |e| return e,
+    };
+    return @intCast(usize, count);
+}
+
+pub fn getCurrentThreadId() u64 {
+    switch (std.Target.current.os.tag) {
+        .linux => {
+            // Use the syscall directly as musl doesn't provide a wrapper.
+            return @bitCast(u32, os.linux.gettid());
+        },
+        .windows => {
+            return os.windows.kernel32.GetCurrentThreadId();
+        },
+        .macos, .ios, .watchos, .tvos => {
+            var thread_id: u64 = undefined;
+            // Pass thread=null to get the current thread ID.
+            assert(c.pthread_threadid_np(null, &thread_id) == 0);
+            return thread_id;
+        },
+        .netbsd => {
+            return @bitCast(u32, c._lwp_self());
+        },
+        .freebsd => {
+            return @bitCast(u32, c.pthread_getthreadid_np());
+        },
+        .openbsd => {
+            return @bitCast(u32, c.getthrid());
+        },
+        else => {
+            @compileError("getCurrentThreadId not implemented for this platform");
+        },
+    }
+}
+
+test "" {
+    std.testing.refAllDecls(@This());
+}
diff --git a/lib/std/Thread/AutoResetEvent.zig b/lib/std/Thread/AutoResetEvent.zig
new file mode 100644
index 0000000000..8b8b5658bf
--- /dev/null
+++ b/lib/std/Thread/AutoResetEvent.zig
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! Similar to `StaticResetEvent` but on `set()` it also (atomically) does `reset()`.
+//! Unlike StaticResetEvent, `wait()` can only be called by one thread (MPSC-like).
+//!
+//! AutoResetEvent has 3 possible states:
+//! - UNSET: the AutoResetEvent is currently unset
+//! - SET: the AutoResetEvent was notified before a wait() was called
+//! - <StaticResetEvent pointer>: there is an active waiter waiting for a notification.
+//!
+//! When attempting to wait:
+//!  if the event is unset, it registers a ResetEvent pointer to be notified when the event is set
+//!  if the event is already set, then it consumes the notification and resets the event.
+//!
+//! When attempting to notify:
+//!  if the event is unset, then we set the event
+//!  if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent
+//!
+//! This ensures that the event is automatically reset after a wait() has been issued
+//! and avoids the race condition when using StaticResetEvent in the following scenario:
+//!  thread 1                  | thread 2
+//!  StaticResetEvent.wait()   |
+//!                            | StaticResetEvent.set()
+//!                            | StaticResetEvent.set()
+//!  StaticResetEvent.reset()  |
+//!  StaticResetEvent.wait()   | (missed the second .set() notification above)
+
+state: usize = UNSET,
+
+const std = @import("../std.zig");
+const builtin = @import("builtin");
+const testing = std.testing;
+const assert = std.debug.assert;
+const StaticResetEvent = std.Thread.StaticResetEvent;
+const AutoResetEvent = @This();
+
+const UNSET = 0;
+const SET = 1;
+
+/// the minimum alignment for the `*StaticResetEvent` created by wait*()
+const event_align = std.math.max(@alignOf(StaticResetEvent), 2);
+
+pub fn wait(self: *AutoResetEvent) void {
+    self.waitFor(null) catch unreachable;
+}
+
+pub fn timedWait(self: *AutoResetEvent, timeout: u64) error{TimedOut}!void {
+    return self.waitFor(timeout);
+}
+
+fn waitFor(self: *AutoResetEvent, timeout: ?u64) error{TimedOut}!void {
+    // lazily initialized StaticResetEvent
+    var reset_event: StaticResetEvent align(event_align) = undefined;
+    var has_reset_event = false;
+
+    var state = @atomicLoad(usize, &self.state, .SeqCst);
+    while (true) {
+        // consume a notification if there is any
+        if (state == SET) {
+            @atomicStore(usize, &self.state, UNSET, .SeqCst);
+            return;
+        }
+
+        // check if theres currently a pending ResetEvent pointer already registered
+        if (state != UNSET) {
+            unreachable; // multiple waiting threads on the same AutoResetEvent
+        }
+
+        // lazily initialize the ResetEvent if it hasn't been already
+        if (!has_reset_event) {
+            has_reset_event = true;
+            reset_event = .{};
+        }
+
+        // Since the AutoResetEvent currently isnt set,
+        // try to register our ResetEvent on it to wait
+        // for a set() call from another thread.
+        if (@cmpxchgWeak(
+            usize,
+            &self.state,
+            UNSET,
+            @ptrToInt(&reset_event),
+            .SeqCst,
+            .SeqCst,
+        )) |new_state| {
+            state = new_state;
+            continue;
+        }
+
+        // if no timeout was specified, then just wait forever
+        const timeout_ns = timeout orelse {
+            reset_event.wait();
+            return;
+        };
+
+        // wait with a timeout and return if signalled via set()
+        switch (reset_event.timedWait(timeout_ns)) {
+            .event_set => return,
+            .timed_out => {},
+        }
+
+        // If we timed out, we need to transition the AutoResetEvent back to UNSET.
+        // If we don't, then when we return, a set() thread could observe a pointer to an invalid ResetEvent.
+        state = @cmpxchgStrong(
+            usize,
+            &self.state,
+            @ptrToInt(&reset_event),
+            UNSET,
+            .SeqCst,
+            .SeqCst,
+        ) orelse return error.TimedOut;
+
+        // We didn't manage to unregister ourselves from the state.
+        if (state == SET) {
+            unreachable; // AutoResetEvent notified without waking up the waiting thread
+        } else if (state != UNSET) {
+            unreachable; // multiple waiting threads on the same AutoResetEvent observed when timing out
+        }
+
+        // This menas a set() thread saw our ResetEvent pointer, acquired it, and is trying to wake it up.
+        // We need to wait for it to wake up our ResetEvent before we can return and invalidate it.
+        // We don't return error.TimedOut here as it technically notified us while we were "timing out".
+        reset_event.wait();
+        return;
+    }
+}
+
+pub fn set(self: *AutoResetEvent) void {
+    var state = @atomicLoad(usize, &self.state, .SeqCst);
+    while (true) {
+        // If the AutoResetEvent is already set, there is nothing else left to do
+        if (state == SET) {
+            return;
+        }
+
+        // If the AutoResetEvent isn't set,
+        // then try to leave a notification for the wait() thread that we set() it.
+        if (state == UNSET) {
+            state = @cmpxchgWeak(
+                usize,
+                &self.state,
+                UNSET,
+                SET,
+                .SeqCst,
+                .SeqCst,
+            ) orelse return;
+            continue;
+        }
+
+        // There is a ResetEvent pointer registered on the AutoResetEvent event thats waiting.
+        // Try to acquire ownership of it so that we can wake it up.
+        // This also resets the AutoResetEvent so that there is no race condition as defined above.
+        if (@cmpxchgWeak(
+            usize,
+            &self.state,
+            state,
+            UNSET,
+            .SeqCst,
+            .SeqCst,
+        )) |new_state| {
+            state = new_state;
+            continue;
+        }
+
+        const reset_event = @intToPtr(*align(event_align) StaticResetEvent, state);
+        reset_event.set();
+        return;
+    }
+}
+
+test "basic usage" {
+    // test local code paths
+    {
+        var event = AutoResetEvent{};
+        testing.expectError(error.TimedOut, event.timedWait(1));
+        event.set();
+        event.wait();
+    }
+
+    // test cross-thread signaling
+    if (builtin.single_threaded)
+        return;
+
+    const Context = struct {
+        value: u128 = 0,
+        in: AutoResetEvent = AutoResetEvent{},
+        out: AutoResetEvent = AutoResetEvent{},
+
+        const Self = @This();
+
+        fn sender(self: *Self) void {
+            testing.expect(self.value == 0);
+            self.value = 1;
+            self.out.set();
+
+            self.in.wait();
+            testing.expect(self.value == 2);
+            self.value = 3;
+            self.out.set();
+
+            self.in.wait();
+            testing.expect(self.value == 4);
+        }
+
+        fn receiver(self: *Self) void {
+            self.out.wait();
+            testing.expect(self.value == 1);
+            self.value = 2;
+            self.in.set();
+
+            self.out.wait();
+            testing.expect(self.value == 3);
+            self.value = 4;
+            self.in.set();
+        }
+    };
+
+    var context = Context{};
+    const send_thread = try std.Thread.spawn(&context, Context.sender);
+    const recv_thread = try std.Thread.spawn(&context, Context.receiver);
+
+    send_thread.wait();
+    recv_thread.wait();
+}
diff --git a/lib/std/Thread/Condition.zig b/lib/std/Thread/Condition.zig
new file mode 100644
index 0000000000..2379d264d1
--- /dev/null
+++ b/lib/std/Thread/Condition.zig
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A condition provides a way for a kernel thread to block until it is signaled
+//! to wake up. Spurious wakeups are possible.
+//! This API supports static initialization and does not require deinitialization.
+
+impl: Impl,
+
+const std = @import("../std.zig");
+const Condition = @This();
+const windows = std.os.windows;
+const linux = std.os.linux;
+const Mutex = std.Thread.Mutex;
+const assert = std.debug.assert;
+
+const Impl = if (std.builtin.single_threaded)
+    SingleThreadedCondition
+else if (std.Target.current.os.tag == .windows)
+    WindowsCondition
+else if (std.Thread.use_pthreads)
+    PthreadCondition
+else
+    AtomicCondition;
+
+pub const SingleThreadedCondition = struct {
+    pub fn wait(cond: *SingleThreadedCondition, mutex: *Mutex) void {
+        unreachable; // deadlock detected
+    }
+
+    pub fn signal(cond: *SingleThreadedCondition) void {}
+
+    pub fn broadcast(cond: *SingleThreadedCondition) void {}
+};
+
+pub const WindowsCondition = struct {
+    cond: windows.CONDITION_VARIABLE = windows.CONDITION_VARIABLE_INIT,
+
+    pub fn wait(cond: *WindowsCondition, mutex: *Mutex) void {
+        const rc = windows.SleepConditionVariableSRW(
+            &cond.cond,
+            &mutex.srwlock,
+            windows.INFINITE,
+            @as(windows.ULONG, 0),
+        );
+        assert(rc != windows.FALSE);
+    }
+
+    pub fn signal(cond: *WindowsCondition) void {
+        windows.WakeConditionVariable(&cond.cond);
+    }
+
+    pub fn broadcast(cond: *WindowsCondition) void {
+        windows.WakeAllConditionVariable(&cond.cond);
+    }
+};
+
+pub const PthreadCondition = struct {
+    cond: std.c.pthread_cond_t = .{},
+
+    pub fn wait(cond: *PthreadCondition, mutex: *Mutex) void {
+        const rc = std.c.pthread_cond_wait(&cond.cond, &mutex.mutex);
+        assert(rc == 0);
+    }
+
+    pub fn signal(cond: *PthreadCondition) void {
+        const rc = std.c.pthread_cond_signal(&cond.cond);
+        assert(rc == 0);
+    }
+
+    pub fn broadcast(cond: *PthreadCondition) void {
+        const rc = std.c.pthread_cond_broadcast(&cond.cond);
+        assert(rc == 0);
+    }
+};
+
+pub const AtomicCondition = struct {
+    pending: bool = false,
+    queue_mutex: Mutex = .{},
+    queue_list: QueueList = .{},
+
+    pub const QueueList = std.SinglyLinkedList(QueueItem);
+
+    pub const QueueItem = struct {
+        futex: i32 = 0,
+
+        fn wait(cond: *@This()) void {
+            while (@atomicLoad(i32, &cond.futex, .Acquire) == 0) {
+                switch (std.Target.current.os.tag) {
+                    .linux => {
+                        switch (linux.getErrno(linux.futex_wait(
+                            &cond.futex,
+                            linux.FUTEX_PRIVATE_FLAG | linux.FUTEX_WAIT,
+                            0,
+                            null,
+                        ))) {
+                            0 => {},
+                            std.os.EINTR => {},
+                            std.os.EAGAIN => {},
+                            else => unreachable,
+                        }
+                    },
+                    else => spinLoopHint(),
+                }
+            }
+        }
+
+        fn notify(cond: *@This()) void {
+            @atomicStore(i32, &cond.futex, 1, .Release);
+
+            switch (std.Target.current.os.tag) {
+                .linux => {
+                    switch (linux.getErrno(linux.futex_wake(
+                        &cond.futex,
+                        linux.FUTEX_PRIVATE_FLAG | linux.FUTEX_WAKE,
+                        1,
+                    ))) {
+                        0 => {},
+                        std.os.EFAULT => {},
+                        else => unreachable,
+                    }
+                },
+                else => {},
+            }
+        }
+    };
+
+    pub fn wait(cond: *AtomicCondition, mutex: *Mutex) void {
+        var waiter = QueueList.Node{ .data = .{} };
+
+        {
+            const held = cond.queue_mutex.acquire();
+            defer held.release();
+
+            cond.queue_list.prepend(&waiter);
+            @atomicStore(bool, &cond.pending, true, .SeqCst);
+        }
+
+        mutex.unlock();
+        waiter.data.wait();
+        mutex.lock();
+    }
+
+    pub fn signal(cond: *AtomicCondition) void {
+        if (@atomicLoad(bool, &cond.pending, .SeqCst) == false)
+            return;
+
+        const maybe_waiter = blk: {
+            const held = cond.queue_mutex.acquire();
+            defer held.release();
+
+            const maybe_waiter = cond.queue_list.popFirst();
+            @atomicStore(bool, &cond.pending, cond.queue_list.first != null, .SeqCst);
+            break :blk maybe_waiter;
+        };
+
+        if (maybe_waiter) |waiter|
+            waiter.data.notify();
+    }
+
+    pub fn broadcast(cond: *AtomicCondition) void {
+        if (@atomicLoad(bool, &cond.pending, .SeqCst) == false)
+            return;
+
+        @atomicStore(bool, &cond.pending, false, .SeqCst);
+
+        var waiters = blk: {
+            const held = cond.queue_mutex.acquire();
+            defer held.release();
+
+            const waiters = cond.queue_list;
+            cond.queue_list = .{};
+            break :blk waiters;
+        };
+
+        while (waiters.popFirst()) |waiter|
+            waiter.data.notify();
+    }
+};
diff --git a/lib/std/Thread/Mutex.zig b/lib/std/Thread/Mutex.zig
new file mode 100644
index 0000000000..128eb0be80
--- /dev/null
+++ b/lib/std/Thread/Mutex.zig
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! Lock may be held only once. If the same thread tries to acquire
+//! the same mutex twice, it deadlocks.  This type supports static
+//! initialization and is at most `@sizeOf(usize)` in size.  When an
+//! application is built in single threaded release mode, all the
+//! functions are no-ops. In single threaded debug mode, there is
+//! deadlock detection.
+//!
+//! Example usage:
+//! var m = Mutex{};
+//!
+//! const lock = m.acquire();
+//! defer lock.release();
+//! ... critical code
+//!
+//! Non-blocking:
+//! if (m.tryAcquire) |lock| {
+//!     defer lock.release();
+//!     // ... critical section
+//! } else {
+//!     // ... lock not acquired
+//! }
+
+impl: Impl = .{},
+
+const Mutex = @This();
+const std = @import("../std.zig");
+const builtin = std.builtin;
+const os = std.os;
+const assert = std.debug.assert;
+const windows = os.windows;
+const linux = os.linux;
+const testing = std.testing;
+const StaticResetEvent = std.thread.StaticResetEvent;
+
+pub const Held = struct {
+    impl: *Impl,
+
+    pub fn release(held: Held) void {
+        held.impl.release();
+    }
+};
+
+/// Try to acquire the mutex without blocking. Returns null if
+/// the mutex is unavailable. Otherwise returns Held. Call
+/// release on Held.
+pub fn tryAcquire(m: *Mutex) ?Held {
+    if (m.impl.tryAcquire()) {
+        return Held{ .impl = &m.impl };
+    } else {
+        return null;
+    }
+}
+
+/// Acquire the mutex. Deadlocks if the mutex is already
+/// held by the calling thread.
+pub fn acquire(m: *Mutex) Held {
+    m.impl.acquire();
+    return .{ .impl = &m.impl };
+}
+
+const Impl = if (builtin.single_threaded)
+    Dummy
+else if (builtin.os.tag == .windows)
+    WindowsMutex
+else if (std.Thread.use_pthreads)
+    PthreadMutex
+else
+    AtomicMutex;
+
+pub const AtomicMutex = struct {
+    state: State = .unlocked,
+
+    const State = enum(i32) {
+        unlocked,
+        locked,
+        waiting,
+    };
+
+    pub fn tryAcquire(self: *AtomicMutex) bool {
+        return @cmpxchgStrong(
+            State,
+            &self.state,
+            .unlocked,
+            .locked,
+            .Acquire,
+            .Monotonic,
+        ) == null;
+    }
+
+    pub fn acquire(self: *AtomicMutex) void {
+        switch (@atomicRmw(State, &self.state, .Xchg, .locked, .Acquire)) {
+            .unlocked => {},
+            else => |s| self.lockSlow(s),
+        }
+    }
+
+    fn lockSlow(self: *AtomicMutex, current_state: State) void {
+        @setCold(true);
+        var new_state = current_state;
+
+        var spin: u8 = 0;
+        while (spin < 100) : (spin += 1) {
+            const state = @cmpxchgWeak(
+                State,
+                &self.state,
+                .unlocked,
+                new_state,
+                .Acquire,
+                .Monotonic,
+            ) orelse return;
+
+            switch (state) {
+                .unlocked => {},
+                .locked => {},
+                .waiting => break,
+            }
+
+            var iter = std.math.min(32, spin + 1);
+            while (iter > 0) : (iter -= 1)
+                std.Thread.spinLoopHint();
+        }
+
+        new_state = .waiting;
+        while (true) {
+            switch (@atomicRmw(State, &self.state, .Xchg, new_state, .Acquire)) {
+                .unlocked => return,
+                else => {},
+            }
+            switch (std.Target.current.os.tag) {
+                .linux => {
+                    switch (linux.getErrno(linux.futex_wait(
+                        @ptrCast(*const i32, &self.state),
+                        linux.FUTEX_PRIVATE_FLAG | linux.FUTEX_WAIT,
+                        @enumToInt(new_state),
+                        null,
+                    ))) {
+                        0 => {},
+                        std.os.EINTR => {},
+                        std.os.EAGAIN => {},
+                        else => unreachable,
+                    }
+                },
+                else => std.Thread.spinLoopHint(),
+            }
+        }
+    }
+
+    pub fn release(self: *AtomicMutex) void {
+        switch (@atomicRmw(State, &self.state, .Xchg, .unlocked, .Release)) {
+            .unlocked => unreachable,
+            .locked => {},
+            .waiting => self.unlockSlow(),
+        }
+    }
+
+    fn unlockSlow(self: *AtomicMutex) void {
+        @setCold(true);
+
+        switch (std.Target.current.os.tag) {
+            .linux => {
+                switch (linux.getErrno(linux.futex_wake(
+                    @ptrCast(*const i32, &self.state),
+                    linux.FUTEX_PRIVATE_FLAG | linux.FUTEX_WAKE,
+                    1,
+                ))) {
+                    0 => {},
+                    std.os.EFAULT => {},
+                    else => unreachable,
+                }
+            },
+            else => {},
+        }
+    }
+};
+
+pub const PthreadMutex = struct {
+    pthread_mutex: std.c.pthread_mutex_t = .{},
+
+    /// Try to acquire the mutex without blocking. Returns null if
+    /// the mutex is unavailable. Otherwise returns Held. Call
+    /// release on Held.
+    pub fn tryAcquire(self: *PthreadMutex) bool {
+        return std.c.pthread_mutex_trylock(&self.pthread_mutex) == 0;
+    }
+
+    /// Acquire the mutex. Will deadlock if the mutex is already
+    /// held by the calling thread.
+    pub fn acquire(self: *PthreadMutex) void {
+        switch (std.c.pthread_mutex_lock(&self.pthread_mutex)) {
+            0 => return,
+            std.c.EINVAL => unreachable,
+            std.c.EBUSY => unreachable,
+            std.c.EAGAIN => unreachable,
+            std.c.EDEADLK => unreachable,
+            std.c.EPERM => unreachable,
+            else => unreachable,
+        }
+    }
+
+    pub fn release(self: *PthreadMutex) void {
+        switch (std.c.pthread_mutex_unlock(&self.pthread_mutex)) {
+            0 => return,
+            std.c.EINVAL => unreachable,
+            std.c.EAGAIN => unreachable,
+            std.c.EPERM => unreachable,
+            else => unreachable,
+        }
+    }
+};
+
+/// This has the sematics as `Mutex`, however it does not actually do any
+/// synchronization. Operations are safety-checked no-ops.
+pub const Dummy = struct {
+    lock: @TypeOf(lock_init) = lock_init,
+
+    const lock_init = if (std.debug.runtime_safety) false else {};
+
+    /// Try to acquire the mutex without blocking. Returns null if
+    /// the mutex is unavailable. Otherwise returns Held. Call
+    /// release on Held.
+    pub fn tryAcquire(self: *Dummy) bool {
+        if (std.debug.runtime_safety) {
+            if (self.lock) return false;
+            self.lock = true;
+        }
+        return true;
+    }
+
+    /// Acquire the mutex. Will deadlock if the mutex is already
+    /// held by the calling thread.
+    pub fn acquire(self: *Dummy) void {
+        return self.tryAcquire() orelse @panic("deadlock detected");
+    }
+
+    pub fn release(self: *Dummy) void {
+        if (std.debug.runtime_safety) {
+            self.mutex.lock = false;
+        }
+    }
+};
+
+const WindowsMutex = struct {
+    srwlock: windows.SRWLOCK = windows.SRWLOCK_INIT,
+
+    pub fn tryAcquire(self: *WindowsMutex) bool {
+        return TryAcquireSRWLockExclusive(&self.srwlock) != system.FALSE;
+    }
+
+    pub fn acquire(self: *WindowsMutex) void {
+        AcquireSRWLockExclusive(&self.srwlock);
+    }
+
+    pub fn release(self: *WindowsMutex) void {
+        ReleaseSRWLockExclusive(&self.srwlock);
+    }
+};
+
+const TestContext = struct {
+    mutex: *Mutex,
+    data: i128,
+
+    const incr_count = 10000;
+};
+
+test "basic usage" {
+    var mutex = Mutex{};
+
+    var context = TestContext{
+        .mutex = &mutex,
+        .data = 0,
+    };
+
+    if (builtin.single_threaded) {
+        worker(&context);
+        testing.expect(context.data == TestContext.incr_count);
+    } else {
+        const thread_count = 10;
+        var threads: [thread_count]*std.Thread = undefined;
+        for (threads) |*t| {
+            t.* = try std.Thread.spawn(&context, worker);
+        }
+        for (threads) |t|
+            t.wait();
+
+        testing.expect(context.data == thread_count * TestContext.incr_count);
+    }
+}
+
+fn worker(ctx: *TestContext) void {
+    var i: usize = 0;
+    while (i != TestContext.incr_count) : (i += 1) {
+        const held = ctx.mutex.acquire();
+        defer held.release();
+
+        ctx.data += 1;
+    }
+}
diff --git a/lib/std/Thread/ResetEvent.zig b/lib/std/Thread/ResetEvent.zig
new file mode 100644
index 0000000000..622f9be98e
--- /dev/null
+++ b/lib/std/Thread/ResetEvent.zig
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A thread-safe resource which supports blocking until signaled.
+//! This API is for kernel threads, not evented I/O.
+//! This API requires being initialized at runtime, and initialization
+//! can fail. Once initialized, the core operations cannot fail.
+//! If you need an abstraction that cannot fail to be initialized, see
+//! `std.Thread.StaticResetEvent`. However if you can handle initialization failure,
+//! it is preferred to use `ResetEvent`.
+
+const ResetEvent = @This();
+const std = @import("../std.zig");
+const builtin = std.builtin;
+const testing = std.testing;
+const assert = std.debug.assert;
+const c = std.c;
+const os = std.os;
+const time = std.time;
+
+impl: Impl,
+
+pub const Impl = if (builtin.single_threaded)
+    std.Thread.StaticResetEvent.DebugEvent
+else if (std.Target.current.isDarwin())
+    DarwinEvent
+else if (std.Thread.use_pthreads)
+    PosixEvent
+else
+    std.Thread.StaticResetEvent.AtomicEvent;
+
+pub const InitError = error{SystemResources};
+
+/// After `init`, it is legal to call any other function.
+pub fn init(ev: *ResetEvent) InitError!void {
+    return ev.impl.init();
+}
+
+/// This function is not thread-safe.
+/// After `deinit`, the only legal function to call is `init`.
+pub fn deinit(ev: *ResetEvent) void {
+    return ev.impl.deinit();
+}
+
+/// Sets the event if not already set and wakes up all the threads waiting on
+/// the event. It is safe to call `set` multiple times before calling `wait`.
+/// However it is illegal to call `set` after `wait` is called until the event
+/// is `reset`. This function is thread-safe.
+pub fn set(ev: *ResetEvent) void {
+    return ev.impl.set();
+}
+
+/// Resets the event to its original, unset state.
+/// This function is *not* thread-safe. It is equivalent to calling
+/// `deinit` followed by `init` but without the possibility of failure.
+pub fn reset(ev: *ResetEvent) void {
+    return ev.impl.reset();
+}
+
+/// Wait for the event to be set by blocking the current thread.
+/// Thread-safe. No spurious wakeups.
+/// Upon return from `wait`, the only functions available to be called
+/// in `ResetEvent` are `reset` and `deinit`.
+pub fn wait(ev: *ResetEvent) void {
+    return ev.impl.wait();
+}
+
+pub const TimedWaitResult = enum { event_set, timed_out };
+
+/// Wait for the event to be set by blocking the current thread.
+/// A timeout in nanoseconds can be provided as a hint for how
+/// long the thread should block on the unset event before returning
+/// `TimedWaitResult.timed_out`.
+/// Thread-safe. No precision of timing is guaranteed.
+/// Upon return from `wait`, the only functions available to be called
+/// in `ResetEvent` are `reset` and `deinit`.
+pub fn timedWait(ev: *ResetEvent, timeout_ns: u64) TimedWaitResult {
+    return ev.impl.timedWait(timeout_ns);
+}
+
+/// Apple has decided to not support POSIX semaphores, so we go with a
+/// different approach using Grand Central Dispatch. This API is exposed
+/// by libSystem so it is guaranteed to be available on all Darwin platforms.
+pub const DarwinEvent = struct {
+    sem: c.dispatch_semaphore_t = undefined,
+
+    pub fn init(ev: *DarwinEvent) !void {
+        ev.* = .{
+            .sem = c.dispatch_semaphore_create(0) orelse return error.SystemResources,
+        };
+    }
+
+    pub fn deinit(ev: *DarwinEvent) void {
+        c.dispatch_release(ev.sem);
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *DarwinEvent) void {
+        // Empirically this returns the numerical value of the semaphore.
+        _ = c.dispatch_semaphore_signal(ev.sem);
+    }
+
+    pub fn wait(ev: *DarwinEvent) void {
+        assert(c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_FOREVER) == 0);
+    }
+
+    pub fn timedWait(ev: *DarwinEvent, timeout_ns: u64) TimedWaitResult {
+        const t = c.dispatch_time(c.DISPATCH_TIME_NOW, @intCast(i64, timeout_ns));
+        if (c.dispatch_semaphore_wait(ev.sem, t) != 0) {
+            return .timed_out;
+        } else {
+            return .event_set;
+        }
+    }
+
+    pub fn reset(ev: *DarwinEvent) void {
+        // Keep calling until the semaphore goes back down to 0.
+        while (c.dispatch_semaphore_wait(ev.sem, c.DISPATCH_TIME_NOW) == 0) {}
+    }
+};
+
+/// POSIX semaphores must be initialized at runtime because they are allowed to
+/// be implemented as file descriptors, in which case initialization would require
+/// a syscall to open the fd.
+pub const PosixEvent = struct {
+    sem: c.sem_t = undefined,
+
+    pub fn init(ev: *PosixEvent) !void {
+        switch (c.getErrno(c.sem_init(&ev.sem, 0, 0))) {
+            0 => return,
+            else => return error.SystemResources,
+        }
+    }
+
+    pub fn deinit(ev: *PosixEvent) void {
+        assert(c.sem_destroy(&ev.sem) == 0);
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *PosixEvent) void {
+        assert(c.sem_post(&ev.sem) == 0);
+    }
+
+    pub fn wait(ev: *PosixEvent) void {
+        while (true) {
+            switch (c.getErrno(c.sem_wait(&ev.sem))) {
+                0 => return,
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                else => unreachable,
+            }
+        }
+    }
+
+    pub fn timedWait(ev: *PosixEvent, timeout_ns: u64) TimedWaitResult {
+        var ts: os.timespec = undefined;
+        var timeout_abs = timeout_ns;
+        os.clock_gettime(os.CLOCK_REALTIME, &ts) catch return .timed_out;
+        timeout_abs += @intCast(u64, ts.tv_sec) * time.ns_per_s;
+        timeout_abs += @intCast(u64, ts.tv_nsec);
+        ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), @divFloor(timeout_abs, time.ns_per_s));
+        ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), @mod(timeout_abs, time.ns_per_s));
+        while (true) {
+            switch (c.getErrno(c.sem_timedwait(&ev.sem, &ts))) {
+                0 => return .event_set,
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                c.ETIMEDOUT => return .timed_out,
+                else => unreachable,
+            }
+        }
+    }
+
+    pub fn reset(ev: *PosixEvent) void {
+        while (true) {
+            switch (c.getErrno(c.sem_trywait(&ev.sem))) {
+                0 => continue, // Need to make it go to zero.
+                c.EINTR => continue,
+                c.EINVAL => unreachable,
+                c.EAGAIN => return, // The semaphore currently has the value zero.
+                else => unreachable,
+            }
+        }
+    }
+};
+
+test "basic usage" {
+    var event: ResetEvent = undefined;
+    try event.init();
+    defer event.deinit();
+
+    // test event setting
+    event.set();
+
+    // test event resetting
+    event.reset();
+
+    // test event waiting (non-blocking)
+    event.set();
+    event.wait();
+    event.reset();
+
+    event.set();
+    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
+
+    // test cross-thread signaling
+    if (builtin.single_threaded)
+        return;
+
+    const Context = struct {
+        const Self = @This();
+
+        value: u128,
+        in: ResetEvent,
+        out: ResetEvent,
+
+        fn init(self: *Self) !void {
+            self.* = .{
+                .value = 0,
+                .in = undefined,
+                .out = undefined,
+            };
+            try self.in.init();
+            try self.out.init();
+        }
+
+        fn deinit(self: *Self) void {
+            self.in.deinit();
+            self.out.deinit();
+            self.* = undefined;
+        }
+
+        fn sender(self: *Self) void {
+            // update value and signal input
+            testing.expect(self.value == 0);
+            self.value = 1;
+            self.in.set();
+
+            // wait for receiver to update value and signal output
+            self.out.wait();
+            testing.expect(self.value == 2);
+
+            // update value and signal final input
+            self.value = 3;
+            self.in.set();
+        }
+
+        fn receiver(self: *Self) void {
+            // wait for sender to update value and signal input
+            self.in.wait();
+            assert(self.value == 1);
+
+            // update value and signal output
+            self.in.reset();
+            self.value = 2;
+            self.out.set();
+
+            // wait for sender to update value and signal final input
+            self.in.wait();
+            assert(self.value == 3);
+        }
+
+        fn sleeper(self: *Self) void {
+            self.in.set();
+            time.sleep(time.ns_per_ms * 2);
+            self.value = 5;
+            self.out.set();
+        }
+
+        fn timedWaiter(self: *Self) !void {
+            self.in.wait();
+            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
+            try self.out.timedWait(time.ns_per_ms * 100);
+            testing.expect(self.value == 5);
+        }
+    };
+
+    var context: Context = undefined;
+    try context.init();
+    defer context.deinit();
+    const receiver = try std.Thread.spawn(&context, Context.receiver);
+    defer receiver.wait();
+    context.sender();
+
+    if (false) {
+        // I have now observed this fail on macOS, Windows, and Linux.
+        // https://github.com/ziglang/zig/issues/7009
+        var timed = Context.init();
+        defer timed.deinit();
+        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
+        defer sleeper.wait();
+        try timed.timedWaiter();
+    }
+}
diff --git a/lib/std/Thread/RwLock.zig b/lib/std/Thread/RwLock.zig
new file mode 100644
index 0000000000..1d606a9cf1
--- /dev/null
+++ b/lib/std/Thread/RwLock.zig
@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A lock that supports one writer or many readers.
+//! This API is for kernel threads, not evented I/O.
+//! This API requires being initialized at runtime, and initialization
+//! can fail. Once initialized, the core operations cannot fail.
+
+impl: Impl,
+
+const RwLock = @This();
+const std = @import("../std.zig");
+const builtin = std.builtin;
+const assert = std.debug.assert;
+const Mutex = std.Thread.Mutex;
+const Semaphore = std.Semaphore;
+const CondVar = std.CondVar;
+
+pub const Impl = if (builtin.single_threaded)
+    SingleThreadedRwLock
+else if (std.Thread.use_pthreads)
+    PthreadRwLock
+else
+    DefaultRwLock;
+
+pub fn init(rwl: *RwLock) void {
+    return rwl.impl.init();
+}
+
+pub fn deinit(rwl: *RwLock) void {
+    return rwl.impl.deinit();
+}
+
+/// Attempts to obtain exclusive lock ownership.
+/// Returns `true` if the lock is obtained, `false` otherwise.
+pub fn tryLock(rwl: *RwLock) bool {
+    return rwl.impl.tryLock();
+}
+
+/// Blocks until exclusive lock ownership is acquired.
+pub fn lock(rwl: *RwLock) void {
+    return rwl.impl.lock();
+}
+
+/// Releases a held exclusive lock.
+/// Asserts the lock is held exclusively.
+pub fn unlock(rwl: *RwLock) void {
+    return rwl.impl.unlock();
+}
+
+/// Attempts to obtain shared lock ownership.
+/// Returns `true` if the lock is obtained, `false` otherwise.
+pub fn tryLockShared(rwl: *RwLock) bool {
+    return rwl.impl.tryLockShared();
+}
+
+/// Blocks until shared lock ownership is acquired.
+pub fn lockShared(rwl: *RwLock) void {
+    return rwl.impl.lockShared();
+}
+
+/// Releases a held shared lock.
+pub fn unlockShared(rwl: *RwLock) void {
+    return rwl.impl.unlockShared();
+}
+
+/// Single-threaded applications use this for deadlock checks in
+/// debug mode, and no-ops in release modes.
+pub const SingleThreadedRwLock = struct {
+    state: enum { unlocked, locked_exclusive, locked_shared },
+    shared_count: usize,
+
+    pub fn init(rwl: *SingleThreadedRwLock) void {
+        rwl.* = .{
+            .state = .unlocked,
+            .shared_count = 0,
+        };
+    }
+
+    pub fn deinit(rwl: *SingleThreadedRwLock) void {
+        assert(rwl.state == .unlocked);
+        assert(rwl.shared_count == 0);
+    }
+
+    /// Attempts to obtain exclusive lock ownership.
+    /// Returns `true` if the lock is obtained, `false` otherwise.
+    pub fn tryLock(rwl: *SingleThreadedRwLock) bool {
+        switch (rwl.state) {
+            .unlocked => {
+                assert(rwl.shared_count == 0);
+                rwl.state = .locked_exclusive;
+                return true;
+            },
+            .locked_exclusive, .locked_shared => return false,
+        }
+    }
+
+    /// Blocks until exclusive lock ownership is acquired.
+    pub fn lock(rwl: *SingleThreadedRwLock) void {
+        assert(rwl.state == .unlocked); // deadlock detected
+        assert(rwl.shared_count == 0); // corrupted state detected
+        rwl.state = .locked_exclusive;
+    }
+
+    /// Releases a held exclusive lock.
+    /// Asserts the lock is held exclusively.
+    pub fn unlock(rwl: *SingleThreadedRwLock) void {
+        assert(rwl.state == .locked_exclusive);
+        assert(rwl.shared_count == 0); // corrupted state detected
+        rwl.state = .unlocked;
+    }
+
+    /// Attempts to obtain shared lock ownership.
+    /// Returns `true` if the lock is obtained, `false` otherwise.
+    pub fn tryLockShared(rwl: *SingleThreadedRwLock) bool {
+        switch (rwl.state) {
+            .unlocked => {
+                rwl.state = .locked_shared;
+                assert(rwl.shared_count == 0);
+                rwl.shared_count = 1;
+                return true;
+            },
+            .locked_exclusive, .locked_shared => return false,
+        }
+    }
+
+    /// Blocks until shared lock ownership is acquired.
+    pub fn lockShared(rwl: *SingleThreadedRwLock) void {
+        switch (rwl.state) {
+            .unlocked => {
+                rwl.state = .locked_shared;
+                assert(rwl.shared_count == 0);
+                rwl.shared_count = 1;
+            },
+            .locked_shared => {
+                rwl.shared_count += 1;
+            },
+            .locked_exclusive => unreachable, // deadlock detected
+        }
+    }
+
+    /// Releases a held shared lock.
+    pub fn unlockShared(rwl: *SingleThreadedRwLock) void {
+        switch (rwl.state) {
+            .unlocked => unreachable, // too many calls to `unlockShared`
+            .locked_exclusive => unreachable, // exclusively held lock
+            .locked_shared => {
+                rwl.shared_count -= 1;
+                if (rwl.shared_count == 0) {
+                    rwl.state = .unlocked;
+                }
+            },
+        }
+    }
+};
+
+pub const PthreadRwLock = struct {
+    rwlock: pthread_rwlock_t,
+
+    pub fn init(rwl: *PthreadRwLock) void {
+        rwl.* = .{ .rwlock = .{} };
+    }
+
+    pub fn deinit(rwl: *PthreadRwLock) void {
+        const safe_rc = switch (std.builtin.os.tag) {
+            .dragonfly, .netbsd => std.os.EAGAIN,
+            else => 0,
+        };
+
+        const rc = std.c.pthread_rwlock_destroy(&rwl.rwlock);
+        assert(rc == 0 or rc == safe_rc);
+
+        rwl.* = undefined;
+    }
+
+    pub fn tryLock(rwl: *PthreadRwLock) bool {
+        return pthread_rwlock_trywrlock(&rwl.rwlock) == 0;
+    }
+
+    pub fn lock(rwl: *PthreadRwLock) void {
+        const rc = pthread_rwlock_wrlock(&rwl.rwlock);
+        assert(rc == 0);
+    }
+
+    pub fn unlock(rwl: *PthreadRwLock) void {
+        const rc = pthread_rwlock_unlock(&rwl.rwlock);
+        assert(rc == 0);
+    }
+
+    pub fn tryLockShared(rwl: *PthreadRwLock) bool {
+        return pthread_rwlock_tryrdlock(&rwl.rwlock) == 0;
+    }
+
+    pub fn lockShared(rwl: *PthreadRwLock) void {
+        const rc = pthread_rwlock_rdlock(&rwl.rwlock);
+        assert(rc == 0);
+    }
+
+    pub fn unlockShared(rwl: *PthreadRwLock) void {
+        const rc = pthread_rwlock_unlock(&rwl.rwlock);
+        assert(rc == 0);
+    }
+};
+
+pub const DefaultRwLock = struct {
+    state: usize,
+    mutex: Mutex,
+    semaphore: Semaphore,
+
+    const IS_WRITING: usize = 1;
+    const WRITER: usize = 1 << 1;
+    const READER: usize = 1 << (1 + std.meta.bitCount(Count));
+    const WRITER_MASK: usize = std.math.maxInt(Count) << @ctz(usize, WRITER);
+    const READER_MASK: usize = std.math.maxInt(Count) << @ctz(usize, READER);
+    const Count = std.meta.Int(.unsigned, @divFloor(std.meta.bitCount(usize) - 1, 2));
+
+    pub fn init(rwl: *DefaultRwLock) void {
+        rwl.* = .{
+            .state = 0,
+            .mutex = Mutex.init(),
+            .semaphore = Semaphore.init(0),
+        };
+    }
+
+    pub fn deinit(rwl: *DefaultRwLock) void {
+        rwl.semaphore.deinit();
+        rwl.mutex.deinit();
+        rwl.* = undefined;
+    }
+
+    pub fn tryLock(rwl: *DefaultRwLock) bool {
+        if (rwl.mutex.tryLock()) {
+            const state = @atomicLoad(usize, &rwl.state, .SeqCst);
+            if (state & READER_MASK == 0) {
+                _ = @atomicRmw(usize, &rwl.state, .Or, IS_WRITING, .SeqCst);
+                return true;
+            }
+
+            rwl.mutex.unlock();
+        }
+
+        return false;
+    }
+
+    pub fn lock(rwl: *DefaultRwLock) void {
+        _ = @atomicRmw(usize, &rwl.state, .Add, WRITER, .SeqCst);
+        rwl.mutex.lock();
+
+        const state = @atomicRmw(usize, &rwl.state, .Or, IS_WRITING, .SeqCst);
+        if (state & READER_MASK != 0)
+            rwl.semaphore.wait();
+    }
+
+    pub fn unlock(rwl: *DefaultRwLock) void {
+        _ = @atomicRmw(usize, &rwl.state, .And, ~IS_WRITING, .SeqCst);
+        rwl.mutex.unlock();
+    }
+
+    pub fn tryLockShared(rwl: *DefaultRwLock) bool {
+        const state = @atomicLoad(usize, &rwl.state, .SeqCst);
+        if (state & (IS_WRITING | WRITER_MASK) == 0) {
+            _ = @cmpxchgStrong(
+                usize,
+                &rwl.state,
+                state,
+                state + READER,
+                .SeqCst,
+                .SeqCst,
+            ) orelse return true;
+        }
+
+        if (rwl.mutex.tryLock()) {
+            _ = @atomicRmw(usize, &rwl.state, .Add, READER, .SeqCst);
+            rwl.mutex.unlock();
+            return true;
+        }
+
+        return false;
+    }
+
+    pub fn lockShared(rwl: *DefaultRwLock) void {
+        var state = @atomicLoad(usize, &rwl.state, .SeqCst);
+        while (state & (IS_WRITING | WRITER_MASK) == 0) {
+            state = @cmpxchgWeak(
+                usize,
+                &rwl.state,
+                state,
+                state + READER,
+                .SeqCst,
+                .SeqCst,
+            ) orelse return;
+        }
+
+        rwl.mutex.lock();
+        _ = @atomicRmw(usize, &rwl.state, .Add, READER, .SeqCst);
+        rwl.mutex.unlock();
+    }
+
+    pub fn unlockShared(rwl: *DefaultRwLock) void {
+        const state = @atomicRmw(usize, &rwl.state, .Sub, READER, .SeqCst);
+
+        if ((state & READER_MASK == READER) and (state & IS_WRITING != 0))
+            rwl.semaphore.post();
+    }
+};
diff --git a/lib/std/Thread/Semaphore.zig b/lib/std/Thread/Semaphore.zig
new file mode 100644
index 0000000000..77a278b355
--- /dev/null
+++ b/lib/std/Thread/Semaphore.zig
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A semaphore is an unsigned integer that blocks the kernel thread if
+//! the number would become negative.
+//! This API supports static initialization and does not require deinitialization.
+
+mutex: Mutex = .{},
+cond: Condition = .{},
+//! It is OK to initialize this field to any value.
+permits: usize = 0,
+
+const RwLock = @This();
+const std = @import("../std.zig");
+const Mutex = std.Thread.Mutex;
+const Condition = std.Thread.Condition;
+
+pub fn wait(sem: *Semaphore) void {
+    const held = sem.mutex.acquire();
+    defer held.release();
+
+    while (sem.permits == 0)
+        sem.cond.wait(&sem.mutex);
+
+    sem.permits -= 1;
+    if (sem.permits > 0)
+        sem.cond.signal();
+}
+
+pub fn post(sem: *Semaphore) void {
+    const held = sem.mutex.acquire();
+    defer held.release();
+
+    sem.permits += 1;
+    sem.cond.signal();
+}
diff --git a/lib/std/Thread/StaticResetEvent.zig b/lib/std/Thread/StaticResetEvent.zig
new file mode 100644
index 0000000000..414583e477
--- /dev/null
+++ b/lib/std/Thread/StaticResetEvent.zig
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2021 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+
+//! A thread-safe resource which supports blocking until signaled.
+//! This API is for kernel threads, not evented I/O.
+//! This API is statically initializable. It cannot fail to be initialized
+//! and it requires no deinitialization. The downside is that it may not
+//! integrate as cleanly into other synchronization APIs, or, in a worst case,
+//! may be forced to fall back on spin locking. As a rule of thumb, prefer
+//! to use `std.Thread.ResetEvent` when possible, and use `StaticResetEvent` when
+//! the logic needs stronger API guarantees.
+
+const std = @import("../std.zig");
+const StaticResetEvent = @This();
+const SpinLock = std.SpinLock;
+const assert = std.debug.assert;
+const os = std.os;
+const time = std.time;
+const linux = std.os.linux;
+const windows = std.os.windows;
+const testing = std.testing;
+
+impl: Impl = .{},
+
+pub const Impl = if (std.builtin.single_threaded)
+    DebugEvent
+else
+    AtomicEvent;
+
+/// Sets the event if not already set and wakes up all the threads waiting on
+/// the event. It is safe to call `set` multiple times before calling `wait`.
+/// However it is illegal to call `set` after `wait` is called until the event
+/// is `reset`. This function is thread-safe.
+pub fn set(ev: *StaticResetEvent) void {
+    return ev.impl.set();
+}
+
+/// Wait for the event to be set by blocking the current thread.
+/// Thread-safe. No spurious wakeups.
+/// Upon return from `wait`, the only function available to be called
+/// in `StaticResetEvent` is `reset`.
+pub fn wait(ev: *StaticResetEvent) void {
+    return ev.impl.wait();
+}
+
+/// Resets the event to its original, unset state.
+/// This function is *not* thread-safe. It is equivalent to calling
+/// `deinit` followed by `init` but without the possibility of failure.
+pub fn reset(ev: *StaticResetEvent) void {
+    return ev.impl.reset();
+}
+
+pub const TimedWaitResult = std.Thread.ResetEvent.TimedWaitResult;
+
+/// Wait for the event to be set by blocking the current thread.
+/// A timeout in nanoseconds can be provided as a hint for how
+/// long the thread should block on the unset event before returning
+/// `TimedWaitResult.timed_out`.
+/// Thread-safe. No precision of timing is guaranteed.
+/// Upon return from `timedWait`, the only function available to be called
+/// in `StaticResetEvent` is `reset`.
+pub fn timedWait(ev: *StaticResetEvent, timeout_ns: u64) TimedWaitResult {
+    return ev.impl.timedWait(timeout_ns);
+}
+
+/// For single-threaded builds, we use this to detect deadlocks.
+/// In unsafe modes this ends up being no-ops.
+pub const DebugEvent = struct {
+    state: State = State.unset,
+
+    const State = enum {
+        unset,
+        set,
+        waited,
+    };
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.Thread.ResetEvent`.
+    pub fn init(ev: *DebugEvent) void {
+        ev.* = .{};
+    }
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.Thread.ResetEvent`.
+    pub fn deinit(ev: *DebugEvent) void {
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *DebugEvent) void {
+        switch (ev.state) {
+            .unset => ev.state = .set,
+            .set => {},
+            .waited => unreachable, // Not allowed to call `set` until `reset`.
+        }
+    }
+
+    pub fn wait(ev: *DebugEvent) void {
+        switch (ev.state) {
+            .unset => unreachable, // Deadlock detected.
+            .set => return,
+            .waited => unreachable, // Not allowed to call `wait` until `reset`.
+        }
+    }
+
+    pub fn timedWait(ev: *DebugEvent, timeout: u64) TimedWaitResult {
+        switch (ev.state) {
+            .unset => return .timed_out,
+            .set => return .event_set,
+            .waited => unreachable, // Not allowed to call `wait` until `reset`.
+        }
+    }
+
+    pub fn reset(ev: *DebugEvent) void {
+        ev.state = .unset;
+    }
+};
+
+pub const AtomicEvent = struct {
+    waiters: u32 = 0,
+
+    const WAKE = 1 << 0;
+    const WAIT = 1 << 1;
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.Thread.ResetEvent`.
+    pub fn init(ev: *AtomicEvent) void {
+        ev.* = .{};
+    }
+
+    /// This function is provided so that this type can be re-used inside
+    /// `std.Thread.ResetEvent`.
+    pub fn deinit(ev: *AtomicEvent) void {
+        ev.* = undefined;
+    }
+
+    pub fn set(ev: *AtomicEvent) void {
+        const waiters = @atomicRmw(u32, &ev.waiters, .Xchg, WAKE, .Release);
+        if (waiters >= WAIT) {
+            return Futex.wake(&ev.waiters, waiters >> 1);
+        }
+    }
+
+    pub fn wait(ev: *AtomicEvent) void {
+        switch (ev.timedWait(null)) {
+            .timed_out => unreachable,
+            .event_set => return,
+        }
+    }
+
+    pub fn timedWait(ev: *AtomicEvent, timeout: ?u64) TimedWaitResult {
+        var waiters = @atomicLoad(u32, &ev.waiters, .Acquire);
+        while (waiters != WAKE) {
+            waiters = @cmpxchgWeak(u32, &ev.waiters, waiters, waiters + WAIT, .Acquire, .Acquire) orelse {
+                if (Futex.wait(&ev.waiters, timeout)) |_| {
+                    return .event_set;
+                } else |_| {
+                    return .timed_out;
+                }
+            };
+        }
+        return .event_set;
+    }
+
+    pub fn reset(ev: *AtomicEvent) void {
+        @atomicStore(u32, &ev.waiters, 0, .Monotonic);
+    }
+
+    pub const Futex = switch (std.Target.current.os.tag) {
+        .windows => WindowsFutex,
+        .linux => LinuxFutex,
+        else => SpinFutex,
+    };
+
+    pub const SpinFutex = struct {
+        fn wake(waiters: *u32, wake_count: u32) void {}
+
+        fn wait(waiters: *u32, timeout: ?u64) !void {
+            var timer: time.Timer = undefined;
+            if (timeout != null)
+                timer = time.Timer.start() catch return error.TimedOut;
+
+            while (@atomicLoad(u32, waiters, .Acquire) != WAKE) {
+                SpinLock.yield();
+                if (timeout) |timeout_ns| {
+                    if (timer.read() >= timeout_ns)
+                        return error.TimedOut;
+                }
+            }
+        }
+    };
+
+    pub const LinuxFutex = struct {
+        fn wake(waiters: *u32, wake_count: u32) void {
+            const waiting = std.math.maxInt(i32); // wake_count
+            const ptr = @ptrCast(*const i32, waiters);
+            const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, waiting);
+            assert(linux.getErrno(rc) == 0);
+        }
+
+        fn wait(waiters: *u32, timeout: ?u64) !void {
+            var ts: linux.timespec = undefined;
+            var ts_ptr: ?*linux.timespec = null;
+            if (timeout) |timeout_ns| {
+                ts_ptr = &ts;
+                ts.tv_sec = @intCast(isize, timeout_ns / time.ns_per_s);
+                ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);
+            }
+
+            while (true) {
+                const waiting = @atomicLoad(u32, waiters, .Acquire);
+                if (waiting == WAKE)
+                    return;
+                const expected = @intCast(i32, waiting);
+                const ptr = @ptrCast(*const i32, waiters);
+                const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);
+                switch (linux.getErrno(rc)) {
+                    0 => continue,
+                    os.ETIMEDOUT => return error.TimedOut,
+                    os.EINTR => continue,
+                    os.EAGAIN => return,
+                    else => unreachable,
+                }
+            }
+        }
+    };
+
+    pub const WindowsFutex = struct {
+        pub fn wake(waiters: *u32, wake_count: u32) void {
+            const handle = getEventHandle() orelse return SpinFutex.wake(waiters, wake_count);
+            const key = @ptrCast(*const c_void, waiters);
+
+            var waiting = wake_count;
+            while (waiting != 0) : (waiting -= 1) {
+                const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
+                assert(rc == .SUCCESS);
+            }
+        }
+
+        pub fn wait(waiters: *u32, timeout: ?u64) !void {
+            const handle = getEventHandle() orelse return SpinFutex.wait(waiters, timeout);
+            const key = @ptrCast(*const c_void, waiters);
+
+            // NT uses timeouts in units of 100ns with negative value being relative
+            var timeout_ptr: ?*windows.LARGE_INTEGER = null;
+            var timeout_value: windows.LARGE_INTEGER = undefined;
+            if (timeout) |timeout_ns| {
+                timeout_ptr = &timeout_value;
+                timeout_value = -@intCast(windows.LARGE_INTEGER, timeout_ns / 100);
+            }
+
+            // NtWaitForKeyedEvent doesnt have spurious wake-ups
+            var rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
+            switch (rc) {
+                .TIMEOUT => {
+                    // update the wait count to signal that we're not waiting anymore.
+                    // if the .set() thread already observed that we are, perform a
+                    // matching NtWaitForKeyedEvent so that the .set() thread doesn't
+                    // deadlock trying to run NtReleaseKeyedEvent above.
+                    var waiting = @atomicLoad(u32, waiters, .Monotonic);
+                    while (true) {
+                        if (waiting == WAKE) {
+                            rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
+                            assert(rc == .WAIT_0);
+                            break;
+                        } else {
+                            waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - WAIT, .Acquire, .Monotonic) orelse break;
+                            continue;
+                        }
+                    }
+                    return error.TimedOut;
+                },
+                .WAIT_0 => {},
+                else => unreachable,
+            }
+        }
+
+        var event_handle: usize = EMPTY;
+        const EMPTY = ~@as(usize, 0);
+        const LOADING = EMPTY - 1;
+
+        pub fn getEventHandle() ?windows.HANDLE {
+            var handle = @atomicLoad(usize, &event_handle, .Monotonic);
+            while (true) {
+                switch (handle) {
+                    EMPTY => handle = @cmpxchgWeak(usize, &event_handle, EMPTY, LOADING, .Acquire, .Monotonic) orelse {
+                        const handle_ptr = @ptrCast(*windows.HANDLE, &handle);
+                        const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
+                        if (windows.ntdll.NtCreateKeyedEvent(handle_ptr, access_mask, null, 0) != .SUCCESS)
+                            handle = 0;
+                        @atomicStore(usize, &event_handle, handle, .Monotonic);
+                        return @intToPtr(?windows.HANDLE, handle);
+                    },
+                    LOADING => {
+                        SpinLock.yield();
+                        handle = @atomicLoad(usize, &event_handle, .Monotonic);
+                    },
+                    else => {
+                        return @intToPtr(?windows.HANDLE, handle);
+                    },
+                }
+            }
+        }
+    };
+};
+
+test "basic usage" {
+    var event = StaticResetEvent{};
+
+    // test event setting
+    event.set();
+
+    // test event resetting
+    event.reset();
+
+    // test event waiting (non-blocking)
+    event.set();
+    event.wait();
+    event.reset();
+
+    event.set();
+    testing.expectEqual(TimedWaitResult.event_set, event.timedWait(1));
+
+    // test cross-thread signaling
+    if (std.builtin.single_threaded)
+        return;
+
+    const Context = struct {
+        const Self = @This();
+
+        value: u128 = 0,
+        in: StaticResetEvent = .{},
+        out: StaticResetEvent = .{},
+
+        fn sender(self: *Self) void {
+            // update value and signal input
+            testing.expect(self.value == 0);
+            self.value = 1;
+            self.in.set();
+
+            // wait for receiver to update value and signal output
+            self.out.wait();
+            testing.expect(self.value == 2);
+
+            // update value and signal final input
+            self.value = 3;
+            self.in.set();
+        }
+
+        fn receiver(self: *Self) void {
+            // wait for sender to update value and signal input
+            self.in.wait();
+            assert(self.value == 1);
+
+            // update value and signal output
+            self.in.reset();
+            self.value = 2;
+            self.out.set();
+
+            // wait for sender to update value and signal final input
+            self.in.wait();
+            assert(self.value == 3);
+        }
+
+        fn sleeper(self: *Self) void {
+            self.in.set();
+            time.sleep(time.ns_per_ms * 2);
+            self.value = 5;
+            self.out.set();
+        }
+
+        fn timedWaiter(self: *Self) !void {
+            self.in.wait();
+            testing.expectEqual(TimedWaitResult.timed_out, self.out.timedWait(time.ns_per_us));
+            try self.out.timedWait(time.ns_per_ms * 100);
+            testing.expect(self.value == 5);
+        }
+    };
+
+    var context = Context{};
+    const receiver = try std.Thread.spawn(&context, Context.receiver);
+    defer receiver.wait();
+    context.sender();
+
+    if (false) {
+        // I have now observed this fail on macOS, Windows, and Linux.
+        // https://github.com/ziglang/zig/issues/7009
+        var timed = Context.init();
+        defer timed.deinit();
+        const sleeper = try std.Thread.spawn(&timed, Context.sleeper);
+        defer sleeper.wait();
+        try timed.timedWaiter();
+    }
+}
diff --git a/lib/std/atomic/queue.zig b/lib/std/atomic/queue.zig
index fa3711cd9f..f5f63944ab 100644
--- a/lib/std/atomic/queue.zig
+++ b/lib/std/atomic/queue.zig
@@ -16,7 +16,7 @@ pub fn Queue(comptime T: type) type {
     return struct {
         head: ?*Node,
         tail: ?*Node,
-        mutex: std.Mutex,
+        mutex: std.Thread.Mutex,
 
         pub const Self = @This();
         pub const Node = std.TailQueue(T).Node;
@@ -27,7 +27,7 @@ pub fn Queue(comptime T: type) type {
             return Self{
                 .head = null,
                 .tail = null,
-                .mutex = std.Mutex{},
+                .mutex = std.Thread.Mutex{},
             };
         }
 
diff --git a/lib/std/auto_reset_event.zig b/lib/std/auto_reset_event.zig
deleted file mode 100644
index 39cd184a68..0000000000
--- a/lib/std/auto_reset_event.zig
+++ /dev/null
@@ -1,226 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const std = @import("std.zig");
-const builtin = @import("builtin");
-const testing = std.testing;
-const assert = std.debug.assert;
-const StaticResetEvent = std.StaticResetEvent;
-
-/// Similar to `StaticResetEvent` but on `set()` it also (atomically) does `reset()`.
-/// Unlike StaticResetEvent, `wait()` can only be called by one thread (MPSC-like).
-pub const AutoResetEvent = struct {
-    /// AutoResetEvent has 3 possible states:
-    /// - UNSET: the AutoResetEvent is currently unset
-    /// - SET: the AutoResetEvent was notified before a wait() was called
-    /// - <StaticResetEvent pointer>: there is an active waiter waiting for a notification.
-    ///
-    /// When attempting to wait:
-    ///  if the event is unset, it registers a ResetEvent pointer to be notified when the event is set
-    ///  if the event is already set, then it consumes the notification and resets the event.
-    ///
-    /// When attempting to notify:
-    ///  if the event is unset, then we set the event
-    ///  if theres a waiting ResetEvent, then we unset the event and notify the ResetEvent
-    ///
-    /// This ensures that the event is automatically reset after a wait() has been issued
-    /// and avoids the race condition when using StaticResetEvent in the following scenario:
-    ///  thread 1                  | thread 2
-    ///  StaticResetEvent.wait()   |
-    ///                            | StaticResetEvent.set()
-    ///                            | StaticResetEvent.set()
-    ///  StaticResetEvent.reset()  |
-    ///  StaticResetEvent.wait()   | (missed the second .set() notification above)
-    state: usize = UNSET,
-
-    const UNSET = 0;
-    const SET = 1;
-
-    /// the minimum alignment for the `*StaticResetEvent` created by wait*()
-    const event_align = std.math.max(@alignOf(StaticResetEvent), 2);
-
-    pub fn wait(self: *AutoResetEvent) void {
-        self.waitFor(null) catch unreachable;
-    }
-
-    pub fn timedWait(self: *AutoResetEvent, timeout: u64) error{TimedOut}!void {
-        return self.waitFor(timeout);
-    }
-
-    fn waitFor(self: *AutoResetEvent, timeout: ?u64) error{TimedOut}!void {
-        // lazily initialized StaticResetEvent
-        var reset_event: StaticResetEvent align(event_align) = undefined;
-        var has_reset_event = false;
-
-        var state = @atomicLoad(usize, &self.state, .SeqCst);
-        while (true) {
-            // consume a notification if there is any
-            if (state == SET) {
-                @atomicStore(usize, &self.state, UNSET, .SeqCst);
-                return;
-            }
-
-            // check if theres currently a pending ResetEvent pointer already registered
-            if (state != UNSET) {
-                unreachable; // multiple waiting threads on the same AutoResetEvent
-            }
-
-            // lazily initialize the ResetEvent if it hasn't been already
-            if (!has_reset_event) {
-                has_reset_event = true;
-                reset_event = .{};
-            }
-
-            // Since the AutoResetEvent currently isnt set,
-            // try to register our ResetEvent on it to wait
-            // for a set() call from another thread.
-            if (@cmpxchgWeak(
-                usize,
-                &self.state,
-                UNSET,
-                @ptrToInt(&reset_event),
-                .SeqCst,
-                .SeqCst,
-            )) |new_state| {
-                state = new_state;
-                continue;
-            }
-
-            // if no timeout was specified, then just wait forever
-            const timeout_ns = timeout orelse {
-                reset_event.wait();
-                return;
-            };
-
-            // wait with a timeout and return if signalled via set()
-            switch (reset_event.timedWait(timeout_ns)) {
-                .event_set => return,
-                .timed_out => {},
-            }
-
-            // If we timed out, we need to transition the AutoResetEvent back to UNSET.
-            // If we don't, then when we return, a set() thread could observe a pointer to an invalid ResetEvent.
-            state = @cmpxchgStrong(
-                usize,
-                &self.state,
-                @ptrToInt(&reset_event),
-                UNSET,
-                .SeqCst,
-                .SeqCst,
-            ) orelse return error.TimedOut;
-
-            // We didn't manage to unregister ourselves from the state.
-            if (state == SET) {
-                unreachable; // AutoResetEvent notified without waking up the waiting thread
-            } else if (state != UNSET) {
-                unreachable; // multiple waiting threads on the same AutoResetEvent observed when timing out
-            }
-
-            // This menas a set() thread saw our ResetEvent pointer, acquired it, and is trying to wake it up.
-            // We need to wait for it to wake up our ResetEvent before we can return and invalidate it.
-            // We don't return error.TimedOut here as it technically notified us while we were "timing out".
-            reset_event.wait();
-            return;
-        }
-    }
-
-    pub fn set(self: *AutoResetEvent) void {
-        var state = @atomicLoad(usize, &self.state, .SeqCst);
-        while (true) {
-            // If the AutoResetEvent is already set, there is nothing else left to do
-            if (state == SET) {
-                return;
-            }
-
-            // If the AutoResetEvent isn't set,
-            // then try to leave a notification for the wait() thread that we set() it.
-            if (state == UNSET) {
-                state = @cmpxchgWeak(
-                    usize,
-                    &self.state,
-                    UNSET,
-                    SET,
-                    .SeqCst,
-                    .SeqCst,
-                ) orelse return;
-                continue;
-            }
-
-            // There is a ResetEvent pointer registered on the AutoResetEvent event thats waiting.
-            // Try to acquire ownership of it so that we can wake it up.
-            // This also resets the AutoResetEvent so that there is no race condition as defined above.
-            if (@cmpxchgWeak(
-                usize,
-                &self.state,
-                state,
-                UNSET,
-                .SeqCst,
-                .SeqCst,
-            )) |new_state| {
-                state = new_state;
-                continue;
-            }
-
-            const reset_event = @intToPtr(*align(event_align) StaticResetEvent, state);
-            reset_event.set();
-            return;
-        }
-    }
-};
-
-test "std.AutoResetEvent" {
-    // test local code paths
-    {
-        var event = AutoResetEvent{};
-        testing.expectError(error.TimedOut, event.timedWait(1));
-        event.set();
-        event.wait();
-    }
-
-    // test cross-thread signaling
-    if (builtin.single_threaded)
-        return;
-
-    const Context = struct {
-        value: u128 = 0,
-        in: AutoResetEvent = AutoResetEvent{},
-        out: AutoResetEvent = AutoResetEvent{},
-
-        const Self = @This();
-
-        fn sender(self: *Self) void {
-            testing.expect(self.value == 0);
-            self.value = 1;
-            self.out.set();
-
-            self.in.wait();
-            testing.expect(self.value == 2);
-            self.value = 3;
-            self.out.set();
-
-            self.in.wait();
-            testing.expect(self.value == 4);
-        }
-
-        fn receiver(self: *Self) void {
-            self.out.wait();
-            testing.expect(self.value == 1);
-            self.value = 2;
-            self.in.set();
-
-            self.out.wait();
-            testing.expect(self.value == 3);
-            self.value = 4;
-            self.in.set();
-        }
-    };
-
-    var context = Context{};
-    const send_thread = try std.Thread.spawn(&context, Context.sender);
-    const recv_thread = try std.Thread.spawn(&context, Context.receiver);
-
-    send_thread.wait();
-    recv_thread.wait();
-}
diff --git a/lib/std/c.zig b/lib/std/c.zig
index b7a412339e..1e86bfbd8c 100644
--- a/lib/std/c.zig
+++ b/lib/std/c.zig
@@ -338,6 +338,13 @@ pub extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int;
 pub extern "c" fn pthread_cond_broadcast(cond: *pthread_cond_t) c_int;
 pub extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int;
 
+pub extern "c" fn pthread_rwlock_destroy(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+pub extern "c" fn pthread_rwlock_rdlock(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+pub extern "c" fn pthread_rwlock_wrlock(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+pub extern "c" fn pthread_rwlock_tryrdlock(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+pub extern "c" fn pthread_rwlock_trywrlock(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+pub extern "c" fn pthread_rwlock_unlock(rwl: *pthread_rwlock_t) callconv(.C) c_int;
+
 pub const pthread_t = *opaque {};
 pub const FILE = opaque {};
 
diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index 7527752527..b5c3fbf977 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -177,6 +177,10 @@ pub const pthread_cond_t = extern struct {
     __sig: c_long = 0x3CB0B1BB,
     __opaque: [__PTHREAD_COND_SIZE__]u8 = [_]u8{0} ** __PTHREAD_COND_SIZE__,
 };
+pub const pthread_rwlock_t = extern struct {
+    __sig: c_long = 0x2DA8B3B4,
+    __opaque: [192]u8 = [_]u8{0} ** 192,
+};
 pub const sem_t = c_int;
 const __PTHREAD_MUTEX_SIZE__ = if (@sizeOf(usize) == 8) 56 else 40;
 const __PTHREAD_COND_SIZE__ = if (@sizeOf(usize) == 8) 40 else 24;
@@ -192,7 +196,7 @@ pub extern "c" fn pthread_threadid_np(thread: ?pthread_t, thread_id: *u64) c_int
 pub extern "c" fn arc4random_buf(buf: [*]u8, len: usize) void;
 
 // Grand Central Dispatch is exposed by libSystem.
-pub const dispatch_semaphore_t = *opaque{};
+pub const dispatch_semaphore_t = *opaque {};
 pub const dispatch_time_t = u64;
 pub const DISPATCH_TIME_NOW = @as(dispatch_time_t, 0);
 pub const DISPATCH_TIME_FOREVER = ~@as(dispatch_time_t, 0);
diff --git a/lib/std/c/emscripten.zig b/lib/std/c/emscripten.zig
index 1652975eb9..526eb9e99c 100644
--- a/lib/std/c/emscripten.zig
+++ b/lib/std/c/emscripten.zig
@@ -9,5 +9,8 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
 };
+pub const pthread_rwlock_t = extern struct {
+    size: [32]u8 align(4) = [_]u8{0} ** 32,
+};
 const __SIZEOF_PTHREAD_COND_T = 48;
 const __SIZEOF_PTHREAD_MUTEX_T = 28;
diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index a8d11b95b2..a6c84c66fa 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -43,6 +43,9 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     inner: ?*c_void = null,
 };
+pub const pthread_rwlock_t = extern struct {
+    ptr: ?*c_void = null,
+};
 
 pub const pthread_attr_t = extern struct {
     __size: [56]u8,
diff --git a/lib/std/c/fuchsia.zig b/lib/std/c/fuchsia.zig
index bc53dc81a6..fc34f49d22 100644
--- a/lib/std/c/fuchsia.zig
+++ b/lib/std/c/fuchsia.zig
@@ -9,5 +9,8 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
 };
+pub const pthread_rwlock_t = extern struct {
+    size: [56]u8 align(@alignOf(usize)) = [_]u8{0} ** 56,
+};
 const __SIZEOF_PTHREAD_COND_T = 48;
 const __SIZEOF_PTHREAD_MUTEX_T = 40;
diff --git a/lib/std/c/haiku.zig b/lib/std/c/haiku.zig
index 438012c3b3..0f695a9446 100644
--- a/lib/std/c/haiku.zig
+++ b/lib/std/c/haiku.zig
@@ -17,3 +17,12 @@ pub const pthread_cond_t = extern struct {
     waiter_count: i32 = 0,
     lock: i32 = 0,
 };
+pub const pthread_rwlock_t = extern struct {
+    flags: u32 = 0,
+    owner: i32 = -1,
+    lock_sem: i32 = 0,
+    lock_count: i32 = 0,
+    reader_count: i32 = 0,
+    writer_count: i32 = 0,
+    waiters: [2]?*c_void = [_]?*c_void{ null, null },
+};
diff --git a/lib/std/c/hermit.zig b/lib/std/c/hermit.zig
index fa351bc0db..a159395ab3 100644
--- a/lib/std/c/hermit.zig
+++ b/lib/std/c/hermit.zig
@@ -9,3 +9,6 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     inner: usize = ~@as(usize, 0),
 };
+pub const pthread_rwlock_t = extern struct {
+    ptr: usize = std.math.maxInt(usize),
+};
diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig
index db464d7a6d..fbfabdd568 100644
--- a/lib/std/c/linux.zig
+++ b/lib/std/c/linux.zig
@@ -123,6 +123,32 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
 };
+pub const pthread_rwlock_t = switch (std.builtin.abi) {
+    .android => switch (@sizeOf(usize)) {
+        4 => extern struct {
+            lock: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER,
+            cond: std.c.pthread_cond_t = std.c.PTHREAD_COND_INITIALIZER,
+            numLocks: c_int = 0,
+            writerThreadId: c_int = 0,
+            pendingReaders: c_int = 0,
+            pendingWriters: c_int = 0,
+            attr: i32 = 0,
+            __reserved: [12]u8 = [_]u8{0} ** 2,
+        },
+        8 => extern struct {
+            numLocks: c_int = 0,
+            writerThreadId: c_int = 0,
+            pendingReaders: c_int = 0,
+            pendingWriters: c_int = 0,
+            attr: i32 = 0,
+            __reserved: [36]u8 = [_]u8{0} ** 36,
+        },
+        else => unreachable,
+    },
+    else => extern struct {
+        size: [56]u8 align(@alignOf(usize)) = [_]u8{0} ** 56,
+    },
+};
 pub const sem_t = extern struct {
     __size: [__SIZEOF_SEM_T]u8 align(@alignOf(usize)),
 };
diff --git a/lib/std/c/netbsd.zig b/lib/std/c/netbsd.zig
index 258ee6a5c6..46a38706f4 100644
--- a/lib/std/c/netbsd.zig
+++ b/lib/std/c/netbsd.zig
@@ -54,6 +54,22 @@ pub const pthread_cond_t = extern struct {
     ptc_private: ?*c_void = null,
 };
 
+pub const pthread_rwlock_t = extern struct {
+    ptr_magic: c_uint = 0x99990009,
+    ptr_interlock: switch (std.builtin.arch) {
+        .aarch64, .sparc, .x86_64, .i386 => u8,
+        .arm, .powerpc => c_int,
+        else => unreachable,
+    } = 0,
+    ptr_rblocked_first: ?*u8 = null,
+    ptr_rblocked_last: ?*u8 = null,
+    ptr_wblocked_first: ?*u8 = null,
+    ptr_wblocked_last: ?*u8 = null,
+    ptr_nreaders: c_uint = 0,
+    ptr_owner: std.c.pthread_t = null,
+    ptr_private: ?*c_void = null,
+};
+
 const pthread_spin_t = switch (builtin.arch) {
     .aarch64, .aarch64_be, .aarch64_32 => u8,
     .mips, .mipsel, .mips64, .mips64el => u32,
diff --git a/lib/std/c/openbsd.zig b/lib/std/c/openbsd.zig
index dd89c837ff..99debf57e7 100644
--- a/lib/std/c/openbsd.zig
+++ b/lib/std/c/openbsd.zig
@@ -27,6 +27,9 @@ pub const pthread_mutex_t = extern struct {
 pub const pthread_cond_t = extern struct {
     inner: ?*c_void = null,
 };
+pub const pthread_rwlock_t = extern struct {
+    ptr: ?*c_void = null,
+};
 pub const pthread_spinlock_t = extern struct {
     inner: ?*c_void = null,
 };
diff --git a/lib/std/debug.zig b/lib/std/debug.zig
index 15d3baa1d0..e4ef25724b 100644
--- a/lib/std/debug.zig
+++ b/lib/std/debug.zig
@@ -50,7 +50,7 @@ pub const LineInfo = struct {
     }
 };
 
-var stderr_mutex = std.Mutex{};
+var stderr_mutex = std.Thread.Mutex{};
 
 /// Deprecated. Use `std.log` functions for logging or `std.debug.print` for
 /// "printf debugging".
@@ -65,7 +65,7 @@ pub fn print(comptime fmt: []const u8, args: anytype) void {
     nosuspend stderr.print(fmt, args) catch return;
 }
 
-pub fn getStderrMutex() *std.Mutex {
+pub fn getStderrMutex() *std.Thread.Mutex {
     return &stderr_mutex;
 }
 
@@ -235,7 +235,7 @@ pub fn panic(comptime format: []const u8, args: anytype) noreturn {
 var panicking: u8 = 0;
 
 // Locked to avoid interleaving panic messages from multiple threads.
-var panic_mutex = std.Mutex{};
+var panic_mutex = std.Thread.Mutex{};
 
 /// Counts how many times the panic handler is invoked by this thread.
 /// This is used to catch and handle panics triggered by the panic handler.
@@ -280,7 +280,7 @@ pub fn panicExtra(trace: ?*const builtin.StackTrace, first_trace_addr: ?usize, c
                 // and call abort()
 
                 // Sleep forever without hammering the CPU
-                var event: std.StaticResetEvent = .{};
+                var event: std.Thread.StaticResetEvent = .{};
                 event.wait();
                 unreachable;
             }
diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig
index 17d79c753b..d48c6c1520 100644
--- a/lib/std/event/lock.zig
+++ b/lib/std/event/lock.zig
@@ -16,7 +16,7 @@ const Loop = std.event.Loop;
 /// Allows only one actor to hold the lock.
 /// TODO: make this API also work in blocking I/O mode.
 pub const Lock = struct {
-    mutex: std.Mutex = std.Mutex{},
+    mutex: std.Thread.Mutex = std.Thread.Mutex{},
     head: usize = UNLOCKED,
 
     const UNLOCKED = 0;
diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig
index 8101d27a55..492b7c1758 100644
--- a/lib/std/event/loop.zig
+++ b/lib/std/event/loop.zig
@@ -29,7 +29,7 @@ pub const Loop = struct {
     fs_thread: *Thread,
     fs_queue: std.atomic.Queue(Request),
     fs_end_request: Request.Node,
-    fs_thread_wakeup: std.ResetEvent,
+    fs_thread_wakeup: std.Thread.ResetEvent,
 
     /// For resources that have the same lifetime as the `Loop`.
     /// This is only used by `Loop` for the thread pool and associated resources.
@@ -785,7 +785,7 @@ pub const Loop = struct {
         timer: std.time.Timer,
         waiters: Waiters,
         thread: *std.Thread,
-        event: std.AutoResetEvent,
+        event: std.Thread.AutoResetEvent,
         is_running: bool,
 
         /// Initialize the delay queue by spawning the timer thread
@@ -796,7 +796,7 @@ pub const Loop = struct {
                 .waiters = DelayQueue.Waiters{
                     .entries = std.atomic.Queue(anyframe).init(),
                 },
-                .event = std.AutoResetEvent{},
+                .event = std.Thread.AutoResetEvent{},
                 .is_running = true,
                 // Must be last so that it can read the other state, such as `is_running`.
                 .thread = try std.Thread.spawn(self, DelayQueue.run),
diff --git a/lib/std/event/wait_group.zig b/lib/std/event/wait_group.zig
index d123f7df27..0b83c18c74 100644
--- a/lib/std/event/wait_group.zig
+++ b/lib/std/event/wait_group.zig
@@ -30,7 +30,7 @@ pub fn WaitGroupGeneric(comptime counter_size: u16) type {
     return struct {
         counter: CounterType = 0,
         max_counter: CounterType = std.math.maxInt(CounterType),
-        mutex: std.Mutex = .{},
+        mutex: std.Thread.Mutex = .{},
         waiters: ?*Waiter = null,
         const Waiter = struct {
             next: ?*Waiter,
diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
index 9afcc50f3c..e9f28a0b60 100644
--- a/lib/std/fs/test.zig
+++ b/lib/std/fs/test.zig
@@ -750,7 +750,7 @@ test "open file with exclusive lock twice, make sure it waits" {
     errdefer file.close();
 
     const S = struct {
-        const C = struct { dir: *fs.Dir, evt: *std.ResetEvent };
+        const C = struct { dir: *fs.Dir, evt: *std.Thread.ResetEvent };
         fn checkFn(ctx: C) !void {
             const file1 = try ctx.dir.createFile(filename, .{ .lock = .Exclusive });
             defer file1.close();
@@ -758,7 +758,7 @@ test "open file with exclusive lock twice, make sure it waits" {
         }
     };
 
-    var evt: std.ResetEvent = undefined;
+    var evt: std.Thread.ResetEvent = undefined;
     try evt.init();
     defer evt.deinit();
 
diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig
index 415b206b6a..05f05c1da3 100644
--- a/lib/std/heap/general_purpose_allocator.zig
+++ b/lib/std/heap/general_purpose_allocator.zig
@@ -149,12 +149,12 @@ pub const Config = struct {
     thread_safe: bool = !std.builtin.single_threaded,
 
     /// What type of mutex you'd like to use, for thread safety.
-    /// when specfied, the mutex type must have the same shape as `std.Mutex` and
+    /// when specfied, the mutex type must have the same shape as `std.Thread.Mutex` and
     /// `std.mutex.Dummy`, and have no required fields. Specifying this field causes
     /// the `thread_safe` field to be ignored.
     ///
     /// when null (default):
-    /// * the mutex type defaults to `std.Mutex` when thread_safe is enabled.
+    /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
     /// * the mutex type defaults to `std.mutex.Dummy` otherwise.
     MutexType: ?type = null,
 
@@ -187,7 +187,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
         const mutex_init = if (config.MutexType) |T|
             T{}
         else if (config.thread_safe)
-            std.Mutex{}
+            std.Thread.Mutex{}
         else
             std.mutex.Dummy{};
 
@@ -869,9 +869,9 @@ test "realloc large object to small object" {
 }
 
 test "overrideable mutexes" {
-    var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Mutex }){
+    var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){
         .backing_allocator = std.testing.allocator,
-        .mutex = std.Mutex{},
+        .mutex = std.Thread.Mutex{},
     };
     defer std.testing.expect(!gpa.deinit());
     const allocator = &gpa.allocator;
diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig
deleted file mode 100644
index 50bbb40bf0..0000000000
--- a/lib/std/mutex.zig
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const std = @import("std.zig");
-const builtin = @import("builtin");
-const os = std.os;
-const assert = std.debug.assert;
-const windows = os.windows;
-const testing = std.testing;
-const SpinLock = std.SpinLock;
-const StaticResetEvent = std.StaticResetEvent;
-
-/// Lock may be held only once. If the same thread tries to acquire
-/// the same mutex twice, it deadlocks.  This type supports static
-/// initialization and is at most `@sizeOf(usize)` in size.  When an
-/// application is built in single threaded release mode, all the
-/// functions are no-ops. In single threaded debug mode, there is
-/// deadlock detection.
-///
-/// Example usage:
-/// var m = Mutex{};
-///
-/// const lock = m.acquire();
-/// defer lock.release();
-/// ... critical code
-///
-/// Non-blocking:
-/// if (m.tryAcquire) |lock| {
-///     defer lock.release();
-///     // ... critical section
-/// } else {
-///     // ... lock not acquired
-/// }
-pub const Mutex = if (builtin.single_threaded)
-    Dummy
-else if (builtin.os.tag == .windows)
-    WindowsMutex
-else if (std.Thread.use_pthreads)
-    PthreadMutex
-else if (builtin.link_libc or builtin.os.tag == .linux)
-    // stack-based version of https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
-    struct {
-        state: usize = 0,
-
-        /// number of times to spin trying to acquire the lock.
-        /// https://webkit.org/blog/6161/locking-in-webkit/
-        const SPIN_COUNT = 40;
-
-        const MUTEX_LOCK: usize = 1 << 0;
-        const QUEUE_LOCK: usize = 1 << 1;
-        const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);
-
-        const Node = struct {
-            next: ?*Node,
-            event: StaticResetEvent,
-        };
-
-        pub fn tryAcquire(self: *Mutex) ?Held {
-            if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic) != null)
-                return null;
-            return Held{ .mutex = self };
-        }
-
-        pub fn acquire(self: *Mutex) Held {
-            return self.tryAcquire() orelse {
-                self.acquireSlow();
-                return Held{ .mutex = self };
-            };
-        }
-
-        fn acquireSlow(self: *Mutex) void {
-            // inlining the fast path and hiding *Slow()
-            // calls behind a @setCold(true) appears to
-            // improve performance in release builds.
-            @setCold(true);
-            while (true) {
-
-                // try and spin for a bit to acquire the mutex if theres currently no queue
-                var spin_count: u32 = SPIN_COUNT;
-                var state = @atomicLoad(usize, &self.state, .Monotonic);
-                while (spin_count != 0) : (spin_count -= 1) {
-                    if (state & MUTEX_LOCK == 0) {
-                        _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
-                    } else if (state & QUEUE_MASK == 0) {
-                        break;
-                    }
-                    SpinLock.yield();
-                    state = @atomicLoad(usize, &self.state, .Monotonic);
-                }
-
-                // create the StaticResetEvent node on the stack
-                // (faster than threadlocal on platforms like OSX)
-                var node: Node = .{
-                    .next = undefined,
-                    .event = .{},
-                };
-
-                // we've spun too long, try and add our node to the LIFO queue.
-                // if the mutex becomes available in the process, try and grab it instead.
-                while (true) {
-                    if (state & MUTEX_LOCK == 0) {
-                        _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
-                    } else {
-                        node.next = @intToPtr(?*Node, state & QUEUE_MASK);
-                        const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
-                        _ = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
-                            node.event.wait();
-                            break;
-                        };
-                    }
-                    SpinLock.yield();
-                    state = @atomicLoad(usize, &self.state, .Monotonic);
-                }
-            }
-        }
-
-        /// Returned when the lock is acquired. Call release to
-        /// release.
-        pub const Held = struct {
-            mutex: *Mutex,
-
-            /// Release the held lock.
-            pub fn release(self: Held) void {
-                // first, remove the lock bit so another possibly parallel acquire() can succeed.
-                // use .Sub since it can be usually compiled down more efficiency
-                // (`lock sub` on x86) vs .And ~MUTEX_LOCK (`lock cmpxchg` loop on x86)
-                const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
-
-                // if the LIFO queue isnt locked and it has a node, try and wake up the node.
-                if ((state & QUEUE_LOCK) == 0 and (state & QUEUE_MASK) != 0)
-                    self.mutex.releaseSlow();
-            }
-        };
-
-        fn releaseSlow(self: *Mutex) void {
-            @setCold(true);
-
-            // try and lock the LFIO queue to pop a node off,
-            // stopping altogether if its already locked or the queue is empty
-            var state = @atomicLoad(usize, &self.state, .Monotonic);
-            while (true) : (SpinLock.loopHint(1)) {
-                if (state & QUEUE_LOCK != 0 or state & QUEUE_MASK == 0)
-                    return;
-                state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;
-            }
-
-            // acquired the QUEUE_LOCK, try and pop a node to wake it.
-            // if the mutex is locked, then unset QUEUE_LOCK and let
-            // the thread who holds the mutex do the wake-up on unlock()
-            while (true) : (SpinLock.loopHint(1)) {
-                if ((state & MUTEX_LOCK) != 0) {
-                    state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Acquire) orelse return;
-                } else {
-                    const node = @intToPtr(*Node, state & QUEUE_MASK);
-                    const new_state = @ptrToInt(node.next);
-                    state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Acquire) orelse {
-                        node.event.set();
-                        return;
-                    };
-                }
-            }
-        }
-    }
-
-    // for platforms without a known OS blocking
-    // primitive, default to SpinLock for correctness
-else
-    SpinLock;
-
-pub const PthreadMutex = struct {
-    pthread_mutex: std.c.pthread_mutex_t = init,
-
-    pub const Held = struct {
-        mutex: *PthreadMutex,
-
-        pub fn release(self: Held) void {
-            switch (std.c.pthread_mutex_unlock(&self.mutex.pthread_mutex)) {
-                0 => return,
-                std.c.EINVAL => unreachable,
-                std.c.EAGAIN => unreachable,
-                std.c.EPERM => unreachable,
-                else => unreachable,
-            }
-        }
-    };
-
-    /// Create a new mutex in unlocked state.
-    pub const init = std.c.PTHREAD_MUTEX_INITIALIZER;
-
-    /// Try to acquire the mutex without blocking. Returns null if
-    /// the mutex is unavailable. Otherwise returns Held. Call
-    /// release on Held.
-    pub fn tryAcquire(self: *PthreadMutex) ?Held {
-        if (std.c.pthread_mutex_trylock(&self.pthread_mutex) == 0) {
-            return Held{ .mutex = self };
-        } else {
-            return null;
-        }
-    }
-
-    /// Acquire the mutex. Will deadlock if the mutex is already
-    /// held by the calling thread.
-    pub fn acquire(self: *PthreadMutex) Held {
-        switch (std.c.pthread_mutex_lock(&self.pthread_mutex)) {
-            0 => return Held{ .mutex = self },
-            std.c.EINVAL => unreachable,
-            std.c.EBUSY => unreachable,
-            std.c.EAGAIN => unreachable,
-            std.c.EDEADLK => unreachable,
-            std.c.EPERM => unreachable,
-            else => unreachable,
-        }
-    }
-};
-
-/// This has the sematics as `Mutex`, however it does not actually do any
-/// synchronization. Operations are safety-checked no-ops.
-pub const Dummy = struct {
-    lock: @TypeOf(lock_init) = lock_init,
-
-    const lock_init = if (std.debug.runtime_safety) false else {};
-
-    pub const Held = struct {
-        mutex: *Dummy,
-
-        pub fn release(self: Held) void {
-            if (std.debug.runtime_safety) {
-                self.mutex.lock = false;
-            }
-        }
-    };
-
-    /// Create a new mutex in unlocked state.
-    pub const init = Dummy{};
-
-    /// Try to acquire the mutex without blocking. Returns null if
-    /// the mutex is unavailable. Otherwise returns Held. Call
-    /// release on Held.
-    pub fn tryAcquire(self: *Dummy) ?Held {
-        if (std.debug.runtime_safety) {
-            if (self.lock) return null;
-            self.lock = true;
-        }
-        return Held{ .mutex = self };
-    }
-
-    /// Acquire the mutex. Will deadlock if the mutex is already
-    /// held by the calling thread.
-    pub fn acquire(self: *Dummy) Held {
-        return self.tryAcquire() orelse @panic("deadlock detected");
-    }
-};
-
-// https://locklessinc.com/articles/keyed_events/
-const WindowsMutex = struct {
-    state: State = State{ .waiters = 0 },
-
-    const State = extern union {
-        locked: u8,
-        waiters: u32,
-    };
-
-    const WAKE = 1 << 8;
-    const WAIT = 1 << 9;
-
-    pub fn tryAcquire(self: *WindowsMutex) ?Held {
-        if (@atomicRmw(u8, &self.state.locked, .Xchg, 1, .Acquire) != 0)
-            return null;
-        return Held{ .mutex = self };
-    }
-
-    pub fn acquire(self: *WindowsMutex) Held {
-        return self.tryAcquire() orelse self.acquireSlow();
-    }
-
-    fn acquireSpinning(self: *WindowsMutex) Held {
-        @setCold(true);
-        while (true) : (SpinLock.yield()) {
-            return self.tryAcquire() orelse continue;
-        }
-    }
-
-    fn acquireSlow(self: *WindowsMutex) Held {
-        // try to use NT keyed events for blocking, falling back to spinlock if unavailable
-        @setCold(true);
-        const handle = StaticResetEvent.Impl.Futex.getEventHandle() orelse return self.acquireSpinning();
-        const key = @ptrCast(*const c_void, &self.state.waiters);
-
-        while (true) : (SpinLock.loopHint(1)) {
-            const waiters = @atomicLoad(u32, &self.state.waiters, .Monotonic);
-
-            // try and take lock if unlocked
-            if ((waiters & 1) == 0) {
-                if (@atomicRmw(u8, &self.state.locked, .Xchg, 1, .Acquire) == 0) {
-                    return Held{ .mutex = self };
-                }
-
-                // otherwise, try and update the waiting count.
-                // then unset the WAKE bit so that another unlocker can wake up a thread.
-            } else if (@cmpxchgWeak(u32, &self.state.waiters, waiters, (waiters + WAIT) | 1, .Monotonic, .Monotonic) == null) {
-                const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
-                assert(rc == .SUCCESS);
-                _ = @atomicRmw(u32, &self.state.waiters, .Sub, WAKE, .Monotonic);
-            }
-        }
-    }
-
-    pub const Held = struct {
-        mutex: *WindowsMutex,
-
-        pub fn release(self: Held) void {
-            // unlock without a rmw/cmpxchg instruction
-            @atomicStore(u8, @ptrCast(*u8, &self.mutex.state.locked), 0, .Release);
-            const handle = StaticResetEvent.Impl.Futex.getEventHandle() orelse return;
-            const key = @ptrCast(*const c_void, &self.mutex.state.waiters);
-
-            while (true) : (SpinLock.loopHint(1)) {
-                const waiters = @atomicLoad(u32, &self.mutex.state.waiters, .Monotonic);
-
-                // no one is waiting
-                if (waiters < WAIT) return;
-                // someone grabbed the lock and will do the wake instead
-                if (waiters & 1 != 0) return;
-                // someone else is currently waking up
-                if (waiters & WAKE != 0) return;
-
-                // try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
-                if (@cmpxchgWeak(u32, &self.mutex.state.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null) {
-                    const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
-                    assert(rc == .SUCCESS);
-                    return;
-                }
-            }
-        }
-    };
-};
-
-const TestContext = struct {
-    mutex: *Mutex,
-    data: i128,
-
-    const incr_count = 10000;
-};
-
-test "std.Mutex" {
-    var mutex = Mutex{};
-
-    var context = TestContext{
-        .mutex = &mutex,
-        .data = 0,
-    };
-
-    if (builtin.single_threaded) {
-        worker(&context);
-        testing.expect(context.data == TestContext.incr_count);
-    } else {
-        const thread_count = 10;
-        var threads: [thread_count]*std.Thread = undefined;
-        for (threads) |*t| {
-            t.* = try std.Thread.spawn(&context, worker);
-        }
-        for (threads) |t|
-            t.wait();
-
-        testing.expect(context.data == thread_count * TestContext.incr_count);
-    }
-}
-
-fn worker(ctx: *TestContext) void {
-    var i: usize = 0;
-    while (i != TestContext.incr_count) : (i += 1) {
-        const held = ctx.mutex.acquire();
-        defer held.release();
-
-        ctx.data += 1;
-    }
-}
diff --git a/lib/std/once.zig b/lib/std/once.zig
index f4ac47f8d8..efa99060d3 100644
--- a/lib/std/once.zig
+++ b/lib/std/once.zig
@@ -15,7 +15,7 @@ pub fn once(comptime f: fn () void) Once(f) {
 pub fn Once(comptime f: fn () void) type {
     return struct {
         done: bool = false,
-        mutex: std.Mutex = std.Mutex{},
+        mutex: std.Thread.Mutex = std.Thread.Mutex{},
 
         /// Call the function `f`.
         /// If `call` is invoked multiple times `f` will be executed only the
diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig
index d8a2fb4a4d..8461378da0 100644
--- a/lib/std/os/windows/bits.zig
+++ b/lib/std/os/windows/bits.zig
@@ -1635,3 +1635,8 @@ pub const OBJECT_NAME_INFORMATION = extern struct {
     Name: UNICODE_STRING,
 };
 pub const POBJECT_NAME_INFORMATION = *OBJECT_NAME_INFORMATION;
+
+pub const SRWLOCK = usize;
+pub const SRWLOCK_INIT: SRWLOCK = 0;
+pub const CONDITION_VARIABLE = usize;
+pub const CONDITION_VARIABLE_INIT: CONDITION_VARIABLE = 0;
diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig
index 78d2a6501e..ec4a75afa9 100644
--- a/lib/std/os/windows/kernel32.zig
+++ b/lib/std/os/windows/kernel32.zig
@@ -289,3 +289,16 @@ pub extern "kernel32" fn K32QueryWorkingSet(hProcess: HANDLE, pv: PVOID, cb: DWO
 pub extern "kernel32" fn K32QueryWorkingSetEx(hProcess: HANDLE, pv: PVOID, cb: DWORD) callconv(WINAPI) BOOL;
 
 pub extern "kernel32" fn FlushFileBuffers(hFile: HANDLE) callconv(WINAPI) BOOL;
+
+pub extern "kernel32" fn WakeAllConditionVariable(c: *CONDITION_VARIABLE) callconv(WINAPI) void;
+pub extern "kernel32" fn WakeConditionVariable(c: *CONDITION_VARIABLE) callconv(WINAPI) void;
+pub extern "kernel32" fn SleepConditionVariableSRW(
+    c: *CONDITION_VARIABLE,
+    s: *SRWLOCK,
+    t: DWORD,
+    f: ULONG,
+) callconv(WINAPI) BOOL;
+
+pub extern "kernel32" fn TryAcquireSRWLockExclusive(s: *SRWLOCK) callconv(WINAPI) BOOL;
+pub extern "kernel32" fn AcquireSRWLockExclusive(s: *SRWLOCK) callconv(WINAPI) void;
+pub extern "kernel32" fn ReleaseSRWLockExclusive(s: *SRWLOCK) callconv(WINAPI) void;
diff --git a/lib/std/std.zig b/lib/std/std.zig
index d736899a45..d085d4fc41 100644
--- a/lib/std/std.zig
+++ b/lib/std/std.zig
@@ -13,7 +13,6 @@ pub const AutoArrayHashMap = array_hash_map.AutoArrayHashMap;
 pub const AutoArrayHashMapUnmanaged = array_hash_map.AutoArrayHashMapUnmanaged;
 pub const AutoHashMap = hash_map.AutoHashMap;
 pub const AutoHashMapUnmanaged = hash_map.AutoHashMapUnmanaged;
-pub const AutoResetEvent = @import("auto_reset_event.zig").AutoResetEvent;
 pub const BufMap = @import("buf_map.zig").BufMap;
 pub const BufSet = @import("buf_set.zig").BufSet;
 pub const ChildProcess = @import("child_process.zig").ChildProcess;
@@ -21,26 +20,21 @@ pub const ComptimeStringMap = @import("comptime_string_map.zig").ComptimeStringM
 pub const DynLib = @import("dynamic_library.zig").DynLib;
 pub const HashMap = hash_map.HashMap;
 pub const HashMapUnmanaged = hash_map.HashMapUnmanaged;
-pub const mutex = @import("mutex.zig");
-pub const Mutex = mutex.Mutex;
 pub const PackedIntArray = @import("packed_int_array.zig").PackedIntArray;
 pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayEndian;
 pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
 pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
 pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
 pub const Progress = @import("Progress.zig");
-pub const ResetEvent = @import("ResetEvent.zig");
 pub const SemanticVersion = @import("SemanticVersion.zig");
 pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
-pub const SpinLock = @import("SpinLock.zig");
-pub const StaticResetEvent = @import("StaticResetEvent.zig");
 pub const StringHashMap = hash_map.StringHashMap;
 pub const StringHashMapUnmanaged = hash_map.StringHashMapUnmanaged;
 pub const StringArrayHashMap = array_hash_map.StringArrayHashMap;
 pub const StringArrayHashMapUnmanaged = array_hash_map.StringArrayHashMapUnmanaged;
 pub const TailQueue = @import("linked_list.zig").TailQueue;
 pub const Target = @import("target.zig").Target;
-pub const Thread = @import("thread.zig").Thread;
+pub const Thread = @import("Thread.zig");
 
 pub const array_hash_map = @import("array_hash_map.zig");
 pub const atomic = @import("atomic.zig");
@@ -98,12 +92,7 @@ test "" {
         // server is hitting OOM. TODO revert this after stage2 arrives.
         _ = ChildProcess;
         _ = DynLib;
-        _ = mutex;
-        _ = Mutex;
         _ = Progress;
-        _ = ResetEvent;
-        _ = SpinLock;
-        _ = StaticResetEvent;
         _ = Target;
         _ = Thread;
 
diff --git a/lib/std/thread.zig b/lib/std/thread.zig
deleted file mode 100644
index 0fee13b057..0000000000
--- a/lib/std/thread.zig
+++ /dev/null
@@ -1,526 +0,0 @@
-// SPDX-License-Identifier: MIT
-// Copyright (c) 2015-2021 Zig Contributors
-// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
-// The MIT license requires this copyright notice to be included in all copies
-// and substantial portions of the software.
-const std = @import("std.zig");
-const builtin = std.builtin;
-const os = std.os;
-const mem = std.mem;
-const windows = std.os.windows;
-const c = std.c;
-const assert = std.debug.assert;
-
-const bad_startfn_ret = "expected return type of startFn to be 'u8', 'noreturn', 'void', or '!void'";
-
-pub const Thread = struct {
-    data: Data,
-
-    pub const use_pthreads = std.Target.current.os.tag != .windows and builtin.link_libc;
-
-    /// Represents a kernel thread handle.
-    /// May be an integer or a pointer depending on the platform.
-    /// On Linux and POSIX, this is the same as Id.
-    pub const Handle = if (use_pthreads)
-        c.pthread_t
-    else switch (std.Target.current.os.tag) {
-        .linux => i32,
-        .windows => windows.HANDLE,
-        else => void,
-    };
-
-    /// Represents a unique ID per thread.
-    /// May be an integer or pointer depending on the platform.
-    /// On Linux and POSIX, this is the same as Handle.
-    pub const Id = switch (std.Target.current.os.tag) {
-        .windows => windows.DWORD,
-        else => Handle,
-    };
-
-    pub const Data = if (use_pthreads)
-        struct {
-            handle: Thread.Handle,
-            memory: []u8,
-        }
-    else switch (std.Target.current.os.tag) {
-        .linux => struct {
-            handle: Thread.Handle,
-            memory: []align(mem.page_size) u8,
-        },
-        .windows => struct {
-            handle: Thread.Handle,
-            alloc_start: *c_void,
-            heap_handle: windows.HANDLE,
-        },
-        else => struct {},
-    };
-
-    /// Returns the ID of the calling thread.
-    /// Makes a syscall every time the function is called.
-    /// On Linux and POSIX, this Id is the same as a Handle.
-    pub fn getCurrentId() Id {
-        if (use_pthreads) {
-            return c.pthread_self();
-        } else
-            return switch (std.Target.current.os.tag) {
-                .linux => os.linux.gettid(),
-                .windows => windows.kernel32.GetCurrentThreadId(),
-                else => @compileError("Unsupported OS"),
-            };
-    }
-
-    /// Returns the handle of this thread.
-    /// On Linux and POSIX, this is the same as Id.
-    /// On Linux, it is possible that the thread spawned with `spawn`
-    /// finishes executing entirely before the clone syscall completes. In this
-    /// case, this function will return 0 rather than the no-longer-existing thread's
-    /// pid.
-    pub fn handle(self: Thread) Handle {
-        return self.data.handle;
-    }
-
-    pub fn wait(self: *Thread) void {
-        if (use_pthreads) {
-            const err = c.pthread_join(self.data.handle, null);
-            switch (err) {
-                0 => {},
-                os.EINVAL => unreachable,
-                os.ESRCH => unreachable,
-                os.EDEADLK => unreachable,
-                else => unreachable,
-            }
-            std.heap.c_allocator.free(self.data.memory);
-            std.heap.c_allocator.destroy(self);
-        } else switch (std.Target.current.os.tag) {
-            .linux => {
-                while (true) {
-                    const pid_value = @atomicLoad(i32, &self.data.handle, .SeqCst);
-                    if (pid_value == 0) break;
-                    const rc = os.linux.futex_wait(&self.data.handle, os.linux.FUTEX_WAIT, pid_value, null);
-                    switch (os.linux.getErrno(rc)) {
-                        0 => continue,
-                        os.EINTR => continue,
-                        os.EAGAIN => continue,
-                        else => unreachable,
-                    }
-                }
-                os.munmap(self.data.memory);
-            },
-            .windows => {
-                windows.WaitForSingleObjectEx(self.data.handle, windows.INFINITE, false) catch unreachable;
-                windows.CloseHandle(self.data.handle);
-                windows.HeapFree(self.data.heap_handle, 0, self.data.alloc_start);
-            },
-            else => @compileError("Unsupported OS"),
-        }
-    }
-
-    pub const SpawnError = error{
-        /// A system-imposed limit on the number of threads was encountered.
-        /// There are a number of limits that may trigger this error:
-        /// *  the  RLIMIT_NPROC soft resource limit (set via setrlimit(2)),
-        ///    which limits the number of processes and threads for  a  real
-        ///    user ID, was reached;
-        /// *  the kernel's system-wide limit on the number of processes and
-        ///    threads,  /proc/sys/kernel/threads-max,  was   reached   (see
-        ///    proc(5));
-        /// *  the  maximum  number  of  PIDs, /proc/sys/kernel/pid_max, was
-        ///    reached (see proc(5)); or
-        /// *  the PID limit (pids.max) imposed by the cgroup "process  num‐
-        ///    ber" (PIDs) controller was reached.
-        ThreadQuotaExceeded,
-
-        /// The kernel cannot allocate sufficient memory to allocate a task structure
-        /// for the child, or to copy those parts of the caller's context that need to
-        /// be copied.
-        SystemResources,
-
-        /// Not enough userland memory to spawn the thread.
-        OutOfMemory,
-
-        /// `mlockall` is enabled, and the memory needed to spawn the thread
-        /// would exceed the limit.
-        LockedMemoryLimitExceeded,
-
-        Unexpected,
-    };
-
-    /// caller must call wait on the returned thread
-    /// fn startFn(@TypeOf(context)) T
-    /// where T is u8, noreturn, void, or !void
-    /// caller must call wait on the returned thread
-    pub fn spawn(context: anytype, comptime startFn: anytype) SpawnError!*Thread {
-        if (builtin.single_threaded) @compileError("cannot spawn thread when building in single-threaded mode");
-        // TODO compile-time call graph analysis to determine stack upper bound
-        // https://github.com/ziglang/zig/issues/157
-        const default_stack_size = 16 * 1024 * 1024;
-
-        const Context = @TypeOf(context);
-        comptime assert(@typeInfo(@TypeOf(startFn)).Fn.args[0].arg_type.? == Context);
-
-        if (std.Target.current.os.tag == .windows) {
-            const WinThread = struct {
-                const OuterContext = struct {
-                    thread: Thread,
-                    inner: Context,
-                };
-                fn threadMain(raw_arg: windows.LPVOID) callconv(.C) windows.DWORD {
-                    const arg = if (@sizeOf(Context) == 0) {} else @ptrCast(*Context, @alignCast(@alignOf(Context), raw_arg)).*;
-
-                    switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
-                        .NoReturn => {
-                            startFn(arg);
-                        },
-                        .Void => {
-                            startFn(arg);
-                            return 0;
-                        },
-                        .Int => |info| {
-                            if (info.bits != 8) {
-                                @compileError(bad_startfn_ret);
-                            }
-                            return startFn(arg);
-                        },
-                        .ErrorUnion => |info| {
-                            if (info.payload != void) {
-                                @compileError(bad_startfn_ret);
-                            }
-                            startFn(arg) catch |err| {
-                                std.debug.warn("error: {s}\n", .{@errorName(err)});
-                                if (@errorReturnTrace()) |trace| {
-                                    std.debug.dumpStackTrace(trace.*);
-                                }
-                            };
-                            return 0;
-                        },
-                        else => @compileError(bad_startfn_ret),
-                    }
-                }
-            };
-
-            const heap_handle = windows.kernel32.GetProcessHeap() orelse return error.OutOfMemory;
-            const byte_count = @alignOf(WinThread.OuterContext) + @sizeOf(WinThread.OuterContext);
-            const bytes_ptr = windows.kernel32.HeapAlloc(heap_handle, 0, byte_count) orelse return error.OutOfMemory;
-            errdefer assert(windows.kernel32.HeapFree(heap_handle, 0, bytes_ptr) != 0);
-            const bytes = @ptrCast([*]u8, bytes_ptr)[0..byte_count];
-            const outer_context = std.heap.FixedBufferAllocator.init(bytes).allocator.create(WinThread.OuterContext) catch unreachable;
-            outer_context.* = WinThread.OuterContext{
-                .thread = Thread{
-                    .data = Thread.Data{
-                        .heap_handle = heap_handle,
-                        .alloc_start = bytes_ptr,
-                        .handle = undefined,
-                    },
-                },
-                .inner = context,
-            };
-
-            const parameter = if (@sizeOf(Context) == 0) null else @ptrCast(*c_void, &outer_context.inner);
-            outer_context.thread.data.handle = windows.kernel32.CreateThread(null, default_stack_size, WinThread.threadMain, parameter, 0, null) orelse {
-                switch (windows.kernel32.GetLastError()) {
-                    else => |err| return windows.unexpectedError(err),
-                }
-            };
-            return &outer_context.thread;
-        }
-
-        const MainFuncs = struct {
-            fn linuxThreadMain(ctx_addr: usize) callconv(.C) u8 {
-                const arg = if (@sizeOf(Context) == 0) {} else @intToPtr(*const Context, ctx_addr).*;
-
-                switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
-                    .NoReturn => {
-                        startFn(arg);
-                    },
-                    .Void => {
-                        startFn(arg);
-                        return 0;
-                    },
-                    .Int => |info| {
-                        if (info.bits != 8) {
-                            @compileError(bad_startfn_ret);
-                        }
-                        return startFn(arg);
-                    },
-                    .ErrorUnion => |info| {
-                        if (info.payload != void) {
-                            @compileError(bad_startfn_ret);
-                        }
-                        startFn(arg) catch |err| {
-                            std.debug.warn("error: {s}\n", .{@errorName(err)});
-                            if (@errorReturnTrace()) |trace| {
-                                std.debug.dumpStackTrace(trace.*);
-                            }
-                        };
-                        return 0;
-                    },
-                    else => @compileError(bad_startfn_ret),
-                }
-            }
-            fn posixThreadMain(ctx: ?*c_void) callconv(.C) ?*c_void {
-                const arg = if (@sizeOf(Context) == 0) {} else @ptrCast(*Context, @alignCast(@alignOf(Context), ctx)).*;
-
-                switch (@typeInfo(@typeInfo(@TypeOf(startFn)).Fn.return_type.?)) {
-                    .NoReturn => {
-                        startFn(arg);
-                    },
-                    .Void => {
-                        startFn(arg);
-                        return null;
-                    },
-                    .Int => |info| {
-                        if (info.bits != 8) {
-                            @compileError(bad_startfn_ret);
-                        }
-                        // pthreads don't support exit status, ignore value
-                        _ = startFn(arg);
-                        return null;
-                    },
-                    .ErrorUnion => |info| {
-                        if (info.payload != void) {
-                            @compileError(bad_startfn_ret);
-                        }
-                        startFn(arg) catch |err| {
-                            std.debug.warn("error: {s}\n", .{@errorName(err)});
-                            if (@errorReturnTrace()) |trace| {
-                                std.debug.dumpStackTrace(trace.*);
-                            }
-                        };
-                        return null;
-                    },
-                    else => @compileError(bad_startfn_ret),
-                }
-            }
-        };
-
-        if (Thread.use_pthreads) {
-            var attr: c.pthread_attr_t = undefined;
-            if (c.pthread_attr_init(&attr) != 0) return error.SystemResources;
-            defer assert(c.pthread_attr_destroy(&attr) == 0);
-
-            const thread_obj = try std.heap.c_allocator.create(Thread);
-            errdefer std.heap.c_allocator.destroy(thread_obj);
-            if (@sizeOf(Context) > 0) {
-                thread_obj.data.memory = try std.heap.c_allocator.allocAdvanced(
-                    u8,
-                    @alignOf(Context),
-                    @sizeOf(Context),
-                    .at_least,
-                );
-                errdefer std.heap.c_allocator.free(thread_obj.data.memory);
-                mem.copy(u8, thread_obj.data.memory, mem.asBytes(&context));
-            } else {
-                thread_obj.data.memory = @as([*]u8, undefined)[0..0];
-            }
-
-            // Use the same set of parameters used by the libc-less impl.
-            assert(c.pthread_attr_setstacksize(&attr, default_stack_size) == 0);
-            assert(c.pthread_attr_setguardsize(&attr, mem.page_size) == 0);
-
-            const err = c.pthread_create(
-                &thread_obj.data.handle,
-                &attr,
-                MainFuncs.posixThreadMain,
-                thread_obj.data.memory.ptr,
-            );
-            switch (err) {
-                0 => return thread_obj,
-                os.EAGAIN => return error.SystemResources,
-                os.EPERM => unreachable,
-                os.EINVAL => unreachable,
-                else => return os.unexpectedErrno(@intCast(usize, err)),
-            }
-
-            return thread_obj;
-        }
-
-        var guard_end_offset: usize = undefined;
-        var stack_end_offset: usize = undefined;
-        var thread_start_offset: usize = undefined;
-        var context_start_offset: usize = undefined;
-        var tls_start_offset: usize = undefined;
-        const mmap_len = blk: {
-            var l: usize = mem.page_size;
-            // Allocate a guard page right after the end of the stack region
-            guard_end_offset = l;
-            // The stack itself, which grows downwards.
-            l = mem.alignForward(l + default_stack_size, mem.page_size);
-            stack_end_offset = l;
-            // Above the stack, so that it can be in the same mmap call, put the Thread object.
-            l = mem.alignForward(l, @alignOf(Thread));
-            thread_start_offset = l;
-            l += @sizeOf(Thread);
-            // Next, the Context object.
-            if (@sizeOf(Context) != 0) {
-                l = mem.alignForward(l, @alignOf(Context));
-                context_start_offset = l;
-                l += @sizeOf(Context);
-            }
-            // Finally, the Thread Local Storage, if any.
-            l = mem.alignForward(l, os.linux.tls.tls_image.alloc_align);
-            tls_start_offset = l;
-            l += os.linux.tls.tls_image.alloc_size;
-            // Round the size to the page size.
-            break :blk mem.alignForward(l, mem.page_size);
-        };
-
-        const mmap_slice = mem: {
-            // Map the whole stack with no rw permissions to avoid
-            // committing the whole region right away
-            const mmap_slice = os.mmap(
-                null,
-                mmap_len,
-                os.PROT_NONE,
-                os.MAP_PRIVATE | os.MAP_ANONYMOUS,
-                -1,
-                0,
-            ) catch |err| switch (err) {
-                error.MemoryMappingNotSupported => unreachable,
-                error.AccessDenied => unreachable,
-                error.PermissionDenied => unreachable,
-                else => |e| return e,
-            };
-            errdefer os.munmap(mmap_slice);
-
-            // Map everything but the guard page as rw
-            os.mprotect(
-                mmap_slice[guard_end_offset..],
-                os.PROT_READ | os.PROT_WRITE,
-            ) catch |err| switch (err) {
-                error.AccessDenied => unreachable,
-                else => |e| return e,
-            };
-
-            break :mem mmap_slice;
-        };
-
-        const mmap_addr = @ptrToInt(mmap_slice.ptr);
-
-        const thread_ptr = @alignCast(@alignOf(Thread), @intToPtr(*Thread, mmap_addr + thread_start_offset));
-        thread_ptr.data.memory = mmap_slice;
-
-        var arg: usize = undefined;
-        if (@sizeOf(Context) != 0) {
-            arg = mmap_addr + context_start_offset;
-            const context_ptr = @alignCast(@alignOf(Context), @intToPtr(*Context, arg));
-            context_ptr.* = context;
-        }
-
-        if (std.Target.current.os.tag == .linux) {
-            const flags: u32 = os.CLONE_VM | os.CLONE_FS | os.CLONE_FILES |
-                os.CLONE_SIGHAND | os.CLONE_THREAD | os.CLONE_SYSVSEM |
-                os.CLONE_PARENT_SETTID | os.CLONE_CHILD_CLEARTID |
-                os.CLONE_DETACHED | os.CLONE_SETTLS;
-            // This structure is only needed when targeting i386
-            var user_desc: if (std.Target.current.cpu.arch == .i386) os.linux.user_desc else void = undefined;
-
-            const tls_area = mmap_slice[tls_start_offset..];
-            const tp_value = os.linux.tls.prepareTLS(tls_area);
-
-            const newtls = blk: {
-                if (std.Target.current.cpu.arch == .i386) {
-                    user_desc = os.linux.user_desc{
-                        .entry_number = os.linux.tls.tls_image.gdt_entry_number,
-                        .base_addr = tp_value,
-                        .limit = 0xfffff,
-                        .seg_32bit = 1,
-                        .contents = 0, // Data
-                        .read_exec_only = 0,
-                        .limit_in_pages = 1,
-                        .seg_not_present = 0,
-                        .useable = 1,
-                    };
-                    break :blk @ptrToInt(&user_desc);
-                } else {
-                    break :blk tp_value;
-                }
-            };
-
-            const rc = os.linux.clone(
-                MainFuncs.linuxThreadMain,
-                mmap_addr + stack_end_offset,
-                flags,
-                arg,
-                &thread_ptr.data.handle,
-                newtls,
-                &thread_ptr.data.handle,
-            );
-            switch (os.errno(rc)) {
-                0 => return thread_ptr,
-                os.EAGAIN => return error.ThreadQuotaExceeded,
-                os.EINVAL => unreachable,
-                os.ENOMEM => return error.SystemResources,
-                os.ENOSPC => unreachable,
-                os.EPERM => unreachable,
-                os.EUSERS => unreachable,
-                else => |err| return os.unexpectedErrno(err),
-            }
-        } else {
-            @compileError("Unsupported OS");
-        }
-    }
-
-    pub const CpuCountError = error{
-        PermissionDenied,
-        SystemResources,
-        Unexpected,
-    };
-
-    pub fn cpuCount() CpuCountError!usize {
-        if (std.Target.current.os.tag == .linux) {
-            const cpu_set = try os.sched_getaffinity(0);
-            return @as(usize, os.CPU_COUNT(cpu_set)); // TODO should not need this usize cast
-        }
-        if (std.Target.current.os.tag == .windows) {
-            return os.windows.peb().NumberOfProcessors;
-        }
-        if (std.Target.current.os.tag == .openbsd) {
-            var count: c_int = undefined;
-            var count_size: usize = @sizeOf(c_int);
-            const mib = [_]c_int{ os.CTL_HW, os.HW_NCPUONLINE };
-            os.sysctl(&mib, &count, &count_size, null, 0) catch |err| switch (err) {
-                error.NameTooLong, error.UnknownName => unreachable,
-                else => |e| return e,
-            };
-            return @intCast(usize, count);
-        }
-        var count: c_int = undefined;
-        var count_len: usize = @sizeOf(c_int);
-        const name = if (comptime std.Target.current.isDarwin()) "hw.logicalcpu" else "hw.ncpu";
-        os.sysctlbynameZ(name, &count, &count_len, null, 0) catch |err| switch (err) {
-            error.NameTooLong, error.UnknownName => unreachable,
-            else => |e| return e,
-        };
-        return @intCast(usize, count);
-    }
-
-    pub fn getCurrentThreadId() u64 {
-        switch (std.Target.current.os.tag) {
-            .linux => {
-                // Use the syscall directly as musl doesn't provide a wrapper.
-                return @bitCast(u32, os.linux.gettid());
-            },
-            .windows => {
-                return os.windows.kernel32.GetCurrentThreadId();
-            },
-            .macos, .ios, .watchos, .tvos => {
-                var thread_id: u64 = undefined;
-                // Pass thread=null to get the current thread ID.
-                assert(c.pthread_threadid_np(null, &thread_id) == 0);
-                return thread_id;
-            },
-            .netbsd => {
-                return @bitCast(u32, c._lwp_self());
-            },
-            .freebsd => {
-                return @bitCast(u32, c.pthread_getthreadid_np());
-            },
-            .openbsd => {
-                return @bitCast(u32, c.getthrid());
-            },
-            else => {
-                @compileError("getCurrentThreadId not implemented for this platform");
-            },
-        }
-    }
-};
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 0efad3362d..225a91e5d2 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -125,7 +125,7 @@ owned_link_dir: ?std.fs.Dir,
 color: @import("main.zig").Color = .auto,
 
 /// This mutex guards all `Compilation` mutable state.
-mutex: std.Mutex = .{},
+mutex: std.Thread.Mutex = .{},
 
 test_filter: ?[]const u8,
 test_name_prefix: ?[]const u8,
diff --git a/src/ThreadPool.zig b/src/ThreadPool.zig
index 1e91d3f731..ec580210e9 100644
--- a/src/ThreadPool.zig
+++ b/src/ThreadPool.zig
@@ -6,14 +6,14 @@
 const std = @import("std");
 const ThreadPool = @This();
 
-lock: std.Mutex = .{},
+lock: std.Thread.Mutex = .{},
 is_running: bool = true,
 allocator: *std.mem.Allocator,
 workers: []Worker,
 run_queue: RunQueue = .{},
 idle_queue: IdleQueue = .{},
 
-const IdleQueue = std.SinglyLinkedList(std.ResetEvent);
+const IdleQueue = std.SinglyLinkedList(std.Thread.ResetEvent);
 const RunQueue = std.SinglyLinkedList(Runnable);
 const Runnable = struct {
     runFn: fn (*Runnable) void,
diff --git a/src/WaitGroup.zig b/src/WaitGroup.zig
index bd6274c10a..d690143710 100644
--- a/src/WaitGroup.zig
+++ b/src/WaitGroup.zig
@@ -6,9 +6,9 @@
 const std = @import("std");
 const WaitGroup = @This();
 
-lock: std.Mutex = .{},
+lock: std.Thread.Mutex = .{},
 counter: usize = 0,
-event: std.ResetEvent,
+event: std.Thread.ResetEvent,
 
 pub fn init(self: *WaitGroup) !void {
     self.* = .{
-- 
cgit v1.2.3


From 8c9ac4db978c80246b4872c899b1618b1b195ec2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 16 Jan 2021 22:51:01 -0700
Subject: stage2: implement error notes and regress -femit-zir

 * Implement error notes
   - note: other symbol exported here
   - note: previous else prong is here
   - note: previous '_' prong is here
 * Add Compilation.CObject.ErrorMsg. This object properly converts to
   AllErrors.Message when the time comes.
 * Add Compilation.CObject.failure_retryable. Properly handles
   out-of-memory and other transient failures.
 * Introduce Module.SrcLoc which has not only a byte offset but also
   references the file which the byte offset applies to.
 * Scope.Block now contains both a pointer to the "owner" Decl and the
   "source" Decl. As an example, during inline function call, the
   "owner" will be the Decl of the caller and the "source" will be the
   Decl of the callee.
 * Module.ErrorMsg now sports a `file_scope` field so that notes can
   refer to source locations in a file other than the parent error
   message.
 * Some instances where a `*Scope` was stored, now store a
   `*Scope.Container`.
 * Some methods in the `Scope` namespace were moved to the more specific
   type, since there was only an implementation for one particular tag.
   - `removeDecl` moved to `Scope.Container`
   - `destroy` moved to `Scope.File`
 * Two kinds of Scope deleted:
   - zir_module
   - decl
 * astgen: properly use DeclVal / DeclRef. DeclVal was incorrectly
   changed to be a reference; this commit fixes it. Fewer ZIR
   instructions processed as a result.
   - declval_in_module is renamed to declval
   - previous declval ZIR instruction is deleted; it was only for .zir
     files.
 * Test harness: friendlier diagnostics when an unexpected set of errors
   is encountered.
 * zir_sema: fix analyzeInstBlockFlat by properly calling resolvingInst
   on the last zir instruction in the block.

Compile log implementation:
 * Write to a buffer rather than directly to stderr.
 * Only keep track of 1 callsite per Decl.
 * No longer mutate the ZIR Inst struct data.
 * "Compile log statement found" errors are only emitted when there are
   no other compile errors.

-femit-zir and support for .zir source files is regressed. If we wanted
to support this again, outputting .zir would need to be done as yet
another backend rather than in the haphazard way it was previously
implemented.

For parsing .zir, it was implemented previously in a way that was not
helpful for debugging. We need tighter integration with the test harness
for it to be useful; so clearly a rewrite is needed. Given that a
rewrite is needed, and it was getting in the way of progress and
organization of the rest of stage2, I regressed the feature.
---
 src/Compilation.zig             |  305 +++++---
 src/Module.zig                  |  764 +++++++------------
 src/astgen.zig                  |   77 +-
 src/codegen.zig                 |  166 ++--
 src/codegen/c.zig               |    7 +-
 src/codegen/llvm.zig            |   13 +-
 src/link/Coff.zig               |    6 +-
 src/link/Elf.zig                |   33 +-
 src/link/MachO.zig              |    8 +-
 src/link/MachO/DebugSymbols.zig |   27 +-
 src/main.zig                    |   47 +-
 src/test.zig                    |  212 +++---
 src/type/Enum.zig               |    2 +-
 src/type/Struct.zig             |    2 +-
 src/type/Union.zig              |    2 +-
 src/zir.zig                     | 1610 +--------------------------------------
 src/zir_sema.zig                |  179 ++---
 test/stage2/test.zig            |   62 +-
 18 files changed, 904 insertions(+), 2618 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 225a91e5d2..ad99e40541 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -51,7 +51,7 @@ c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic),
 
 /// The ErrorMsg memory is owned by the `CObject`, using Compilation's general purpose allocator.
 /// This data is accessed by multiple threads and is protected by `mutex`.
-failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *ErrorMsg) = .{},
+failed_c_objects: std.AutoArrayHashMapUnmanaged(*CObject, *CObject.ErrorMsg) = .{},
 
 keep_source_files_loaded: bool,
 use_clang: bool,
@@ -215,13 +215,29 @@ pub const CObject = struct {
         },
         /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects.
         failure,
+        /// A transient failure happened when trying to compile the C Object; it may
+        /// succeed if we try again. There may be a corresponding ErrorMsg in
+        /// Compilation.failed_c_objects. If there is not, the failure is out of memory.
+        failure_retryable,
     },
 
+    pub const ErrorMsg = struct {
+        msg: []const u8,
+        line: u32,
+        column: u32,
+
+        pub fn destroy(em: *ErrorMsg, gpa: *Allocator) void {
+            gpa.free(em.msg);
+            gpa.destroy(em);
+            em.* = undefined;
+        }
+    };
+
     /// Returns if there was failure.
     pub fn clearStatus(self: *CObject, gpa: *Allocator) bool {
         switch (self.status) {
             .new => return false,
-            .failure => {
+            .failure, .failure_retryable => {
                 self.status = .new;
                 return true;
             },
@@ -240,6 +256,11 @@ pub const CObject = struct {
     }
 };
 
+/// To support incremental compilation, errors are stored in various places
+/// so that they can be created and destroyed appropriately. This structure
+/// is used to collect all the errors from the various places into one
+/// convenient place for API users to consume. It is allocated into 1 heap
+/// and freed all at once.
 pub const AllErrors = struct {
     arena: std.heap.ArenaAllocator.State,
     list: []const Message,
@@ -251,23 +272,32 @@ pub const AllErrors = struct {
             column: usize,
             byte_offset: usize,
             msg: []const u8,
+            notes: []Message = &.{},
         },
         plain: struct {
             msg: []const u8,
         },
 
-        pub fn renderToStdErr(self: Message) void {
-            switch (self) {
+        pub fn renderToStdErr(msg: Message) void {
+            return msg.renderToStdErrInner("error");
+        }
+
+        fn renderToStdErrInner(msg: Message, kind: []const u8) void {
+            switch (msg) {
                 .src => |src| {
-                    std.debug.print("{s}:{d}:{d}: error: {s}\n", .{
+                    std.debug.print("{s}:{d}:{d}: {s}: {s}\n", .{
                         src.src_path,
                         src.line + 1,
                         src.column + 1,
+                        kind,
                         src.msg,
                     });
+                    for (src.notes) |note| {
+                        note.renderToStdErrInner("note");
+                    }
                 },
                 .plain => |plain| {
-                    std.debug.print("error: {s}\n", .{plain.msg});
+                    std.debug.print("{s}: {s}\n", .{ kind, plain.msg });
                 },
             }
         }
@@ -278,20 +308,38 @@ pub const AllErrors = struct {
     }
 
     fn add(
+        module: *Module,
         arena: *std.heap.ArenaAllocator,
         errors: *std.ArrayList(Message),
-        sub_file_path: []const u8,
-        source: []const u8,
-        simple_err_msg: ErrorMsg,
+        module_err_msg: Module.ErrorMsg,
     ) !void {
-        const loc = std.zig.findLineColumn(source, simple_err_msg.byte_offset);
+        const notes = try arena.allocator.alloc(Message, module_err_msg.notes.len);
+        for (notes) |*note, i| {
+            const module_note = module_err_msg.notes[i];
+            const source = try module_note.src_loc.file_scope.getSource(module);
+            const loc = std.zig.findLineColumn(source, module_note.src_loc.byte_offset);
+            const sub_file_path = module_note.src_loc.file_scope.sub_file_path;
+            note.* = .{
+                .src = .{
+                    .src_path = try arena.allocator.dupe(u8, sub_file_path),
+                    .msg = try arena.allocator.dupe(u8, module_note.msg),
+                    .byte_offset = module_note.src_loc.byte_offset,
+                    .line = loc.line,
+                    .column = loc.column,
+                },
+            };
+        }
+        const source = try module_err_msg.src_loc.file_scope.getSource(module);
+        const loc = std.zig.findLineColumn(source, module_err_msg.src_loc.byte_offset);
+        const sub_file_path = module_err_msg.src_loc.file_scope.sub_file_path;
         try errors.append(.{
             .src = .{
                 .src_path = try arena.allocator.dupe(u8, sub_file_path),
-                .msg = try arena.allocator.dupe(u8, simple_err_msg.msg),
-                .byte_offset = simple_err_msg.byte_offset,
+                .msg = try arena.allocator.dupe(u8, module_err_msg.msg),
+                .byte_offset = module_err_msg.src_loc.byte_offset,
                 .line = loc.line,
                 .column = loc.column,
+                .notes = notes,
             },
         });
     }
@@ -849,17 +897,9 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                             .ty = struct_ty,
                         },
                     };
-                    break :rs &root_scope.base;
+                    break :rs root_scope;
                 } else if (mem.endsWith(u8, root_pkg.root_src_path, ".zir")) {
-                    const root_scope = try gpa.create(Module.Scope.ZIRModule);
-                    root_scope.* = .{
-                        .sub_file_path = root_pkg.root_src_path,
-                        .source = .{ .unloaded = {} },
-                        .contents = .{ .not_available = {} },
-                        .status = .never_loaded,
-                        .decls = .{},
-                    };
-                    break :rs &root_scope.base;
+                    return error.ZirFilesUnsupported;
                 } else {
                     unreachable;
                 }
@@ -1258,32 +1298,23 @@ pub fn update(self: *Compilation) !void {
     const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
     if (!use_stage1) {
         if (self.bin_file.options.module) |module| {
+            module.compile_log_text.shrinkAndFree(module.gpa, 0);
             module.generation += 1;
 
             // TODO Detect which source files changed.
             // Until then we simulate a full cache miss. Source files could have been loaded for any reason;
             // to force a refresh we unload now.
-            if (module.root_scope.cast(Module.Scope.File)) |zig_file| {
-                zig_file.unload(module.gpa);
-                module.failed_root_src_file = null;
-                module.analyzeContainer(&zig_file.root_container) catch |err| switch (err) {
-                    error.AnalysisFail => {
-                        assert(self.totalErrorCount() != 0);
-                    },
-                    error.OutOfMemory => return error.OutOfMemory,
-                    else => |e| {
-                        module.failed_root_src_file = e;
-                    },
-                };
-            } else if (module.root_scope.cast(Module.Scope.ZIRModule)) |zir_module| {
-                zir_module.unload(module.gpa);
-                module.analyzeRootZIRModule(zir_module) catch |err| switch (err) {
-                    error.AnalysisFail => {
-                        assert(self.totalErrorCount() != 0);
-                    },
-                    else => |e| return e,
-                };
-            }
+            module.root_scope.unload(module.gpa);
+            module.failed_root_src_file = null;
+            module.analyzeContainer(&module.root_scope.root_container) catch |err| switch (err) {
+                error.AnalysisFail => {
+                    assert(self.totalErrorCount() != 0);
+                },
+                error.OutOfMemory => return error.OutOfMemory,
+                else => |e| {
+                    module.failed_root_src_file = e;
+                },
+            };
 
             // TODO only analyze imports if they are still referenced
             for (module.import_table.items()) |entry| {
@@ -1359,14 +1390,18 @@ pub fn totalErrorCount(self: *Compilation) usize {
             module.failed_exports.items().len +
             module.failed_files.items().len +
             @boolToInt(module.failed_root_src_file != null);
-        for (module.compile_log_decls.items()) |entry| {
-            total += entry.value.items.len;
-        }
     }
 
     // The "no entry point found" error only counts if there are no other errors.
     if (total == 0) {
-        return @boolToInt(self.link_error_flags.no_entry_point_found);
+        total += @boolToInt(self.link_error_flags.no_entry_point_found);
+    }
+
+    // Compile log errors only count if there are no other errors.
+    if (total == 0) {
+        if (self.bin_file.options.module) |module| {
+            total += @boolToInt(module.compile_log_decls.items().len != 0);
+        }
     }
 
     return total;
@@ -1382,32 +1417,32 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     for (self.failed_c_objects.items()) |entry| {
         const c_object = entry.key;
         const err_msg = entry.value;
-        try AllErrors.add(&arena, &errors, c_object.src.src_path, "", err_msg.*);
+        // TODO these fields will need to be adjusted when we have proper
+        // C error reporting bubbling up.
+        try errors.append(.{
+            .src = .{
+                .src_path = try arena.allocator.dupe(u8, c_object.src.src_path),
+                .msg = try std.fmt.allocPrint(&arena.allocator, "unable to build C object: {s}", .{
+                    err_msg.msg,
+                }),
+                .byte_offset = 0,
+                .line = err_msg.line,
+                .column = err_msg.column,
+            },
+        });
     }
     if (self.bin_file.options.module) |module| {
         for (module.failed_files.items()) |entry| {
-            const scope = entry.key;
-            const err_msg = entry.value;
-            const source = try scope.getSource(module);
-            try AllErrors.add(&arena, &errors, scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.failed_decls.items()) |entry| {
-            const decl = entry.key;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.emit_h_failed_decls.items()) |entry| {
-            const decl = entry.key;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         for (module.failed_exports.items()) |entry| {
-            const decl = entry.key.owner_decl;
-            const err_msg = entry.value;
-            const source = try decl.scope.getSource(module);
-            try AllErrors.add(&arena, &errors, decl.scope.subFilePath(), source, err_msg.*);
+            try AllErrors.add(module, &arena, &errors, entry.value.*);
         }
         if (module.failed_root_src_file) |err| {
             const file_path = try module.root_pkg.root_src_directory.join(&arena.allocator, &[_][]const u8{
@@ -1418,15 +1453,6 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
             });
             try AllErrors.addPlain(&arena, &errors, msg);
         }
-        for (module.compile_log_decls.items()) |entry| {
-            const decl = entry.key;
-            const path = decl.scope.subFilePath();
-            const source = try decl.scope.getSource(module);
-            for (entry.value.items) |src_loc| {
-                const err_msg = ErrorMsg{ .byte_offset = src_loc, .msg = "found compile log statement" };
-                try AllErrors.add(&arena, &errors, path, source, err_msg);
-            }
-        }
     }
 
     if (errors.items.len == 0 and self.link_error_flags.no_entry_point_found) {
@@ -1437,6 +1463,28 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
         });
     }
 
+    if (self.bin_file.options.module) |module| {
+        const compile_log_items = module.compile_log_decls.items();
+        if (errors.items.len == 0 and compile_log_items.len != 0) {
+            // First one will be the error; subsequent ones will be notes.
+            const err_msg = Module.ErrorMsg{
+                .src_loc = compile_log_items[0].value,
+                .msg = "found compile log statement",
+                .notes = try self.gpa.alloc(Module.ErrorMsg, compile_log_items.len - 1),
+            };
+            defer self.gpa.free(err_msg.notes);
+
+            for (compile_log_items[1..]) |entry, i| {
+                err_msg.notes[i] = .{
+                    .src_loc = entry.value,
+                    .msg = "also here",
+                };
+            }
+
+            try AllErrors.add(module, &arena, &errors, err_msg);
+        }
+    }
+
     assert(errors.items.len == self.totalErrorCount());
 
     return AllErrors{
@@ -1445,6 +1493,11 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     };
 }
 
+pub fn getCompileLogOutput(self: *Compilation) []const u8 {
+    const module = self.bin_file.options.module orelse return &[0]u8{};
+    return module.compile_log_text.items;
+}
+
 pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
     var progress: std.Progress = .{};
     var main_progress_node = try progress.start("", 0);
@@ -1517,9 +1570,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     },
                     else => {
                         try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                        module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                        module.failed_decls.putAssumeCapacityNoClobber(decl, try Module.ErrorMsg.create(
                             module.gpa,
-                            decl.src(),
+                            decl.srcLoc(),
                             "unable to codegen: {s}",
                             .{@errorName(err)},
                         ));
@@ -1586,9 +1639,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
             const module = self.bin_file.options.module.?;
             self.bin_file.updateDeclLineNumber(module, decl) catch |err| {
                 try module.failed_decls.ensureCapacity(module.gpa, module.failed_decls.items().len + 1);
-                module.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                module.failed_decls.putAssumeCapacityNoClobber(decl, try Module.ErrorMsg.create(
                     module.gpa,
-                    decl.src(),
+                    decl.srcLoc(),
                     "unable to update line number: {s}",
                     .{@errorName(err)},
                 ));
@@ -1858,26 +1911,38 @@ fn workerUpdateCObject(
     comp.updateCObject(c_object, progress_node) catch |err| switch (err) {
         error.AnalysisFail => return,
         else => {
-            {
-                const lock = comp.mutex.acquire();
-                defer lock.release();
-                comp.failed_c_objects.ensureCapacity(comp.gpa, comp.failed_c_objects.items().len + 1) catch {
-                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
-                };
-                comp.failed_c_objects.putAssumeCapacityNoClobber(c_object, ErrorMsg.create(
-                    comp.gpa,
-                    0,
-                    "unable to build C object: {s}",
-                    .{@errorName(err)},
-                ) catch {
-                    fatal("TODO handle this by setting c_object.status = oom failure", .{});
-                });
-            }
-            c_object.status = .{ .failure = {} };
+            comp.reportRetryableCObjectError(c_object, err) catch |oom| switch (oom) {
+                // Swallowing this error is OK because it's implied to be OOM when
+                // there is a missing failed_c_objects error message.
+                error.OutOfMemory => {},
+            };
         },
     };
 }
 
+fn reportRetryableCObjectError(
+    comp: *Compilation,
+    c_object: *CObject,
+    err: anyerror,
+) error{OutOfMemory}!void {
+    c_object.status = .failure_retryable;
+
+    const c_obj_err_msg = try comp.gpa.create(CObject.ErrorMsg);
+    errdefer comp.gpa.destroy(c_obj_err_msg);
+    const msg = try std.fmt.allocPrint(comp.gpa, "unable to build C object: {s}", .{@errorName(err)});
+    errdefer comp.gpa.free(msg);
+    c_obj_err_msg.* = .{
+        .msg = msg,
+        .line = 0,
+        .column = 0,
+    };
+    {
+        const lock = comp.mutex.acquire();
+        defer lock.release();
+        try comp.failed_c_objects.putNoClobber(comp.gpa, c_object, c_obj_err_msg);
+    }
+}
+
 fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *std.Progress.Node) !void {
     if (!build_options.have_llvm) {
         return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
@@ -1892,7 +1957,9 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
         // There was previous failure.
         const lock = comp.mutex.acquire();
         defer lock.release();
-        comp.failed_c_objects.removeAssertDiscard(c_object);
+        // If the failure was OOM, there will not be an entry here, so we do
+        // not assert discard.
+        _ = comp.failed_c_objects.swapRemove(c_object);
     }
 
     var man = comp.obtainCObjectCacheManifest();
@@ -2343,11 +2410,27 @@ pub fn addCCArgs(
 
 fn failCObj(comp: *Compilation, c_object: *CObject, comptime format: []const u8, args: anytype) InnerError {
     @setCold(true);
-    const err_msg = try ErrorMsg.create(comp.gpa, 0, "unable to build C object: " ++ format, args);
+    const err_msg = blk: {
+        const msg = try std.fmt.allocPrint(comp.gpa, format, args);
+        errdefer comp.gpa.free(msg);
+        const err_msg = try comp.gpa.create(CObject.ErrorMsg);
+        errdefer comp.gpa.destroy(err_msg);
+        err_msg.* = .{
+            .msg = msg,
+            .line = 0,
+            .column = 0,
+        };
+        break :blk err_msg;
+    };
     return comp.failCObjWithOwnedErrorMsg(c_object, err_msg);
 }
 
-fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *ErrorMsg) InnerError {
+fn failCObjWithOwnedErrorMsg(
+    comp: *Compilation,
+    c_object: *CObject,
+    err_msg: *CObject.ErrorMsg,
+) InnerError {
+    @setCold(true);
     {
         const lock = comp.mutex.acquire();
         defer lock.release();
@@ -2361,36 +2444,6 @@ fn failCObjWithOwnedErrorMsg(comp: *Compilation, c_object: *CObject, err_msg: *E
     return error.AnalysisFail;
 }
 
-pub const ErrorMsg = struct {
-    byte_offset: usize,
-    msg: []const u8,
-
-    pub fn create(gpa: *Allocator, byte_offset: usize, comptime format: []const u8, args: anytype) !*ErrorMsg {
-        const self = try gpa.create(ErrorMsg);
-        errdefer gpa.destroy(self);
-        self.* = try init(gpa, byte_offset, format, args);
-        return self;
-    }
-
-    /// Assumes the ErrorMsg struct and msg were both allocated with allocator.
-    pub fn destroy(self: *ErrorMsg, gpa: *Allocator) void {
-        self.deinit(gpa);
-        gpa.destroy(self);
-    }
-
-    pub fn init(gpa: *Allocator, byte_offset: usize, comptime format: []const u8, args: anytype) !ErrorMsg {
-        return ErrorMsg{
-            .byte_offset = byte_offset,
-            .msg = try std.fmt.allocPrint(gpa, format, args),
-        };
-    }
-
-    pub fn deinit(self: *ErrorMsg, gpa: *Allocator) void {
-        gpa.free(self.msg);
-        self.* = undefined;
-    }
-};
-
 pub const FileExt = enum {
     c,
     cpp,
diff --git a/src/Module.zig b/src/Module.zig
index 0bdeab68d0..e612f8f759 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -35,8 +35,7 @@ zig_cache_artifact_directory: Compilation.Directory,
 /// Pointer to externally managed resource. `null` if there is no zig file being compiled.
 root_pkg: *Package,
 /// Module owns this resource.
-/// The `Scope` is either a `Scope.ZIRModule` or `Scope.File`.
-root_scope: *Scope,
+root_scope: *Scope.File,
 /// It's rare for a decl to be exported, so we save memory by having a sparse map of
 /// Decl pointers to details about them being exported.
 /// The Export memory is owned by the `export_owners` table; the slice itself is owned by this table.
@@ -57,19 +56,19 @@ decl_table: std.ArrayHashMapUnmanaged(Scope.NameHash, *Decl, Scope.name_hash_has
 /// The ErrorMsg memory is owned by the decl, using Module's general purpose allocator.
 /// Note that a Decl can succeed but the Fn it represents can fail. In this case,
 /// a Decl can have a failed_decls entry but have analysis status of success.
-failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
+failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *ErrorMsg) = .{},
 /// When emit_h is non-null, each Decl gets one more compile error slot for
 /// emit-h failing for that Decl. This table is also how we tell if a Decl has
 /// failed emit-h or succeeded.
-emit_h_failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *Compilation.ErrorMsg) = .{},
-/// A Decl can have multiple compileLogs, but only one error, so we map a Decl to a the src locs of all the compileLogs
-compile_log_decls: std.AutoArrayHashMapUnmanaged(*Decl, ArrayListUnmanaged(usize)) = .{},
+emit_h_failed_decls: std.AutoArrayHashMapUnmanaged(*Decl, *ErrorMsg) = .{},
+/// Keep track of one `@compileLog` callsite per owner Decl.
+compile_log_decls: std.AutoArrayHashMapUnmanaged(*Decl, SrcLoc) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Scope`, using Module's general purpose allocator.
-failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *Compilation.ErrorMsg) = .{},
+failed_files: std.AutoArrayHashMapUnmanaged(*Scope, *ErrorMsg) = .{},
 /// Using a map here for consistency with the other fields here.
 /// The ErrorMsg memory is owned by the `Export`, using Module's general purpose allocator.
-failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *Compilation.ErrorMsg) = .{},
+failed_exports: std.AutoArrayHashMapUnmanaged(*Export, *ErrorMsg) = .{},
 
 next_anon_name_index: usize = 0,
 
@@ -103,6 +102,8 @@ stage1_flags: packed struct {
 
 emit_h: ?Compilation.EmitLoc,
 
+compile_log_text: std.ArrayListUnmanaged(u8) = .{},
+
 pub const Export = struct {
     options: std.builtin.ExportOptions,
     /// Byte offset into the file that contains the export directive.
@@ -138,9 +139,9 @@ pub const Decl = struct {
     /// mapping them to an address in the output file.
     /// Memory owned by this decl, using Module's allocator.
     name: [*:0]const u8,
-    /// The direct parent container of the Decl. This is either a `Scope.Container` or `Scope.ZIRModule`.
+    /// The direct parent container of the Decl.
     /// Reference to externally owned memory.
-    scope: *Scope,
+    container: *Scope.Container,
     /// The AST Node decl index or ZIR Inst index that contains this declaration.
     /// Must be recomputed when the corresponding source file is modified.
     src_index: usize,
@@ -235,31 +236,21 @@ pub const Decl = struct {
         }
     }
 
+    pub fn srcLoc(self: Decl) SrcLoc {
+        return .{
+            .byte_offset = self.src(),
+            .file_scope = self.getFileScope(),
+        };
+    }
+
     pub fn src(self: Decl) usize {
-        switch (self.scope.tag) {
-            .container => {
-                const container = @fieldParentPtr(Scope.Container, "base", self.scope);
-                const tree = container.file_scope.contents.tree;
-                // TODO Container should have its own decls()
-                const decl_node = tree.root_node.decls()[self.src_index];
-                return tree.token_locs[decl_node.firstToken()].start;
-            },
-            .zir_module => {
-                const zir_module = @fieldParentPtr(Scope.ZIRModule, "base", self.scope);
-                const module = zir_module.contents.module;
-                const src_decl = module.decls[self.src_index];
-                return src_decl.inst.src;
-            },
-            .file, .block => unreachable,
-            .gen_zir => unreachable,
-            .local_val => unreachable,
-            .local_ptr => unreachable,
-            .decl => unreachable,
-        }
+        const tree = self.container.file_scope.contents.tree;
+        const decl_node = tree.root_node.decls()[self.src_index];
+        return tree.token_locs[decl_node.firstToken()].start;
     }
 
     pub fn fullyQualifiedNameHash(self: Decl) Scope.NameHash {
-        return self.scope.fullyQualifiedNameHash(mem.spanZ(self.name));
+        return self.container.fullyQualifiedNameHash(mem.spanZ(self.name));
     }
 
     pub fn typedValue(self: *Decl) error{AnalysisFail}!TypedValue {
@@ -293,9 +284,8 @@ pub const Decl = struct {
         }
     }
 
-    /// Asserts that the `Decl` is part of AST and not ZIRModule.
-    pub fn getFileScope(self: *Decl) *Scope.File {
-        return self.scope.cast(Scope.Container).?.file_scope;
+    pub fn getFileScope(self: Decl) *Scope.File {
+        return self.container.file_scope;
     }
 
     pub fn getEmitH(decl: *Decl, module: *Module) *EmitH {
@@ -326,7 +316,7 @@ pub const Fn = struct {
     /// Contains un-analyzed ZIR instructions generated from Zig source AST.
     /// Even after we finish analysis, the ZIR is kept in memory, so that
     /// comptime and inline function calls can happen.
-    zir: zir.Module.Body,
+    zir: zir.Body,
     /// undefined unless analysis state is `success`.
     body: Body,
     state: Analysis,
@@ -373,47 +363,49 @@ pub const Scope = struct {
         return @fieldParentPtr(T, "base", base);
     }
 
-    /// Asserts the scope has a parent which is a DeclAnalysis and
-    /// returns the arena Allocator.
+    /// Returns the arena Allocator associated with the Decl of the Scope.
     pub fn arena(self: *Scope) *Allocator {
         switch (self.tag) {
             .block => return self.cast(Block).?.arena,
-            .decl => return &self.cast(DeclAnalysis).?.arena.allocator,
             .gen_zir => return self.cast(GenZIR).?.arena,
             .local_val => return self.cast(LocalVal).?.gen_zir.arena,
             .local_ptr => return self.cast(LocalPtr).?.gen_zir.arena,
-            .zir_module => return &self.cast(ZIRModule).?.contents.module.arena.allocator,
             .file => unreachable,
             .container => unreachable,
         }
     }
 
-    /// If the scope has a parent which is a `DeclAnalysis`,
-    /// returns the `Decl`, otherwise returns `null`.
-    pub fn decl(self: *Scope) ?*Decl {
+    pub fn ownerDecl(self: *Scope) ?*Decl {
+        return switch (self.tag) {
+            .block => self.cast(Block).?.owner_decl,
+            .gen_zir => self.cast(GenZIR).?.decl,
+            .local_val => self.cast(LocalVal).?.gen_zir.decl,
+            .local_ptr => self.cast(LocalPtr).?.gen_zir.decl,
+            .file => null,
+            .container => null,
+        };
+    }
+
+    pub fn srcDecl(self: *Scope) ?*Decl {
         return switch (self.tag) {
-            .block => self.cast(Block).?.decl,
+            .block => self.cast(Block).?.src_decl,
             .gen_zir => self.cast(GenZIR).?.decl,
             .local_val => self.cast(LocalVal).?.gen_zir.decl,
             .local_ptr => self.cast(LocalPtr).?.gen_zir.decl,
-            .decl => self.cast(DeclAnalysis).?.decl,
-            .zir_module => null,
             .file => null,
             .container => null,
         };
     }
 
-    /// Asserts the scope has a parent which is a ZIRModule or Container and
-    /// returns it.
-    pub fn namespace(self: *Scope) *Scope {
+    /// Asserts the scope has a parent which is a Container and returns it.
+    pub fn namespace(self: *Scope) *Container {
         switch (self.tag) {
-            .block => return self.cast(Block).?.decl.scope,
-            .gen_zir => return self.cast(GenZIR).?.decl.scope,
-            .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope,
-            .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope,
-            .decl => return self.cast(DeclAnalysis).?.decl.scope,
-            .file => return &self.cast(File).?.root_container.base,
-            .zir_module, .container => return self,
+            .block => return self.cast(Block).?.owner_decl.container,
+            .gen_zir => return self.cast(GenZIR).?.decl.container,
+            .local_val => return self.cast(LocalVal).?.gen_zir.decl.container,
+            .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.container,
+            .file => return &self.cast(File).?.root_container,
+            .container => return self.cast(Container).?,
         }
     }
 
@@ -426,9 +418,7 @@ pub const Scope = struct {
             .gen_zir => unreachable,
             .local_val => unreachable,
             .local_ptr => unreachable,
-            .decl => unreachable,
             .file => unreachable,
-            .zir_module => return self.cast(ZIRModule).?.fullyQualifiedNameHash(name),
             .container => return self.cast(Container).?.fullyQualifiedNameHash(name),
         }
     }
@@ -437,12 +427,10 @@ pub const Scope = struct {
     pub fn tree(self: *Scope) *ast.Tree {
         switch (self.tag) {
             .file => return self.cast(File).?.contents.tree,
-            .zir_module => unreachable,
-            .decl => return self.cast(DeclAnalysis).?.decl.scope.cast(Container).?.file_scope.contents.tree,
-            .block => return self.cast(Block).?.decl.scope.cast(Container).?.file_scope.contents.tree,
-            .gen_zir => return self.cast(GenZIR).?.decl.scope.cast(Container).?.file_scope.contents.tree,
-            .local_val => return self.cast(LocalVal).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree,
-            .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.scope.cast(Container).?.file_scope.contents.tree,
+            .block => return self.cast(Block).?.src_decl.container.file_scope.contents.tree,
+            .gen_zir => return self.cast(GenZIR).?.decl.container.file_scope.contents.tree,
+            .local_val => return self.cast(LocalVal).?.gen_zir.decl.container.file_scope.contents.tree,
+            .local_ptr => return self.cast(LocalPtr).?.gen_zir.decl.container.file_scope.contents.tree,
             .container => return self.cast(Container).?.file_scope.contents.tree,
         }
     }
@@ -454,38 +442,21 @@ pub const Scope = struct {
             .gen_zir => self.cast(GenZIR).?,
             .local_val => return self.cast(LocalVal).?.gen_zir,
             .local_ptr => return self.cast(LocalPtr).?.gen_zir,
-            .decl => unreachable,
-            .zir_module => unreachable,
             .file => unreachable,
             .container => unreachable,
         };
     }
 
-    /// Asserts the scope has a parent which is a ZIRModule, Container or File and
+    /// Asserts the scope has a parent which is a Container or File and
     /// returns the sub_file_path field.
     pub fn subFilePath(base: *Scope) []const u8 {
         switch (base.tag) {
             .container => return @fieldParentPtr(Container, "base", base).file_scope.sub_file_path,
             .file => return @fieldParentPtr(File, "base", base).sub_file_path,
-            .zir_module => return @fieldParentPtr(ZIRModule, "base", base).sub_file_path,
             .block => unreachable,
             .gen_zir => unreachable,
             .local_val => unreachable,
             .local_ptr => unreachable,
-            .decl => unreachable,
-        }
-    }
-
-    pub fn unload(base: *Scope, gpa: *Allocator) void {
-        switch (base.tag) {
-            .file => return @fieldParentPtr(File, "base", base).unload(gpa),
-            .zir_module => return @fieldParentPtr(ZIRModule, "base", base).unload(gpa),
-            .block => unreachable,
-            .gen_zir => unreachable,
-            .local_val => unreachable,
-            .local_ptr => unreachable,
-            .decl => unreachable,
-            .container => unreachable,
         }
     }
 
@@ -493,67 +464,28 @@ pub const Scope = struct {
         switch (base.tag) {
             .container => return @fieldParentPtr(Container, "base", base).file_scope.getSource(module),
             .file => return @fieldParentPtr(File, "base", base).getSource(module),
-            .zir_module => return @fieldParentPtr(ZIRModule, "base", base).getSource(module),
             .gen_zir => unreachable,
             .local_val => unreachable,
             .local_ptr => unreachable,
             .block => unreachable,
-            .decl => unreachable,
         }
     }
 
+    /// When called from inside a Block Scope, chases the src_decl, not the owner_decl.
     pub fn getFileScope(base: *Scope) *Scope.File {
         var cur = base;
         while (true) {
             cur = switch (cur.tag) {
                 .container => return @fieldParentPtr(Container, "base", cur).file_scope,
                 .file => return @fieldParentPtr(File, "base", cur),
-                .zir_module => unreachable, // TODO are zir modules allowed to import packages?
                 .gen_zir => @fieldParentPtr(GenZIR, "base", cur).parent,
                 .local_val => @fieldParentPtr(LocalVal, "base", cur).parent,
                 .local_ptr => @fieldParentPtr(LocalPtr, "base", cur).parent,
-                .block => @fieldParentPtr(Block, "base", cur).decl.scope,
-                .decl => @fieldParentPtr(DeclAnalysis, "base", cur).decl.scope,
+                .block => return @fieldParentPtr(Block, "base", cur).src_decl.container.file_scope,
             };
         }
     }
 
-    /// Asserts the scope is a namespace Scope and removes the Decl from the namespace.
-    pub fn removeDecl(base: *Scope, child: *Decl) void {
-        switch (base.tag) {
-            .container => return @fieldParentPtr(Container, "base", base).removeDecl(child),
-            .zir_module => return @fieldParentPtr(ZIRModule, "base", base).removeDecl(child),
-            .file => unreachable,
-            .block => unreachable,
-            .gen_zir => unreachable,
-            .local_val => unreachable,
-            .local_ptr => unreachable,
-            .decl => unreachable,
-        }
-    }
-
-    /// Asserts the scope is a File or ZIRModule and deinitializes it, then deallocates it.
-    pub fn destroy(base: *Scope, gpa: *Allocator) void {
-        switch (base.tag) {
-            .file => {
-                const scope_file = @fieldParentPtr(File, "base", base);
-                scope_file.deinit(gpa);
-                gpa.destroy(scope_file);
-            },
-            .zir_module => {
-                const scope_zir_module = @fieldParentPtr(ZIRModule, "base", base);
-                scope_zir_module.deinit(gpa);
-                gpa.destroy(scope_zir_module);
-            },
-            .block => unreachable,
-            .gen_zir => unreachable,
-            .local_val => unreachable,
-            .local_ptr => unreachable,
-            .decl => unreachable,
-            .container => unreachable,
-        }
-    }
-
     fn name_hash_hash(x: NameHash) u32 {
         return @truncate(u32, @bitCast(u128, x));
     }
@@ -563,14 +495,11 @@ pub const Scope = struct {
     }
 
     pub const Tag = enum {
-        /// .zir source code.
-        zir_module,
         /// .zig source code.
         file,
         /// struct, enum or union, every .file contains one of these.
         container,
         block,
-        decl,
         gen_zir,
         local_val,
         local_ptr,
@@ -657,6 +586,11 @@ pub const Scope = struct {
             self.* = undefined;
         }
 
+        pub fn destroy(self: *File, gpa: *Allocator) void {
+            self.deinit(gpa);
+            gpa.destroy(self);
+        }
+
         pub fn dumpSrc(self: *File, src: usize) void {
             const loc = std.zig.findLineColumn(self.source.bytes, src);
             std.debug.print("{s}:{d}:{d}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
@@ -681,109 +615,6 @@ pub const Scope = struct {
         }
     };
 
-    pub const ZIRModule = struct {
-        pub const base_tag: Tag = .zir_module;
-        base: Scope = Scope{ .tag = base_tag },
-        /// Relative to the owning package's root_src_dir.
-        /// Reference to external memory, not owned by ZIRModule.
-        sub_file_path: []const u8,
-        source: union(enum) {
-            unloaded: void,
-            bytes: [:0]const u8,
-        },
-        contents: union {
-            not_available: void,
-            module: *zir.Module,
-        },
-        status: enum {
-            never_loaded,
-            unloaded_success,
-            unloaded_parse_failure,
-            unloaded_sema_failure,
-
-            loaded_sema_failure,
-            loaded_success,
-        },
-
-        /// Even though .zir files only have 1 module, this set is still needed
-        /// because of anonymous Decls, which can exist in the global set, but
-        /// not this one.
-        decls: ArrayListUnmanaged(*Decl),
-
-        pub fn unload(self: *ZIRModule, gpa: *Allocator) void {
-            switch (self.status) {
-                .never_loaded,
-                .unloaded_parse_failure,
-                .unloaded_sema_failure,
-                .unloaded_success,
-                => {},
-
-                .loaded_success => {
-                    self.contents.module.deinit(gpa);
-                    gpa.destroy(self.contents.module);
-                    self.contents = .{ .not_available = {} };
-                    self.status = .unloaded_success;
-                },
-                .loaded_sema_failure => {
-                    self.contents.module.deinit(gpa);
-                    gpa.destroy(self.contents.module);
-                    self.contents = .{ .not_available = {} };
-                    self.status = .unloaded_sema_failure;
-                },
-            }
-            switch (self.source) {
-                .bytes => |bytes| {
-                    gpa.free(bytes);
-                    self.source = .{ .unloaded = {} };
-                },
-                .unloaded => {},
-            }
-        }
-
-        pub fn deinit(self: *ZIRModule, gpa: *Allocator) void {
-            self.decls.deinit(gpa);
-            self.unload(gpa);
-            self.* = undefined;
-        }
-
-        pub fn removeDecl(self: *ZIRModule, child: *Decl) void {
-            for (self.decls.items) |item, i| {
-                if (item == child) {
-                    _ = self.decls.swapRemove(i);
-                    return;
-                }
-            }
-        }
-
-        pub fn dumpSrc(self: *ZIRModule, src: usize) void {
-            const loc = std.zig.findLineColumn(self.source.bytes, src);
-            std.debug.print("{s}:{d}:{d}\n", .{ self.sub_file_path, loc.line + 1, loc.column + 1 });
-        }
-
-        pub fn getSource(self: *ZIRModule, module: *Module) ![:0]const u8 {
-            switch (self.source) {
-                .unloaded => {
-                    const source = try module.root_pkg.root_src_directory.handle.readFileAllocOptions(
-                        module.gpa,
-                        self.sub_file_path,
-                        std.math.maxInt(u32),
-                        null,
-                        1,
-                        0,
-                    );
-                    self.source = .{ .bytes = source };
-                    return source;
-                },
-                .bytes => |bytes| return bytes,
-            }
-        }
-
-        pub fn fullyQualifiedNameHash(self: *ZIRModule, name: []const u8) NameHash {
-            // ZIR modules only have 1 file with all decls global in the same namespace.
-            return std.zig.hashSrc(name);
-        }
-    };
-
     /// This is a temporary structure, references to it are valid only
     /// during semantic analysis of the block.
     pub const Block = struct {
@@ -794,9 +625,14 @@ pub const Scope = struct {
         /// Maps ZIR to TZIR. Shared to sub-blocks.
         inst_table: *InstTable,
         func: ?*Fn,
-        decl: *Decl,
+        /// When analyzing an inline function call, owner_decl is the Decl of the caller
+        /// and src_decl is the Decl of the callee.
+        /// This Decl owns the arena memory of this Block.
+        owner_decl: *Decl,
+        /// This Decl is the Decl according to the Zig source code corresponding to this Block.
+        src_decl: *Decl,
         instructions: ArrayListUnmanaged(*Inst),
-        /// Points to the arena allocator of DeclAnalysis
+        /// Points to the arena allocator of the Decl.
         arena: *Allocator,
         label: ?Label = null,
         inlining: ?*Inlining,
@@ -845,21 +681,12 @@ pub const Scope = struct {
         }
     };
 
-    /// This is a temporary structure, references to it are valid only
-    /// during semantic analysis of the decl.
-    pub const DeclAnalysis = struct {
-        pub const base_tag: Tag = .decl;
-        base: Scope = Scope{ .tag = base_tag },
-        decl: *Decl,
-        arena: std.heap.ArenaAllocator,
-    };
-
     /// This is a temporary structure, references to it are valid only
     /// during semantic analysis of the decl.
     pub const GenZIR = struct {
         pub const base_tag: Tag = .gen_zir;
         base: Scope = Scope{ .tag = base_tag },
-        /// Parents can be: `GenZIR`, `ZIRModule`, `File`
+        /// Parents can be: `GenZIR`, `File`
         parent: *Scope,
         decl: *Decl,
         arena: *Allocator,
@@ -905,11 +732,73 @@ pub const Scope = struct {
     };
 };
 
+/// This struct holds data necessary to construct API-facing `AllErrors.Message`.
+/// Its memory is managed with the general purpose allocator so that they
+/// can be created and destroyed in response to incremental updates.
+/// In some cases, the Scope.File could have been inferred from where the ErrorMsg
+/// is stored. For example, if it is stored in Module.failed_decls, then the Scope.File
+/// would be determined by the Decl Scope. However, the data structure contains the field
+/// anyway so that `ErrorMsg` can be reused for error notes, which may be in a different
+/// file than the parent error message. It also simplifies processing of error messages.
+pub const ErrorMsg = struct {
+    src_loc: SrcLoc,
+    msg: []const u8,
+    notes: []ErrorMsg = &.{},
+
+    pub fn create(
+        gpa: *Allocator,
+        src_loc: SrcLoc,
+        comptime format: []const u8,
+        args: anytype,
+    ) !*ErrorMsg {
+        const self = try gpa.create(ErrorMsg);
+        errdefer gpa.destroy(self);
+        self.* = try init(gpa, src_loc, format, args);
+        return self;
+    }
+
+    /// Assumes the ErrorMsg struct and msg were both allocated with `gpa`,
+    /// as well as all notes.
+    pub fn destroy(self: *ErrorMsg, gpa: *Allocator) void {
+        self.deinit(gpa);
+        gpa.destroy(self);
+    }
+
+    pub fn init(
+        gpa: *Allocator,
+        src_loc: SrcLoc,
+        comptime format: []const u8,
+        args: anytype,
+    ) !ErrorMsg {
+        return ErrorMsg{
+            .src_loc = src_loc,
+            .msg = try std.fmt.allocPrint(gpa, format, args),
+        };
+    }
+
+    pub fn deinit(self: *ErrorMsg, gpa: *Allocator) void {
+        for (self.notes) |*note| {
+            note.deinit(gpa);
+        }
+        gpa.free(self.notes);
+        gpa.free(self.msg);
+        self.* = undefined;
+    }
+};
+
+/// Canonical reference to a position within a source file.
+pub const SrcLoc = struct {
+    file_scope: *Scope.File,
+    byte_offset: usize,
+};
+
 pub const InnerError = error{ OutOfMemory, AnalysisFail };
 
 pub fn deinit(self: *Module) void {
     const gpa = self.gpa;
 
+    self.compile_log_text.deinit(gpa);
+
     self.zig_cache_artifact_directory.handle.close();
 
     self.deletion_set.deinit(gpa);
@@ -939,9 +828,6 @@ pub fn deinit(self: *Module) void {
     }
     self.failed_exports.deinit(gpa);
 
-    for (self.compile_log_decls.items()) |*entry| {
-        entry.value.deinit(gpa);
-    }
     self.compile_log_decls.deinit(gpa);
 
     for (self.decl_exports.items()) |entry| {
@@ -965,7 +851,7 @@ pub fn deinit(self: *Module) void {
     self.global_error_set.deinit(gpa);
 
     for (self.import_table.items()) |entry| {
-        entry.value.base.destroy(gpa);
+        entry.value.destroy(gpa);
     }
     self.import_table.deinit(gpa);
 }
@@ -978,7 +864,7 @@ fn freeExportList(gpa: *Allocator, export_list: []*Export) void {
     gpa.free(export_list);
 }
 
-pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
+pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -999,7 +885,7 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
 
             // The exports this Decl performs will be re-discovered, so we remove them here
             // prior to re-analysis.
-            self.deleteDeclExports(decl);
+            mod.deleteDeclExports(decl);
             // Dependencies will be re-discovered, so we remove them here prior to re-analysis.
             for (decl.dependencies.items()) |entry| {
                 const dep = entry.key;
@@ -1008,7 +894,7 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
                     // We don't perform a deletion here, because this Decl or another one
                     // may end up referencing it before the update is complete.
                     dep.deletion_flag = true;
-                    try self.deletion_set.append(self.gpa, dep);
+                    try mod.deletion_set.append(mod.gpa, dep);
                 }
             }
             decl.dependencies.clearRetainingCapacity();
@@ -1019,24 +905,21 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
         .unreferenced => false,
     };
 
-    const type_changed = if (self.root_scope.cast(Scope.ZIRModule)) |zir_module|
-        try zir_sema.analyzeZirDecl(self, decl, zir_module.contents.module.decls[decl.src_index])
-    else
-        self.astGenAndAnalyzeDecl(decl) catch |err| switch (err) {
-            error.OutOfMemory => return error.OutOfMemory,
-            error.AnalysisFail => return error.AnalysisFail,
-            else => {
-                try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
-                self.failed_decls.putAssumeCapacityNoClobber(decl, try Compilation.ErrorMsg.create(
-                    self.gpa,
-                    decl.src(),
-                    "unable to analyze: {s}",
-                    .{@errorName(err)},
-                ));
-                decl.analysis = .sema_failure_retryable;
-                return error.AnalysisFail;
-            },
-        };
+    const type_changed = mod.astGenAndAnalyzeDecl(decl) catch |err| switch (err) {
+        error.OutOfMemory => return error.OutOfMemory,
+        error.AnalysisFail => return error.AnalysisFail,
+        else => {
+            decl.analysis = .sema_failure_retryable;
+            try mod.failed_decls.ensureCapacity(mod.gpa, mod.failed_decls.items().len + 1);
+            mod.failed_decls.putAssumeCapacityNoClobber(decl, try ErrorMsg.create(
+                mod.gpa,
+                decl.srcLoc(),
+                "unable to analyze: {s}",
+                .{@errorName(err)},
+            ));
+            return error.AnalysisFail;
+        },
+    };
 
     if (subsequent_analysis) {
         // We may need to chase the dependants and re-analyze them.
@@ -1055,8 +938,8 @@ pub fn ensureDeclAnalyzed(self: *Module, decl: *Decl) InnerError!void {
                     .codegen_failure,
                     .codegen_failure_retryable,
                     .complete,
-                    => if (dep.generation != self.generation) {
-                        try self.markOutdatedDecl(dep);
+                    => if (dep.generation != mod.generation) {
+                        try mod.markOutdatedDecl(dep);
                     },
                 }
             }
@@ -1068,8 +951,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
     const tracy = trace(@src());
     defer tracy.end();
 
-    const container_scope = decl.scope.cast(Scope.Container).?;
-    const tree = try self.getAstTree(container_scope.file_scope);
+    const tree = try self.getAstTree(decl.container.file_scope);
     const ast_node = tree.root_node.decls()[decl.src_index];
     switch (ast_node.tag) {
         .FnProto => {
@@ -1085,7 +967,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             var fn_type_scope: Scope.GenZIR = .{
                 .decl = decl,
                 .arena = &fn_type_scope_arena.allocator,
-                .parent = decl.scope,
+                .parent = &decl.container.base,
             };
             defer fn_type_scope.instructions.deinit(self.gpa);
 
@@ -1197,7 +1079,8 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .parent = null,
                 .inst_table = &inst_table,
                 .func = null,
-                .decl = decl,
+                .owner_decl = decl,
+                .src_decl = decl,
                 .instructions = .{},
                 .arena = &decl_arena.allocator,
                 .inlining = null,
@@ -1242,12 +1125,12 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             const new_func = try decl_arena.allocator.create(Fn);
             const fn_payload = try decl_arena.allocator.create(Value.Payload.Function);
 
-            const fn_zir: zir.Module.Body = blk: {
+            const fn_zir: zir.Body = blk: {
                 // We put the ZIR inside the Decl arena.
                 var gen_scope: Scope.GenZIR = .{
                     .decl = decl,
                     .arena = &decl_arena.allocator,
-                    .parent = decl.scope,
+                    .parent = &decl.container.base,
                 };
                 defer gen_scope.instructions.deinit(self.gpa);
 
@@ -1400,7 +1283,8 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .parent = null,
                 .inst_table = &decl_inst_table,
                 .func = null,
-                .decl = decl,
+                .owner_decl = decl,
+                .src_decl = decl,
                 .instructions = .{},
                 .arena = &decl_arena.allocator,
                 .inlining = null,
@@ -1444,7 +1328,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 var gen_scope: Scope.GenZIR = .{
                     .decl = decl,
                     .arena = &gen_scope_arena.allocator,
-                    .parent = decl.scope,
+                    .parent = &decl.container.base,
                 };
                 defer gen_scope.instructions.deinit(self.gpa);
 
@@ -1472,7 +1356,8 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                     .parent = null,
                     .inst_table = &var_inst_table,
                     .func = null,
-                    .decl = decl,
+                    .owner_decl = decl,
+                    .src_decl = decl,
                     .instructions = .{},
                     .arena = &gen_scope_arena.allocator,
                     .inlining = null,
@@ -1503,7 +1388,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 var type_scope: Scope.GenZIR = .{
                     .decl = decl,
                     .arena = &type_scope_arena.allocator,
-                    .parent = decl.scope,
+                    .parent = &decl.container.base,
                 };
                 defer type_scope.instructions.deinit(self.gpa);
 
@@ -1584,7 +1469,7 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
             var gen_scope: Scope.GenZIR = .{
                 .decl = decl,
                 .arena = &analysis_arena.allocator,
-                .parent = decl.scope,
+                .parent = &decl.container.base,
             };
             defer gen_scope.instructions.deinit(self.gpa);
 
@@ -1602,7 +1487,8 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .parent = null,
                 .inst_table = &inst_table,
                 .func = null,
-                .decl = decl,
+                .owner_decl = decl,
+                .src_decl = decl,
                 .instructions = .{},
                 .arena = &analysis_arena.allocator,
                 .inlining = null,
@@ -1632,44 +1518,6 @@ fn declareDeclDependency(self: *Module, depender: *Decl, dependee: *Decl) !void
     dependee.dependants.putAssumeCapacity(depender, {});
 }
 
-fn getSrcModule(self: *Module, root_scope: *Scope.ZIRModule) !*zir.Module {
-    switch (root_scope.status) {
-        .never_loaded, .unloaded_success => {
-            try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1);
-
-            const source = try root_scope.getSource(self);
-
-            var keep_zir_module = false;
-            const zir_module = try self.gpa.create(zir.Module);
-            defer if (!keep_zir_module) self.gpa.destroy(zir_module);
-
-            zir_module.* = try zir.parse(self.gpa, source);
-            defer if (!keep_zir_module) zir_module.deinit(self.gpa);
-
-            if (zir_module.error_msg) |src_err_msg| {
-                self.failed_files.putAssumeCapacityNoClobber(
-                    &root_scope.base,
-                    try Compilation.ErrorMsg.create(self.gpa, src_err_msg.byte_offset, "{s}", .{src_err_msg.msg}),
-                );
-                root_scope.status = .unloaded_parse_failure;
-                return error.AnalysisFail;
-            }
-
-            root_scope.status = .loaded_success;
-            root_scope.contents = .{ .module = zir_module };
-            keep_zir_module = true;
-
-            return zir_module;
-        },
-
-        .unloaded_parse_failure,
-        .unloaded_sema_failure,
-        => return error.AnalysisFail,
-
-        .loaded_success, .loaded_sema_failure => return root_scope.contents.module,
-    }
-}
-
 pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
     const tracy = trace(@src());
     defer tracy.end();
@@ -1691,10 +1539,13 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
                 defer msg.deinit();
 
                 try parse_err.render(tree.token_ids, msg.writer());
-                const err_msg = try self.gpa.create(Compilation.ErrorMsg);
+                const err_msg = try self.gpa.create(ErrorMsg);
                 err_msg.* = .{
+                    .src_loc = .{
+                        .file_scope = root_scope,
+                        .byte_offset = tree.token_locs[parse_err.loc()].start,
+                    },
                     .msg = msg.toOwnedSlice(),
-                    .byte_offset = tree.token_locs[parse_err.loc()].start,
                 };
 
                 self.failed_files.putAssumeCapacityNoClobber(&root_scope.base, err_msg);
@@ -1753,9 +1604,12 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                 decl.src_index = decl_i;
                 if (deleted_decls.swapRemove(decl) == null) {
                     decl.analysis = .sema_failure;
-                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, tree.token_locs[name_tok].start, "redefinition of '{s}'", .{decl.name});
-                    errdefer err_msg.destroy(self.gpa);
-                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
+                    const msg = try ErrorMsg.create(self.gpa, .{
+                        .file_scope = container_scope.file_scope,
+                        .byte_offset = tree.token_locs[name_tok].start,
+                    }, "redefinition of '{s}'", .{decl.name});
+                    errdefer msg.destroy(self.gpa);
+                    try self.failed_decls.putNoClobber(self.gpa, decl, msg);
                 } else {
                     if (!srcHashEql(decl.contents_hash, contents_hash)) {
                         try self.markOutdatedDecl(decl);
@@ -1795,7 +1649,10 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
                 decl.src_index = decl_i;
                 if (deleted_decls.swapRemove(decl) == null) {
                     decl.analysis = .sema_failure;
-                    const err_msg = try Compilation.ErrorMsg.create(self.gpa, name_loc.start, "redefinition of '{s}'", .{decl.name});
+                    const err_msg = try ErrorMsg.create(self.gpa, .{
+                        .file_scope = container_scope.file_scope,
+                        .byte_offset = name_loc.start,
+                    }, "redefinition of '{s}'", .{decl.name});
                     errdefer err_msg.destroy(self.gpa);
                     try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
                 } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
@@ -1840,65 +1697,12 @@ pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void
     }
 }
 
-pub fn analyzeRootZIRModule(self: *Module, root_scope: *Scope.ZIRModule) !void {
-    // We may be analyzing it for the first time, or this may be
-    // an incremental update. This code handles both cases.
-    const src_module = try self.getSrcModule(root_scope);
-
-    try self.comp.work_queue.ensureUnusedCapacity(src_module.decls.len);
-    try root_scope.decls.ensureCapacity(self.gpa, src_module.decls.len);
-
-    var exports_to_resolve = std.ArrayList(*zir.Decl).init(self.gpa);
-    defer exports_to_resolve.deinit();
-
-    // Keep track of the decls that we expect to see in this file so that
-    // we know which ones have been deleted.
-    var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa);
-    defer deleted_decls.deinit();
-    try deleted_decls.ensureCapacity(self.decl_table.items().len);
-    for (self.decl_table.items()) |entry| {
-        deleted_decls.putAssumeCapacityNoClobber(entry.value, {});
-    }
-
-    for (src_module.decls) |src_decl, decl_i| {
-        const name_hash = root_scope.fullyQualifiedNameHash(src_decl.name);
-        if (self.decl_table.get(name_hash)) |decl| {
-            deleted_decls.removeAssertDiscard(decl);
-            if (!srcHashEql(src_decl.contents_hash, decl.contents_hash)) {
-                try self.markOutdatedDecl(decl);
-                decl.contents_hash = src_decl.contents_hash;
-            }
-        } else {
-            const new_decl = try self.createNewDecl(
-                &root_scope.base,
-                src_decl.name,
-                decl_i,
-                name_hash,
-                src_decl.contents_hash,
-            );
-            root_scope.decls.appendAssumeCapacity(new_decl);
-            if (src_decl.inst.cast(zir.Inst.Export)) |export_inst| {
-                try exports_to_resolve.append(src_decl);
-            }
-        }
-    }
-    for (exports_to_resolve.items) |export_decl| {
-        _ = try zir_sema.resolveZirDecl(self, &root_scope.base, export_decl);
-    }
-    // Handle explicitly deleted decls from the source code. Not to be confused
-    // with when we delete decls because they are no longer referenced.
-    for (deleted_decls.items()) |entry| {
-        log.debug("noticed '{s}' deleted from source\n", .{entry.key.name});
-        try self.deleteDecl(entry.key);
-    }
-}
-
 pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     try self.deletion_set.ensureCapacity(self.gpa, self.deletion_set.items.len + decl.dependencies.items().len);
 
     // Remove from the namespace it resides in. In the case of an anonymous Decl it will
     // not be present in the set, and this does nothing.
-    decl.scope.removeDecl(decl);
+    decl.container.removeDecl(decl);
 
     log.debug("deleting decl '{s}'\n", .{decl.name});
     const name_hash = decl.fullyQualifiedNameHash();
@@ -1929,9 +1733,7 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     if (self.emit_h_failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.compile_log_decls.swapRemove(decl)) |*entry| {
-        entry.value.deinit(self.gpa);
-    }
+    _ = self.compile_log_decls.swapRemove(decl);
     self.deleteDeclExports(decl);
     self.comp.bin_file.freeDecl(decl);
 
@@ -1993,7 +1795,8 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
         .parent = null,
         .inst_table = &inst_table,
         .func = func,
-        .decl = decl,
+        .owner_decl = decl,
+        .src_decl = decl,
         .instructions = .{},
         .arena = &arena.allocator,
         .inlining = null,
@@ -2022,9 +1825,7 @@ fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
     if (self.emit_h_failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
     }
-    if (self.compile_log_decls.swapRemove(decl)) |*entry| {
-        entry.value.deinit(self.gpa);
-    }
+    _ = self.compile_log_decls.swapRemove(decl);
     decl.analysis = .outdated;
 }
 
@@ -2046,7 +1847,7 @@ fn allocateNewDecl(
 
     new_decl.* = .{
         .name = "",
-        .scope = scope.namespace(),
+        .container = scope.namespace(),
         .src_index = src_index,
         .typed_value = .{ .never_succeeded = {} },
         .analysis = .unreferenced,
@@ -2129,34 +1930,34 @@ pub fn resolveDefinedValue(self: *Module, scope: *Scope, base: *Inst) !?Value {
 }
 
 pub fn analyzeExport(
-    self: *Module,
+    mod: *Module,
     scope: *Scope,
     src: usize,
     borrowed_symbol_name: []const u8,
     exported_decl: *Decl,
 ) !void {
-    try self.ensureDeclAnalyzed(exported_decl);
+    try mod.ensureDeclAnalyzed(exported_decl);
     const typed_value = exported_decl.typed_value.most_recent.typed_value;
     switch (typed_value.ty.zigTypeTag()) {
         .Fn => {},
-        else => return self.fail(scope, src, "unable to export type '{}'", .{typed_value.ty}),
+        else => return mod.fail(scope, src, "unable to export type '{}'", .{typed_value.ty}),
     }
 
-    try self.decl_exports.ensureCapacity(self.gpa, self.decl_exports.items().len + 1);
-    try self.export_owners.ensureCapacity(self.gpa, self.export_owners.items().len + 1);
+    try mod.decl_exports.ensureCapacity(mod.gpa, mod.decl_exports.items().len + 1);
+    try mod.export_owners.ensureCapacity(mod.gpa, mod.export_owners.items().len + 1);
 
-    const new_export = try self.gpa.create(Export);
-    errdefer self.gpa.destroy(new_export);
+    const new_export = try mod.gpa.create(Export);
+    errdefer mod.gpa.destroy(new_export);
 
-    const symbol_name = try self.gpa.dupe(u8, borrowed_symbol_name);
-    errdefer self.gpa.free(symbol_name);
+    const symbol_name = try mod.gpa.dupe(u8, borrowed_symbol_name);
+    errdefer mod.gpa.free(symbol_name);
 
-    const owner_decl = scope.decl().?;
+    const owner_decl = scope.ownerDecl().?;
 
     new_export.* = .{
         .options = .{ .name = symbol_name },
         .src = src,
-        .link = switch (self.comp.bin_file.tag) {
+        .link = switch (mod.comp.bin_file.tag) {
             .coff => .{ .coff = {} },
             .elf => .{ .elf = link.File.Elf.Export{} },
             .macho => .{ .macho = link.File.MachO.Export{} },
@@ -2169,48 +1970,53 @@ pub fn analyzeExport(
     };
 
     // Add to export_owners table.
-    const eo_gop = self.export_owners.getOrPutAssumeCapacity(owner_decl);
+    const eo_gop = mod.export_owners.getOrPutAssumeCapacity(owner_decl);
     if (!eo_gop.found_existing) {
         eo_gop.entry.value = &[0]*Export{};
     }
-    eo_gop.entry.value = try self.gpa.realloc(eo_gop.entry.value, eo_gop.entry.value.len + 1);
+    eo_gop.entry.value = try mod.gpa.realloc(eo_gop.entry.value, eo_gop.entry.value.len + 1);
     eo_gop.entry.value[eo_gop.entry.value.len - 1] = new_export;
-    errdefer eo_gop.entry.value = self.gpa.shrink(eo_gop.entry.value, eo_gop.entry.value.len - 1);
+    errdefer eo_gop.entry.value = mod.gpa.shrink(eo_gop.entry.value, eo_gop.entry.value.len - 1);
 
     // Add to exported_decl table.
-    const de_gop = self.decl_exports.getOrPutAssumeCapacity(exported_decl);
+    const de_gop = mod.decl_exports.getOrPutAssumeCapacity(exported_decl);
     if (!de_gop.found_existing) {
         de_gop.entry.value = &[0]*Export{};
     }
-    de_gop.entry.value = try self.gpa.realloc(de_gop.entry.value, de_gop.entry.value.len + 1);
+    de_gop.entry.value = try mod.gpa.realloc(de_gop.entry.value, de_gop.entry.value.len + 1);
     de_gop.entry.value[de_gop.entry.value.len - 1] = new_export;
-    errdefer de_gop.entry.value = self.gpa.shrink(de_gop.entry.value, de_gop.entry.value.len - 1);
+    errdefer de_gop.entry.value = mod.gpa.shrink(de_gop.entry.value, de_gop.entry.value.len - 1);
 
-    if (self.symbol_exports.get(symbol_name)) |_| {
-        try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1);
-        self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create(
-            self.gpa,
+    if (mod.symbol_exports.get(symbol_name)) |other_export| {
+        new_export.status = .failed_retryable;
+        try mod.failed_exports.ensureCapacity(mod.gpa, mod.failed_exports.items().len + 1);
+        const msg = try mod.errMsg(
+            scope,
             src,
             "exported symbol collision: {s}",
             .{symbol_name},
-        ));
-        // TODO: add a note
+        );
+        errdefer msg.destroy(mod.gpa);
+        try mod.errNote(
+            &other_export.owner_decl.container.base,
+            other_export.src,
+            msg,
+            "other symbol here",
+            .{},
+        );
+        mod.failed_exports.putAssumeCapacityNoClobber(new_export, msg);
         new_export.status = .failed;
         return;
     }
 
-    try self.symbol_exports.putNoClobber(self.gpa, symbol_name, new_export);
-    self.comp.bin_file.updateDeclExports(self, exported_decl, de_gop.entry.value) catch |err| switch (err) {
+    try mod.symbol_exports.putNoClobber(mod.gpa, symbol_name, new_export);
+    mod.comp.bin_file.updateDeclExports(mod, exported_decl, de_gop.entry.value) catch |err| switch (err) {
         error.OutOfMemory => return error.OutOfMemory,
         else => {
-            try self.failed_exports.ensureCapacity(self.gpa, self.failed_exports.items().len + 1);
-            self.failed_exports.putAssumeCapacityNoClobber(new_export, try Compilation.ErrorMsg.create(
-                self.gpa,
-                src,
-                "unable to export: {s}",
-                .{@errorName(err)},
-            ));
             new_export.status = .failed_retryable;
+            try mod.failed_exports.ensureCapacity(mod.gpa, mod.failed_exports.items().len + 1);
+            const msg = try mod.errMsg(scope, src, "unable to export: {s}", .{@errorName(err)});
+            mod.failed_exports.putAssumeCapacityNoClobber(new_export, msg);
         },
     };
 }
@@ -2476,7 +2282,7 @@ pub fn createAnonymousDecl(
     typed_value: TypedValue,
 ) !*Decl {
     const name_index = self.getNextAnonNameIndex();
-    const scope_decl = scope.decl().?;
+    const scope_decl = scope.ownerDecl().?;
     const name = try std.fmt.allocPrint(self.gpa, "{s}__anon_{d}", .{ scope_decl.name, name_index });
     defer self.gpa.free(name);
     const name_hash = scope.namespace().fullyQualifiedNameHash(name);
@@ -2512,7 +2318,7 @@ pub fn createContainerDecl(
     decl_arena: *std.heap.ArenaAllocator,
     typed_value: TypedValue,
 ) !*Decl {
-    const scope_decl = scope.decl().?;
+    const scope_decl = scope.ownerDecl().?;
     const name = try self.getAnonTypeName(scope, base_token);
     defer self.gpa.free(name);
     const name_hash = scope.namespace().fullyQualifiedNameHash(name);
@@ -2558,14 +2364,14 @@ pub fn lookupDeclName(self: *Module, scope: *Scope, ident_name: []const u8) ?*De
 }
 
 pub fn analyzeDeclRef(self: *Module, scope: *Scope, src: usize, decl: *Decl) InnerError!*Inst {
-    const scope_decl = scope.decl().?;
+    const scope_decl = scope.ownerDecl().?;
     try self.declareDeclDependency(scope_decl, decl);
     self.ensureDeclAnalyzed(decl) catch |err| {
         if (scope.cast(Scope.Block)) |block| {
             if (block.func) |func| {
                 func.state = .dependency_failure;
             } else {
-                block.decl.analysis = .dependency_failure;
+                block.owner_decl.analysis = .dependency_failure;
             }
         } else {
             scope_decl.analysis = .dependency_failure;
@@ -3217,10 +3023,51 @@ fn coerceArrayPtrToMany(self: *Module, scope: *Scope, dest_type: Type, inst: *In
     return self.fail(scope, inst.src, "TODO implement coerceArrayPtrToMany runtime instruction", .{});
 }
 
-pub fn fail(self: *Module, scope: *Scope, src: usize, comptime format: []const u8, args: anytype) InnerError {
-    @setCold(true);
-    const err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
-    return self.failWithOwnedErrorMsg(scope, src, err_msg);
+/// We don't return a pointer to the new error note because the pointer
+/// becomes invalid when you add another one.
+pub fn errNote(
+    mod: *Module,
+    scope: *Scope,
+    src: usize,
+    parent: *ErrorMsg,
+    comptime format: []const u8,
+    args: anytype,
+) error{OutOfMemory}!void {
+    const msg = try std.fmt.allocPrint(mod.gpa, format, args);
+    errdefer mod.gpa.free(msg);
+
+    parent.notes = try mod.gpa.realloc(parent.notes, parent.notes.len + 1);
+    parent.notes[parent.notes.len - 1] = .{
+        .src_loc = .{
+            .file_scope = scope.getFileScope(),
+            .byte_offset = src,
+        },
+        .msg = msg,
+    };
+}
+
+pub fn errMsg(
+    mod: *Module,
+    scope: *Scope,
+    src_byte_offset: usize,
+    comptime format: []const u8,
+    args: anytype,
+) error{OutOfMemory}!*ErrorMsg {
+    return ErrorMsg.create(mod.gpa, .{
+        .file_scope = scope.getFileScope(),
+        .byte_offset = src_byte_offset,
+    }, format, args);
+}
+
+pub fn fail(
+    mod: *Module,
+    scope: *Scope,
+    src_byte_offset: usize,
+    comptime format: []const u8,
+    args: anytype,
+) InnerError {
+    const err_msg = try mod.errMsg(scope, src_byte_offset, format, args);
+    return mod.failWithOwnedErrorMsg(scope, err_msg);
 }
 
 pub fn failTok(
@@ -3230,7 +3077,6 @@ pub fn failTok(
     comptime format: []const u8,
     args: anytype,
 ) InnerError {
-    @setCold(true);
     const src = scope.tree().token_locs[token_index].start;
     return self.fail(scope, src, format, args);
 }
@@ -3242,80 +3088,36 @@ pub fn failNode(
     comptime format: []const u8,
     args: anytype,
 ) InnerError {
-    @setCold(true);
     const src = scope.tree().token_locs[ast_node.firstToken()].start;
     return self.fail(scope, src, format, args);
 }
 
-fn addCompileLog(self: *Module, decl: *Decl, src: usize) error{OutOfMemory}!void {
-    const entry = try self.compile_log_decls.getOrPutValue(self.gpa, decl, .{});
-    try entry.value.append(self.gpa, src);
-}
-
-pub fn failCompileLog(
-    self: *Module,
-    scope: *Scope,
-    src: usize,
-) InnerError!void {
-    switch (scope.tag) {
-        .decl => {
-            const decl = scope.cast(Scope.DeclAnalysis).?.decl;
-            try self.addCompileLog(decl, src);
-        },
-        .block => {
-            const block = scope.cast(Scope.Block).?;
-            try self.addCompileLog(block.decl, src);
-        },
-        .gen_zir => {
-            const gen_zir = scope.cast(Scope.GenZIR).?;
-            try self.addCompileLog(gen_zir.decl, src);
-        },
-        .local_val => {
-            const gen_zir = scope.cast(Scope.LocalVal).?.gen_zir;
-            try self.addCompileLog(gen_zir.decl, src);
-        },
-        .local_ptr => {
-            const gen_zir = scope.cast(Scope.LocalPtr).?.gen_zir;
-            try self.addCompileLog(gen_zir.decl, src);
-        },
-        .zir_module,
-        .file,
-        .container,
-        => unreachable,
-    }
-}
-
-fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Compilation.ErrorMsg) InnerError {
+pub fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, err_msg: *ErrorMsg) InnerError {
+    @setCold(true);
     {
         errdefer err_msg.destroy(self.gpa);
         try self.failed_decls.ensureCapacity(self.gpa, self.failed_decls.items().len + 1);
         try self.failed_files.ensureCapacity(self.gpa, self.failed_files.items().len + 1);
     }
     switch (scope.tag) {
-        .decl => {
-            const decl = scope.cast(Scope.DeclAnalysis).?.decl;
-            decl.analysis = .sema_failure;
-            decl.generation = self.generation;
-            self.failed_decls.putAssumeCapacityNoClobber(decl, err_msg);
-        },
         .block => {
             const block = scope.cast(Scope.Block).?;
             if (block.inlining) |inlining| {
                 if (inlining.shared.caller) |func| {
                     func.state = .sema_failure;
                 } else {
-                    block.decl.analysis = .sema_failure;
-                    block.decl.generation = self.generation;
+                    block.owner_decl.analysis = .sema_failure;
+                    block.owner_decl.generation = self.generation;
                 }
             } else {
                 if (block.func) |func| {
                     func.state = .sema_failure;
                 } else {
-                    block.decl.analysis = .sema_failure;
-                    block.decl.generation = self.generation;
+                    block.owner_decl.analysis = .sema_failure;
+                    block.owner_decl.generation = self.generation;
                 }
             }
-            self.failed_decls.putAssumeCapacityNoClobber(block.decl, err_msg);
+            self.failed_decls.putAssumeCapacityNoClobber(block.owner_decl, err_msg);
         },
         .gen_zir => {
             const gen_zir = scope.cast(Scope.GenZIR).?;
@@ -3335,11 +3137,6 @@ fn failWithOwnedErrorMsg(self: *Module, scope: *Scope, src: usize, err_msg: *Com
             gen_zir.decl.generation = self.generation;
             self.failed_decls.putAssumeCapacityNoClobber(gen_zir.decl, err_msg);
         },
-        .zir_module => {
-            const zir_module = scope.cast(Scope.ZIRModule).?;
-            zir_module.status = .loaded_sema_failure;
-            self.failed_files.putAssumeCapacityNoClobber(scope, err_msg);
-        },
         .file => unreachable,
         .container => unreachable,
     }
@@ -3671,7 +3468,8 @@ pub fn addSafetyCheck(mod: *Module, parent_block: *Scope.Block, ok: *Inst, panic
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .inlining = parent_block.inlining,
diff --git a/src/astgen.zig b/src/astgen.zig
index 53503a0467..4631e46b5d 100644
--- a/src/astgen.zig
+++ b/src/astgen.zig
@@ -318,7 +318,7 @@ pub fn comptimeExpr(mod: *Module, parent_scope: *Scope, rl: ResultLoc, node: *as
     // Make a scope to collect generated instructions in the sub-expression.
     var block_scope: Scope.GenZIR = .{
         .parent = parent_scope,
-        .decl = parent_scope.decl().?,
+        .decl = parent_scope.ownerDecl().?,
         .arena = parent_scope.arena(),
         .instructions = .{},
     };
@@ -474,7 +474,7 @@ fn labeledBlockExpr(
 
     var block_scope: Scope.GenZIR = .{
         .parent = parent_scope,
-        .decl = parent_scope.decl().?,
+        .decl = parent_scope.ownerDecl().?,
         .arena = gen_zir.arena,
         .instructions = .{},
         .break_result_loc = rl,
@@ -899,7 +899,7 @@ fn containerDecl(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node.Con
 
     var gen_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1028,7 +1028,13 @@ fn containerDecl(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node.Con
         .ty = Type.initTag(.type),
         .val = val,
     });
-    return rlWrapPtr(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.DeclValInModule, .{ .decl = decl }, .{}));
+    if (rl == .ref) {
+        return addZIRInst(mod, scope, src, zir.Inst.DeclRef, .{ .decl = decl }, .{});
+    } else {
+        return rlWrap(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.DeclVal, .{
+            .decl = decl,
+        }, .{}));
+    }
 }
 
 fn errorSetDecl(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node.ErrorSetDecl) InnerError!*zir.Inst {
@@ -1084,7 +1090,7 @@ fn orelseCatchExpr(
 
     var block_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1266,7 +1272,7 @@ fn boolBinOp(
 
     var block_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1412,7 +1418,7 @@ fn ifExpr(mod: *Module, scope: *Scope, rl: ResultLoc, if_node: *ast.Node.If) Inn
     }
     var block_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1513,7 +1519,7 @@ fn whileExpr(mod: *Module, scope: *Scope, rl: ResultLoc, while_node: *ast.Node.W
 
     var expr_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1649,7 +1655,7 @@ fn forExpr(mod: *Module, scope: *Scope, rl: ResultLoc, for_node: *ast.Node.For)
 
     var for_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1843,7 +1849,7 @@ fn getRangeNode(node: *ast.Node) ?*ast.Node.SimpleInfixOp {
 fn switchExpr(mod: *Module, scope: *Scope, rl: ResultLoc, switch_node: *ast.Node.Switch) InnerError!*zir.Inst {
     var block_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1885,7 +1891,7 @@ fn switchExpr(mod: *Module, scope: *Scope, rl: ResultLoc, switch_node: *ast.Node
 
     var item_scope: Scope.GenZIR = .{
         .parent = scope,
-        .decl = scope.decl().?,
+        .decl = scope.ownerDecl().?,
         .arena = scope.arena(),
         .instructions = .{},
     };
@@ -1922,8 +1928,15 @@ fn switchExpr(mod: *Module, scope: *Scope, rl: ResultLoc, switch_node: *ast.Node
         // Check for else/_ prong, those are handled last.
         if (case.items_len == 1 and case.items()[0].tag == .SwitchElse) {
             if (else_src) |src| {
-                return mod.fail(scope, case_src, "multiple else prongs in switch expression", .{});
-                // TODO notes "previous else prong is here"
+                const msg = try mod.errMsg(
+                    scope,
+                    case_src,
+                    "multiple else prongs in switch expression",
+                    .{},
+                );
+                errdefer msg.destroy(mod.gpa);
+                try mod.errNote(scope, src, msg, "previous else prong is here", .{});
+                return mod.failWithOwnedErrorMsg(scope, msg);
             }
             else_src = case_src;
             special_case = case;
@@ -1932,8 +1945,15 @@ fn switchExpr(mod: *Module, scope: *Scope, rl: ResultLoc, switch_node: *ast.Node
             mem.eql(u8, tree.tokenSlice(case.items()[0].firstToken()), "_"))
         {
             if (underscore_src) |src| {
-                return mod.fail(scope, case_src, "multiple '_' prongs in switch expression", .{});
-                // TODO notes "previous '_' prong is here"
+                const msg = try mod.errMsg(
+                    scope,
+                    case_src,
+                    "multiple '_' prongs in switch expression",
+                    .{},
+                );
+                errdefer msg.destroy(mod.gpa);
+                try mod.errNote(scope, src, msg, "previous '_' prong is here", .{});
+                return mod.failWithOwnedErrorMsg(scope, msg);
             }
             underscore_src = case_src;
             special_case = case;
@@ -1942,9 +1962,16 @@ fn switchExpr(mod: *Module, scope: *Scope, rl: ResultLoc, switch_node: *ast.Node
 
         if (else_src) |some_else| {
             if (underscore_src) |some_underscore| {
-                return mod.fail(scope, switch_src, "else and '_' prong in switch expression", .{});
-                // TODO notes "else prong is here"
-                // TODO notes "'_' prong is here"
+                const msg = try mod.errMsg(
+                    scope,
+                    switch_src,
+                    "else and '_' prong in switch expression",
+                    .{},
+                );
+                errdefer msg.destroy(mod.gpa);
+                try mod.errNote(scope, some_else, msg, "else prong is here", .{});
+                try mod.errNote(scope, some_underscore, msg, "'_' prong is here", .{});
+                return mod.failWithOwnedErrorMsg(scope, msg);
             }
         }
 
@@ -2162,7 +2189,13 @@ fn identifier(mod: *Module, scope: *Scope, rl: ResultLoc, ident: *ast.Node.OneTo
     }
 
     if (mod.lookupDeclName(scope, ident_name)) |decl| {
-        return rlWrapPtr(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.DeclValInModule, .{ .decl = decl }, .{}));
+        if (rl == .ref) {
+            return addZIRInst(mod, scope, src, zir.Inst.DeclRef, .{ .decl = decl }, .{});
+        } else {
+            return rlWrap(mod, scope, rl, try addZIRInst(mod, scope, src, zir.Inst.DeclVal, .{
+                .decl = decl,
+            }, .{}));
+        }
     }
 
     return mod.failNode(scope, &ident.base, "use of undeclared identifier '{s}'", .{ident_name});
@@ -2927,6 +2960,8 @@ fn rlWrapVoid(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node, resul
     return rlWrap(mod, scope, rl, void_inst);
 }
 
+/// TODO go over all the callsites and see where we can introduce "by-value" ZIR instructions
+/// to save ZIR memory. For example, see DeclVal vs DeclRef.
 fn rlWrapPtr(mod: *Module, scope: *Scope, rl: ResultLoc, ptr: *zir.Inst) InnerError!*zir.Inst {
     if (rl == .ref) return ptr;
 
@@ -3032,7 +3067,7 @@ pub fn addZIRInstBlock(
     scope: *Scope,
     src: usize,
     tag: zir.Inst.Tag,
-    body: zir.Module.Body,
+    body: zir.Body,
 ) !*zir.Inst.Block {
     const gen_zir = scope.getGenZIR();
     try gen_zir.instructions.ensureCapacity(mod.gpa, gen_zir.instructions.items.len + 1);
@@ -3070,7 +3105,7 @@ pub fn addZIRInstConst(mod: *Module, scope: *Scope, src: usize, typed_value: Typ
 }
 
 /// TODO The existence of this function is a workaround for a bug in stage1.
-pub fn addZIRInstLoop(mod: *Module, scope: *Scope, src: usize, body: zir.Module.Body) !*zir.Inst.Loop {
+pub fn addZIRInstLoop(mod: *Module, scope: *Scope, src: usize, body: zir.Body) !*zir.Inst.Loop {
     const P = std.meta.fieldInfo(zir.Inst.Loop, .positionals).field_type;
     return addZIRInstSpecial(mod, scope, src, zir.Inst.Loop, P{ .body = body }, .{});
 }
diff --git a/src/codegen.zig b/src/codegen.zig
index 709c91a635..9f2fbaab78 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -9,7 +9,7 @@ const TypedValue = @import("TypedValue.zig");
 const link = @import("link.zig");
 const Module = @import("Module.zig");
 const Compilation = @import("Compilation.zig");
-const ErrorMsg = Compilation.ErrorMsg;
+const ErrorMsg = Module.ErrorMsg;
 const Target = std.Target;
 const Allocator = mem.Allocator;
 const trace = @import("tracy.zig").trace;
@@ -74,7 +74,7 @@ pub const DebugInfoOutput = union(enum) {
 
 pub fn generateSymbol(
     bin_file: *link.File,
-    src: usize,
+    src_loc: Module.SrcLoc,
     typed_value: TypedValue,
     code: *std.ArrayList(u8),
     debug_output: DebugInfoOutput,
@@ -87,56 +87,56 @@ pub fn generateSymbol(
             switch (bin_file.options.target.cpu.arch) {
                 .wasm32 => unreachable, // has its own code path
                 .wasm64 => unreachable, // has its own code path
-                .arm => return Function(.arm).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .armeb => return Function(.armeb).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .aarch64 => return Function(.aarch64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.arc => return Function(.arc).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.avr => return Function(.avr).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.mips => return Function(.mips).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.mips64 => return Function(.mips64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.msp430 => return Function(.msp430).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.r600 => return Function(.r600).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .riscv64 => return Function(.riscv64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.sparc => return Function(.sparc).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.s390x => return Function(.s390x).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .spu_2 => return Function(.spu_2).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.tce => return Function(.tce).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.tcele => return Function(.tcele).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.thumb => return Function(.thumb).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.i386 => return Function(.i386).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.xcore => return Function(.xcore).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.le32 => return Function(.le32).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.le64 => return Function(.le64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.amdil => return Function(.amdil).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.hsail => return Function(.hsail).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.spir => return Function(.spir).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.spir64 => return Function(.spir64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.shave => return Function(.shave).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.lanai => return Function(.lanai).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src, typed_value, code, debug_output),
-                //.ve => return Function(.ve).generateSymbol(bin_file, src, typed_value, code, debug_output),
+                .arm => return Function(.arm).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .armeb => return Function(.armeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .aarch64 => return Function(.aarch64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .aarch64_be => return Function(.aarch64_be).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .aarch64_32 => return Function(.aarch64_32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.arc => return Function(.arc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.avr => return Function(.avr).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.bpfel => return Function(.bpfel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.bpfeb => return Function(.bpfeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.hexagon => return Function(.hexagon).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.mips => return Function(.mips).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.mipsel => return Function(.mipsel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.mips64 => return Function(.mips64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.mips64el => return Function(.mips64el).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.msp430 => return Function(.msp430).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.powerpc => return Function(.powerpc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.powerpc64 => return Function(.powerpc64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.powerpc64le => return Function(.powerpc64le).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.r600 => return Function(.r600).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.amdgcn => return Function(.amdgcn).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.riscv32 => return Function(.riscv32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .riscv64 => return Function(.riscv64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.sparc => return Function(.sparc).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.sparcv9 => return Function(.sparcv9).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.sparcel => return Function(.sparcel).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.s390x => return Function(.s390x).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .spu_2 => return Function(.spu_2).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.tce => return Function(.tce).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.tcele => return Function(.tcele).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.thumb => return Function(.thumb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.thumbeb => return Function(.thumbeb).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.i386 => return Function(.i386).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                .x86_64 => return Function(.x86_64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.xcore => return Function(.xcore).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.nvptx => return Function(.nvptx).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.nvptx64 => return Function(.nvptx64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.le32 => return Function(.le32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.le64 => return Function(.le64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.amdil => return Function(.amdil).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.amdil64 => return Function(.amdil64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.hsail => return Function(.hsail).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.hsail64 => return Function(.hsail64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.spir => return Function(.spir).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.spir64 => return Function(.spir64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.kalimba => return Function(.kalimba).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.shave => return Function(.shave).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.lanai => return Function(.lanai).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.renderscript32 => return Function(.renderscript32).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.renderscript64 => return Function(.renderscript64).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
+                //.ve => return Function(.ve).generateSymbol(bin_file, src_loc, typed_value, code, debug_output),
                 else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."),
             }
         },
@@ -147,7 +147,7 @@ pub fn generateSymbol(
                     try code.ensureCapacity(code.items.len + payload.data.len + 1);
                     code.appendSliceAssumeCapacity(payload.data);
                     const prev_len = code.items.len;
-                    switch (try generateSymbol(bin_file, src, .{
+                    switch (try generateSymbol(bin_file, src_loc, .{
                         .ty = typed_value.ty.elemType(),
                         .val = sentinel,
                     }, code, debug_output)) {
@@ -165,7 +165,7 @@ pub fn generateSymbol(
             return Result{
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
-                    src,
+                    src_loc,
                     "TODO implement generateSymbol for more kinds of arrays",
                     .{},
                 ),
@@ -200,7 +200,7 @@ pub fn generateSymbol(
             return Result{
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
-                    src,
+                    src_loc,
                     "TODO implement generateSymbol for pointer {}",
                     .{typed_value.val},
                 ),
@@ -217,7 +217,7 @@ pub fn generateSymbol(
             return Result{
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
-                    src,
+                    src_loc,
                     "TODO implement generateSymbol for int type '{}'",
                     .{typed_value.ty},
                 ),
@@ -227,7 +227,7 @@ pub fn generateSymbol(
             return Result{
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
-                    src,
+                    src_loc,
                     "TODO implement generateSymbol for type '{s}'",
                     .{@tagName(t)},
                 ),
@@ -259,7 +259,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
         ret_mcv: MCValue,
         fn_type: Type,
         arg_index: usize,
-        src: usize,
+        src_loc: Module.SrcLoc,
         stack_align: u32,
 
         /// Byte offset within the source file.
@@ -428,7 +428,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
         fn generateSymbol(
             bin_file: *link.File,
-            src: usize,
+            src_loc: Module.SrcLoc,
             typed_value: TypedValue,
             code: *std.ArrayList(u8),
             debug_output: DebugInfoOutput,
@@ -450,19 +450,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
             try branch_stack.append(.{});
 
             const src_data: struct { lbrace_src: usize, rbrace_src: usize, source: []const u8 } = blk: {
-                if (module_fn.owner_decl.scope.cast(Module.Scope.Container)) |container_scope| {
-                    const tree = container_scope.file_scope.contents.tree;
-                    const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?;
-                    const block = fn_proto.getBodyNode().?.castTag(.Block).?;
-                    const lbrace_src = tree.token_locs[block.lbrace].start;
-                    const rbrace_src = tree.token_locs[block.rbrace].start;
-                    break :blk .{ .lbrace_src = lbrace_src, .rbrace_src = rbrace_src, .source = tree.source };
-                } else if (module_fn.owner_decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| {
-                    const byte_off = zir_module.contents.module.decls[module_fn.owner_decl.src_index].inst.src;
-                    break :blk .{ .lbrace_src = byte_off, .rbrace_src = byte_off, .source = zir_module.source.bytes };
-                } else {
-                    unreachable;
-                }
+                const container_scope = module_fn.owner_decl.container;
+                const tree = container_scope.file_scope.contents.tree;
+                const fn_proto = tree.root_node.decls()[module_fn.owner_decl.src_index].castTag(.FnProto).?;
+                const block = fn_proto.getBodyNode().?.castTag(.Block).?;
+                const lbrace_src = tree.token_locs[block.lbrace].start;
+                const rbrace_src = tree.token_locs[block.rbrace].start;
+                break :blk .{
+                    .lbrace_src = lbrace_src,
+                    .rbrace_src = rbrace_src,
+                    .source = tree.source,
+                };
             };
 
             var function = Self{
@@ -478,7 +476,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                 .fn_type = fn_type,
                 .arg_index = 0,
                 .branch_stack = &branch_stack,
-                .src = src,
+                .src_loc = src_loc,
                 .stack_align = undefined,
                 .prev_di_pc = 0,
                 .prev_di_src = src_data.lbrace_src,
@@ -489,7 +487,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
             defer function.stack.deinit(bin_file.allocator);
             defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
 
-            var call_info = function.resolveCallingConventionValues(src, fn_type) catch |err| switch (err) {
+            var call_info = function.resolveCallingConventionValues(src_loc.byte_offset, fn_type) catch |err| switch (err) {
                 error.CodegenFail => return Result{ .fail = function.err_msg.? },
                 else => |e| return e,
             };
@@ -536,12 +534,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
 
                         const stack_end = self.max_end_stack;
                         if (stack_end > math.maxInt(i32))
-                            return self.fail(self.src, "too much stack used in call parameters", .{});
+                            return self.failSymbol("too much stack used in call parameters", .{});
                         const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
                         mem.writeIntLittle(u32, self.code.items[reloc_index..][0..4], @intCast(u32, aligned_stack_end));
 
                         if (self.code.items.len >= math.maxInt(i32)) {
-                            return self.fail(self.src, "unable to perform relocation: jump too far", .{});
+                            return self.failSymbol("unable to perform relocation: jump too far", .{});
                         }
                         if (self.exitlude_jump_relocs.items.len == 1) {
                             self.code.items.len -= 5;
@@ -598,7 +596,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                         if (Instruction.Operand.fromU32(@intCast(u32, aligned_stack_end))) |op| {
                             writeInt(u32, self.code.items[backpatch_reloc..][0..4], Instruction.sub(.al, .sp, .sp, op).toU32());
                         } else {
-                            return self.fail(self.src, "TODO ARM: allow larger stacks", .{});
+                            return self.failSymbol("TODO ARM: allow larger stacks", .{});
                         }
 
                         try self.dbgSetEpilogueBegin();
@@ -624,7 +622,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
                                 if (math.cast(i26, amt)) |offset| {
                                     writeInt(u32, self.code.items[jmp_reloc..][0..4], Instruction.b(.al, offset).toU32());
                                 } else |err| {
-                                    return self.fail(self.src, "exitlude jump is too large", .{});
+                                    return self.failSymbol("exitlude jump is too large", .{});
                                 }
                             }
                         }
@@ -3678,7 +3676,17 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
         fn fail(self: *Self, src: usize, comptime format: []const u8, args: anytype) InnerError {
             @setCold(true);
             assert(self.err_msg == null);
-            self.err_msg = try ErrorMsg.create(self.bin_file.allocator, src, format, args);
+            self.err_msg = try ErrorMsg.create(self.bin_file.allocator, .{
+                .file_scope = self.src_loc.file_scope,
+                .byte_offset = src,
+            }, format, args);
+            return error.CodegenFail;
+        }
+
+        fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
+            @setCold(true);
+            assert(self.err_msg == null);
+            self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
             return error.CodegenFail;
         }
 
diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index 8c85f482fd..b26f753757 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -114,10 +114,13 @@ pub const DeclGen = struct {
     module: *Module,
     decl: *Decl,
     fwd_decl: std.ArrayList(u8),
-    error_msg: ?*Compilation.ErrorMsg,
+    error_msg: ?*Module.ErrorMsg,
 
     fn fail(dg: *DeclGen, src: usize, comptime format: []const u8, args: anytype) error{ AnalysisFail, OutOfMemory } {
-        dg.error_msg = try Compilation.ErrorMsg.create(dg.module.gpa, src, format, args);
+        dg.error_msg = try Module.ErrorMsg.create(dg.module.gpa, .{
+            .file_scope = dg.decl.getFileScope(),
+            .byte_offset = src,
+        }, format, args);
         return error.AnalysisFail;
     }
 
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
index 5d753c41cb..1edd466d54 100644
--- a/src/codegen/llvm.zig
+++ b/src/codegen/llvm.zig
@@ -148,7 +148,7 @@ pub const LLVMIRModule = struct {
     object_path: []const u8,
 
     gpa: *Allocator,
-    err_msg: ?*Compilation.ErrorMsg = null,
+    err_msg: ?*Module.ErrorMsg = null,
 
     // TODO: The fields below should really move into a different struct,
     //       because they are only valid when generating a function
@@ -177,6 +177,8 @@ pub const LLVMIRModule = struct {
         break_vals: *BreakValues,
     }) = .{},
 
+    src_loc: Module.SrcLoc,
+
     const BreakBasicBlocks = std.ArrayListUnmanaged(*const llvm.BasicBlock);
     const BreakValues = std.ArrayListUnmanaged(*const llvm.Value);
 
@@ -254,6 +256,8 @@ pub const LLVMIRModule = struct {
             .builder = builder,
             .object_path = object_path,
             .gpa = gpa,
+            // TODO move this field into a struct that is only instantiated per gen() call
+            .src_loc = undefined,
         };
         return self;
     }
@@ -335,6 +339,8 @@ pub const LLVMIRModule = struct {
         const typed_value = decl.typed_value.most_recent.typed_value;
         const src = decl.src();
 
+        self.src_loc = decl.srcLoc();
+
         log.debug("gen: {s} type: {}, value: {}", .{ decl.name, typed_value.ty, typed_value.val });
 
         if (typed_value.val.castTag(.function)) |func_payload| {
@@ -853,7 +859,10 @@ pub const LLVMIRModule = struct {
     pub fn fail(self: *LLVMIRModule, src: usize, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
         @setCold(true);
         assert(self.err_msg == null);
-        self.err_msg = try Compilation.ErrorMsg.create(self.gpa, src, format, args);
+        self.err_msg = try Module.ErrorMsg.create(self.gpa, .{
+            .file_scope = self.src_loc.file_scope,
+            .byte_offset = src,
+        }, format, args);
         return error.CodegenFail;
     }
 };
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index f7cd9b69ce..981d4ec3a3 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -670,7 +670,7 @@ pub fn updateDecl(self: *Coff, module: *Module, decl: *Module.Decl) !void {
     var code_buffer = std.ArrayList(u8).init(self.base.allocator);
     defer code_buffer.deinit();
 
-    const res = try codegen.generateSymbol(&self.base, decl.src(), typed_value, &code_buffer, .none);
+    const res = try codegen.generateSymbol(&self.base, decl.srcLoc(), typed_value, &code_buffer, .none);
     const code = switch (res) {
         .externally_managed => |x| x,
         .appended => code_buffer.items,
@@ -732,7 +732,7 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
-                    try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}),
+                    try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: ExportOptions.section", .{}),
                 );
                 continue;
             }
@@ -743,7 +743,7 @@ pub fn updateDeclExports(self: *Coff, module: *Module, decl: *const Module.Decl,
             try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
             module.failed_exports.putAssumeCapacityNoClobber(
                 exp,
-                try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: Exports other than '_start'", .{}),
+                try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: Exports other than '_start'", .{}),
             );
             continue;
         }
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 8c76a4e967..ee50eb5d94 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2189,22 +2189,14 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
         try dbg_line_buffer.ensureCapacity(26);
 
         const line_off: u28 = blk: {
-            if (decl.scope.cast(Module.Scope.Container)) |container_scope| {
-                const tree = container_scope.file_scope.contents.tree;
-                const file_ast_decls = tree.root_node.decls();
-                // TODO Look into improving the performance here by adding a token-index-to-line
-                // lookup table. Currently this involves scanning over the source code for newlines.
-                const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
-                const block = fn_proto.getBodyNode().?.castTag(.Block).?;
-                const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start);
-                break :blk @intCast(u28, line_delta);
-            } else if (decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| {
-                const byte_off = zir_module.contents.module.decls[decl.src_index].inst.src;
-                const line_delta = std.zig.lineDelta(zir_module.source.bytes, 0, byte_off);
-                break :blk @intCast(u28, line_delta);
-            } else {
-                unreachable;
-            }
+            const tree = decl.container.file_scope.contents.tree;
+            const file_ast_decls = tree.root_node.decls();
+            // TODO Look into improving the performance here by adding a token-index-to-line
+            // lookup table. Currently this involves scanning over the source code for newlines.
+            const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
+            const block = fn_proto.getBodyNode().?.castTag(.Block).?;
+            const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start);
+            break :blk @intCast(u28, line_delta);
         };
 
         const ptr_width_bytes = self.ptrWidthBytes();
@@ -2268,7 +2260,7 @@ pub fn updateDecl(self: *Elf, module: *Module, decl: *Module.Decl) !void {
     } else {
         // TODO implement .debug_info for global variables
     }
-    const res = try codegen.generateSymbol(&self.base, decl.src(), typed_value, &code_buffer, .{
+    const res = try codegen.generateSymbol(&self.base, decl.srcLoc(), typed_value, &code_buffer, .{
         .dwarf = .{
             .dbg_line = &dbg_line_buffer,
             .dbg_info = &dbg_info_buffer,
@@ -2642,7 +2634,7 @@ pub fn updateDeclExports(
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
-                    try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}),
+                    try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: ExportOptions.section", .{}),
                 );
                 continue;
             }
@@ -2660,7 +2652,7 @@ pub fn updateDeclExports(
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
-                    try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}),
+                    try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: GlobalLinkage.LinkOnce", .{}),
                 );
                 continue;
             },
@@ -2703,8 +2695,7 @@ pub fn updateDeclLineNumber(self: *Elf, module: *Module, decl: *const Module.Dec
 
     if (self.llvm_ir_module) |_| return;
 
-    const container_scope = decl.scope.cast(Module.Scope.Container).?;
-    const tree = container_scope.file_scope.contents.tree;
+    const tree = decl.container.file_scope.contents.tree;
     const file_ast_decls = tree.root_node.decls();
     // TODO Look into improving the performance here by adding a token-index-to-line
     // lookup table. Currently this involves scanning over the source code for newlines.
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index d913a82328..1017405255 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -1148,7 +1148,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
     }
 
     const res = if (debug_buffers) |*dbg|
-        try codegen.generateSymbol(&self.base, decl.src(), typed_value, &code_buffer, .{
+        try codegen.generateSymbol(&self.base, decl.srcLoc(), typed_value, &code_buffer, .{
             .dwarf = .{
                 .dbg_line = &dbg.dbg_line_buffer,
                 .dbg_info = &dbg.dbg_info_buffer,
@@ -1156,7 +1156,7 @@ pub fn updateDecl(self: *MachO, module: *Module, decl: *Module.Decl) !void {
             },
         })
     else
-        try codegen.generateSymbol(&self.base, decl.src(), typed_value, &code_buffer, .none);
+        try codegen.generateSymbol(&self.base, decl.srcLoc(), typed_value, &code_buffer, .none);
 
     const code = switch (res) {
         .externally_managed => |x| x,
@@ -1316,7 +1316,7 @@ pub fn updateDeclExports(
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
-                    try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: ExportOptions.section", .{}),
+                    try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: ExportOptions.section", .{}),
                 );
                 continue;
             }
@@ -1334,7 +1334,7 @@ pub fn updateDeclExports(
                 try module.failed_exports.ensureCapacity(module.gpa, module.failed_exports.items().len + 1);
                 module.failed_exports.putAssumeCapacityNoClobber(
                     exp,
-                    try Compilation.ErrorMsg.create(self.base.allocator, 0, "Unimplemented: GlobalLinkage.LinkOnce", .{}),
+                    try Module.ErrorMsg.create(self.base.allocator, decl.srcLoc(), "Unimplemented: GlobalLinkage.LinkOnce", .{}),
                 );
                 continue;
             },
diff --git a/src/link/MachO/DebugSymbols.zig b/src/link/MachO/DebugSymbols.zig
index 81a016ce42..fb7488a12c 100644
--- a/src/link/MachO/DebugSymbols.zig
+++ b/src/link/MachO/DebugSymbols.zig
@@ -906,8 +906,7 @@ pub fn updateDeclLineNumber(self: *DebugSymbols, module: *Module, decl: *const M
     const tracy = trace(@src());
     defer tracy.end();
 
-    const container_scope = decl.scope.cast(Module.Scope.Container).?;
-    const tree = container_scope.file_scope.contents.tree;
+    const tree = decl.container.file_scope.contents.tree;
     const file_ast_decls = tree.root_node.decls();
     // TODO Look into improving the performance here by adding a token-index-to-line
     // lookup table. Currently this involves scanning over the source code for newlines.
@@ -951,22 +950,14 @@ pub fn initDeclDebugBuffers(
             try dbg_line_buffer.ensureCapacity(26);
 
             const line_off: u28 = blk: {
-                if (decl.scope.cast(Module.Scope.Container)) |container_scope| {
-                    const tree = container_scope.file_scope.contents.tree;
-                    const file_ast_decls = tree.root_node.decls();
-                    // TODO Look into improving the performance here by adding a token-index-to-line
-                    // lookup table. Currently this involves scanning over the source code for newlines.
-                    const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
-                    const block = fn_proto.getBodyNode().?.castTag(.Block).?;
-                    const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start);
-                    break :blk @intCast(u28, line_delta);
-                } else if (decl.scope.cast(Module.Scope.ZIRModule)) |zir_module| {
-                    const byte_off = zir_module.contents.module.decls[decl.src_index].inst.src;
-                    const line_delta = std.zig.lineDelta(zir_module.source.bytes, 0, byte_off);
-                    break :blk @intCast(u28, line_delta);
-                } else {
-                    unreachable;
-                }
+                const tree = decl.container.file_scope.contents.tree;
+                const file_ast_decls = tree.root_node.decls();
+                // TODO Look into improving the performance here by adding a token-index-to-line
+                // lookup table. Currently this involves scanning over the source code for newlines.
+                const fn_proto = file_ast_decls[decl.src_index].castTag(.FnProto).?;
+                const block = fn_proto.getBodyNode().?.castTag(.Block).?;
+                const line_delta = std.zig.lineDelta(tree.source, 0, tree.token_locs[block.lbrace].start);
+                break :blk @intCast(u28, line_delta);
             };
 
             dbg_line_buffer.appendSliceAssumeCapacity(&[_]u8{
diff --git a/src/main.zig b/src/main.zig
index 867aa348b1..13bea13a5e 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -221,7 +221,6 @@ const usage_build_generic =
     \\
     \\Supported file types:
     \\                    .zig    Zig source code
-    \\                    .zir    Zig Intermediate Representation code
     \\                      .o    ELF object file
     \\                      .o    MACH-O (macOS) object file
     \\                    .obj    COFF (Windows) object file
@@ -245,8 +244,6 @@ const usage_build_generic =
     \\  -fno-emit-bin             Do not output machine code
     \\  -femit-asm[=path]         Output .s (assembly code)
     \\  -fno-emit-asm             (default) Do not output .s (assembly code)
-    \\  -femit-zir[=path]         Produce a .zir file with Zig IR
-    \\  -fno-emit-zir             (default) Do not produce a .zir file with Zig IR
     \\  -femit-llvm-ir[=path]     Produce a .ll file with LLVM IR (requires LLVM extensions)
     \\  -fno-emit-llvm-ir         (default) Do not produce a .ll file with LLVM IR
     \\  -femit-h[=path]           Generate a C header file (.h)
@@ -1631,18 +1628,12 @@ fn buildOutputType(
     var emit_docs_resolved = try emit_docs.resolve("docs");
     defer emit_docs_resolved.deinit();
 
-    const zir_out_path: ?[]const u8 = switch (emit_zir) {
-        .no => null,
-        .yes_default_path => blk: {
-            if (root_src_file) |rsf| {
-                if (mem.endsWith(u8, rsf, ".zir")) {
-                    break :blk try std.fmt.allocPrint(arena, "{s}.out.zir", .{root_name});
-                }
-            }
-            break :blk try std.fmt.allocPrint(arena, "{s}.zir", .{root_name});
+    switch (emit_zir) {
+        .no => {},
+        .yes_default_path, .yes => {
+            fatal("The -femit-zir implementation has been intentionally deleted so that it can be rewritten as a proper backend.", .{});
         },
-        .yes => |p| p,
-    };
+    }
 
     const root_pkg: ?*Package = if (root_src_file) |src_path| blk: {
         if (main_pkg_path) |p| {
@@ -1753,7 +1744,7 @@ fn buildOutputType(
         .dll_export_fns = dll_export_fns,
         .object_format = object_format,
         .optimize_mode = optimize_mode,
-        .keep_source_files_loaded = zir_out_path != null,
+        .keep_source_files_loaded = false,
         .clang_argv = clang_argv.items,
         .lld_argv = lld_argv.items,
         .lib_dirs = lib_dirs.items,
@@ -1845,7 +1836,7 @@ fn buildOutputType(
         }
     };
 
-    updateModule(gpa, comp, zir_out_path, hook) catch |err| switch (err) {
+    updateModule(gpa, comp, hook) catch |err| switch (err) {
         error.SemanticAnalyzeFail => if (!watch) process.exit(1),
         else => |e| return e,
     };
@@ -1980,7 +1971,7 @@ fn buildOutputType(
                 if (output_mode == .Exe) {
                     try comp.makeBinFileWritable();
                 }
-                updateModule(gpa, comp, zir_out_path, hook) catch |err| switch (err) {
+                updateModule(gpa, comp, hook) catch |err| switch (err) {
                     error.SemanticAnalyzeFail => continue,
                     else => |e| return e,
                 };
@@ -2003,7 +1994,7 @@ const AfterUpdateHook = union(enum) {
     update: []const u8,
 };
 
-fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8, hook: AfterUpdateHook) !void {
+fn updateModule(gpa: *Allocator, comp: *Compilation, hook: AfterUpdateHook) !void {
     try comp.update();
 
     var errors = try comp.getAllErrorsAlloc();
@@ -2013,6 +2004,10 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8,
         for (errors.list) |full_err_msg| {
             full_err_msg.renderToStdErr();
         }
+        const log_text = comp.getCompileLogOutput();
+        if (log_text.len != 0) {
+            std.debug.print("\nCompile Log Output:\n{s}", .{log_text});
+        }
         return error.SemanticAnalyzeFail;
     } else switch (hook) {
         .none => {},
@@ -2024,20 +2019,6 @@ fn updateModule(gpa: *Allocator, comp: *Compilation, zir_out_path: ?[]const u8,
             .{},
         ),
     }
-
-    if (zir_out_path) |zop| {
-        const module = comp.bin_file.options.module orelse
-            fatal("-femit-zir with no zig source code", .{});
-        var new_zir_module = try zir.emit(gpa, module);
-        defer new_zir_module.deinit(gpa);
-
-        const baf = try io.BufferedAtomicFile.create(gpa, fs.cwd(), zop, .{});
-        defer baf.destroy();
-
-        try new_zir_module.writeToStream(gpa, baf.writer());
-
-        try baf.finish();
-    }
 }
 
 fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !void {
@@ -2506,7 +2487,7 @@ pub fn cmdBuild(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
         };
         defer comp.destroy();
 
-        try updateModule(gpa, comp, null, .none);
+        try updateModule(gpa, comp, .none);
         try comp.makeBinFileExecutable();
 
         child_argv.items[argv_index_exe] = try comp.bin_file.options.emit.?.directory.join(
diff --git a/src/test.zig b/src/test.zig
index 59927525df..1c9fb57f01 100644
--- a/src/test.zig
+++ b/src/test.zig
@@ -15,6 +15,8 @@ const CrossTarget = std.zig.CrossTarget;
 
 const zig_h = link.File.C.zig_h;
 
+const hr = "=" ** 40;
+
 test "self-hosted" {
     var ctx = TestContext.init();
     defer ctx.deinit();
@@ -29,23 +31,32 @@ const ErrorMsg = union(enum) {
         msg: []const u8,
         line: u32,
         column: u32,
+        kind: Kind,
     },
     plain: struct {
         msg: []const u8,
+        kind: Kind,
     },
 
-    fn init(other: Compilation.AllErrors.Message) ErrorMsg {
+    const Kind = enum {
+        @"error",
+        note,
+    };
+
+    fn init(other: Compilation.AllErrors.Message, kind: Kind) ErrorMsg {
         switch (other) {
             .src => |src| return .{
                 .src = .{
                     .msg = src.msg,
                     .line = @intCast(u32, src.line),
                     .column = @intCast(u32, src.column),
+                    .kind = kind,
                 },
             },
             .plain => |plain| return .{
                 .plain = .{
                     .msg = plain.msg,
+                    .kind = kind,
                 },
             },
         }
@@ -59,14 +70,15 @@ const ErrorMsg = union(enum) {
     ) !void {
         switch (self) {
             .src => |src| {
-                return writer.print(":{d}:{d}: error: {s}", .{
+                return writer.print(":{d}:{d}: {s}: {s}", .{
                     src.line + 1,
                     src.column + 1,
+                    @tagName(src.kind),
                     src.msg,
                 });
             },
             .plain => |plain| {
-                return writer.print("error: {s}", .{plain.msg});
+                return writer.print("{s}: {s}", .{ plain.msg, @tagName(plain.kind) });
             },
         }
     }
@@ -86,9 +98,6 @@ pub const TestContext = struct {
         /// effects of the incremental compilation.
         src: [:0]const u8,
         case: union(enum) {
-            /// A transformation update transforms the input and tests against
-            /// the expected output ZIR.
-            Transformation: [:0]const u8,
             /// Check the main binary output file against an expected set of bytes.
             /// This is most useful with, for example, `-ofmt=c`.
             CompareObjectFile: []const u8,
@@ -139,15 +148,6 @@ pub const TestContext = struct {
 
         files: std.ArrayList(File),
 
-        /// Adds a subcase in which the module is updated with `src`, and the
-        /// resulting ZIR is validated against `result`.
-        pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) void {
-            self.updates.append(.{
-                .src = src,
-                .case = .{ .Transformation = result },
-            }) catch unreachable;
-        }
-
         /// Adds a subcase in which the module is updated with `src`, and a C
         /// header is generated.
         pub fn addHeader(self: *Case, src: [:0]const u8, result: [:0]const u8) void {
@@ -182,31 +182,37 @@ pub const TestContext = struct {
         /// the form `:line:column: error: message`.
         pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
             var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable;
-            for (errors) |e, i| {
-                if (e[0] != ':') {
-                    array[i] = .{ .plain = .{ .msg = e } };
+            for (errors) |err_msg_line, i| {
+                if (std.mem.startsWith(u8, err_msg_line, "error: ")) {
+                    array[i] = .{
+                        .plain = .{ .msg = err_msg_line["error: ".len..], .kind = .@"error" },
+                    };
+                    continue;
+                } else if (std.mem.startsWith(u8, err_msg_line, "note: ")) {
+                    array[i] = .{
+                        .plain = .{ .msg = err_msg_line["note: ".len..], .kind = .note },
+                    };
                     continue;
                 }
-                var cur = e[1..];
-                var line_index = std.mem.indexOf(u8, cur, ":");
-                if (line_index == null) {
-                    @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
-                }
-                const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number");
-                cur = cur[line_index.? + 1 ..];
-                const column_index = std.mem.indexOf(u8, cur, ":");
-                if (column_index == null) {
-                    @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
-                }
-                const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number");
-                cur = cur[column_index.? + 2 ..];
-                if (!std.mem.eql(u8, cur[0..7], "error: ")) {
-                    @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
-                }
-                const msg = cur[7..];
+                // example: ":1:2: error: bad thing happened"
+                var it = std.mem.split(err_msg_line, ":");
+                _ = it.next() orelse @panic("missing colon");
+                const line_text = it.next() orelse @panic("missing line");
+                const col_text = it.next() orelse @panic("missing column");
+                const kind_text = it.next() orelse @panic("missing 'error'/'note'");
+                const msg = it.rest()[1..]; // skip over the space at end of "error: "
+
+                const line = std.fmt.parseInt(u32, line_text, 10) catch @panic("bad line number");
+                const column = std.fmt.parseInt(u32, col_text, 10) catch @panic("bad column number");
+                const kind: ErrorMsg.Kind = if (std.mem.eql(u8, kind_text, " error"))
+                    .@"error"
+                else if (std.mem.eql(u8, kind_text, " note"))
+                    .note
+                else
+                    @panic("expected 'error'/'note'");
 
                 if (line == 0 or column == 0) {
-                    @panic("Invalid test: error line and column must be specified starting at one!");
+                    @panic("line and column must be specified starting at one");
                 }
 
                 array[i] = .{
@@ -214,6 +220,7 @@ pub const TestContext = struct {
                         .msg = msg,
                         .line = line - 1,
                         .column = column - 1,
+                        .kind = kind,
                     },
                 };
             }
@@ -689,25 +696,20 @@ pub const TestContext = struct {
                 var all_errors = try comp.getAllErrorsAlloc();
                 defer all_errors.deinit(allocator);
                 if (all_errors.list.len != 0) {
-                    std.debug.print("\nErrors occurred updating the compilation:\n================\n", .{});
+                    std.debug.print("\nErrors occurred updating the compilation:\n{s}\n", .{hr});
                     for (all_errors.list) |err_msg| {
                         switch (err_msg) {
                             .src => |src| {
-                                std.debug.print(":{d}:{d}: error: {s}\n================\n", .{
-                                    src.line + 1, src.column + 1, src.msg,
+                                std.debug.print(":{d}:{d}: error: {s}\n{s}\n", .{
+                                    src.line + 1, src.column + 1, src.msg, hr,
                                 });
                             },
                             .plain => |plain| {
-                                std.debug.print("error: {s}\n================\n", .{plain.msg});
+                                std.debug.print("error: {s}\n{s}\n", .{ plain.msg, hr });
                             },
                         }
                     }
                     // TODO print generated C code
-                    //if (comp.bin_file.cast(link.File.C)) |c_file| {
-                    //    std.debug.print("Generated C: \n===============\n{s}\n\n===========\n\n", .{
-                    //        c_file.main.items,
-                    //    });
-                    //}
                     std.debug.print("Test failed.\n", .{});
                     std.process.exit(1);
                 }
@@ -728,48 +730,74 @@ pub const TestContext = struct {
 
                     std.testing.expectEqualStrings(expected_output, out);
                 },
-                .Transformation => |expected_output| {
-                    update_node.setEstimatedTotalItems(5);
-                    var emit_node = update_node.start("emit", 0);
-                    emit_node.activate();
-                    var new_zir_module = try zir.emit(allocator, comp.bin_file.options.module.?);
-                    defer new_zir_module.deinit(allocator);
-                    emit_node.end();
-
-                    var write_node = update_node.start("write", 0);
-                    write_node.activate();
-                    var out_zir = std.ArrayList(u8).init(allocator);
-                    defer out_zir.deinit();
-                    try new_zir_module.writeToStream(allocator, out_zir.writer());
-                    write_node.end();
-
+                .Error => |case_error_list| {
                     var test_node = update_node.start("assert", 0);
                     test_node.activate();
                     defer test_node.end();
 
-                    std.testing.expectEqualStrings(expected_output, out_zir.items);
-                },
-                .Error => |e| {
-                    var test_node = update_node.start("assert", 0);
-                    test_node.activate();
-                    defer test_node.end();
-                    var handled_errors = try arena.alloc(bool, e.len);
-                    for (handled_errors) |*handled| {
-                        handled.* = false;
+                    const handled_errors = try arena.alloc(bool, case_error_list.len);
+                    std.mem.set(bool, handled_errors, false);
+
+                    var actual_errors = try comp.getAllErrorsAlloc();
+                    defer actual_errors.deinit(allocator);
+
+                    var any_failed = false;
+                    var notes_to_check = std.ArrayList(*const Compilation.AllErrors.Message).init(allocator);
+                    defer notes_to_check.deinit();
+
+                    for (actual_errors.list) |actual_error| {
+                        for (case_error_list) |case_msg, i| {
+                            const ex_tag: @TagType(@TypeOf(case_msg)) = case_msg;
+                            switch (actual_error) {
+                                .src => |actual_msg| {
+                                    for (actual_msg.notes) |*note| {
+                                        try notes_to_check.append(note);
+                                    }
+
+                                    if (ex_tag != .src) continue;
+
+                                    if (actual_msg.line == case_msg.src.line and
+                                        actual_msg.column == case_msg.src.column and
+                                        std.mem.eql(u8, case_msg.src.msg, actual_msg.msg) and
+                                        case_msg.src.kind == .@"error")
+                                    {
+                                        handled_errors[i] = true;
+                                        break;
+                                    }
+                                },
+                                .plain => |plain| {
+                                    if (ex_tag != .plain) continue;
+
+                                    if (std.mem.eql(u8, case_msg.plain.msg, plain.msg) and
+                                        case_msg.plain.kind == .@"error")
+                                    {
+                                        handled_errors[i] = true;
+                                        break;
+                                    }
+                                },
+                            }
+                        } else {
+                            std.debug.print(
+                                "\nUnexpected error:\n{s}\n{}\n{s}",
+                                .{ hr, ErrorMsg.init(actual_error, .@"error"), hr },
+                            );
+                            any_failed = true;
+                        }
                     }
-                    var all_errors = try comp.getAllErrorsAlloc();
-                    defer all_errors.deinit(allocator);
-                    for (all_errors.list) |a| {
-                        for (e) |ex, i| {
-                            const a_tag: @TagType(@TypeOf(a)) = a;
-                            const ex_tag: @TagType(@TypeOf(ex)) = ex;
-                            switch (a) {
-                                .src => |src| {
+                    while (notes_to_check.popOrNull()) |note| {
+                        for (case_error_list) |case_msg, i| {
+                            const ex_tag: @TagType(@TypeOf(case_msg)) = case_msg;
+                            switch (note.*) {
+                                .src => |actual_msg| {
+                                    for (actual_msg.notes) |*sub_note| {
+                                        try notes_to_check.append(sub_note);
+                                    }
                                     if (ex_tag != .src) continue;
 
-                                    if (src.line == ex.src.line and
-                                        src.column == ex.src.column and
-                                        std.mem.eql(u8, ex.src.msg, src.msg))
+                                    if (actual_msg.line == case_msg.src.line and
+                                        actual_msg.column == case_msg.src.column and
+                                        std.mem.eql(u8, case_msg.src.msg, actual_msg.msg) and
+                                        case_msg.src.kind == .note)
                                     {
                                         handled_errors[i] = true;
                                         break;
@@ -778,7 +806,9 @@ pub const TestContext = struct {
                                 .plain => |plain| {
                                     if (ex_tag != .plain) continue;
 
-                                    if (std.mem.eql(u8, ex.plain.msg, plain.msg)) {
+                                    if (std.mem.eql(u8, case_msg.plain.msg, plain.msg) and
+                                        case_msg.plain.kind == .note)
+                                    {
                                         handled_errors[i] = true;
                                         break;
                                     }
@@ -786,23 +816,29 @@ pub const TestContext = struct {
                             }
                         } else {
                             std.debug.print(
-                                "{s}\nUnexpected error:\n================\n{}\n================\nTest failed.\n",
-                                .{ case.name, ErrorMsg.init(a) },
+                                "\nUnexpected note:\n{s}\n{}\n{s}",
+                                .{ hr, ErrorMsg.init(note.*, .note), hr },
                             );
-                            std.process.exit(1);
+                            any_failed = true;
                         }
                     }
 
                     for (handled_errors) |handled, i| {
                         if (!handled) {
-                            const er = e[i];
                             std.debug.print(
-                                "{s}\nDid not receive error:\n================\n{}\n================\nTest failed.\n",
-                                .{ case.name, er },
+                                "\nExpected error not found:\n{s}\n{}\n{s}",
+                                .{ hr, case_error_list[i], hr },
                             );
-                            std.process.exit(1);
+                            any_failed = true;
                         }
                     }
+
+                    if (any_failed) {
+                        std.debug.print("\nTest case '{s}' failed, update_index={d}.\n", .{
+                            case.name, update_index,
+                        });
+                        std.process.exit(1);
+                    }
                 },
                 .Execution => |expected_stdout| {
                     update_node.setEstimatedTotalItems(4);
diff --git a/src/type/Enum.zig b/src/type/Enum.zig
index 9b9ec5b319..4dfd5f6e44 100644
--- a/src/type/Enum.zig
+++ b/src/type/Enum.zig
@@ -21,7 +21,7 @@ pub const Field = struct {
 };
 
 pub const Zir = struct {
-    body: zir.Module.Body,
+    body: zir.Body,
     inst: *zir.Inst,
 };
 
diff --git a/src/type/Struct.zig b/src/type/Struct.zig
index d6a591c95e..24e3a0dcad 100644
--- a/src/type/Struct.zig
+++ b/src/type/Struct.zig
@@ -24,7 +24,7 @@ pub const Field = struct {
 };
 
 pub const Zir = struct {
-    body: zir.Module.Body,
+    body: zir.Body,
     inst: *zir.Inst,
 };
 
diff --git a/src/type/Union.zig b/src/type/Union.zig
index 26cc1796c6..5c7acf7d36 100644
--- a/src/type/Union.zig
+++ b/src/type/Union.zig
@@ -24,7 +24,7 @@ pub const Field = struct {
 };
 
 pub const Zir = struct {
-    body: zir.Module.Body,
+    body: zir.Body,
     inst: *zir.Inst,
 };
 
diff --git a/src/zir.zig b/src/zir.zig
index 0ccb09efac..0e7b3a3520 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -12,17 +12,6 @@ const TypedValue = @import("TypedValue.zig");
 const ir = @import("ir.zig");
 const IrModule = @import("Module.zig");
 
-/// This struct is relevent only for the ZIR Module text format. It is not used for
-/// semantic analysis of Zig source code.
-pub const Decl = struct {
-    name: []const u8,
-
-    /// Hash of slice into the source of the part after the = and before the next instruction.
-    contents_hash: std.zig.SrcHash,
-
-    inst: *Inst,
-};
-
 /// These are instructions that correspond to the ZIR text format. See `ir.Inst` for
 /// in-memory, analyzed instructions with types and values.
 /// We use a table to map these instruction to their respective semantically analyzed
@@ -141,15 +130,12 @@ pub const Inst = struct {
         container_field,
         /// Declares the beginning of a statement. Used for debug info.
         dbg_stmt,
-        /// Represents a pointer to a global decl by name.
+        /// Represents a pointer to a global decl.
         declref,
         /// Represents a pointer to a global decl by string name.
         declref_str,
-        /// The syntax `@foo` is equivalent to `declval("foo")`.
-        /// declval is equivalent to declref followed by deref.
+        /// Equivalent to a declref followed by deref.
         declval,
-        /// Same as declval but the parameter is a `*Module.Decl` rather than a name.
-        declval_in_module,
         /// Load the value from a pointer.
         deref,
         /// Arithmetic division. Asserts no integer overflow.
@@ -419,7 +405,6 @@ pub const Inst = struct {
                 .declref => DeclRef,
                 .declref_str => DeclRefStr,
                 .declval => DeclVal,
-                .declval_in_module => DeclValInModule,
                 .coerce_result_block_ptr => CoerceResultBlockPtr,
                 .compilelog => CompileLog,
                 .loop => Loop,
@@ -496,7 +481,6 @@ pub const Inst = struct {
                 .declref,
                 .declref_str,
                 .declval,
-                .declval_in_module,
                 .deref,
                 .div,
                 .elemptr,
@@ -650,7 +634,7 @@ pub const Inst = struct {
         base: Inst,
 
         positionals: struct {
-            body: Module.Body,
+            body: Body,
         },
         kw_args: struct {},
     };
@@ -705,7 +689,7 @@ pub const Inst = struct {
         base: Inst,
 
         positionals: struct {
-            name: []const u8,
+            decl: *IrModule.Decl,
         },
         kw_args: struct {},
     };
@@ -724,16 +708,6 @@ pub const Inst = struct {
         pub const base_tag = Tag.declval;
         base: Inst,
 
-        positionals: struct {
-            name: []const u8,
-        },
-        kw_args: struct {},
-    };
-
-    pub const DeclValInModule = struct {
-        pub const base_tag = Tag.declval_in_module;
-        base: Inst,
-
         positionals: struct {
             decl: *IrModule.Decl,
         },
@@ -758,10 +732,7 @@ pub const Inst = struct {
         positionals: struct {
             to_log: []*Inst,
         },
-        kw_args: struct {
-            /// If we have seen it already so don't make another error
-            seen: bool = false,
-        },
+        kw_args: struct {},
     };
 
     pub const Const = struct {
@@ -799,7 +770,7 @@ pub const Inst = struct {
         base: Inst,
 
         positionals: struct {
-            body: Module.Body,
+            body: Body,
         },
         kw_args: struct {},
     };
@@ -838,7 +809,7 @@ pub const Inst = struct {
 
         positionals: struct {
             fn_type: *Inst,
-            body: Module.Body,
+            body: Body,
         },
         kw_args: struct {
             is_inline: bool = false,
@@ -998,8 +969,8 @@ pub const Inst = struct {
 
         positionals: struct {
             condition: *Inst,
-            then_body: Module.Body,
-            else_body: Module.Body,
+            then_body: Body,
+            else_body: Body,
         },
         kw_args: struct {},
     };
@@ -1078,7 +1049,7 @@ pub const Inst = struct {
             /// List of all individual items and ranges
             items: []*Inst,
             cases: []Case,
-            else_body: Module.Body,
+            else_body: Body,
         },
         kw_args: struct {
             /// Pointer to first range if such exists.
@@ -1092,7 +1063,7 @@ pub const Inst = struct {
 
         pub const Case = struct {
             item: *Inst,
-            body: Module.Body,
+            body: Body,
         };
     };
     pub const TypeOfPeer = struct {
@@ -1192,6 +1163,10 @@ pub const ErrorMsg = struct {
     msg: []const u8,
 };
 
+pub const Body = struct {
+    instructions: []*Inst,
+};
+
 pub const Module = struct {
     decls: []*Decl,
     arena: std.heap.ArenaAllocator,
@@ -1199,6 +1174,15 @@ pub const Module = struct {
     metadata: std.AutoHashMap(*Inst, MetaData),
     body_metadata: std.AutoHashMap(*Body, BodyMetaData),
 
+    pub const Decl = struct {
+        name: []const u8,
+
+        /// Hash of slice into the source of the part after the = and before the next instruction.
+        contents_hash: std.zig.SrcHash,
+
+        inst: *Inst,
+    };
+
     pub const MetaData = struct {
         deaths: ir.Inst.DeathsInt,
         addr: usize,
@@ -1208,10 +1192,6 @@ pub const Module = struct {
         deaths: []*Inst,
     };
 
-    pub const Body = struct {
-        instructions: []*Inst,
-    };
-
     pub fn deinit(self: *Module, allocator: *Allocator) void {
         self.metadata.deinit();
         self.body_metadata.deinit();
@@ -1369,7 +1349,7 @@ const Writer = struct {
                 }
                 try stream.writeByte(']');
             },
-            Module.Body => {
+            Body => {
                 try stream.writeAll("{\n");
                 if (self.module.body_metadata.get(param_ptr)) |metadata| {
                     if (metadata.deaths.len > 0) {
@@ -1468,8 +1448,6 @@ const Writer = struct {
                 try stream.print("@{s}", .{info.name});
             }
         } else if (inst.cast(Inst.DeclVal)) |decl_val| {
-            try stream.print("@{s}", .{decl_val.positionals.name});
-        } else if (inst.cast(Inst.DeclValInModule)) |decl_val| {
             try stream.print("@{s}", .{decl_val.positionals.decl.name});
         } else {
             // This should be unreachable in theory, but since ZIR is used for debugging the compiler
@@ -1479,502 +1457,6 @@ const Writer = struct {
     }
 };
 
-pub fn parse(allocator: *Allocator, source: [:0]const u8) Allocator.Error!Module {
-    var global_name_map = std.StringHashMap(*Inst).init(allocator);
-    defer global_name_map.deinit();
-
-    var parser: Parser = .{
-        .allocator = allocator,
-        .arena = std.heap.ArenaAllocator.init(allocator),
-        .i = 0,
-        .source = source,
-        .global_name_map = &global_name_map,
-        .decls = .{},
-        .unnamed_index = 0,
-        .block_table = std.StringHashMap(*Inst.Block).init(allocator),
-        .loop_table = std.StringHashMap(*Inst.Loop).init(allocator),
-    };
-    defer parser.block_table.deinit();
-    defer parser.loop_table.deinit();
-    errdefer parser.arena.deinit();
-
-    parser.parseRoot() catch |err| switch (err) {
-        error.ParseFailure => {
-            assert(parser.error_msg != null);
-        },
-        else => |e| return e,
-    };
-
-    return Module{
-        .decls = parser.decls.toOwnedSlice(allocator),
-        .arena = parser.arena,
-        .error_msg = parser.error_msg,
-        .metadata = std.AutoHashMap(*Inst, Module.MetaData).init(allocator),
-        .body_metadata = std.AutoHashMap(*Module.Body, Module.BodyMetaData).init(allocator),
-    };
-}
-
-const Parser = struct {
-    allocator: *Allocator,
-    arena: std.heap.ArenaAllocator,
-    i: usize,
-    source: [:0]const u8,
-    decls: std.ArrayListUnmanaged(*Decl),
-    global_name_map: *std.StringHashMap(*Inst),
-    error_msg: ?ErrorMsg = null,
-    unnamed_index: usize,
-    block_table: std.StringHashMap(*Inst.Block),
-    loop_table: std.StringHashMap(*Inst.Loop),
-
-    const Body = struct {
-        instructions: std.ArrayList(*Inst),
-        name_map: *std.StringHashMap(*Inst),
-    };
-
-    fn parseBody(self: *Parser, body_ctx: ?*Body) !Module.Body {
-        var name_map = std.StringHashMap(*Inst).init(self.allocator);
-        defer name_map.deinit();
-
-        var body_context = Body{
-            .instructions = std.ArrayList(*Inst).init(self.allocator),
-            .name_map = if (body_ctx) |bctx| bctx.name_map else &name_map,
-        };
-        defer body_context.instructions.deinit();
-
-        try requireEatBytes(self, "{");
-        skipSpace(self);
-
-        while (true) : (self.i += 1) switch (self.source[self.i]) {
-            ';' => _ = try skipToAndOver(self, '\n'),
-            '%' => {
-                self.i += 1;
-                const ident = try skipToAndOver(self, ' ');
-                skipSpace(self);
-                try requireEatBytes(self, "=");
-                skipSpace(self);
-                const decl = try parseInstruction(self, &body_context, ident);
-                const ident_index = body_context.instructions.items.len;
-                if (try body_context.name_map.fetchPut(ident, decl.inst)) |_| {
-                    return self.fail("redefinition of identifier '{s}'", .{ident});
-                }
-                try body_context.instructions.append(decl.inst);
-                continue;
-            },
-            ' ', '\n' => continue,
-            '}' => {
-                self.i += 1;
-                break;
-            },
-            else => |byte| return self.failByte(byte),
-        };
-
-        // Move the instructions to the arena
-        const instrs = try self.arena.allocator.alloc(*Inst, body_context.instructions.items.len);
-        mem.copy(*Inst, instrs, body_context.instructions.items);
-        return Module.Body{ .instructions = instrs };
-    }
-
-    fn parseStringLiteral(self: *Parser) ![]u8 {
-        const start = self.i;
-        try self.requireEatBytes("\"");
-
-        while (true) : (self.i += 1) switch (self.source[self.i]) {
-            '"' => {
-                self.i += 1;
-                const span = self.source[start..self.i];
-                var bad_index: usize = undefined;
-                const parsed = std.zig.parseStringLiteral(&self.arena.allocator, span, &bad_index) catch |err| switch (err) {
-                    error.InvalidCharacter => {
-                        self.i = start + bad_index;
-                        const bad_byte = self.source[self.i];
-                        return self.fail("invalid string literal character: '{c}'\n", .{bad_byte});
-                    },
-                    else => |e| return e,
-                };
-                return parsed;
-            },
-            '\\' => {
-                self.i += 1;
-                continue;
-            },
-            0 => return self.failByte(0),
-            else => continue,
-        };
-    }
-
-    fn parseIntegerLiteral(self: *Parser) !BigIntConst {
-        const start = self.i;
-        if (self.source[self.i] == '-') self.i += 1;
-        while (true) : (self.i += 1) switch (self.source[self.i]) {
-            '0'...'9' => continue,
-            else => break,
-        };
-        const number_text = self.source[start..self.i];
-        const base = 10;
-        // TODO reuse the same array list for this
-        const limbs_buffer_len = std.math.big.int.calcSetStringLimbsBufferLen(base, number_text.len);
-        const limbs_buffer = try self.allocator.alloc(std.math.big.Limb, limbs_buffer_len);
-        defer self.allocator.free(limbs_buffer);
-        const limb_len = std.math.big.int.calcSetStringLimbCount(base, number_text.len);
-        const limbs = try self.arena.allocator.alloc(std.math.big.Limb, limb_len);
-        var result = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined };
-        result.setString(base, number_text, limbs_buffer, self.allocator) catch |err| switch (err) {
-            error.InvalidCharacter => {
-                self.i = start;
-                return self.fail("invalid digit in integer literal", .{});
-            },
-        };
-        return result.toConst();
-    }
-
-    fn parseRoot(self: *Parser) !void {
-        // The IR format is designed so that it can be tokenized and parsed at the same time.
-        while (true) {
-            switch (self.source[self.i]) {
-                ';' => _ = try skipToAndOver(self, '\n'),
-                '@' => {
-                    self.i += 1;
-                    const ident = try skipToAndOver(self, ' ');
-                    skipSpace(self);
-                    try requireEatBytes(self, "=");
-                    skipSpace(self);
-                    const decl = try parseInstruction(self, null, ident);
-                    const ident_index = self.decls.items.len;
-                    if (try self.global_name_map.fetchPut(ident, decl.inst)) |_| {
-                        return self.fail("redefinition of identifier '{s}'", .{ident});
-                    }
-                    try self.decls.append(self.allocator, decl);
-                },
-                ' ', '\n' => self.i += 1,
-                0 => break,
-                else => |byte| return self.fail("unexpected byte: '{c}'", .{byte}),
-            }
-        }
-    }
-
-    fn eatByte(self: *Parser, byte: u8) bool {
-        if (self.source[self.i] != byte) return false;
-        self.i += 1;
-        return true;
-    }
-
-    fn skipSpace(self: *Parser) void {
-        while (self.source[self.i] == ' ' or self.source[self.i] == '\n') {
-            self.i += 1;
-        }
-    }
-
-    fn requireEatBytes(self: *Parser, bytes: []const u8) !void {
-        const start = self.i;
-        for (bytes) |byte| {
-            if (self.source[self.i] != byte) {
-                self.i = start;
-                return self.fail("expected '{s}'", .{bytes});
-            }
-            self.i += 1;
-        }
-    }
-
-    fn skipToAndOver(self: *Parser, byte: u8) ![]const u8 {
-        const start_i = self.i;
-        while (self.source[self.i] != 0) : (self.i += 1) {
-            if (self.source[self.i] == byte) {
-                const result = self.source[start_i..self.i];
-                self.i += 1;
-                return result;
-            }
-        }
-        return self.fail("unexpected EOF", .{});
-    }
-
-    /// ParseFailure is an internal error code; handled in `parse`.
-    const InnerError = error{ ParseFailure, OutOfMemory };
-
-    fn failByte(self: *Parser, byte: u8) InnerError {
-        if (byte == 0) {
-            return self.fail("unexpected EOF", .{});
-        } else {
-            return self.fail("unexpected byte: '{c}'", .{byte});
-        }
-    }
-
-    fn fail(self: *Parser, comptime format: []const u8, args: anytype) InnerError {
-        @setCold(true);
-        self.error_msg = ErrorMsg{
-            .byte_offset = self.i,
-            .msg = try std.fmt.allocPrint(&self.arena.allocator, format, args),
-        };
-        return error.ParseFailure;
-    }
-
-    fn parseInstruction(self: *Parser, body_ctx: ?*Body, name: []const u8) InnerError!*Decl {
-        const contents_start = self.i;
-        const fn_name = try skipToAndOver(self, '(');
-        inline for (@typeInfo(Inst.Tag).Enum.fields) |field| {
-            if (mem.eql(u8, field.name, fn_name)) {
-                const tag = @field(Inst.Tag, field.name);
-                return parseInstructionGeneric(self, field.name, tag.Type(), tag, body_ctx, name, contents_start);
-            }
-        }
-        return self.fail("unknown instruction '{s}'", .{fn_name});
-    }
-
-    fn parseInstructionGeneric(
-        self: *Parser,
-        comptime fn_name: []const u8,
-        comptime InstType: type,
-        tag: Inst.Tag,
-        body_ctx: ?*Body,
-        inst_name: []const u8,
-        contents_start: usize,
-    ) InnerError!*Decl {
-        const inst_specific = try self.arena.allocator.create(InstType);
-        inst_specific.base = .{
-            .src = self.i,
-            .tag = tag,
-        };
-
-        if (InstType == Inst.Block) {
-            try self.block_table.put(inst_name, inst_specific);
-        } else if (InstType == Inst.Loop) {
-            try self.loop_table.put(inst_name, inst_specific);
-        }
-
-        if (@hasField(InstType, "ty")) {
-            inst_specific.ty = opt_type orelse {
-                return self.fail("instruction '" ++ fn_name ++ "' requires type", .{});
-            };
-        }
-
-        const Positionals = @TypeOf(inst_specific.positionals);
-        inline for (@typeInfo(Positionals).Struct.fields) |arg_field| {
-            if (self.source[self.i] == ',') {
-                self.i += 1;
-                skipSpace(self);
-            } else if (self.source[self.i] == ')') {
-                return self.fail("expected positional parameter '{s}'", .{arg_field.name});
-            }
-            @field(inst_specific.positionals, arg_field.name) = try parseParameterGeneric(
-                self,
-                arg_field.field_type,
-                body_ctx,
-            );
-            skipSpace(self);
-        }
-
-        const KW_Args = @TypeOf(inst_specific.kw_args);
-        inst_specific.kw_args = .{}; // assign defaults
-        skipSpace(self);
-        while (eatByte(self, ',')) {
-            skipSpace(self);
-            const name = try skipToAndOver(self, '=');
-            inline for (@typeInfo(KW_Args).Struct.fields) |arg_field| {
-                const field_name = arg_field.name;
-                if (mem.eql(u8, name, field_name)) {
-                    const NonOptional = switch (@typeInfo(arg_field.field_type)) {
-                        .Optional => |info| info.child,
-                        else => arg_field.field_type,
-                    };
-                    @field(inst_specific.kw_args, field_name) = try parseParameterGeneric(self, NonOptional, body_ctx);
-                    break;
-                }
-            } else {
-                return self.fail("unrecognized keyword parameter: '{s}'", .{name});
-            }
-            skipSpace(self);
-        }
-        try requireEatBytes(self, ")");
-
-        const decl = try self.arena.allocator.create(Decl);
-        decl.* = .{
-            .name = inst_name,
-            .contents_hash = std.zig.hashSrc(self.source[contents_start..self.i]),
-            .inst = &inst_specific.base,
-        };
-
-        return decl;
-    }
-
-    fn parseParameterGeneric(self: *Parser, comptime T: type, body_ctx: ?*Body) !T {
-        if (@typeInfo(T) == .Enum) {
-            const start = self.i;
-            while (true) : (self.i += 1) switch (self.source[self.i]) {
-                ' ', '\n', ',', ')' => {
-                    const enum_name = self.source[start..self.i];
-                    return std.meta.stringToEnum(T, enum_name) orelse {
-                        return self.fail("tag '{s}' not a member of enum '{s}'", .{ enum_name, @typeName(T) });
-                    };
-                },
-                0 => return self.failByte(0),
-                else => continue,
-            };
-        }
-        switch (T) {
-            Module.Body => return parseBody(self, body_ctx),
-            bool => {
-                const bool_value = switch (self.source[self.i]) {
-                    '0' => false,
-                    '1' => true,
-                    else => |byte| return self.fail("expected '0' or '1' for boolean value, found {c}", .{byte}),
-                };
-                self.i += 1;
-                return bool_value;
-            },
-            []*Inst => {
-                try requireEatBytes(self, "[");
-                skipSpace(self);
-                if (eatByte(self, ']')) return &[0]*Inst{};
-
-                var instructions = std.ArrayList(*Inst).init(&self.arena.allocator);
-                while (true) {
-                    skipSpace(self);
-                    try instructions.append(try parseParameterInst(self, body_ctx));
-                    skipSpace(self);
-                    if (!eatByte(self, ',')) break;
-                }
-                try requireEatBytes(self, "]");
-                return instructions.toOwnedSlice();
-            },
-            *Inst => return parseParameterInst(self, body_ctx),
-            []u8, []const u8 => return self.parseStringLiteral(),
-            BigIntConst => return self.parseIntegerLiteral(),
-            usize => {
-                const big_int = try self.parseIntegerLiteral();
-                return big_int.to(usize) catch |err| return self.fail("integer literal: {s}", .{@errorName(err)});
-            },
-            TypedValue => return self.fail("'const' is a special instruction; not legal in ZIR text", .{}),
-            *IrModule.Decl => return self.fail("'declval_in_module' is a special instruction; not legal in ZIR text", .{}),
-            *Inst.Block => {
-                const name = try self.parseStringLiteral();
-                return self.block_table.get(name).?;
-            },
-            *Inst.Loop => {
-                const name = try self.parseStringLiteral();
-                return self.loop_table.get(name).?;
-            },
-            [][]const u8 => {
-                try requireEatBytes(self, "[");
-                skipSpace(self);
-                if (eatByte(self, ']')) return &[0][]const u8{};
-
-                var strings = std.ArrayList([]const u8).init(&self.arena.allocator);
-                while (true) {
-                    skipSpace(self);
-                    try strings.append(try self.parseStringLiteral());
-                    skipSpace(self);
-                    if (!eatByte(self, ',')) break;
-                }
-                try requireEatBytes(self, "]");
-                return strings.toOwnedSlice();
-            },
-            []Inst.SwitchBr.Case => {
-                try requireEatBytes(self, "{");
-                skipSpace(self);
-                if (eatByte(self, '}')) return &[0]Inst.SwitchBr.Case{};
-
-                var cases = std.ArrayList(Inst.SwitchBr.Case).init(&self.arena.allocator);
-                while (true) {
-                    const cur = try cases.addOne();
-                    skipSpace(self);
-                    cur.item = try self.parseParameterGeneric(*Inst, body_ctx);
-                    skipSpace(self);
-                    try requireEatBytes(self, "=>");
-                    cur.body = try self.parseBody(body_ctx);
-                    skipSpace(self);
-                    if (!eatByte(self, ',')) break;
-                }
-                skipSpace(self);
-                try requireEatBytes(self, "}");
-                return cases.toOwnedSlice();
-            },
-            else => @compileError("Unimplemented: ir parseParameterGeneric for type " ++ @typeName(T)),
-        }
-        return self.fail("TODO parse parameter {s}", .{@typeName(T)});
-    }
-
-    fn parseParameterInst(self: *Parser, body_ctx: ?*Body) !*Inst {
-        const local_ref = switch (self.source[self.i]) {
-            '@' => false,
-            '%' => true,
-            else => |byte| return self.fail("unexpected byte: '{c}'", .{byte}),
-        };
-        const map = if (local_ref)
-            if (body_ctx) |bc|
-                bc.name_map
-            else
-                return self.fail("referencing a % instruction in global scope", .{})
-        else
-            self.global_name_map;
-
-        self.i += 1;
-        const name_start = self.i;
-        while (true) : (self.i += 1) switch (self.source[self.i]) {
-            0, ' ', '\n', ',', ')', ']' => break,
-            else => continue,
-        };
-        const ident = self.source[name_start..self.i];
-        return map.get(ident) orelse {
-            const bad_name = self.source[name_start - 1 .. self.i];
-            const src = name_start - 1;
-            if (local_ref) {
-                self.i = src;
-                return self.fail("unrecognized identifier: {s}", .{bad_name});
-            } else {
-                const declval = try self.arena.allocator.create(Inst.DeclVal);
-                declval.* = .{
-                    .base = .{
-                        .src = src,
-                        .tag = Inst.DeclVal.base_tag,
-                    },
-                    .positionals = .{ .name = ident },
-                    .kw_args = .{},
-                };
-                return &declval.base;
-            }
-        };
-    }
-
-    fn generateName(self: *Parser) ![]u8 {
-        const result = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${d}", .{self.unnamed_index});
-        self.unnamed_index += 1;
-        return result;
-    }
-};
-
-pub fn emit(allocator: *Allocator, old_module: *IrModule) !Module {
-    var ctx: EmitZIR = .{
-        .allocator = allocator,
-        .decls = .{},
-        .arena = std.heap.ArenaAllocator.init(allocator),
-        .old_module = old_module,
-        .next_auto_name = 0,
-        .names = std.StringArrayHashMap(void).init(allocator),
-        .primitive_table = std.AutoHashMap(Inst.Primitive.Builtin, *Decl).init(allocator),
-        .indent = 0,
-        .block_table = std.AutoHashMap(*ir.Inst.Block, *Inst.Block).init(allocator),
-        .loop_table = std.AutoHashMap(*ir.Inst.Loop, *Inst.Loop).init(allocator),
-        .metadata = std.AutoHashMap(*Inst, Module.MetaData).init(allocator),
-        .body_metadata = std.AutoHashMap(*Module.Body, Module.BodyMetaData).init(allocator),
-    };
-    errdefer ctx.metadata.deinit();
-    errdefer ctx.body_metadata.deinit();
-    defer ctx.block_table.deinit();
-    defer ctx.loop_table.deinit();
-    defer ctx.decls.deinit(allocator);
-    defer ctx.names.deinit();
-    defer ctx.primitive_table.deinit();
-    errdefer ctx.arena.deinit();
-
-    try ctx.emit();
-
-    return Module{
-        .decls = ctx.decls.toOwnedSlice(allocator),
-        .arena = ctx.arena,
-        .metadata = ctx.metadata,
-        .body_metadata = ctx.body_metadata,
-    };
-}
-
 /// For debugging purposes, prints a function representation to stderr.
 pub fn dumpFn(old_module: IrModule, module_fn: *IrModule.Fn) void {
     const allocator = old_module.gpa;
@@ -2374,1052 +1856,14 @@ const DumpTzir = struct {
     }
 };
 
-const EmitZIR = struct {
-    allocator: *Allocator,
-    arena: std.heap.ArenaAllocator,
-    old_module: *const IrModule,
-    decls: std.ArrayListUnmanaged(*Decl),
-    names: std.StringArrayHashMap(void),
-    next_auto_name: usize,
-    primitive_table: std.AutoHashMap(Inst.Primitive.Builtin, *Decl),
-    indent: usize,
-    block_table: std.AutoHashMap(*ir.Inst.Block, *Inst.Block),
-    loop_table: std.AutoHashMap(*ir.Inst.Loop, *Inst.Loop),
-    metadata: std.AutoHashMap(*Inst, Module.MetaData),
-    body_metadata: std.AutoHashMap(*Module.Body, Module.BodyMetaData),
-
-    fn emit(self: *EmitZIR) !void {
-        // Put all the Decls in a list and sort them by name to avoid nondeterminism introduced
-        // by the hash table.
-        var src_decls = std.ArrayList(*IrModule.Decl).init(self.allocator);
-        defer src_decls.deinit();
-        try src_decls.ensureCapacity(self.old_module.decl_table.items().len);
-        try self.decls.ensureCapacity(self.allocator, self.old_module.decl_table.items().len);
-        try self.names.ensureCapacity(self.old_module.decl_table.items().len);
-
-        for (self.old_module.decl_table.items()) |entry| {
-            const decl = entry.value;
-            src_decls.appendAssumeCapacity(decl);
-            self.names.putAssumeCapacityNoClobber(mem.spanZ(decl.name), {});
-        }
-        std.sort.sort(*IrModule.Decl, src_decls.items, {}, (struct {
-            fn lessThan(context: void, a: *IrModule.Decl, b: *IrModule.Decl) bool {
-                return a.src_index < b.src_index;
-            }
-        }).lessThan);
-
-        // Emit all the decls.
-        for (src_decls.items) |ir_decl| {
-            switch (ir_decl.analysis) {
-                .unreferenced => continue,
-
-                .complete => {},
-                .codegen_failure => {}, // We still can emit the ZIR.
-                .codegen_failure_retryable => {}, // We still can emit the ZIR.
-
-                .in_progress => unreachable,
-                .outdated => unreachable,
-
-                .sema_failure,
-                .sema_failure_retryable,
-                .dependency_failure,
-                => if (self.old_module.failed_decls.get(ir_decl)) |err_msg| {
-                    const fail_inst = try self.arena.allocator.create(Inst.UnOp);
-                    fail_inst.* = .{
-                        .base = .{
-                            .src = ir_decl.src(),
-                            .tag = .compileerror,
-                        },
-                        .positionals = .{
-                            .operand = blk: {
-                                const msg_str = try self.arena.allocator.dupe(u8, err_msg.msg);
-
-                                const str_inst = try self.arena.allocator.create(Inst.Str);
-                                str_inst.* = .{
-                                    .base = .{
-                                        .src = ir_decl.src(),
-                                        .tag = Inst.Str.base_tag,
-                                    },
-                                    .positionals = .{
-                                        .bytes = err_msg.msg,
-                                    },
-                                    .kw_args = .{},
-                                };
-                                break :blk &str_inst.base;
-                            },
-                        },
-                        .kw_args = .{},
-                    };
-                    const decl = try self.arena.allocator.create(Decl);
-                    decl.* = .{
-                        .name = mem.spanZ(ir_decl.name),
-                        .contents_hash = undefined,
-                        .inst = &fail_inst.base,
-                    };
-                    try self.decls.append(self.allocator, decl);
-                    continue;
-                },
-            }
-            if (self.old_module.export_owners.get(ir_decl)) |exports| {
-                for (exports) |module_export| {
-                    const symbol_name = try self.emitStringLiteral(module_export.src, module_export.options.name);
-                    const export_inst = try self.arena.allocator.create(Inst.Export);
-                    export_inst.* = .{
-                        .base = .{
-                            .src = module_export.src,
-                            .tag = Inst.Export.base_tag,
-                        },
-                        .positionals = .{
-                            .symbol_name = symbol_name.inst,
-                            .decl_name = mem.spanZ(module_export.exported_decl.name),
-                        },
-                        .kw_args = .{},
-                    };
-                    _ = try self.emitUnnamedDecl(&export_inst.base);
-                }
-            }
-            const new_decl = try self.emitTypedValue(ir_decl.src(), ir_decl.typed_value.most_recent.typed_value);
-            new_decl.name = try self.arena.allocator.dupe(u8, mem.spanZ(ir_decl.name));
-        }
-    }
-
-    const ZirBody = struct {
-        inst_table: *std.AutoHashMap(*ir.Inst, *Inst),
-        instructions: *std.ArrayList(*Inst),
-    };
-
-    fn resolveInst(self: *EmitZIR, new_body: ZirBody, inst: *ir.Inst) !*Inst {
-        if (inst.cast(ir.Inst.Constant)) |const_inst| {
-            const new_inst = if (const_inst.val.castTag(.function)) |func_pl| blk: {
-                const owner_decl = func_pl.data.owner_decl;
-                break :blk try self.emitDeclVal(inst.src, mem.spanZ(owner_decl.name));
-            } else if (const_inst.val.castTag(.decl_ref)) |declref| blk: {
-                const decl_ref = try self.emitDeclRef(inst.src, declref.data);
-                try new_body.instructions.append(decl_ref);
-                break :blk decl_ref;
-            } else if (const_inst.val.castTag(.variable)) |var_pl| blk: {
-                const owner_decl = var_pl.data.owner_decl;
-                break :blk try self.emitDeclVal(inst.src, mem.spanZ(owner_decl.name));
-            } else blk: {
-                break :blk (try self.emitTypedValue(inst.src, .{ .ty = inst.ty, .val = const_inst.val })).inst;
-            };
-            _ = try new_body.inst_table.put(inst, new_inst);
-            return new_inst;
-        } else {
-            return new_body.inst_table.get(inst).?;
-        }
-    }
-
-    fn emitDeclVal(self: *EmitZIR, src: usize, decl_name: []const u8) !*Inst {
-        const declval = try self.arena.allocator.create(Inst.DeclVal);
-        declval.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.DeclVal.base_tag,
-            },
-            .positionals = .{ .name = try self.arena.allocator.dupe(u8, decl_name) },
-            .kw_args = .{},
-        };
-        return &declval.base;
-    }
-
-    fn emitComptimeIntVal(self: *EmitZIR, src: usize, val: Value) !*Decl {
-        const big_int_space = try self.arena.allocator.create(Value.BigIntSpace);
-        const int_inst = try self.arena.allocator.create(Inst.Int);
-        int_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.Int.base_tag,
-            },
-            .positionals = .{
-                .int = val.toBigInt(big_int_space),
-            },
-            .kw_args = .{},
-        };
-        return self.emitUnnamedDecl(&int_inst.base);
-    }
-
-    fn emitDeclRef(self: *EmitZIR, src: usize, module_decl: *IrModule.Decl) !*Inst {
-        const declref_inst = try self.arena.allocator.create(Inst.DeclRef);
-        declref_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.DeclRef.base_tag,
-            },
-            .positionals = .{
-                .name = mem.spanZ(module_decl.name),
-            },
-            .kw_args = .{},
-        };
-        return &declref_inst.base;
-    }
-
-    fn emitFn(self: *EmitZIR, module_fn: *IrModule.Fn, src: usize, ty: Type) Allocator.Error!*Decl {
-        var inst_table = std.AutoHashMap(*ir.Inst, *Inst).init(self.allocator);
-        defer inst_table.deinit();
-
-        var instructions = std.ArrayList(*Inst).init(self.allocator);
-        defer instructions.deinit();
-
-        switch (module_fn.state) {
-            .queued => unreachable,
-            .in_progress => unreachable,
-            .inline_only => unreachable,
-            .success => {
-                try self.emitBody(module_fn.body, &inst_table, &instructions);
-            },
-            .sema_failure => {
-                const err_msg = self.old_module.failed_decls.get(module_fn.owner_decl).?;
-                const fail_inst = try self.arena.allocator.create(Inst.UnOp);
-                fail_inst.* = .{
-                    .base = .{
-                        .src = src,
-                        .tag = .compileerror,
-                    },
-                    .positionals = .{
-                        .operand = blk: {
-                            const msg_str = try self.arena.allocator.dupe(u8, err_msg.msg);
-
-                            const str_inst = try self.arena.allocator.create(Inst.Str);
-                            str_inst.* = .{
-                                .base = .{
-                                    .src = src,
-                                    .tag = Inst.Str.base_tag,
-                                },
-                                .positionals = .{
-                                    .bytes = msg_str,
-                                },
-                                .kw_args = .{},
-                            };
-                            break :blk &str_inst.base;
-                        },
-                    },
-                    .kw_args = .{},
-                };
-                try instructions.append(&fail_inst.base);
-            },
-            .dependency_failure => {
-                const fail_inst = try self.arena.allocator.create(Inst.UnOp);
-                fail_inst.* = .{
-                    .base = .{
-                        .src = src,
-                        .tag = .compileerror,
-                    },
-                    .positionals = .{
-                        .operand = blk: {
-                            const msg_str = try self.arena.allocator.dupe(u8, "depends on another failed Decl");
-
-                            const str_inst = try self.arena.allocator.create(Inst.Str);
-                            str_inst.* = .{
-                                .base = .{
-                                    .src = src,
-                                    .tag = Inst.Str.base_tag,
-                                },
-                                .positionals = .{
-                                    .bytes = msg_str,
-                                },
-                                .kw_args = .{},
-                            };
-                            break :blk &str_inst.base;
-                        },
-                    },
-                    .kw_args = .{},
-                };
-                try instructions.append(&fail_inst.base);
-            },
-        }
-
-        const fn_type = try self.emitType(src, ty);
-
-        const arena_instrs = try self.arena.allocator.alloc(*Inst, instructions.items.len);
-        mem.copy(*Inst, arena_instrs, instructions.items);
-
-        const fn_inst = try self.arena.allocator.create(Inst.Fn);
-        fn_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.Fn.base_tag,
-            },
-            .positionals = .{
-                .fn_type = fn_type.inst,
-                .body = .{ .instructions = arena_instrs },
-            },
-            .kw_args = .{
-                .is_inline = module_fn.state == .inline_only,
-            },
-        };
-        return self.emitUnnamedDecl(&fn_inst.base);
-    }
-
-    fn emitTypedValue(self: *EmitZIR, src: usize, typed_value: TypedValue) Allocator.Error!*Decl {
-        const allocator = &self.arena.allocator;
-        if (typed_value.val.castTag(.decl_ref)) |decl_ref| {
-            const decl = decl_ref.data;
-            return try self.emitUnnamedDecl(try self.emitDeclRef(src, decl));
-        } else if (typed_value.val.castTag(.variable)) |variable| {
-            return self.emitTypedValue(src, .{
-                .ty = typed_value.ty,
-                .val = variable.data.init,
-            });
-        }
-        if (typed_value.val.isUndef()) {
-            const as_inst = try self.arena.allocator.create(Inst.BinOp);
-            as_inst.* = .{
-                .base = .{
-                    .tag = .as,
-                    .src = src,
-                },
-                .positionals = .{
-                    .lhs = (try self.emitType(src, typed_value.ty)).inst,
-                    .rhs = (try self.emitPrimitive(src, .@"undefined")).inst,
-                },
-                .kw_args = .{},
-            };
-            return self.emitUnnamedDecl(&as_inst.base);
-        }
-        switch (typed_value.ty.zigTypeTag()) {
-            .Pointer => {
-                const ptr_elem_type = typed_value.ty.elemType();
-                switch (ptr_elem_type.zigTypeTag()) {
-                    .Array => {
-                        // TODO more checks to make sure this can be emitted as a string literal
-                        //const array_elem_type = ptr_elem_type.elemType();
-                        //if (array_elem_type.eql(Type.initTag(.u8)) and
-                        //    ptr_elem_type.hasSentinel(Value.initTag(.zero)))
-                        //{
-                        //}
-                        const bytes = typed_value.val.toAllocatedBytes(allocator) catch |err| switch (err) {
-                            error.AnalysisFail => unreachable,
-                            else => |e| return e,
-                        };
-                        return self.emitStringLiteral(src, bytes);
-                    },
-                    else => |t| std.debug.panic("TODO implement emitTypedValue for pointer to {s}", .{@tagName(t)}),
-                }
-            },
-            .ComptimeInt => return self.emitComptimeIntVal(src, typed_value.val),
-            .Int => {
-                const as_inst = try self.arena.allocator.create(Inst.BinOp);
-                as_inst.* = .{
-                    .base = .{
-                        .tag = .as,
-                        .src = src,
-                    },
-                    .positionals = .{
-                        .lhs = (try self.emitType(src, typed_value.ty)).inst,
-                        .rhs = (try self.emitComptimeIntVal(src, typed_value.val)).inst,
-                    },
-                    .kw_args = .{},
-                };
-                return self.emitUnnamedDecl(&as_inst.base);
-            },
-            .Type => {
-                const ty = try typed_value.val.toType(&self.arena.allocator);
-                return self.emitType(src, ty);
-            },
-            .Fn => {
-                const module_fn = typed_value.val.castTag(.function).?.data;
-                return self.emitFn(module_fn, src, typed_value.ty);
-            },
-            .Array => {
-                // TODO more checks to make sure this can be emitted as a string literal
-                //const array_elem_type = ptr_elem_type.elemType();
-                //if (array_elem_type.eql(Type.initTag(.u8)) and
-                //    ptr_elem_type.hasSentinel(Value.initTag(.zero)))
-                //{
-                //}
-                const bytes = typed_value.val.toAllocatedBytes(allocator) catch |err| switch (err) {
-                    error.AnalysisFail => unreachable,
-                    else => |e| return e,
-                };
-                const str_inst = try self.arena.allocator.create(Inst.Str);
-                str_inst.* = .{
-                    .base = .{
-                        .src = src,
-                        .tag = Inst.Str.base_tag,
-                    },
-                    .positionals = .{
-                        .bytes = bytes,
-                    },
-                    .kw_args = .{},
-                };
-                return self.emitUnnamedDecl(&str_inst.base);
-            },
-            .Void => return self.emitPrimitive(src, .void_value),
-            .Bool => if (typed_value.val.toBool())
-                return self.emitPrimitive(src, .@"true")
-            else
-                return self.emitPrimitive(src, .@"false"),
-            .EnumLiteral => {
-                const enum_literal = typed_value.val.castTag(.enum_literal).?;
-                const inst = try self.arena.allocator.create(Inst.Str);
-                inst.* = .{
-                    .base = .{
-                        .src = src,
-                        .tag = .enum_literal,
-                    },
-                    .positionals = .{
-                        .bytes = enum_literal.data,
-                    },
-                    .kw_args = .{},
-                };
-                return self.emitUnnamedDecl(&inst.base);
-            },
-            else => |t| std.debug.panic("TODO implement emitTypedValue for {s}", .{@tagName(t)}),
-        }
-    }
-
-    fn emitNoOp(self: *EmitZIR, src: usize, old_inst: *ir.Inst.NoOp, tag: Inst.Tag) Allocator.Error!*Inst {
-        const new_inst = try self.arena.allocator.create(Inst.NoOp);
-        new_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = tag,
-            },
-            .positionals = .{},
-            .kw_args = .{},
-        };
-        return &new_inst.base;
-    }
-
-    fn emitUnOp(
-        self: *EmitZIR,
-        src: usize,
-        new_body: ZirBody,
-        old_inst: *ir.Inst.UnOp,
-        tag: Inst.Tag,
-    ) Allocator.Error!*Inst {
-        const new_inst = try self.arena.allocator.create(Inst.UnOp);
-        new_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = tag,
-            },
-            .positionals = .{
-                .operand = try self.resolveInst(new_body, old_inst.operand),
-            },
-            .kw_args = .{},
-        };
-        return &new_inst.base;
-    }
-
-    fn emitBinOp(
-        self: *EmitZIR,
-        src: usize,
-        new_body: ZirBody,
-        old_inst: *ir.Inst.BinOp,
-        tag: Inst.Tag,
-    ) Allocator.Error!*Inst {
-        const new_inst = try self.arena.allocator.create(Inst.BinOp);
-        new_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = tag,
-            },
-            .positionals = .{
-                .lhs = try self.resolveInst(new_body, old_inst.lhs),
-                .rhs = try self.resolveInst(new_body, old_inst.rhs),
-            },
-            .kw_args = .{},
-        };
-        return &new_inst.base;
-    }
-
-    fn emitCast(
-        self: *EmitZIR,
-        src: usize,
-        new_body: ZirBody,
-        old_inst: *ir.Inst.UnOp,
-        tag: Inst.Tag,
-    ) Allocator.Error!*Inst {
-        const new_inst = try self.arena.allocator.create(Inst.BinOp);
-        new_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = tag,
-            },
-            .positionals = .{
-                .lhs = (try self.emitType(old_inst.base.src, old_inst.base.ty)).inst,
-                .rhs = try self.resolveInst(new_body, old_inst.operand),
-            },
-            .kw_args = .{},
-        };
-        return &new_inst.base;
-    }
-
-    fn emitBody(
-        self: *EmitZIR,
-        body: ir.Body,
-        inst_table: *std.AutoHashMap(*ir.Inst, *Inst),
-        instructions: *std.ArrayList(*Inst),
-    ) Allocator.Error!void {
-        const new_body = ZirBody{
-            .inst_table = inst_table,
-            .instructions = instructions,
-        };
-        for (body.instructions) |inst| {
-            const new_inst = switch (inst.tag) {
-                .constant => unreachable, // excluded from function bodies
-
-                .breakpoint => try self.emitNoOp(inst.src, inst.castTag(.breakpoint).?, .breakpoint),
-                .unreach => try self.emitNoOp(inst.src, inst.castTag(.unreach).?, .unreach_nocheck),
-                .retvoid => try self.emitNoOp(inst.src, inst.castTag(.retvoid).?, .returnvoid),
-                .dbg_stmt => try self.emitNoOp(inst.src, inst.castTag(.dbg_stmt).?, .dbg_stmt),
-
-                .not => try self.emitUnOp(inst.src, new_body, inst.castTag(.not).?, .boolnot),
-                .ret => try self.emitUnOp(inst.src, new_body, inst.castTag(.ret).?, .@"return"),
-                .ptrtoint => try self.emitUnOp(inst.src, new_body, inst.castTag(.ptrtoint).?, .ptrtoint),
-                .isnull => try self.emitUnOp(inst.src, new_body, inst.castTag(.isnull).?, .isnull),
-                .isnonnull => try self.emitUnOp(inst.src, new_body, inst.castTag(.isnonnull).?, .isnonnull),
-                .iserr => try self.emitUnOp(inst.src, new_body, inst.castTag(.iserr).?, .iserr),
-                .load => try self.emitUnOp(inst.src, new_body, inst.castTag(.load).?, .deref),
-                .ref => try self.emitUnOp(inst.src, new_body, inst.castTag(.ref).?, .ref),
-                .unwrap_optional => try self.emitUnOp(inst.src, new_body, inst.castTag(.unwrap_optional).?, .unwrap_optional_unsafe),
-                .wrap_optional => try self.emitCast(inst.src, new_body, inst.castTag(.wrap_optional).?, .as),
-
-                .add => try self.emitBinOp(inst.src, new_body, inst.castTag(.add).?, .add),
-                .sub => try self.emitBinOp(inst.src, new_body, inst.castTag(.sub).?, .sub),
-                .store => try self.emitBinOp(inst.src, new_body, inst.castTag(.store).?, .store),
-                .cmp_lt => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_lt).?, .cmp_lt),
-                .cmp_lte => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_lte).?, .cmp_lte),
-                .cmp_eq => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_eq).?, .cmp_eq),
-                .cmp_gte => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_gte).?, .cmp_gte),
-                .cmp_gt => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_gt).?, .cmp_gt),
-                .cmp_neq => try self.emitBinOp(inst.src, new_body, inst.castTag(.cmp_neq).?, .cmp_neq),
-                .booland => try self.emitBinOp(inst.src, new_body, inst.castTag(.booland).?, .booland),
-                .boolor => try self.emitBinOp(inst.src, new_body, inst.castTag(.boolor).?, .boolor),
-                .bitand => try self.emitBinOp(inst.src, new_body, inst.castTag(.bitand).?, .bitand),
-                .bitor => try self.emitBinOp(inst.src, new_body, inst.castTag(.bitor).?, .bitor),
-                .xor => try self.emitBinOp(inst.src, new_body, inst.castTag(.xor).?, .xor),
-
-                .bitcast => try self.emitCast(inst.src, new_body, inst.castTag(.bitcast).?, .bitcast),
-                .intcast => try self.emitCast(inst.src, new_body, inst.castTag(.intcast).?, .intcast),
-                .floatcast => try self.emitCast(inst.src, new_body, inst.castTag(.floatcast).?, .floatcast),
-
-                .alloc => blk: {
-                    const new_inst = try self.arena.allocator.create(Inst.UnOp);
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = .alloc,
-                        },
-                        .positionals = .{
-                            .operand = (try self.emitType(inst.src, inst.ty)).inst,
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .arg => blk: {
-                    const old_inst = inst.castTag(.arg).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Arg);
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = .arg,
-                        },
-                        .positionals = .{
-                            .name = try self.arena.allocator.dupe(u8, mem.spanZ(old_inst.name)),
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .block => blk: {
-                    const old_inst = inst.castTag(.block).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Block);
-
-                    try self.block_table.put(old_inst, new_inst);
-
-                    var block_body = std.ArrayList(*Inst).init(self.allocator);
-                    defer block_body.deinit();
-
-                    try self.emitBody(old_inst.body, inst_table, &block_body);
-
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.Block.base_tag,
-                        },
-                        .positionals = .{
-                            .body = .{ .instructions = block_body.toOwnedSlice() },
-                        },
-                        .kw_args = .{},
-                    };
-
-                    break :blk &new_inst.base;
-                },
-
-                .loop => blk: {
-                    const old_inst = inst.castTag(.loop).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Loop);
-
-                    try self.loop_table.put(old_inst, new_inst);
-
-                    var loop_body = std.ArrayList(*Inst).init(self.allocator);
-                    defer loop_body.deinit();
-
-                    try self.emitBody(old_inst.body, inst_table, &loop_body);
-
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.Loop.base_tag,
-                        },
-                        .positionals = .{
-                            .body = .{ .instructions = loop_body.toOwnedSlice() },
-                        },
-                        .kw_args = .{},
-                    };
-
-                    break :blk &new_inst.base;
-                },
-
-                .brvoid => blk: {
-                    const old_inst = inst.cast(ir.Inst.BrVoid).?;
-                    const new_block = self.block_table.get(old_inst.block).?;
-                    const new_inst = try self.arena.allocator.create(Inst.BreakVoid);
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.BreakVoid.base_tag,
-                        },
-                        .positionals = .{
-                            .block = new_block,
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .br => blk: {
-                    const old_inst = inst.castTag(.br).?;
-                    const new_block = self.block_table.get(old_inst.block).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Break);
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.Break.base_tag,
-                        },
-                        .positionals = .{
-                            .block = new_block,
-                            .operand = try self.resolveInst(new_body, old_inst.operand),
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .call => blk: {
-                    const old_inst = inst.castTag(.call).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Call);
-
-                    const args = try self.arena.allocator.alloc(*Inst, old_inst.args.len);
-                    for (args) |*elem, i| {
-                        elem.* = try self.resolveInst(new_body, old_inst.args[i]);
-                    }
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.Call.base_tag,
-                        },
-                        .positionals = .{
-                            .func = try self.resolveInst(new_body, old_inst.func),
-                            .args = args,
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .assembly => blk: {
-                    const old_inst = inst.castTag(.assembly).?;
-                    const new_inst = try self.arena.allocator.create(Inst.Asm);
-
-                    const inputs = try self.arena.allocator.alloc(*Inst, old_inst.inputs.len);
-                    for (inputs) |*elem, i| {
-                        elem.* = (try self.emitStringLiteral(inst.src, old_inst.inputs[i])).inst;
-                    }
-
-                    const clobbers = try self.arena.allocator.alloc(*Inst, old_inst.clobbers.len);
-                    for (clobbers) |*elem, i| {
-                        elem.* = (try self.emitStringLiteral(inst.src, old_inst.clobbers[i])).inst;
-                    }
-
-                    const args = try self.arena.allocator.alloc(*Inst, old_inst.args.len);
-                    for (args) |*elem, i| {
-                        elem.* = try self.resolveInst(new_body, old_inst.args[i]);
-                    }
-
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.Asm.base_tag,
-                        },
-                        .positionals = .{
-                            .asm_source = (try self.emitStringLiteral(inst.src, old_inst.asm_source)).inst,
-                            .return_type = (try self.emitType(inst.src, inst.ty)).inst,
-                        },
-                        .kw_args = .{
-                            .@"volatile" = old_inst.is_volatile,
-                            .output = if (old_inst.output) |o|
-                                (try self.emitStringLiteral(inst.src, o)).inst
-                            else
-                                null,
-                            .inputs = inputs,
-                            .clobbers = clobbers,
-                            .args = args,
-                        },
-                    };
-                    break :blk &new_inst.base;
-                },
-
-                .condbr => blk: {
-                    const old_inst = inst.castTag(.condbr).?;
-
-                    var then_body = std.ArrayList(*Inst).init(self.allocator);
-                    var else_body = std.ArrayList(*Inst).init(self.allocator);
-
-                    defer then_body.deinit();
-                    defer else_body.deinit();
-
-                    const then_deaths = try self.arena.allocator.alloc(*Inst, old_inst.thenDeaths().len);
-                    const else_deaths = try self.arena.allocator.alloc(*Inst, old_inst.elseDeaths().len);
-
-                    for (old_inst.thenDeaths()) |death, i| {
-                        then_deaths[i] = try self.resolveInst(new_body, death);
-                    }
-                    for (old_inst.elseDeaths()) |death, i| {
-                        else_deaths[i] = try self.resolveInst(new_body, death);
-                    }
-
-                    try self.emitBody(old_inst.then_body, inst_table, &then_body);
-                    try self.emitBody(old_inst.else_body, inst_table, &else_body);
-
-                    const new_inst = try self.arena.allocator.create(Inst.CondBr);
-
-                    try self.body_metadata.put(&new_inst.positionals.then_body, .{ .deaths = then_deaths });
-                    try self.body_metadata.put(&new_inst.positionals.else_body, .{ .deaths = else_deaths });
-
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.CondBr.base_tag,
-                        },
-                        .positionals = .{
-                            .condition = try self.resolveInst(new_body, old_inst.condition),
-                            .then_body = .{ .instructions = then_body.toOwnedSlice() },
-                            .else_body = .{ .instructions = else_body.toOwnedSlice() },
-                        },
-                        .kw_args = .{},
-                    };
-                    break :blk &new_inst.base;
-                },
-                .switchbr => blk: {
-                    const old_inst = inst.castTag(.switchbr).?;
-                    const cases = try self.arena.allocator.alloc(Inst.SwitchBr.Case, old_inst.cases.len);
-                    const new_inst = try self.arena.allocator.create(Inst.SwitchBr);
-                    new_inst.* = .{
-                        .base = .{
-                            .src = inst.src,
-                            .tag = Inst.SwitchBr.base_tag,
-                        },
-                        .positionals = .{
-                            .target_ptr = try self.resolveInst(new_body, old_inst.target_ptr),
-                            .cases = cases,
-                            .items = &[_]*Inst{}, // TODO this should actually be populated
-                            .else_body = undefined, // populated below
-                        },
-                        .kw_args = .{},
-                    };
-
-                    var body_tmp = std.ArrayList(*Inst).init(self.allocator);
-                    defer body_tmp.deinit();
-
-                    for (old_inst.cases) |*case, i| {
-                        body_tmp.items.len = 0;
-
-                        const case_deaths = try self.arena.allocator.alloc(*Inst, old_inst.caseDeaths(i).len);
-                        for (old_inst.caseDeaths(i)) |death, j| {
-                            case_deaths[j] = try self.resolveInst(new_body, death);
-                        }
-                        try self.body_metadata.put(&cases[i].body, .{ .deaths = case_deaths });
-
-                        try self.emitBody(case.body, inst_table, &body_tmp);
-                        const item = (try self.emitTypedValue(inst.src, .{
-                            .ty = old_inst.target_ptr.ty.elemType(),
-                            .val = case.item,
-                        })).inst;
-
-                        cases[i] = .{
-                            .item = item,
-                            .body = .{ .instructions = try self.arena.allocator.dupe(*Inst, body_tmp.items) },
-                        };
-                    }
-                    { // else
-                        const else_deaths = try self.arena.allocator.alloc(*Inst, old_inst.elseDeaths().len);
-                        for (old_inst.elseDeaths()) |death, j| {
-                            else_deaths[j] = try self.resolveInst(new_body, death);
-                        }
-                        try self.body_metadata.put(&new_inst.positionals.else_body, .{ .deaths = else_deaths });
-
-                        body_tmp.items.len = 0;
-                        try self.emitBody(old_inst.else_body, inst_table, &body_tmp);
-                        new_inst.positionals.else_body = .{
-                            .instructions = try self.arena.allocator.dupe(*Inst, body_tmp.items),
-                        };
-                    }
-
-                    break :blk &new_inst.base;
-                },
-                .varptr => @panic("TODO"),
-            };
-            try self.metadata.put(new_inst, .{
-                .deaths = inst.deaths,
-                .addr = @ptrToInt(inst),
-            });
-            try instructions.append(new_inst);
-            try inst_table.put(inst, new_inst);
-        }
-    }
-
-    fn emitType(self: *EmitZIR, src: usize, ty: Type) Allocator.Error!*Decl {
-        switch (ty.tag()) {
-            .i8 => return self.emitPrimitive(src, .i8),
-            .u8 => return self.emitPrimitive(src, .u8),
-            .i16 => return self.emitPrimitive(src, .i16),
-            .u16 => return self.emitPrimitive(src, .u16),
-            .i32 => return self.emitPrimitive(src, .i32),
-            .u32 => return self.emitPrimitive(src, .u32),
-            .i64 => return self.emitPrimitive(src, .i64),
-            .u64 => return self.emitPrimitive(src, .u64),
-            .isize => return self.emitPrimitive(src, .isize),
-            .usize => return self.emitPrimitive(src, .usize),
-            .c_short => return self.emitPrimitive(src, .c_short),
-            .c_ushort => return self.emitPrimitive(src, .c_ushort),
-            .c_int => return self.emitPrimitive(src, .c_int),
-            .c_uint => return self.emitPrimitive(src, .c_uint),
-            .c_long => return self.emitPrimitive(src, .c_long),
-            .c_ulong => return self.emitPrimitive(src, .c_ulong),
-            .c_longlong => return self.emitPrimitive(src, .c_longlong),
-            .c_ulonglong => return self.emitPrimitive(src, .c_ulonglong),
-            .c_longdouble => return self.emitPrimitive(src, .c_longdouble),
-            .c_void => return self.emitPrimitive(src, .c_void),
-            .f16 => return self.emitPrimitive(src, .f16),
-            .f32 => return self.emitPrimitive(src, .f32),
-            .f64 => return self.emitPrimitive(src, .f64),
-            .f128 => return self.emitPrimitive(src, .f128),
-            .anyerror => return self.emitPrimitive(src, .anyerror),
-            else => switch (ty.zigTypeTag()) {
-                .Bool => return self.emitPrimitive(src, .bool),
-                .Void => return self.emitPrimitive(src, .void),
-                .NoReturn => return self.emitPrimitive(src, .noreturn),
-                .Type => return self.emitPrimitive(src, .type),
-                .ComptimeInt => return self.emitPrimitive(src, .comptime_int),
-                .ComptimeFloat => return self.emitPrimitive(src, .comptime_float),
-                .Fn => {
-                    const param_types = try self.allocator.alloc(Type, ty.fnParamLen());
-                    defer self.allocator.free(param_types);
-
-                    ty.fnParamTypes(param_types);
-                    const emitted_params = try self.arena.allocator.alloc(*Inst, param_types.len);
-                    for (param_types) |param_type, i| {
-                        emitted_params[i] = (try self.emitType(src, param_type)).inst;
-                    }
-
-                    const fntype_inst = try self.arena.allocator.create(Inst.FnType);
-                    fntype_inst.* = .{
-                        .base = .{
-                            .src = src,
-                            .tag = Inst.FnType.base_tag,
-                        },
-                        .positionals = .{
-                            .param_types = emitted_params,
-                            .return_type = (try self.emitType(src, ty.fnReturnType())).inst,
-                        },
-                        .kw_args = .{
-                            .cc = ty.fnCallingConvention(),
-                        },
-                    };
-                    return self.emitUnnamedDecl(&fntype_inst.base);
-                },
-                .Int => {
-                    const info = ty.intInfo(self.old_module.getTarget());
-                    const signed = try self.emitPrimitive(src, switch (info.signedness) {
-                        .signed => .@"true",
-                        .unsigned => .@"false",
-                    });
-                    const bits_val = try Value.Tag.int_u64.create(&self.arena.allocator, info.bits);
-                    const bits = try self.emitComptimeIntVal(src, bits_val);
-                    const inttype_inst = try self.arena.allocator.create(Inst.IntType);
-                    inttype_inst.* = .{
-                        .base = .{
-                            .src = src,
-                            .tag = Inst.IntType.base_tag,
-                        },
-                        .positionals = .{
-                            .signed = signed.inst,
-                            .bits = bits.inst,
-                        },
-                        .kw_args = .{},
-                    };
-                    return self.emitUnnamedDecl(&inttype_inst.base);
-                },
-                .Pointer => {
-                    if (ty.isSinglePointer()) {
-                        const inst = try self.arena.allocator.create(Inst.UnOp);
-                        const tag: Inst.Tag = if (ty.isConstPtr()) .single_const_ptr_type else .single_mut_ptr_type;
-                        inst.* = .{
-                            .base = .{
-                                .src = src,
-                                .tag = tag,
-                            },
-                            .positionals = .{
-                                .operand = (try self.emitType(src, ty.elemType())).inst,
-                            },
-                            .kw_args = .{},
-                        };
-                        return self.emitUnnamedDecl(&inst.base);
-                    } else {
-                        std.debug.panic("TODO implement emitType for {}", .{ty});
-                    }
-                },
-                .Optional => {
-                    var buf: Type.Payload.ElemType = undefined;
-                    const inst = try self.arena.allocator.create(Inst.UnOp);
-                    inst.* = .{
-                        .base = .{
-                            .src = src,
-                            .tag = .optional_type,
-                        },
-                        .positionals = .{
-                            .operand = (try self.emitType(src, ty.optionalChild(&buf))).inst,
-                        },
-                        .kw_args = .{},
-                    };
-                    return self.emitUnnamedDecl(&inst.base);
-                },
-                .Array => {
-                    var len_pl = Value.Payload.U64{
-                        .base = .{ .tag = .int_u64 },
-                        .data = ty.arrayLen(),
-                    };
-                    const len = Value.initPayload(&len_pl.base);
-
-                    const inst = if (ty.sentinel()) |sentinel| blk: {
-                        const inst = try self.arena.allocator.create(Inst.ArrayTypeSentinel);
-                        inst.* = .{
-                            .base = .{
-                                .src = src,
-                                .tag = .array_type,
-                            },
-                            .positionals = .{
-                                .len = (try self.emitTypedValue(src, .{
-                                    .ty = Type.initTag(.usize),
-                                    .val = len,
-                                })).inst,
-                                .sentinel = (try self.emitTypedValue(src, .{
-                                    .ty = ty.elemType(),
-                                    .val = sentinel,
-                                })).inst,
-                                .elem_type = (try self.emitType(src, ty.elemType())).inst,
-                            },
-                            .kw_args = .{},
-                        };
-                        break :blk &inst.base;
-                    } else blk: {
-                        const inst = try self.arena.allocator.create(Inst.BinOp);
-                        inst.* = .{
-                            .base = .{
-                                .src = src,
-                                .tag = .array_type,
-                            },
-                            .positionals = .{
-                                .lhs = (try self.emitTypedValue(src, .{
-                                    .ty = Type.initTag(.usize),
-                                    .val = len,
-                                })).inst,
-                                .rhs = (try self.emitType(src, ty.elemType())).inst,
-                            },
-                            .kw_args = .{},
-                        };
-                        break :blk &inst.base;
-                    };
-                    return self.emitUnnamedDecl(inst);
-                },
-                else => std.debug.panic("TODO implement emitType for {}", .{ty}),
-            },
-        }
-    }
-
-    fn autoName(self: *EmitZIR) ![]u8 {
-        while (true) {
-            const proposed_name = try std.fmt.allocPrint(&self.arena.allocator, "unnamed${d}", .{self.next_auto_name});
-            self.next_auto_name += 1;
-            const gop = try self.names.getOrPut(proposed_name);
-            if (!gop.found_existing) {
-                gop.entry.value = {};
-                return proposed_name;
-            }
-        }
-    }
-
-    fn emitPrimitive(self: *EmitZIR, src: usize, tag: Inst.Primitive.Builtin) !*Decl {
-        const gop = try self.primitive_table.getOrPut(tag);
-        if (!gop.found_existing) {
-            const primitive_inst = try self.arena.allocator.create(Inst.Primitive);
-            primitive_inst.* = .{
-                .base = .{
-                    .src = src,
-                    .tag = Inst.Primitive.base_tag,
-                },
-                .positionals = .{
-                    .tag = tag,
-                },
-                .kw_args = .{},
-            };
-            gop.entry.value = try self.emitUnnamedDecl(&primitive_inst.base);
-        }
-        return gop.entry.value;
-    }
-
-    fn emitStringLiteral(self: *EmitZIR, src: usize, str: []const u8) !*Decl {
-        const str_inst = try self.arena.allocator.create(Inst.Str);
-        str_inst.* = .{
-            .base = .{
-                .src = src,
-                .tag = Inst.Str.base_tag,
-            },
-            .positionals = .{
-                .bytes = str,
-            },
-            .kw_args = .{},
-        };
-        return self.emitUnnamedDecl(&str_inst.base);
-    }
-
-    fn emitUnnamedDecl(self: *EmitZIR, inst: *Inst) !*Decl {
-        const decl = try self.arena.allocator.create(Decl);
-        decl.* = .{
-            .name = try self.autoName(),
-            .contents_hash = undefined,
-            .inst = inst,
-        };
-        try self.decls.append(self.allocator, decl);
-        return decl;
-    }
-};
-
 /// For debugging purposes, like dumpFn but for unanalyzed zir blocks
 pub fn dumpZir(allocator: *Allocator, kind: []const u8, decl_name: [*:0]const u8, instructions: []*Inst) !void {
     var fib = std.heap.FixedBufferAllocator.init(&[_]u8{});
     var module = Module{
-        .decls = &[_]*Decl{},
+        .decls = &[_]*Module.Decl{},
         .arena = std.heap.ArenaAllocator.init(&fib.allocator),
         .metadata = std.AutoHashMap(*Inst, Module.MetaData).init(&fib.allocator),
-        .body_metadata = std.AutoHashMap(*Module.Body, Module.BodyMetaData).init(&fib.allocator),
+        .body_metadata = std.AutoHashMap(*Body, Module.BodyMetaData).init(&fib.allocator),
     };
     var write = Writer{
         .module = &module,
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index 6302ab29e4..36eb5f4239 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -63,7 +63,6 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
         .declref => return analyzeInstDeclRef(mod, scope, old_inst.castTag(.declref).?),
         .declref_str => return analyzeInstDeclRefStr(mod, scope, old_inst.castTag(.declref_str).?),
         .declval => return analyzeInstDeclVal(mod, scope, old_inst.castTag(.declval).?),
-        .declval_in_module => return analyzeInstDeclValInModule(mod, scope, old_inst.castTag(.declval_in_module).?),
         .ensure_result_used => return analyzeInstEnsureResultUsed(mod, scope, old_inst.castTag(.ensure_result_used).?),
         .ensure_result_non_error => return analyzeInstEnsureResultNonError(mod, scope, old_inst.castTag(.ensure_result_non_error).?),
         .ensure_indexable => return analyzeInstEnsureIndexable(mod, scope, old_inst.castTag(.ensure_indexable).?),
@@ -166,7 +165,7 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError!
     }
 }
 
-pub fn analyzeBody(mod: *Module, block: *Scope.Block, body: zir.Module.Body) !void {
+pub fn analyzeBody(mod: *Module, block: *Scope.Block, body: zir.Body) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -183,7 +182,7 @@ pub fn analyzeBodyValueAsType(
     mod: *Module,
     block_scope: *Scope.Block,
     zir_result_inst: *zir.Inst,
-    body: zir.Module.Body,
+    body: zir.Body,
 ) !Type {
     try analyzeBody(mod, block_scope, body);
     const result_inst = block_scope.inst_table.get(zir_result_inst).?;
@@ -191,84 +190,6 @@ pub fn analyzeBodyValueAsType(
     return val.toType(block_scope.base.arena());
 }
 
-pub fn analyzeZirDecl(mod: *Module, decl: *Decl, src_decl: *zir.Decl) InnerError!bool {
-    var decl_scope: Scope.DeclAnalysis = .{
-        .decl = decl,
-        .arena = std.heap.ArenaAllocator.init(mod.gpa),
-    };
-    errdefer decl_scope.arena.deinit();
-
-    decl.analysis = .in_progress;
-
-    const typed_value = try analyzeConstInst(mod, &decl_scope.base, src_decl.inst);
-    const arena_state = try decl_scope.arena.allocator.create(std.heap.ArenaAllocator.State);
-
-    var prev_type_has_bits = false;
-    var type_changed = true;
-
-    if (decl.typedValueManaged()) |tvm| {
-        prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits();
-        type_changed = !tvm.typed_value.ty.eql(typed_value.ty);
-
-        tvm.deinit(mod.gpa);
-    }
-
-    arena_state.* = decl_scope.arena.state;
-    decl.typed_value = .{
-        .most_recent = .{
-            .typed_value = typed_value,
-            .arena = arena_state,
-        },
-    };
-    decl.analysis = .complete;
-    decl.generation = mod.generation;
-    if (typed_value.ty.hasCodeGenBits()) {
-        // We don't fully codegen the decl until later, but we do need to reserve a global
-        // offset table index for it. This allows us to codegen decls out of dependency order,
-        // increasing how many computations can be done in parallel.
-        try mod.comp.bin_file.allocateDeclIndexes(decl);
-        try mod.comp.work_queue.writeItem(.{ .codegen_decl = decl });
-    } else if (prev_type_has_bits) {
-        mod.comp.bin_file.freeDecl(decl);
-    }
-
-    return type_changed;
-}
-
-pub fn resolveZirDecl(mod: *Module, scope: *Scope, src_decl: *zir.Decl) InnerError!*Decl {
-    const zir_module = mod.root_scope.cast(Scope.ZIRModule).?;
-    const entry = zir_module.contents.module.findDecl(src_decl.name).?;
-    return resolveZirDeclHavingIndex(mod, scope, src_decl, entry.index);
-}
-
-fn resolveZirDeclHavingIndex(mod: *Module, scope: *Scope, src_decl: *zir.Decl, src_index: usize) InnerError!*Decl {
-    const name_hash = scope.namespace().fullyQualifiedNameHash(src_decl.name);
-    const decl = mod.decl_table.get(name_hash).?;
-    decl.src_index = src_index;
-    try mod.ensureDeclAnalyzed(decl);
-    return decl;
-}
-
-/// Declares a dependency on the decl.
-fn resolveCompleteZirDecl(mod: *Module, scope: *Scope, src_decl: *zir.Decl) InnerError!*Decl {
-    const decl = try resolveZirDecl(mod, scope, src_decl);
-    switch (decl.analysis) {
-        .unreferenced => unreachable,
-        .in_progress => unreachable,
-        .outdated => unreachable,
-
-        .dependency_failure,
-        .sema_failure,
-        .sema_failure_retryable,
-        .codegen_failure,
-        .codegen_failure_retryable,
-        => return error.AnalysisFail,
-
-        .complete => {},
-    }
-    return decl;
-}
-
 pub fn resolveInst(mod: *Module, scope: *Scope, zir_inst: *zir.Inst) InnerError!*Inst {
     const block = scope.cast(Scope.Block).?;
     return block.inst_table.get(zir_inst).?; // Instruction does not dominate all uses!
@@ -640,22 +561,28 @@ fn analyzeInstCompileError(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) In
 }
 
 fn analyzeInstCompileLog(mod: *Module, scope: *Scope, inst: *zir.Inst.CompileLog) InnerError!*Inst {
-    std.debug.print("| ", .{});
-    for (inst.positionals.to_log) |item, i| {
-        const to_log = try resolveInst(mod, scope, item);
-        if (to_log.value()) |val| {
-            std.debug.print("{}", .{val});
+    var managed = mod.compile_log_text.toManaged(mod.gpa);
+    defer mod.compile_log_text = managed.moveToUnmanaged();
+    const writer = managed.writer();
+
+    for (inst.positionals.to_log) |arg_inst, i| {
+        if (i != 0) try writer.print(", ", .{});
+
+        const arg = try resolveInst(mod, scope, arg_inst);
+        if (arg.value()) |val| {
+            try writer.print("@as({}, {})", .{ arg.ty, val });
         } else {
-            std.debug.print("(runtime value)", .{});
+            try writer.print("@as({}, [runtime value])", .{arg.ty});
         }
-        if (i != inst.positionals.to_log.len - 1) std.debug.print(", ", .{});
     }
-    std.debug.print("\n", .{});
-    if (!inst.kw_args.seen) {
+    try writer.print("\n", .{});
 
-        // so that we do not give multiple compile errors if it gets evaled twice
-        inst.kw_args.seen = true;
-        try mod.failCompileLog(scope, inst.base.src);
+    const gop = try mod.compile_log_decls.getOrPut(mod.gpa, scope.ownerDecl().?);
+    if (!gop.found_existing) {
+        gop.entry.value = .{
+            .file_scope = scope.getFileScope(),
+            .byte_offset = inst.base.src,
+        };
     }
     return mod.constVoid(scope, inst.base.src);
 }
@@ -705,7 +632,8 @@ fn analyzeInstLoop(mod: *Module, scope: *Scope, inst: *zir.Inst.Loop) InnerError
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .inlining = parent_block.inlining,
@@ -732,7 +660,8 @@ fn analyzeInstBlockFlat(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_c
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .label = null,
@@ -744,13 +673,14 @@ fn analyzeInstBlockFlat(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_c
 
     try analyzeBody(mod, &child_block, inst.positionals.body);
 
-    try parent_block.instructions.appendSlice(mod.gpa, child_block.instructions.items);
-
-    // comptime blocks won't generate any runtime values
-    if (child_block.instructions.items.len == 0)
-        return mod.constVoid(scope, inst.base.src);
+    // Move the analyzed instructions into the parent block arena.
+    const copied_instructions = try parent_block.arena.dupe(*Inst, child_block.instructions.items);
+    try parent_block.instructions.appendSlice(mod.gpa, copied_instructions);
 
-    return parent_block.instructions.items[parent_block.instructions.items.len - 1];
+    // The result of a flat block is the last instruction.
+    const zir_inst_list = inst.positionals.body.instructions;
+    const last_zir_inst = zir_inst_list[zir_inst_list.len - 1];
+    return resolveInst(mod, scope, last_zir_inst);
 }
 
 fn analyzeInstBlock(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_comptime: bool) InnerError!*Inst {
@@ -775,7 +705,8 @@ fn analyzeInstBlock(mod: *Module, scope: *Scope, inst: *zir.Inst.Block, is_compt
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         // TODO @as here is working around a stage1 miscompilation bug :(
@@ -890,22 +821,15 @@ fn analyzeInstDeclRefStr(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclRefStr
 fn analyzeInstDeclRef(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclRef) InnerError!*Inst {
     const tracy = trace(@src());
     defer tracy.end();
-    return mod.analyzeDeclRefByName(scope, inst.base.src, inst.positionals.name);
+    return mod.analyzeDeclRef(scope, inst.base.src, inst.positionals.decl);
 }
 
 fn analyzeInstDeclVal(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclVal) InnerError!*Inst {
     const tracy = trace(@src());
     defer tracy.end();
-    const decl = try analyzeDeclVal(mod, scope, inst);
-    const ptr = try mod.analyzeDeclRef(scope, inst.base.src, decl);
-    return mod.analyzeDeref(scope, inst.base.src, ptr, inst.base.src);
-}
-
-fn analyzeInstDeclValInModule(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclValInModule) InnerError!*Inst {
-    const tracy = trace(@src());
-    defer tracy.end();
-    const decl = inst.positionals.decl;
-    return mod.analyzeDeclRef(scope, inst.base.src, decl);
+    const decl_ref = try mod.analyzeDeclRef(scope, inst.base.src, inst.positionals.decl);
+    // TODO look into avoiding the call to analyzeDeref here
+    return mod.analyzeDeref(scope, inst.base.src, decl_ref, inst.base.src);
 }
 
 fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError!*Inst {
@@ -1032,9 +956,8 @@ fn analyzeInstCall(mod: *Module, scope: *Scope, inst: *zir.Inst.Call) InnerError
             .parent = null,
             .inst_table = &inst_table,
             .func = module_fn,
-            // Note that we pass the caller's Decl, not the callee. This causes
-            // compile errors to be attached (correctly) to the caller's Decl.
-            .decl = scope.decl().?,
+            .owner_decl = scope.ownerDecl().?,
+            .src_decl = module_fn.owner_decl,
             .instructions = .{},
             .arena = scope.arena(),
             .label = null,
@@ -1069,7 +992,7 @@ fn analyzeInstFn(mod: *Module, scope: *Scope, fn_inst: *zir.Inst.Fn) InnerError!
         .state = if (fn_inst.kw_args.is_inline) .inline_only else .queued,
         .zir = fn_inst.positionals.body,
         .body = undefined,
-        .owner_decl = scope.decl().?,
+        .owner_decl = scope.ownerDecl().?,
     };
     return mod.constInst(scope, fn_inst.base.src, .{
         .ty = fn_type,
@@ -1391,7 +1314,7 @@ fn analyzeInstFieldPtr(mod: *Module, scope: *Scope, fieldptr: *zir.Inst.FieldPtr
                         return mod.analyzeDeclRef(scope, fieldptr.base.src, decl);
                     }
 
-                    if (&container_scope.file_scope.base == mod.root_scope) {
+                    if (container_scope.file_scope == mod.root_scope) {
                         return mod.fail(scope, fieldptr.base.src, "root source file has no member called '{s}'", .{field_name});
                     } else {
                         return mod.fail(scope, fieldptr.base.src, "container '{}' has no member called '{s}'", .{ child_type, field_name });
@@ -1606,7 +1529,8 @@ fn analyzeInstSwitchBr(mod: *Module, scope: *Scope, inst: *zir.Inst.SwitchBr) In
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .inlining = parent_block.inlining,
@@ -2182,7 +2106,8 @@ fn analyzeInstCondBr(mod: *Module, scope: *Scope, inst: *zir.Inst.CondBr) InnerE
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .inlining = parent_block.inlining,
@@ -2196,7 +2121,8 @@ fn analyzeInstCondBr(mod: *Module, scope: *Scope, inst: *zir.Inst.CondBr) InnerE
         .parent = parent_block,
         .inst_table = parent_block.inst_table,
         .func = parent_block.func,
-        .decl = parent_block.decl,
+        .owner_decl = parent_block.owner_decl,
+        .src_decl = parent_block.src_decl,
         .instructions = .{},
         .arena = parent_block.arena,
         .inlining = parent_block.inlining,
@@ -2294,17 +2220,6 @@ fn analyzeBreak(
     } else unreachable;
 }
 
-fn analyzeDeclVal(mod: *Module, scope: *Scope, inst: *zir.Inst.DeclVal) InnerError!*Decl {
-    const decl_name = inst.positionals.name;
-    const zir_module = scope.namespace().cast(Scope.ZIRModule).?;
-    const src_decl = zir_module.contents.module.findDecl(decl_name) orelse
-        return mod.fail(scope, inst.base.src, "use of undeclared identifier '{s}'", .{decl_name});
-
-    const decl = try resolveCompleteZirDecl(mod, scope, src_decl.decl);
-
-    return decl;
-}
-
 fn analyzeInstSimplePtrType(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp, mutable: bool, size: std.builtin.TypeInfo.Pointer.Size) InnerError!*Inst {
     const tracy = trace(@src());
     defer tracy.end();
diff --git a/test/stage2/test.zig b/test/stage2/test.zig
index f25f07adbf..cb5e9bebbb 100644
--- a/test/stage2/test.zig
+++ b/test/stage2/test.zig
@@ -36,7 +36,7 @@ pub fn addCases(ctx: *TestContext) !void {
     {
         var case = ctx.exe("hello world with updates", linux_x64);
 
-        case.addError("", &[_][]const u8{"no entry point found"});
+        case.addError("", &[_][]const u8{"error: no entry point found"});
 
         // Incorrect return type
         case.addError(
@@ -147,7 +147,7 @@ pub fn addCases(ctx: *TestContext) !void {
 
     {
         var case = ctx.exe("hello world with updates", macosx_x64);
-        case.addError("", &[_][]const u8{"no entry point found"});
+        case.addError("", &[_][]const u8{"error: no entry point found"});
 
         // Incorrect return type
         case.addError(
@@ -1243,24 +1243,46 @@ pub fn addCases(ctx: *TestContext) !void {
             \\}
         , &[_][]const u8{":3:9: error: redefinition of 'testing'"});
     }
-    ctx.compileError("compileLog", linux_x64,
-        \\export fn _start() noreturn {
-        \\  const b = true;
-        \\  var f: u32 = 1;
-        \\  @compileLog(b, 20, f, x);
-        \\  @compileLog(1000);
-        \\  var bruh: usize = true;
-        \\  unreachable;
-        \\}
-        \\fn x() void {}
-    , &[_][]const u8{
-        ":4:3: error: found compile log statement",
-        ":5:3: error: found compile log statement",
-        ":6:21: error: expected usize, found bool",
-    });
-    // TODO if this is here it invalidates the compile error checker:
-    // "| true, 20, (runtime value), (function)"
-    // "| 1000"
+
+    {
+        // TODO make the test harness support checking the compile log output too
+        var case = ctx.obj("@compileLog", linux_x64);
+        // The other compile error prevents emission of a "found compile log" statement.
+        case.addError(
+            \\export fn _start() noreturn {
+            \\    const b = true;
+            \\    var f: u32 = 1;
+            \\    @compileLog(b, 20, f, x);
+            \\    @compileLog(1000);
+            \\    var bruh: usize = true;
+            \\    unreachable;
+            \\}
+            \\export fn other() void {
+            \\    @compileLog(1234);
+            \\}
+            \\fn x() void {}
+        , &[_][]const u8{
+            ":6:23: error: expected usize, found bool",
+        });
+
+        // Now only compile log statements remain. One per Decl.
+        case.addError(
+            \\export fn _start() noreturn {
+            \\    const b = true;
+            \\    var f: u32 = 1;
+            \\    @compileLog(b, 20, f, x);
+            \\    @compileLog(1000);
+            \\    unreachable;
+            \\}
+            \\export fn other() void {
+            \\    @compileLog(1234);
+            \\}
+            \\fn x() void {}
+        , &[_][]const u8{
+            ":11:8: error: found compile log statement",
+            ":4:5: note: also here",
+        });
+    }
 
     {
         var case = ctx.obj("extern variable has no type", linux_x64);
-- 
cgit v1.2.3


From 1af31baf0ba447b6ea5a1456df5ba2d82dc26e56 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 19 Jan 2021 15:49:08 -0700
Subject: stage2: -Dlog enables all logging, log scopes can be set at runtime

Previously you had to recompile if you wanted to change the log scopes
that get printed. Now, log scopes can be set at runtime, and -Dlog
controls whether all logging is available at runtime.

Purpose here is a nicer development experience. Most likely stage2
developers will always want -Dlog enabled and then pass --debug-log
scopes when debugging particular issues.
---
 build.zig           |  4 ++--
 src/Compilation.zig |  3 +++
 src/config.zig.in   |  2 +-
 src/main.zig        | 19 ++++++++++++++++---
 4 files changed, 22 insertions(+), 6 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/build.zig b/build.zig
index 77d1b573fd..f53fab73bf 100644
--- a/build.zig
+++ b/build.zig
@@ -134,7 +134,7 @@ pub fn build(b: *Builder) !void {
         test_stage2.linkLibC();
     }
 
-    const log_scopes = b.option([]const []const u8, "log", "Which log scopes to enable") orelse &[0][]const u8{};
+    const enable_logging = b.option(bool, "log", "Whether to enable logging") orelse false;
 
     const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git.");
     const version = if (opt_version_string) |version| version else v: {
@@ -190,7 +190,7 @@ pub fn build(b: *Builder) !void {
     const semver = try std.SemanticVersion.parse(version);
     exe.addBuildOption(std.SemanticVersion, "semver", semver);
 
-    exe.addBuildOption([]const []const u8, "log_scopes", log_scopes);
+    exe.addBuildOption(bool, "enable_logging", enable_logging);
     exe.addBuildOption(bool, "enable_tracy", tracy != null);
     exe.addBuildOption(bool, "is_stage1", is_stage1);
     exe.addBuildOption(bool, "omit_stage2", false);
diff --git a/src/Compilation.zig b/src/Compilation.zig
index ad99e40541..7700fcdc4a 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1560,6 +1560,9 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     }
                 }
 
+                log.debug("calling updateDecl on '{s}', type={}", .{
+                    decl.name, decl.typed_value.most_recent.typed_value.ty,
+                });
                 assert(decl.typed_value.most_recent.typed_value.ty.hasCodeGenBits());
 
                 self.bin_file.updateDecl(module, decl) catch |err| switch (err) {
diff --git a/src/config.zig.in b/src/config.zig.in
index b672581ea4..1e9d1eecdf 100644
--- a/src/config.zig.in
+++ b/src/config.zig.in
@@ -1,7 +1,7 @@
 pub const have_llvm = true;
 pub const version: [:0]const u8 = "@ZIG_VERSION@";
 pub const semver = try @import("std").SemanticVersion.parse(version);
-pub const log_scopes: []const []const u8 = &[_][]const u8{};
+pub const enable_logging: bool = false;
 pub const enable_tracy = false;
 pub const is_stage1 = true;
 pub const skip_non_native = false;
diff --git a/src/main.zig b/src/main.zig
index 13bea13a5e..8a40c1441c 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -70,20 +70,26 @@ pub const log_level: std.log.Level = switch (std.builtin.mode) {
     .ReleaseSmall => .crit,
 };
 
+var log_scopes: std.ArrayListUnmanaged([]const u8) = .{};
+
 pub fn log(
     comptime level: std.log.Level,
     comptime scope: @TypeOf(.EnumLiteral),
     comptime format: []const u8,
     args: anytype,
 ) void {
-    // Hide debug messages unless added with `-Dlog=foo`.
+    // Hide debug messages unless:
+    // * logging enabled with `-Dlog`.
+    // * the --debug-log arg for the scope has been provided
     if (@enumToInt(level) > @enumToInt(std.log.level) or
         @enumToInt(level) > @enumToInt(std.log.Level.info))
     {
+        if (!build_options.enable_logging) return;
+
         const scope_name = @tagName(scope);
-        const ok = comptime for (build_options.log_scopes) |log_scope| {
+        for (log_scopes.items) |log_scope| {
             if (mem.eql(u8, log_scope, scope_name))
-                break true;
+                break;
         } else return;
     }
 
@@ -156,6 +162,8 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
         }
     }
 
+    defer log_scopes.deinit(gpa);
+
     const cmd = args[1];
     const cmd_args = args[2..];
     if (mem.eql(u8, cmd, "build-exe")) {
@@ -358,6 +366,7 @@ const usage_build_generic =
     \\  --verbose-llvm-ir            Enable compiler debug output for LLVM IR
     \\  --verbose-cimport            Enable compiler debug output for C imports
     \\  --verbose-llvm-cpu-features  Enable compiler debug output for LLVM CPU features
+    \\  --debug-log [scope]          Enable printing debug/info log messages for scope
     \\
 ;
 
@@ -811,6 +820,10 @@ fn buildOutputType(
                         if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
                         i += 1;
                         override_lib_dir = args[i];
+                    } else if (mem.eql(u8, arg, "--debug-log")) {
+                        if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg});
+                        i += 1;
+                        try log_scopes.append(gpa, args[i]);
                     } else if (mem.eql(u8, arg, "-fcompiler-rt")) {
                         want_compiler_rt = true;
                     } else if (mem.eql(u8, arg, "-fno-compiler-rt")) {
-- 
cgit v1.2.3


From 072d1e088c3e383c2303b873f55399d374d9808c Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 19 Jan 2021 16:25:20 -0700
Subject: stage2: fix anonymous Decl ty/val wrong arena

string literals and error set types were allocating the ty/val fields of
the anonymous Decl into the owner Decl's arena, rather than the
new anonymous Decl's arena as intended. This caused use of undefined
value later on in the pipeline.
---
 src/Compilation.zig | 2 +-
 src/zir_sema.zig    | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 7700fcdc4a..fae725a1ac 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1552,7 +1552,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
                     // are lifetime annotations in the ZIR.
                     var decl_arena = decl.typed_value.most_recent.arena.?.promote(module.gpa);
                     defer decl.typed_value.most_recent.arena.?.* = decl_arena.state;
-                    log.debug("analyze liveness of {s}\n", .{decl.name});
+                    log.debug("analyze liveness of {s}", .{decl.name});
                     try liveness.analyze(module.gpa, &decl_arena.allocator, func.body);
 
                     if (std.builtin.mode == .Debug and self.verbose_ir) {
diff --git a/src/zir_sema.zig b/src/zir_sema.zig
index b5b256afd3..0caaa2a03f 100644
--- a/src/zir_sema.zig
+++ b/src/zir_sema.zig
@@ -547,9 +547,12 @@ fn analyzeInstStr(mod: *Module, scope: *Scope, str_inst: *zir.Inst.Str) InnerErr
     errdefer new_decl_arena.deinit();
     const arena_bytes = try new_decl_arena.allocator.dupe(u8, str_inst.positionals.bytes);
 
+    const decl_ty = try Type.Tag.array_u8_sentinel_0.create(&new_decl_arena.allocator, arena_bytes.len);
+    const decl_val = try Value.Tag.bytes.create(&new_decl_arena.allocator, arena_bytes);
+
     const new_decl = try mod.createAnonymousDecl(scope, &new_decl_arena, .{
-        .ty = try Type.Tag.array_u8_sentinel_0.create(scope.arena(), arena_bytes.len),
-        .val = try Value.Tag.bytes.create(scope.arena(), arena_bytes),
+        .ty = decl_ty,
+        .val = decl_val,
     });
     return mod.analyzeDeclRef(scope, str_inst.base.src, new_decl);
 }
@@ -1079,7 +1082,7 @@ fn analyzeInstErrorSet(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorSet) In
     var new_decl_arena = std.heap.ArenaAllocator.init(mod.gpa);
     errdefer new_decl_arena.deinit();
 
-    const payload = try scope.arena().create(Value.Payload.ErrorSet);
+    const payload = try new_decl_arena.allocator.create(Value.Payload.ErrorSet);
     payload.* = .{
         .base = .{ .tag = .error_set },
         .data = .{
-- 
cgit v1.2.3


From 0d4b6ac7417d1094ac972981b0241444ce2380ba Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Fri, 22 Jan 2021 23:36:30 -0700
Subject: add LTO support

The CLI gains -flto and -fno-lto options to override the default.
However, the cool thing about this is that the defaults are great! In
general when you use build-exe in release mode, Zig will enable LTO if
it would work and it would help.

zig cc supports detecting and honoring the -flto and -fno-lto flags as
well. The linkWithLld functions are improved to all be the same with
regards to copying the artifact instead of trying to pass single objects
through LLD with -r. There is possibly a future improvement here as
well; see the respective TODOs.

stage1 is updated to support outputting LLVM bitcode instead of machine
code when lto is enabled. This allows LLVM to optimize across the Zig and
C/C++ code boundary.

closes #2845
---
 src/Compilation.zig            |  37 +++
 src/clang_options_data.zig     |  27 +-
 src/codegen/llvm/bindings.zig  |   8 +-
 src/link.zig                   |   1 +
 src/link/Coff.zig              |   7 +
 src/link/Elf.zig               | 604 ++++++++++++++++++++++-------------------
 src/link/MachO.zig             |   7 +
 src/link/Wasm.zig              | 237 +++++++++-------
 src/main.zig                   |  12 +
 src/stage1.zig                 |   1 +
 src/stage1/all_types.hpp       |   1 +
 src/stage1/codegen.cpp         |  10 +-
 src/stage1/stage1.cpp          |   1 +
 src/stage1/stage1.h            |   1 +
 src/zig_llvm.cpp               |  10 +-
 src/zig_llvm.h                 |   2 +-
 tools/update_clang_options.zig |   8 +
 17 files changed, 576 insertions(+), 398 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index fae725a1ac..b6c82c2dbb 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -444,6 +444,7 @@ pub const InitOptions = struct {
     want_valgrind: ?bool = null,
     want_tsan: ?bool = null,
     want_compiler_rt: ?bool = null,
+    want_lto: ?bool = null,
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
     use_clang: ?bool = null,
@@ -602,6 +603,12 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             if (ofmt == .c)
                 break :blk false;
 
+            if (options.want_lto) |lto| {
+                if (lto) {
+                    break :blk true;
+                }
+            }
+
             // Our linker can't handle objects or most advanced options yet.
             if (options.link_objects.len != 0 or
                 options.c_source_files.len != 0 or
@@ -647,6 +654,26 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             break :outer opts;
         } else .{};
 
+        const lto = blk: {
+            if (options.want_lto) |explicit| {
+                if (!use_lld)
+                    return error.LtoUnavailableWithoutLld;
+                break :blk explicit;
+            } else if (!use_lld) {
+                break :blk false;
+            } else if (options.c_source_files.len == 0) {
+                break :blk false;
+            } else if (darwin_options.system_linker_hack) {
+                break :blk false;
+            } else switch (options.output_mode) {
+                .Lib, .Obj => break :blk false,
+                .Exe => switch (options.optimize_mode) {
+                    .Debug => break :blk false,
+                    .ReleaseSafe, .ReleaseFast, .ReleaseSmall => break :blk true,
+                },
+            }
+        };
+
         const tsan = options.want_tsan orelse false;
 
         const link_libc = options.link_libc or target_util.osRequiresLibC(options.target) or tsan;
@@ -821,6 +848,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         cache.hash.add(ofmt);
         cache.hash.add(pic);
         cache.hash.add(pie);
+        cache.hash.add(lto);
         cache.hash.add(tsan);
         cache.hash.add(stack_check);
         cache.hash.add(red_zone);
@@ -1022,6 +1050,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .libc_installation = libc_dirs.libc_installation,
             .pic = pic,
             .pie = pie,
+            .lto = lto,
             .valgrind = valgrind,
             .tsan = tsan,
             .stack_check = stack_check,
@@ -2233,6 +2262,9 @@ pub fn addCCArgs(
                 "-nostdinc",
                 "-fno-spell-checking",
             });
+            if (comp.bin_file.options.lto) {
+                try argv.append("-flto");
+            }
 
             // According to Rich Felker libc headers are supposed to go before C language headers.
             // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics
@@ -3255,6 +3287,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
         .err_color = @enumToInt(comp.color),
         .pic = comp.bin_file.options.pic,
         .pie = comp.bin_file.options.pie,
+        .lto = comp.bin_file.options.lto,
         .link_libc = comp.bin_file.options.link_libc,
         .link_libcpp = comp.bin_file.options.link_libcpp,
         .strip = comp.bin_file.options.strip,
@@ -3415,6 +3448,10 @@ pub fn build_crt_file(
         .want_tsan = false,
         .want_pic = comp.bin_file.options.pic,
         .want_pie = comp.bin_file.options.pie,
+        .want_lto = switch (output_mode) {
+            .Lib => comp.bin_file.options.lto,
+            .Obj, .Exe => false,
+        },
         .emit_h = null,
         .strip = comp.compilerRtStrip(),
         .is_native_os = comp.bin_file.options.is_native_os,
diff --git a/src/clang_options_data.zig b/src/clang_options_data.zig
index c901c6045f..4d89308545 100644
--- a/src/clang_options_data.zig
+++ b/src/clang_options_data.zig
@@ -2732,7 +2732,14 @@ flagpd1("fkeep-static-consts"),
 flagpd1("flat_namespace"),
 flagpd1("flax-vector-conversions"),
 flagpd1("flimit-debug-info"),
-flagpd1("flto"),
+.{
+    .name = "flto",
+    .syntax = .flag,
+    .zig_equivalent = .lto,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 flagpd1("flto-unit"),
 flagpd1("flto-visibility-public-std"),
 sepd1("fmacro-backtrace-limit"),
@@ -2942,7 +2949,14 @@ flagpd1("fno-jump-tables"),
 flagpd1("fno-keep-static-consts"),
 flagpd1("fno-lax-vector-conversions"),
 flagpd1("fno-limit-debug-info"),
-flagpd1("fno-lto"),
+.{
+    .name = "fno-lto",
+    .syntax = .flag,
+    .zig_equivalent = .no_lto,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 flagpd1("fno-lto-unit"),
 flagpd1("fno-math-builtin"),
 flagpd1("fno-math-errno"),
@@ -5638,7 +5652,14 @@ jspd1("Ttext"),
     .pd2 = true,
     .psl = false,
 },
-joinpd1("flto="),
+.{
+    .name = "flto=",
+    .syntax = .joined,
+    .zig_equivalent = .lto,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 joinpd1("gcoff"),
 joinpd1("mabi="),
 joinpd1("mabs="),
diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig
index 8774260a08..6474957b1c 100644
--- a/src/codegen/llvm/bindings.zig
+++ b/src/codegen/llvm/bindings.zig
@@ -243,7 +243,13 @@ pub const TargetMachine = opaque {
     extern fn LLVMDisposeTargetMachine(T: *const TargetMachine) void;
 
     pub const emitToFile = LLVMTargetMachineEmitToFile;
-    extern fn LLVMTargetMachineEmitToFile(*const TargetMachine, M: *const Module, Filename: [*:0]const u8, codegen: CodeGenFileType, ErrorMessage: *[*:0]const u8) LLVMBool;
+    extern fn LLVMTargetMachineEmitToFile(
+        *const TargetMachine,
+        M: *const Module,
+        Filename: [*:0]const u8,
+        codegen: CodeGenFileType,
+        ErrorMessage: *[*:0]const u8,
+    ) LLVMBool;
 };
 
 pub const CodeMode = extern enum {
diff --git a/src/link.zig b/src/link.zig
index 6914131bea..16222b5a79 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -74,6 +74,7 @@ pub const Options = struct {
     is_native_abi: bool,
     pic: bool,
     pie: bool,
+    lto: bool,
     valgrind: bool,
     tsan: bool,
     stack_check: bool,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 981d4ec3a3..1acf09a1dc 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -945,6 +945,13 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         if (!self.base.options.strip) {
             try argv.append("-DEBUG");
         }
+        if (self.base.options.lto) {
+            switch (self.base.options.optimize_mode) {
+                .Debug => {},
+                .ReleaseSmall => try argv.append("-OPT:lldlto=2"),
+                .ReleaseFast, .ReleaseSafe => try argv.append("-OPT:lldlto=3"),
+            }
+        }
         if (self.base.options.output_mode == .Exe) {
             const stack_size = self.base.options.stack_size_override orelse 16777216;
             try argv.append(try allocPrint(arena, "-STACK:{d}", .{stack_size}));
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index fffe5628fe..18f3f57712 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1384,351 +1384,385 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
         };
     }
 
-    // Create an LLD command line and invoke it.
-    var argv = std.ArrayList([]const u8).init(self.base.allocator);
-    defer argv.deinit();
-    // We will invoke ourselves as a child process to gain access to LLD.
-    // This is necessary because LLD does not behave properly as a library -
-    // it calls exit() and does not reset all global data between invocations.
-    try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "ld.lld" });
-    if (is_obj) {
-        try argv.append("-r");
-    }
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
+    if (self.base.options.output_mode == .Obj and self.base.options.lto) {
+        // In this case we must do a simple file copy
+        // here. TODO: think carefully about how we can avoid this redundant operation when doing
+        // build-obj. See also the corresponding TODO in linkAsArchive.
+        const the_object_path = blk: {
+            if (self.base.options.objects.len != 0)
+                break :blk self.base.options.objects[0];
+
+            if (comp.c_object_table.count() != 0)
+                break :blk comp.c_object_table.items()[0].key.status.success.object_path;
+
+            if (module_obj_path) |p|
+                break :blk p;
+
+            // TODO I think this is unreachable. Audit this situation when solving the above TODO
+            // regarding eliding redundant object -> object transformations.
+            return error.NoObjectsToLink;
+        };
+        // This can happen when using --enable-cache and using the stage1 backend. In this case
+        // we can skip the file copy.
+        if (!mem.eql(u8, the_object_path, full_out_path)) {
+            try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
+        }
+    } else {
 
-    try argv.append("-error-limit=0");
+        // Create an LLD command line and invoke it.
+        var argv = std.ArrayList([]const u8).init(self.base.allocator);
+        defer argv.deinit();
+        // We will invoke ourselves as a child process to gain access to LLD.
+        // This is necessary because LLD does not behave properly as a library -
+        // it calls exit() and does not reset all global data between invocations.
+        try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "ld.lld" });
+        if (is_obj) {
+            try argv.append("-r");
+        }
 
-    if (self.base.options.output_mode == .Exe) {
-        try argv.append("-z");
-        try argv.append(try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size}));
-    }
+        try argv.append("-error-limit=0");
 
-    if (self.base.options.image_base_override) |image_base| {
-        try argv.append(try std.fmt.allocPrint(arena, "--image-base={d}", .{image_base}));
-    }
+        if (self.base.options.lto) {
+            switch (self.base.options.optimize_mode) {
+                .Debug => {},
+                .ReleaseSmall => try argv.append("-O2"),
+                .ReleaseFast, .ReleaseSafe => try argv.append("-O3"),
+            }
+        }
 
-    if (self.base.options.linker_script) |linker_script| {
-        try argv.append("-T");
-        try argv.append(linker_script);
-    }
+        if (self.base.options.output_mode == .Exe) {
+            try argv.append("-z");
+            try argv.append(try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size}));
+        }
 
-    if (gc_sections) {
-        try argv.append("--gc-sections");
-    }
+        if (self.base.options.image_base_override) |image_base| {
+            try argv.append(try std.fmt.allocPrint(arena, "--image-base={d}", .{image_base}));
+        }
 
-    if (self.base.options.eh_frame_hdr) {
-        try argv.append("--eh-frame-hdr");
-    }
+        if (self.base.options.linker_script) |linker_script| {
+            try argv.append("-T");
+            try argv.append(linker_script);
+        }
 
-    if (self.base.options.emit_relocs) {
-        try argv.append("--emit-relocs");
-    }
+        if (gc_sections) {
+            try argv.append("--gc-sections");
+        }
 
-    if (self.base.options.rdynamic) {
-        try argv.append("--export-dynamic");
-    }
+        if (self.base.options.eh_frame_hdr) {
+            try argv.append("--eh-frame-hdr");
+        }
 
-    try argv.appendSlice(self.base.options.extra_lld_args);
+        if (self.base.options.emit_relocs) {
+            try argv.append("--emit-relocs");
+        }
 
-    if (self.base.options.z_nodelete) {
-        try argv.append("-z");
-        try argv.append("nodelete");
-    }
-    if (self.base.options.z_defs) {
-        try argv.append("-z");
-        try argv.append("defs");
-    }
+        if (self.base.options.rdynamic) {
+            try argv.append("--export-dynamic");
+        }
 
-    if (getLDMOption(target)) |ldm| {
-        // Any target ELF will use the freebsd osabi if suffixed with "_fbsd".
-        const arg = if (target.os.tag == .freebsd)
-            try std.fmt.allocPrint(arena, "{s}_fbsd", .{ldm})
-        else
-            ldm;
-        try argv.append("-m");
-        try argv.append(arg);
-    }
+        try argv.appendSlice(self.base.options.extra_lld_args);
 
-    if (self.base.options.link_mode == .Static) {
-        if (target.cpu.arch.isARM() or target.cpu.arch.isThumb()) {
-            try argv.append("-Bstatic");
-        } else {
-            try argv.append("-static");
+        if (self.base.options.z_nodelete) {
+            try argv.append("-z");
+            try argv.append("nodelete");
+        }
+        if (self.base.options.z_defs) {
+            try argv.append("-z");
+            try argv.append("defs");
         }
-    } else if (is_dyn_lib) {
-        try argv.append("-shared");
-    }
 
-    if (self.base.options.pie and self.base.options.output_mode == .Exe) {
-        try argv.append("-pie");
-    }
+        if (getLDMOption(target)) |ldm| {
+            // Any target ELF will use the freebsd osabi if suffixed with "_fbsd".
+            const arg = if (target.os.tag == .freebsd)
+                try std.fmt.allocPrint(arena, "{s}_fbsd", .{ldm})
+            else
+                ldm;
+            try argv.append("-m");
+            try argv.append(arg);
+        }
 
-    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
-    try argv.append("-o");
-    try argv.append(full_out_path);
+        if (self.base.options.link_mode == .Static) {
+            if (target.cpu.arch.isARM() or target.cpu.arch.isThumb()) {
+                try argv.append("-Bstatic");
+            } else {
+                try argv.append("-static");
+            }
+        } else if (is_dyn_lib) {
+            try argv.append("-shared");
+        }
 
-    if (link_in_crt) {
-        const crt1o: []const u8 = o: {
-            if (target.os.tag == .netbsd) {
-                break :o "crt0.o";
-            } else if (target.os.tag == .openbsd) {
-                if (self.base.options.link_mode == .Static) {
-                    break :o "rcrt0.o";
-                } else {
+        if (self.base.options.pie and self.base.options.output_mode == .Exe) {
+            try argv.append("-pie");
+        }
+
+        try argv.append("-o");
+        try argv.append(full_out_path);
+
+        if (link_in_crt) {
+            const crt1o: []const u8 = o: {
+                if (target.os.tag == .netbsd) {
                     break :o "crt0.o";
-                }
-            } else if (target.isAndroid()) {
-                if (self.base.options.link_mode == .Dynamic) {
-                    break :o "crtbegin_dynamic.o";
-                } else {
-                    break :o "crtbegin_static.o";
-                }
-            } else if (self.base.options.link_mode == .Static) {
-                if (self.base.options.pie) {
-                    break :o "rcrt1.o";
+                } else if (target.os.tag == .openbsd) {
+                    if (self.base.options.link_mode == .Static) {
+                        break :o "rcrt0.o";
+                    } else {
+                        break :o "crt0.o";
+                    }
+                } else if (target.isAndroid()) {
+                    if (self.base.options.link_mode == .Dynamic) {
+                        break :o "crtbegin_dynamic.o";
+                    } else {
+                        break :o "crtbegin_static.o";
+                    }
+                } else if (self.base.options.link_mode == .Static) {
+                    if (self.base.options.pie) {
+                        break :o "rcrt1.o";
+                    } else {
+                        break :o "crt1.o";
+                    }
                 } else {
-                    break :o "crt1.o";
+                    break :o "Scrt1.o";
                 }
-            } else {
-                break :o "Scrt1.o";
+            };
+            try argv.append(try comp.get_libc_crt_file(arena, crt1o));
+            if (target_util.libc_needs_crti_crtn(target)) {
+                try argv.append(try comp.get_libc_crt_file(arena, "crti.o"));
+            }
+            if (target.os.tag == .openbsd) {
+                try argv.append(try comp.get_libc_crt_file(arena, "crtbegin.o"));
             }
-        };
-        try argv.append(try comp.get_libc_crt_file(arena, crt1o));
-        if (target_util.libc_needs_crti_crtn(target)) {
-            try argv.append(try comp.get_libc_crt_file(arena, "crti.o"));
-        }
-        if (target.os.tag == .openbsd) {
-            try argv.append(try comp.get_libc_crt_file(arena, "crtbegin.o"));
         }
-    }
 
-    // rpaths
-    var rpath_table = std.StringHashMap(void).init(self.base.allocator);
-    defer rpath_table.deinit();
-    for (self.base.options.rpath_list) |rpath| {
-        if ((try rpath_table.fetchPut(rpath, {})) == null) {
-            try argv.append("-rpath");
-            try argv.append(rpath);
+        // rpaths
+        var rpath_table = std.StringHashMap(void).init(self.base.allocator);
+        defer rpath_table.deinit();
+        for (self.base.options.rpath_list) |rpath| {
+            if ((try rpath_table.fetchPut(rpath, {})) == null) {
+                try argv.append("-rpath");
+                try argv.append(rpath);
+            }
         }
-    }
-    if (self.base.options.each_lib_rpath) {
-        var test_path = std.ArrayList(u8).init(self.base.allocator);
-        defer test_path.deinit();
-        for (self.base.options.lib_dirs) |lib_dir_path| {
-            for (self.base.options.system_libs.items()) |entry| {
-                const link_lib = entry.key;
-                test_path.shrinkRetainingCapacity(0);
-                const sep = fs.path.sep_str;
-                try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, link_lib });
-                fs.cwd().access(test_path.items, .{}) catch |err| switch (err) {
-                    error.FileNotFound => continue,
-                    else => |e| return e,
-                };
-                if ((try rpath_table.fetchPut(lib_dir_path, {})) == null) {
-                    try argv.append("-rpath");
-                    try argv.append(lib_dir_path);
+        if (self.base.options.each_lib_rpath) {
+            var test_path = std.ArrayList(u8).init(self.base.allocator);
+            defer test_path.deinit();
+            for (self.base.options.lib_dirs) |lib_dir_path| {
+                for (self.base.options.system_libs.items()) |entry| {
+                    const link_lib = entry.key;
+                    test_path.shrinkRetainingCapacity(0);
+                    const sep = fs.path.sep_str;
+                    try test_path.writer().print("{s}" ++ sep ++ "lib{s}.so", .{ lib_dir_path, link_lib });
+                    fs.cwd().access(test_path.items, .{}) catch |err| switch (err) {
+                        error.FileNotFound => continue,
+                        else => |e| return e,
+                    };
+                    if ((try rpath_table.fetchPut(lib_dir_path, {})) == null) {
+                        try argv.append("-rpath");
+                        try argv.append(lib_dir_path);
+                    }
                 }
             }
         }
-    }
 
-    for (self.base.options.lib_dirs) |lib_dir| {
-        try argv.append("-L");
-        try argv.append(lib_dir);
-    }
-
-    if (self.base.options.link_libc) {
-        if (self.base.options.libc_installation) |libc_installation| {
+        for (self.base.options.lib_dirs) |lib_dir| {
             try argv.append("-L");
-            try argv.append(libc_installation.crt_dir.?);
+            try argv.append(lib_dir);
         }
 
-        if (have_dynamic_linker) {
-            if (self.base.options.dynamic_linker) |dynamic_linker| {
-                try argv.append("-dynamic-linker");
-                try argv.append(dynamic_linker);
+        if (self.base.options.link_libc) {
+            if (self.base.options.libc_installation) |libc_installation| {
+                try argv.append("-L");
+                try argv.append(libc_installation.crt_dir.?);
             }
-        }
-    }
 
-    if (is_dyn_lib) {
-        if (self.base.options.soname) |soname| {
-            try argv.append("-soname");
-            try argv.append(soname);
-        }
-        if (self.base.options.version_script) |version_script| {
-            try argv.append("-version-script");
-            try argv.append(version_script);
+            if (have_dynamic_linker) {
+                if (self.base.options.dynamic_linker) |dynamic_linker| {
+                    try argv.append("-dynamic-linker");
+                    try argv.append(dynamic_linker);
+                }
+            }
         }
-    }
-
-    // Positional arguments to the linker such as object files.
-    try argv.appendSlice(self.base.options.objects);
 
-    for (comp.c_object_table.items()) |entry| {
-        try argv.append(entry.key.status.success.object_path);
-    }
+        if (is_dyn_lib) {
+            if (self.base.options.soname) |soname| {
+                try argv.append("-soname");
+                try argv.append(soname);
+            }
+            if (self.base.options.version_script) |version_script| {
+                try argv.append("-version-script");
+                try argv.append(version_script);
+            }
+        }
 
-    if (module_obj_path) |p| {
-        try argv.append(p);
-    }
+        // Positional arguments to the linker such as object files.
+        try argv.appendSlice(self.base.options.objects);
 
-    // TSAN
-    if (self.base.options.tsan) {
-        try argv.append(comp.tsan_static_lib.?.full_object_path);
-    }
+        for (comp.c_object_table.items()) |entry| {
+            try argv.append(entry.key.status.success.object_path);
+        }
 
-    // libc
-    // TODO: enable when stage2 can build c.zig
-    if (is_exe_or_dyn_lib and
-        !self.base.options.skip_linker_dependencies and
-        !self.base.options.link_libc and
-        build_options.is_stage1)
-    {
-        try argv.append(comp.libc_static_lib.?.full_object_path);
-    }
+        if (module_obj_path) |p| {
+            try argv.append(p);
+        }
 
-    // compiler-rt
-    if (compiler_rt_path) |p| {
-        try argv.append(p);
-    }
+        // TSAN
+        if (self.base.options.tsan) {
+            try argv.append(comp.tsan_static_lib.?.full_object_path);
+        }
 
-    // Shared libraries.
-    if (is_exe_or_dyn_lib) {
-        const system_libs = self.base.options.system_libs.items();
-        try argv.ensureCapacity(argv.items.len + system_libs.len);
-        for (system_libs) |entry| {
-            const link_lib = entry.key;
-            // By this time, we depend on these libs being dynamically linked libraries and not static libraries
-            // (the check for that needs to be earlier), but they could be full paths to .so files, in which
-            // case we want to avoid prepending "-l".
-            const ext = Compilation.classifyFileExt(link_lib);
-            const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
-            argv.appendAssumeCapacity(arg);
+        // libc
+        // TODO: enable when stage2 can build c.zig
+        if (is_exe_or_dyn_lib and
+            !self.base.options.skip_linker_dependencies and
+            !self.base.options.link_libc and
+            build_options.is_stage1)
+        {
+            try argv.append(comp.libc_static_lib.?.full_object_path);
         }
 
-        // libc++ dep
-        if (self.base.options.link_libcpp) {
-            try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
-            try argv.append(comp.libcxx_static_lib.?.full_object_path);
+        // compiler-rt
+        if (compiler_rt_path) |p| {
+            try argv.append(p);
         }
 
-        // libc dep
-        if (self.base.options.link_libc) {
-            if (self.base.options.libc_installation != null) {
-                if (self.base.options.link_mode == .Static) {
-                    try argv.append("--start-group");
-                    try argv.append("-lc");
-                    try argv.append("-lm");
-                    try argv.append("--end-group");
-                } else {
-                    try argv.append("-lc");
-                    try argv.append("-lm");
-                }
+        // Shared libraries.
+        if (is_exe_or_dyn_lib) {
+            const system_libs = self.base.options.system_libs.items();
+            try argv.ensureCapacity(argv.items.len + system_libs.len);
+            for (system_libs) |entry| {
+                const link_lib = entry.key;
+                // By this time, we depend on these libs being dynamically linked libraries and not static libraries
+                // (the check for that needs to be earlier), but they could be full paths to .so files, in which
+                // case we want to avoid prepending "-l".
+                const ext = Compilation.classifyFileExt(link_lib);
+                const arg = if (ext == .shared_library) link_lib else try std.fmt.allocPrint(arena, "-l{s}", .{link_lib});
+                argv.appendAssumeCapacity(arg);
+            }
 
-                if (target.os.tag == .freebsd or target.os.tag == .netbsd or target.os.tag == .openbsd) {
-                    try argv.append("-lpthread");
-                }
-            } else if (target.isGnuLibC()) {
-                try argv.append(comp.libunwind_static_lib.?.full_object_path);
-                for (glibc.libs) |lib| {
-                    const lib_path = try std.fmt.allocPrint(arena, "{s}{c}lib{s}.so.{d}", .{
-                        comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover,
-                    });
-                    try argv.append(lib_path);
+            // libc++ dep
+            if (self.base.options.link_libcpp) {
+                try argv.append(comp.libcxxabi_static_lib.?.full_object_path);
+                try argv.append(comp.libcxx_static_lib.?.full_object_path);
+            }
+
+            // libc dep
+            if (self.base.options.link_libc) {
+                if (self.base.options.libc_installation != null) {
+                    if (self.base.options.link_mode == .Static) {
+                        try argv.append("--start-group");
+                        try argv.append("-lc");
+                        try argv.append("-lm");
+                        try argv.append("--end-group");
+                    } else {
+                        try argv.append("-lc");
+                        try argv.append("-lm");
+                    }
+
+                    if (target.os.tag == .freebsd or target.os.tag == .netbsd or target.os.tag == .openbsd) {
+                        try argv.append("-lpthread");
+                    }
+                } else if (target.isGnuLibC()) {
+                    try argv.append(comp.libunwind_static_lib.?.full_object_path);
+                    for (glibc.libs) |lib| {
+                        const lib_path = try std.fmt.allocPrint(arena, "{s}{c}lib{s}.so.{d}", .{
+                            comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover,
+                        });
+                        try argv.append(lib_path);
+                    }
+                    try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a"));
+                } else if (target.isMusl()) {
+                    try argv.append(comp.libunwind_static_lib.?.full_object_path);
+                    try argv.append(try comp.get_libc_crt_file(arena, switch (self.base.options.link_mode) {
+                        .Static => "libc.a",
+                        .Dynamic => "libc.so",
+                    }));
+                } else if (self.base.options.link_libcpp) {
+                    try argv.append(comp.libunwind_static_lib.?.full_object_path);
+                } else {
+                    unreachable; // Compiler was supposed to emit an error for not being able to provide libc.
                 }
-                try argv.append(try comp.get_libc_crt_file(arena, "libc_nonshared.a"));
-            } else if (target.isMusl()) {
-                try argv.append(comp.libunwind_static_lib.?.full_object_path);
-                try argv.append(try comp.get_libc_crt_file(arena, switch (self.base.options.link_mode) {
-                    .Static => "libc.a",
-                    .Dynamic => "libc.so",
-                }));
-            } else if (self.base.options.link_libcpp) {
-                try argv.append(comp.libunwind_static_lib.?.full_object_path);
-            } else {
-                unreachable; // Compiler was supposed to emit an error for not being able to provide libc.
             }
         }
-    }
 
-    // crt end
-    if (link_in_crt) {
-        if (target.isAndroid()) {
-            try argv.append(try comp.get_libc_crt_file(arena, "crtend_android.o"));
-        } else if (target.os.tag == .openbsd) {
-            try argv.append(try comp.get_libc_crt_file(arena, "crtend.o"));
-        } else if (target_util.libc_needs_crti_crtn(target)) {
-            try argv.append(try comp.get_libc_crt_file(arena, "crtn.o"));
+        // crt end
+        if (link_in_crt) {
+            if (target.isAndroid()) {
+                try argv.append(try comp.get_libc_crt_file(arena, "crtend_android.o"));
+            } else if (target.os.tag == .openbsd) {
+                try argv.append(try comp.get_libc_crt_file(arena, "crtend.o"));
+            } else if (target_util.libc_needs_crti_crtn(target)) {
+                try argv.append(try comp.get_libc_crt_file(arena, "crtn.o"));
+            }
         }
-    }
 
-    if (allow_shlib_undefined) {
-        try argv.append("--allow-shlib-undefined");
-    }
+        if (allow_shlib_undefined) {
+            try argv.append("--allow-shlib-undefined");
+        }
 
-    if (self.base.options.bind_global_refs_locally) {
-        try argv.append("-Bsymbolic");
-    }
+        if (self.base.options.bind_global_refs_locally) {
+            try argv.append("-Bsymbolic");
+        }
 
-    if (self.base.options.verbose_link) {
-        // Skip over our own name so that the LLD linker name is the first argv item.
-        Compilation.dump_argv(argv.items[1..]);
-    }
+        if (self.base.options.verbose_link) {
+            // Skip over our own name so that the LLD linker name is the first argv item.
+            Compilation.dump_argv(argv.items[1..]);
+        }
 
-    // Sadly, we must run LLD as a child process because it does not behave
-    // properly as a library.
-    const child = try std.ChildProcess.init(argv.items, arena);
-    defer child.deinit();
+        // Sadly, we must run LLD as a child process because it does not behave
+        // properly as a library.
+        const child = try std.ChildProcess.init(argv.items, arena);
+        defer child.deinit();
 
-    if (comp.clang_passthrough_mode) {
-        child.stdin_behavior = .Inherit;
-        child.stdout_behavior = .Inherit;
-        child.stderr_behavior = .Inherit;
+        if (comp.clang_passthrough_mode) {
+            child.stdin_behavior = .Inherit;
+            child.stdout_behavior = .Inherit;
+            child.stderr_behavior = .Inherit;
 
-        const term = child.spawnAndWait() catch |err| {
-            log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
-            return error.UnableToSpawnSelf;
-        };
-        switch (term) {
-            .Exited => |code| {
-                if (code != 0) {
-                    // TODO https://github.com/ziglang/zig/issues/6342
-                    std.process.exit(1);
-                }
-            },
-            else => std.process.abort(),
-        }
-    } else {
-        child.stdin_behavior = .Ignore;
-        child.stdout_behavior = .Ignore;
-        child.stderr_behavior = .Pipe;
+            const term = child.spawnAndWait() catch |err| {
+                log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
+                return error.UnableToSpawnSelf;
+            };
+            switch (term) {
+                .Exited => |code| {
+                    if (code != 0) {
+                        // TODO https://github.com/ziglang/zig/issues/6342
+                        std.process.exit(1);
+                    }
+                },
+                else => std.process.abort(),
+            }
+        } else {
+            child.stdin_behavior = .Ignore;
+            child.stdout_behavior = .Ignore;
+            child.stderr_behavior = .Pipe;
 
-        try child.spawn();
+            try child.spawn();
 
-        const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024);
+            const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024);
 
-        const term = child.wait() catch |err| {
-            log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
-            return error.UnableToSpawnSelf;
-        };
+            const term = child.wait() catch |err| {
+                log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
+                return error.UnableToSpawnSelf;
+            };
 
-        switch (term) {
-            .Exited => |code| {
-                if (code != 0) {
-                    // TODO parse this output and surface with the Compilation API rather than
-                    // directly outputting to stderr here.
-                    std.debug.print("{s}", .{stderr});
-                    return error.LLDReportedFailure;
-                }
-            },
-            else => {
-                log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr });
-                return error.LLDCrashed;
-            },
-        }
+            switch (term) {
+                .Exited => |code| {
+                    if (code != 0) {
+                        // TODO parse this output and surface with the Compilation API rather than
+                        // directly outputting to stderr here.
+                        std.debug.print("{s}", .{stderr});
+                        return error.LLDReportedFailure;
+                    }
+                },
+                else => {
+                    log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr });
+                    return error.LLDCrashed;
+                },
+            }
 
-        if (stderr.len != 0) {
-            log.warn("unexpected LLD stderr:\n{s}", .{stderr});
+            if (stderr.len != 0) {
+                log.warn("unexpected LLD stderr:\n{s}", .{stderr});
+            }
         }
     }
 
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 6c2059fcdc..eee5841903 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -620,6 +620,13 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
             try argv.append("0");
         }
 
+        if (self.base.options.lto) {
+            switch (self.base.options.optimize_mode) {
+                .Debug => {},
+                .ReleaseSmall => try argv.append("-O2"),
+                .ReleaseFast, .ReleaseSafe => try argv.append("-O3"),
+            }
+        }
         try argv.append("-demangle");
 
         if (self.base.options.rdynamic and !self.base.options.system_linker_hack) {
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
index 4640c9f1af..fb356ad1aa 100644
--- a/src/link/Wasm.zig
+++ b/src/link/Wasm.zig
@@ -362,122 +362,157 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
         };
     }
 
-    const is_obj = self.base.options.output_mode == .Obj;
-
-    // Create an LLD command line and invoke it.
-    var argv = std.ArrayList([]const u8).init(self.base.allocator);
-    defer argv.deinit();
-    // We will invoke ourselves as a child process to gain access to LLD.
-    // This is necessary because LLD does not behave properly as a library -
-    // it calls exit() and does not reset all global data between invocations.
-    try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "wasm-ld" });
-    if (is_obj) {
-        try argv.append("-r");
-    }
-
-    try argv.append("-error-limit=0");
+    const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
 
-    if (self.base.options.output_mode == .Exe) {
-        // Increase the default stack size to a more reasonable value of 1MB instead of
-        // the default of 1 Wasm page being 64KB, unless overriden by the user.
-        try argv.append("-z");
-        const stack_size = self.base.options.stack_size_override orelse 1048576;
-        const arg = try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size});
-        try argv.append(arg);
-
-        // Put stack before globals so that stack overflow results in segfault immediately
-        // before corrupting globals. See https://github.com/ziglang/zig/issues/4496
-        try argv.append("--stack-first");
-    } else {
-        try argv.append("--no-entry"); // So lld doesn't look for _start.
-        try argv.append("--export-all");
-    }
-    try argv.appendSlice(&[_][]const u8{
-        "--allow-undefined",
-        "-o",
-        try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path}),
-    });
+    if (self.base.options.output_mode == .Obj) {
+        // LLD's WASM driver does not support the equvialent of `-r` so we do a simple file copy
+        // here. TODO: think carefully about how we can avoid this redundant operation when doing
+        // build-obj. See also the corresponding TODO in linkAsArchive.
+        const the_object_path = blk: {
+            if (self.base.options.objects.len != 0)
+                break :blk self.base.options.objects[0];
 
-    // Positional arguments to the linker such as object files.
-    try argv.appendSlice(self.base.options.objects);
+            if (comp.c_object_table.count() != 0)
+                break :blk comp.c_object_table.items()[0].key.status.success.object_path;
 
-    for (comp.c_object_table.items()) |entry| {
-        try argv.append(entry.key.status.success.object_path);
-    }
-    if (module_obj_path) |p| {
-        try argv.append(p);
-    }
+            if (module_obj_path) |p|
+                break :blk p;
 
-    if (self.base.options.output_mode != .Obj and
-        !self.base.options.skip_linker_dependencies and
-        !self.base.options.link_libc)
-    {
-        try argv.append(comp.libc_static_lib.?.full_object_path);
-    }
+            // TODO I think this is unreachable. Audit this situation when solving the above TODO
+            // regarding eliding redundant object -> object transformations.
+            return error.NoObjectsToLink;
+        };
+        // This can happen when using --enable-cache and using the stage1 backend. In this case
+        // we can skip the file copy.
+        if (!mem.eql(u8, the_object_path, full_out_path)) {
+            try fs.cwd().copyFile(the_object_path, fs.cwd(), full_out_path, .{});
+        }
+    } else {
+        const is_obj = self.base.options.output_mode == .Obj;
+
+        // Create an LLD command line and invoke it.
+        var argv = std.ArrayList([]const u8).init(self.base.allocator);
+        defer argv.deinit();
+        // We will invoke ourselves as a child process to gain access to LLD.
+        // This is necessary because LLD does not behave properly as a library -
+        // it calls exit() and does not reset all global data between invocations.
+        try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "wasm-ld" });
+        if (is_obj) {
+            try argv.append("-r");
+        }
 
-    if (compiler_rt_path) |p| {
-        try argv.append(p);
-    }
+        try argv.append("-error-limit=0");
 
-    if (self.base.options.verbose_link) {
-        // Skip over our own name so that the LLD linker name is the first argv item.
-        Compilation.dump_argv(argv.items[1..]);
-    }
+        if (self.base.options.lto) {
+            switch (self.base.options.optimize_mode) {
+                .Debug => {},
+                .ReleaseSmall => try argv.append("-O2"),
+                .ReleaseFast, .ReleaseSafe => try argv.append("-O3"),
+            }
+        }
 
-    // Sadly, we must run LLD as a child process because it does not behave
-    // properly as a library.
-    const child = try std.ChildProcess.init(argv.items, arena);
-    defer child.deinit();
+        if (self.base.options.output_mode == .Exe) {
+            // Increase the default stack size to a more reasonable value of 1MB instead of
+            // the default of 1 Wasm page being 64KB, unless overriden by the user.
+            try argv.append("-z");
+            const stack_size = self.base.options.stack_size_override orelse 1048576;
+            const arg = try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size});
+            try argv.append(arg);
+
+            // Put stack before globals so that stack overflow results in segfault immediately
+            // before corrupting globals. See https://github.com/ziglang/zig/issues/4496
+            try argv.append("--stack-first");
+        } else {
+            try argv.append("--no-entry"); // So lld doesn't look for _start.
+            try argv.append("--export-all");
+        }
+        try argv.appendSlice(&[_][]const u8{
+            "--allow-undefined",
+            "-o",
+            full_out_path,
+        });
 
-    if (comp.clang_passthrough_mode) {
-        child.stdin_behavior = .Inherit;
-        child.stdout_behavior = .Inherit;
-        child.stderr_behavior = .Inherit;
+        // Positional arguments to the linker such as object files.
+        try argv.appendSlice(self.base.options.objects);
 
-        const term = child.spawnAndWait() catch |err| {
-            log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
-            return error.UnableToSpawnSelf;
-        };
-        switch (term) {
-            .Exited => |code| {
-                if (code != 0) {
-                    // TODO https://github.com/ziglang/zig/issues/6342
-                    std.process.exit(1);
-                }
-            },
-            else => std.process.abort(),
+        for (comp.c_object_table.items()) |entry| {
+            try argv.append(entry.key.status.success.object_path);
+        }
+        if (module_obj_path) |p| {
+            try argv.append(p);
         }
-    } else {
-        child.stdin_behavior = .Ignore;
-        child.stdout_behavior = .Ignore;
-        child.stderr_behavior = .Pipe;
-
-        try child.spawn();
 
-        const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024);
+        if (self.base.options.output_mode != .Obj and
+            !self.base.options.skip_linker_dependencies and
+            !self.base.options.link_libc)
+        {
+            try argv.append(comp.libc_static_lib.?.full_object_path);
+        }
 
-        const term = child.wait() catch |err| {
-            log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
-            return error.UnableToSpawnSelf;
-        };
+        if (compiler_rt_path) |p| {
+            try argv.append(p);
+        }
 
-        switch (term) {
-            .Exited => |code| {
-                if (code != 0) {
-                    // TODO parse this output and surface with the Compilation API rather than
-                    // directly outputting to stderr here.
-                    std.debug.print("{s}", .{stderr});
-                    return error.LLDReportedFailure;
-                }
-            },
-            else => {
-                log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr });
-                return error.LLDCrashed;
-            },
+        if (self.base.options.verbose_link) {
+            // Skip over our own name so that the LLD linker name is the first argv item.
+            Compilation.dump_argv(argv.items[1..]);
         }
 
-        if (stderr.len != 0) {
-            log.warn("unexpected LLD stderr:\n{s}", .{stderr});
+        // Sadly, we must run LLD as a child process because it does not behave
+        // properly as a library.
+        const child = try std.ChildProcess.init(argv.items, arena);
+        defer child.deinit();
+
+        if (comp.clang_passthrough_mode) {
+            child.stdin_behavior = .Inherit;
+            child.stdout_behavior = .Inherit;
+            child.stderr_behavior = .Inherit;
+
+            const term = child.spawnAndWait() catch |err| {
+                log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
+                return error.UnableToSpawnSelf;
+            };
+            switch (term) {
+                .Exited => |code| {
+                    if (code != 0) {
+                        // TODO https://github.com/ziglang/zig/issues/6342
+                        std.process.exit(1);
+                    }
+                },
+                else => std.process.abort(),
+            }
+        } else {
+            child.stdin_behavior = .Ignore;
+            child.stdout_behavior = .Ignore;
+            child.stderr_behavior = .Pipe;
+
+            try child.spawn();
+
+            const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024);
+
+            const term = child.wait() catch |err| {
+                log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) });
+                return error.UnableToSpawnSelf;
+            };
+
+            switch (term) {
+                .Exited => |code| {
+                    if (code != 0) {
+                        // TODO parse this output and surface with the Compilation API rather than
+                        // directly outputting to stderr here.
+                        std.debug.print("{s}", .{stderr});
+                        return error.LLDReportedFailure;
+                    }
+                },
+                else => {
+                    log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr });
+                    return error.LLDCrashed;
+                },
+            }
+
+            if (stderr.len != 0) {
+                log.warn("unexpected LLD stderr:\n{s}", .{stderr});
+            }
         }
     }
 
diff --git a/src/main.zig b/src/main.zig
index ceada7e5d2..32c35739cd 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -287,6 +287,8 @@ const usage_build_generic =
     \\  -fno-PIC                  Force-disable Position Independent Code
     \\  -fPIE                     Force-enable Position Independent Executable
     \\  -fno-PIE                  Force-disable Position Independent Executable
+    \\  -flto                     Force-enable Link Time Optimization (requires LLVM extensions)
+    \\  -fno-lto                  Force-disable Link Time Optimization
     \\  -fstack-check             Enable stack probing in unsafe builds
     \\  -fno-stack-check          Disable stack probing in safe builds
     \\  -fsanitize-c              Enable C undefined behavior detection in unsafe builds
@@ -511,6 +513,7 @@ fn buildOutputType(
     var enable_cache: ?bool = null;
     var want_pic: ?bool = null;
     var want_pie: ?bool = null;
+    var want_lto: ?bool = null;
     var want_sanitize_c: ?bool = null;
     var want_stack_check: ?bool = null;
     var want_red_zone: ?bool = null;
@@ -852,6 +855,10 @@ fn buildOutputType(
                         want_pie = true;
                     } else if (mem.eql(u8, arg, "-fno-PIE")) {
                         want_pie = false;
+                    } else if (mem.eql(u8, arg, "-flto")) {
+                        want_lto = true;
+                    } else if (mem.eql(u8, arg, "-fno-lto")) {
+                        want_lto = false;
                     } else if (mem.eql(u8, arg, "-fstack-check")) {
                         want_stack_check = true;
                     } else if (mem.eql(u8, arg, "-fno-stack-check")) {
@@ -1085,6 +1092,8 @@ fn buildOutputType(
                     .no_pic => want_pic = false,
                     .pie => want_pie = true,
                     .no_pie => want_pie = false,
+                    .lto => want_lto = true,
+                    .no_lto => want_lto = false,
                     .red_zone => want_red_zone = true,
                     .no_red_zone => want_red_zone = false,
                     .nostdlib => ensure_libc_on_non_freestanding = false,
@@ -1771,6 +1780,7 @@ fn buildOutputType(
         .link_libcpp = link_libcpp,
         .want_pic = want_pic,
         .want_pie = want_pie,
+        .want_lto = want_lto,
         .want_sanitize_c = want_sanitize_c,
         .want_stack_check = want_stack_check,
         .want_red_zone = want_red_zone,
@@ -2952,6 +2962,8 @@ pub const ClangArgIterator = struct {
         no_pic,
         pie,
         no_pie,
+        lto,
+        no_lto,
         nostdlib,
         nostdlib_cpp,
         shared,
diff --git a/src/stage1.zig b/src/stage1.zig
index 44fd8e109e..8ab3b1d94d 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -109,6 +109,7 @@ pub const Module = extern struct {
     err_color: ErrColor,
     pic: bool,
     pie: bool,
+    lto: bool,
     link_libc: bool,
     link_libcpp: bool,
     strip: bool,
diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp
index c235bb362d..726d1a3aaf 100644
--- a/src/stage1/all_types.hpp
+++ b/src/stage1/all_types.hpp
@@ -2192,6 +2192,7 @@ struct CodeGen {
     bool is_single_threaded;
     bool have_pic;
     bool have_pie;
+    bool have_lto;
     bool link_mode_dynamic;
     bool dll_export_fns;
     bool have_stack_probing;
diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp
index 7a973285df..d850b3ee31 100644
--- a/src/stage1/codegen.cpp
+++ b/src/stage1/codegen.cpp
@@ -8449,8 +8449,9 @@ static void zig_llvm_emit_output(CodeGen *g) {
     // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. So we call the entire
     // pipeline multiple times if this is requested.
     if (asm_filename != nullptr && bin_filename != nullptr) {
-        if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg, g->build_mode == BuildModeDebug,
-            is_small, g->enable_time_report, g->tsan_enabled, nullptr, bin_filename, llvm_ir_filename))
+        if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg,
+            g->build_mode == BuildModeDebug, is_small, g->enable_time_report, g->tsan_enabled,
+            g->have_lto, nullptr, bin_filename, llvm_ir_filename))
         {
             fprintf(stderr, "LLVM failed to emit file: %s\n", err_msg);
             exit(1);
@@ -8459,8 +8460,9 @@ static void zig_llvm_emit_output(CodeGen *g) {
         llvm_ir_filename = nullptr;
     }
 
-    if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg, g->build_mode == BuildModeDebug,
-        is_small, g->enable_time_report, g->tsan_enabled, asm_filename, bin_filename, llvm_ir_filename))
+    if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, &err_msg,
+        g->build_mode == BuildModeDebug, is_small, g->enable_time_report, g->tsan_enabled,
+        g->have_lto, asm_filename, bin_filename, llvm_ir_filename))
     {
         fprintf(stderr, "LLVM failed to emit file: %s\n", err_msg);
         exit(1);
diff --git a/src/stage1/stage1.cpp b/src/stage1/stage1.cpp
index d680121577..39e3bad1a5 100644
--- a/src/stage1/stage1.cpp
+++ b/src/stage1/stage1.cpp
@@ -90,6 +90,7 @@ void zig_stage1_build_object(struct ZigStage1 *stage1) {
     g->dll_export_fns = stage1->dll_export_fns;
     g->have_pic = stage1->pic;
     g->have_pie = stage1->pie;
+    g->have_lto = stage1->lto;
     g->have_stack_probing = stage1->enable_stack_probing;
     g->red_zone = stage1->red_zone;
     g->is_single_threaded = stage1->is_single_threaded;
diff --git a/src/stage1/stage1.h b/src/stage1/stage1.h
index 6db3621cf5..7fa1576d64 100644
--- a/src/stage1/stage1.h
+++ b/src/stage1/stage1.h
@@ -178,6 +178,7 @@ struct ZigStage1 {
 
     bool pic;
     bool pie;
+    bool lto;
     bool link_libc;
     bool link_libcpp;
     bool strip;
diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp
index 51af5e06d1..280920ac74 100644
--- a/src/zig_llvm.cpp
+++ b/src/zig_llvm.cpp
@@ -22,6 +22,7 @@
 
 #include <llvm/Analysis/TargetLibraryInfo.h>
 #include <llvm/Analysis/TargetTransformInfo.h>
+#include <llvm/Bitcode/BitcodeWriter.h>
 #include <llvm/IR/DIBuilder.h>
 #include <llvm/IR/DiagnosticInfo.h>
 #include <llvm/IR/IRBuilder.h>
@@ -184,7 +185,7 @@ unsigned ZigLLVMDataLayoutGetProgramAddressSpace(LLVMTargetDataRef TD) {
 
 bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
         char **error_message, bool is_debug,
-        bool is_small, bool time_report, bool tsan,
+        bool is_small, bool time_report, bool tsan, bool lto,
         const char *asm_filename, const char *bin_filename, const char *llvm_ir_filename)
 {
     TimePassesIsEnabled = time_report;
@@ -234,7 +235,7 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
     PMBuilder->VerifyInput = assertions_on;
     PMBuilder->VerifyOutput = assertions_on;
     PMBuilder->MergeFunctions = !is_debug;
-    PMBuilder->PrepareForLTO = false;
+    PMBuilder->PrepareForLTO = lto;
     PMBuilder->PrepareForThinLTO = false;
     PMBuilder->PerformThinLTO = false;
 
@@ -272,7 +273,7 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
         PMBuilder->populateModulePassManager(MPM);
 
         // Set output passes.
-        if (dest_bin) {
+        if (dest_bin && !lto) {
             if (target_machine->addPassesToEmitFile(MPM, *dest_bin, nullptr, CGFT_ObjectFile)) {
                 *error_message = strdup("TargetMachine can't emit an object file");
                 return true;
@@ -299,6 +300,9 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
                 return true;
             }
         }
+        if (dest_bin && lto) {
+            WriteBitcodeToFile(*module, *dest_bin);
+        }
 
         if (time_report) {
             TimerGroup::printAll(errs());
diff --git a/src/zig_llvm.h b/src/zig_llvm.h
index 9a31092265..504ea4ec01 100644
--- a/src/zig_llvm.h
+++ b/src/zig_llvm.h
@@ -48,7 +48,7 @@ ZIG_EXTERN_C char *ZigLLVMGetNativeFeatures(void);
 
 ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
         char **error_message, bool is_debug,
-        bool is_small, bool time_report, bool tsan,
+        bool is_small, bool time_report, bool tsan, bool lto,
         const char *asm_filename, const char *bin_filename, const char *llvm_ir_filename);
 
 
diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig
index 189a2ed7fa..2aeee6845b 100644
--- a/tools/update_clang_options.zig
+++ b/tools/update_clang_options.zig
@@ -62,6 +62,14 @@ const known_options = [_]KnownOpt{
         .name = "fno-PIE",
         .ident = "no_pie",
     },
+    .{
+        .name = "flto",
+        .ident = "lto",
+    },
+    .{
+        .name = "fno-lto",
+        .ident = "no_lto",
+    },
     .{
         .name = "nolibc",
         .ident = "nostdlib",
-- 
cgit v1.2.3


From 0cfa39304b18c6a04689bd789f5dc4d035ec43b0 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 24 Jan 2021 14:30:05 -0700
Subject: zig cc: recognize more coff linker options

Related: #7874
---
 src/Compilation.zig | 10 ++++++++++
 src/link.zig        |  5 +++++
 src/link/Coff.zig   | 57 +++++++++++++++++++++++++++++++++++++++++++++--------
 src/main.zig        | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 120 insertions(+), 8 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index b6c82c2dbb..c7bb260aa7 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -468,6 +468,11 @@ pub const InitOptions = struct {
     disable_c_depfile: bool = false,
     linker_z_nodelete: bool = false,
     linker_z_defs: bool = false,
+    linker_tsaware: bool = false,
+    linker_nxcompat: bool = false,
+    linker_dynamicbase: bool = false,
+    major_subsystem_version: ?u32 = null,
+    minor_subsystem_version: ?u32 = null,
     clang_passthrough_mode: bool = false,
     verbose_cc: bool = false,
     verbose_link: bool = false,
@@ -1035,6 +1040,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
             .bind_global_refs_locally = options.linker_bind_global_refs_locally orelse false,
             .z_nodelete = options.linker_z_nodelete,
             .z_defs = options.linker_z_defs,
+            .tsaware = options.linker_tsaware,
+            .nxcompat = options.linker_nxcompat,
+            .dynamicbase = options.linker_dynamicbase,
+            .major_subsystem_version = options.major_subsystem_version,
+            .minor_subsystem_version = options.minor_subsystem_version,
             .stack_size_override = options.stack_size_override,
             .image_base_override = options.image_base_override,
             .include_compiler_rt = include_compiler_rt,
diff --git a/src/link.zig b/src/link.zig
index 16222b5a79..82ff0bb3f8 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -69,6 +69,9 @@ pub const Options = struct {
     rdynamic: bool,
     z_nodelete: bool,
     z_defs: bool,
+    tsaware: bool,
+    nxcompat: bool,
+    dynamicbase: bool,
     bind_global_refs_locally: bool,
     is_native_os: bool,
     is_native_abi: bool,
@@ -88,6 +91,8 @@ pub const Options = struct {
     each_lib_rpath: bool,
     disable_lld_caching: bool,
     is_test: bool,
+    major_subsystem_version: ?u32,
+    minor_subsystem_version: ?u32,
     gc_sections: ?bool = null,
     allow_shlib_undefined: ?bool,
     subsystem: ?std.Target.SubSystem,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 1acf09a1dc..0dfa3d54a1 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -877,6 +877,11 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         man.hash.addStringSet(self.base.options.system_libs);
         man.hash.addOptional(self.base.options.subsystem);
         man.hash.add(self.base.options.is_test);
+        man.hash.add(self.base.options.tsaware);
+        man.hash.add(self.base.options.nxcompat);
+        man.hash.add(self.base.options.dynamicbase);
+        man.hash.addOptional(self.base.options.major_subsystem_version);
+        man.hash.addOptional(self.base.options.minor_subsystem_version);
 
         // We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
         _ = try man.hit();
@@ -976,6 +981,26 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
             try argv.append("-DLL");
         }
 
+        if (self.base.options.tsaware) {
+            try argv.append("-tsaware");
+        }
+        if (self.base.options.nxcompat) {
+            try argv.append("-nxcompat");
+        }
+        if (self.base.options.dynamicbase) {
+            try argv.append("-dynamicbase");
+        }
+        const subsystem_suffix = ss: {
+            if (self.base.options.major_subsystem_version) |major| {
+                if (self.base.options.minor_subsystem_version) |minor| {
+                    break :ss try allocPrint(arena, ",{d}.{d}", .{ major, minor });
+                } else {
+                    break :ss try allocPrint(arena, ",{d}", .{major});
+                }
+            }
+            break :ss "";
+        };
+
         try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path}));
 
         if (self.base.options.link_libc) {
@@ -1029,35 +1054,51 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
         const mode: Mode = mode: {
             if (resolved_subsystem) |subsystem| switch (subsystem) {
                 .Console => {
-                    try argv.append("-SUBSYSTEM:console");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:console{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .win32;
                 },
                 .EfiApplication => {
-                    try argv.append("-SUBSYSTEM:efi_application");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:efi_application{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .uefi;
                 },
                 .EfiBootServiceDriver => {
-                    try argv.append("-SUBSYSTEM:efi_boot_service_driver");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:efi_boot_service_driver{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .uefi;
                 },
                 .EfiRom => {
-                    try argv.append("-SUBSYSTEM:efi_rom");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:efi_rom{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .uefi;
                 },
                 .EfiRuntimeDriver => {
-                    try argv.append("-SUBSYSTEM:efi_runtime_driver");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:efi_runtime_driver{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .uefi;
                 },
                 .Native => {
-                    try argv.append("-SUBSYSTEM:native");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:native{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .win32;
                 },
                 .Posix => {
-                    try argv.append("-SUBSYSTEM:posix");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:posix{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .win32;
                 },
                 .Windows => {
-                    try argv.append("-SUBSYSTEM:windows");
+                    try argv.append(try allocPrint(arena, "-SUBSYSTEM:windows{s}", .{
+                        subsystem_suffix,
+                    }));
                     break :mode .win32;
                 },
             } else if (target.os.tag == .uefi) {
diff --git a/src/main.zig b/src/main.zig
index 32c35739cd..26e222016a 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -529,6 +529,9 @@ fn buildOutputType(
     var linker_bind_global_refs_locally: ?bool = null;
     var linker_z_nodelete = false;
     var linker_z_defs = false;
+    var linker_tsaware = false;
+    var linker_nxcompat = false;
+    var linker_dynamicbase = false;
     var test_evented_io = false;
     var stack_size_override: ?u64 = null;
     var image_base_override: ?u64 = null;
@@ -549,6 +552,8 @@ fn buildOutputType(
     var main_pkg_path: ?[]const u8 = null;
     var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no;
     var subsystem: ?std.Target.SubSystem = null;
+    var major_subsystem_version: ?u32 = null;
+    var minor_subsystem_version: ?u32 = null;
 
     var system_libs = std.ArrayList([]const u8).init(gpa);
     defer system_libs.deinit();
@@ -1307,10 +1312,56 @@ fn buildOutputType(
                     image_base_override = std.fmt.parseUnsigned(u64, linker_args.items[i], 0) catch |err| {
                         fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
                     };
+                } else if (mem.eql(u8, arg, "-T")) {
+                    i += 1;
+                    if (i >= linker_args.items.len) {
+                        fatal("expected linker arg after '{s}'", .{arg});
+                    }
+                    linker_script = linker_args.items[i];
                 } else if (mem.eql(u8, arg, "--eh-frame-hdr")) {
                     link_eh_frame_hdr = true;
                 } else if (mem.eql(u8, arg, "--no-eh-frame-hdr")) {
                     link_eh_frame_hdr = false;
+                } else if (mem.eql(u8, arg, "--tsaware")) {
+                    linker_tsaware = true;
+                } else if (mem.eql(u8, arg, "--nxcompat")) {
+                    linker_nxcompat = true;
+                } else if (mem.eql(u8, arg, "--dynamicbase")) {
+                    linker_dynamicbase = true;
+                } else if (mem.eql(u8, arg, "--high-entropy-va")) {
+                    // This option does not do anything.
+                } else if (mem.eql(u8, arg, "--export-all-symbols")) {
+                    rdynamic = true;
+                } else if (mem.eql(u8, arg, "--start-group") or
+                    mem.eql(u8, arg, "--end-group"))
+                {
+                    // We don't need to care about these because these args are
+                    // for resolving circular dependencies but our linker takes
+                    // care of this without explicit args.
+                } else if (mem.startsWith(u8, arg, "--major-os-version") or
+                    mem.startsWith(u8, arg, "--minor-os-version"))
+                {
+                    // This option does not do anything.
+                } else if (mem.startsWith(u8, arg, "--major-subsystem-version=")) {
+                    major_subsystem_version = std.fmt.parseUnsigned(
+                        u32,
+                        arg["--major-subsystem-version=".len..],
+                        10,
+                    ) catch |err| {
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
+                    };
+                } else if (mem.startsWith(u8, arg, "--minor-subsystem-version=")) {
+                    minor_subsystem_version = std.fmt.parseUnsigned(
+                        u32,
+                        arg["--minor-subsystem-version=".len..],
+                        10,
+                    ) catch |err| {
+                        fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) });
+                    };
+                } else if (mem.startsWith(u8, arg, "--major-os-version=") or
+                    mem.startsWith(u8, arg, "--minor-os-version="))
+                {
+                    // These args do nothing.
                 } else {
                     warn("unsupported linker arg: {s}", .{arg});
                 }
@@ -1800,6 +1851,11 @@ fn buildOutputType(
         .linker_bind_global_refs_locally = linker_bind_global_refs_locally,
         .linker_z_nodelete = linker_z_nodelete,
         .linker_z_defs = linker_z_defs,
+        .linker_tsaware = linker_tsaware,
+        .linker_nxcompat = linker_nxcompat,
+        .linker_dynamicbase = linker_dynamicbase,
+        .major_subsystem_version = major_subsystem_version,
+        .minor_subsystem_version = minor_subsystem_version,
         .link_eh_frame_hdr = link_eh_frame_hdr,
         .link_emit_relocs = link_emit_relocs,
         .stack_size_override = stack_size_override,
-- 
cgit v1.2.3


From e197a03124527035f1a92381eb6ef46d6f6d2c39 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 7 Feb 2021 14:51:27 -0700
Subject: zig cc: recognize the `-s` flag to be "strip"

---
 src/Compilation.zig            | 4 +++-
 src/clang_options_data.zig     | 9 ++++++++-
 src/main.zig                   | 2 ++
 tools/update_clang_options.zig | 4 ++++
 4 files changed, 17 insertions(+), 2 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index c7bb260aa7..268c259c02 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2348,7 +2348,9 @@ pub fn addCCArgs(
                 else => {},
             }
 
-            if (!comp.bin_file.options.strip) {
+            if (comp.bin_file.options.strip) {
+                try argv.append("-s");
+            } else {
                 try argv.append("-g");
                 switch (comp.bin_file.options.object_format) {
                     .coff, .pe => try argv.append("-gcodeview"),
diff --git a/src/clang_options_data.zig b/src/clang_options_data.zig
index 4d89308545..0293a5327e 100644
--- a/src/clang_options_data.zig
+++ b/src/clang_options_data.zig
@@ -4305,7 +4305,14 @@ flagpd1("rewrite-macros"),
 flagpd1("rewrite-objc"),
 flagpd1("rewrite-test"),
 sepd1("rpath"),
-flagpd1("s"),
+.{
+    .name = "s",
+    .syntax = .flag,
+    .zig_equivalent = .strip,
+    .pd1 = true,
+    .pd2 = false,
+    .psl = false,
+},
 .{
     .name = "save-stats",
     .syntax = .flag,
diff --git a/src/main.zig b/src/main.zig
index 14f63d9911..c31252a96a 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -1186,6 +1186,7 @@ fn buildOutputType(
                     .framework_dir => try framework_dirs.append(it.only_arg),
                     .framework => try frameworks.append(it.only_arg),
                     .nostdlibinc => want_native_include_dirs = false,
+                    .strip => strip = true,
                 }
             }
             // Parse linker args.
@@ -3047,6 +3048,7 @@ pub const ClangArgIterator = struct {
         nostdlibinc,
         red_zone,
         no_red_zone,
+        strip,
     };
 
     const Args = struct {
diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig
index 2aeee6845b..1cb134a731 100644
--- a/tools/update_clang_options.zig
+++ b/tools/update_clang_options.zig
@@ -312,6 +312,10 @@ const known_options = [_]KnownOpt{
         .name = "framework",
         .ident = "framework",
     },
+    .{
+        .name = "s",
+        .ident = "strip",
+    },
 };
 
 const blacklisted_options = [_][]const u8{};
-- 
cgit v1.2.3


From 1adac0a55bafbba864228ac38c4684612e84f522 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 7 Feb 2021 15:08:27 -0700
Subject: never pass -s to clang

We only use clang to produce object files; the idea of stripping is not
relevant here.

Fixes regression in previous commit.
---
 src/Compilation.zig | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 268c259c02..c7bb260aa7 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -2348,9 +2348,7 @@ pub fn addCCArgs(
                 else => {},
             }
 
-            if (comp.bin_file.options.strip) {
-                try argv.append("-s");
-            } else {
+            if (!comp.bin_file.options.strip) {
                 try argv.append("-g");
                 switch (comp.bin_file.options.object_format) {
                     .coff, .pe => try argv.append("-gcodeview"),
-- 
cgit v1.2.3


From 3d0f4b90305bc1815ccc86613cb3da715e9b62c0 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Thu, 11 Feb 2021 23:29:55 -0700
Subject: stage2: start reworking Module/astgen for memory layout changes

This commit does not reach any particular milestone, it is
work-in-progress towards getting things to build.

There's a `@panic("TODO")` in translate-c that should be removed when
working on translate-c stuff.
---
 lib/std/zig.zig             |    1 -
 lib/std/zig/ast.zig         |  195 ++++++
 lib/std/zig/parser_test.zig |    2 +-
 lib/std/zig/render.zig      |   53 +-
 src/Compilation.zig         |   13 +-
 src/Module.zig              | 1484 +++++++++++++++++++++++++------------------
 src/main.zig                |   84 +--
 src/translate_c.zig         |   10 +-
 src/zir.zig                 |   15 +-
 9 files changed, 1146 insertions(+), 711 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/zig.zig b/lib/std/zig.zig
index 5b564864ad..7f23405150 100644
--- a/lib/std/zig.zig
+++ b/lib/std/zig.zig
@@ -12,7 +12,6 @@ pub const fmtId = @import("zig/fmt.zig").fmtId;
 pub const fmtEscapes = @import("zig/fmt.zig").fmtEscapes;
 pub const parse = @import("zig/parse.zig").parse;
 pub const parseStringLiteral = @import("zig/string_literal.zig").parse;
-pub const render = @import("zig/render.zig").render;
 pub const ast = @import("zig/ast.zig");
 pub const system = @import("zig/system.zig");
 pub const CrossTarget = @import("zig/cross_target.zig").CrossTarget;
diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig
index 0b6133e789..ab81c3415e 100644
--- a/lib/std/zig/ast.zig
+++ b/lib/std/zig/ast.zig
@@ -45,6 +45,28 @@ pub const Tree = struct {
         tree.* = undefined;
     }
 
+    pub const RenderError = error{
+        /// Ran out of memory allocating call stack frames to complete rendering, or
+        /// ran out of memory allocating space in the output buffer.
+        OutOfMemory,
+    };
+
+    /// `gpa` is used for allocating the resulting formatted source code, as well as
+    /// for allocating extra stack memory if needed, because this function utilizes recursion.
+    /// Note: that's not actually true yet, see https://github.com/ziglang/zig/issues/1006.
+    /// Caller owns the returned slice of bytes, allocated with `gpa`.
+    pub fn render(tree: Tree, gpa: *mem.Allocator) RenderError![]u8 {
+        var buffer = std.ArrayList(u8).init(gpa);
+        defer buffer.deinit();
+
+        try tree.renderToArrayList(&buffer);
+        return buffer.toOwnedSlice();
+    }
+
+    pub fn renderToArrayList(tree: Tree, buffer: *std.ArrayList(u8)) RenderError!void {
+        return @import("./render.zig").renderTree(buffer, tree);
+    }
+
     pub fn tokenLocation(self: Tree, start_offset: ByteOffset, token_index: TokenIndex) Location {
         var loc = Location{
             .line = 0,
@@ -72,6 +94,27 @@ pub const Tree = struct {
         return loc;
     }
 
+    pub fn tokenSlice(tree: Tree, token_index: TokenIndex) []const u8 {
+        const token_starts = tree.tokens.items(.start);
+        const token_tags = tree.tokens.items(.tag);
+        const token_tag = token_tags[token_index];
+
+        // Many tokens can be determined entirely by their tag.
+        if (token_tag.lexeme()) |lexeme| {
+            return lexeme;
+        }
+
+        // For some tokens, re-tokenization is needed to find the end.
+        var tokenizer: std.zig.Tokenizer = .{
+            .buffer = tree.source,
+            .index = token_starts[token_index],
+            .pending_invalid_token = null,
+        };
+        const token = tokenizer.next();
+        assert(token.tag == token_tag);
+        return tree.source[token.loc.start..token.loc.end];
+    }
+
     pub fn extraData(tree: Tree, index: usize, comptime T: type) T {
         const fields = std.meta.fields(T);
         var result: T = undefined;
@@ -82,6 +125,12 @@ pub const Tree = struct {
         return result;
     }
 
+    pub fn rootDecls(tree: Tree) []const Node.Index {
+        // Root is always index 0.
+        const nodes_data = tree.nodes.items(.data);
+        return tree.extra_data[nodes_data[0].lhs..nodes_data[0].rhs];
+    }
+
     pub fn renderError(tree: Tree, parse_error: Error, stream: anytype) !void {
         const tokens = tree.tokens.items(.tag);
         switch (parse_error) {
@@ -966,6 +1015,15 @@ pub const Tree = struct {
         return mem.indexOfScalar(u8, source, '\n') == null;
     }
 
+    pub fn getNodeSource(tree: Tree, node: Node.Index) []const u8 {
+        const token_starts = tree.tokens.items(.start);
+        const first_token = tree.firstToken(node);
+        const last_token = tree.lastToken(node);
+        const start = token_starts[first_token];
+        const len = tree.tokenSlice(last_token).len;
+        return tree.source[start..][0..len];
+    }
+
     pub fn globalVarDecl(tree: Tree, node: Node.Index) full.VarDecl {
         assert(tree.nodes.items(.tag)[node] == .global_var_decl);
         const data = tree.nodes.items(.data)[node];
@@ -1653,7 +1711,31 @@ pub const Tree = struct {
         const token_tags = tree.tokens.items(.tag);
         var result: full.FnProto = .{
             .ast = info,
+            .visib_token = null,
+            .extern_export_token = null,
+            .lib_name = null,
+            .name_token = null,
+            .lparen = undefined,
         };
+        var i = info.fn_token;
+        while (i > 0) {
+            i -= 1;
+            switch (token_tags[i]) {
+                .keyword_extern, .keyword_export => result.extern_export_token = i,
+                .keyword_pub => result.visib_token = i,
+                .string_literal => result.lib_name = i,
+                else => break,
+            }
+        }
+        const after_fn_token = info.fn_token + 1;
+        if (token_tags[after_fn_token] == .identifier) {
+            result.name_token = after_fn_token;
+            result.lparen = after_fn_token + 1;
+        } else {
+            result.lparen = after_fn_token;
+        }
+        assert(token_tags[result.lparen] == .l_paren);
+
         return result;
     }
 
@@ -1924,6 +2006,11 @@ pub const full = struct {
     };
 
     pub const FnProto = struct {
+        visib_token: ?TokenIndex,
+        extern_export_token: ?TokenIndex,
+        lib_name: ?TokenIndex,
+        name_token: ?TokenIndex,
+        lparen: TokenIndex,
         ast: Ast,
 
         pub const Ast = struct {
@@ -1934,6 +2021,114 @@ pub const full = struct {
             section_expr: Node.Index,
             callconv_expr: Node.Index,
         };
+
+        pub const Param = struct {
+            first_doc_comment: ?TokenIndex,
+            name_token: ?TokenIndex,
+            comptime_noalias: ?TokenIndex,
+            anytype_ellipsis3: ?TokenIndex,
+            type_expr: Node.Index,
+        };
+
+        /// Abstracts over the fact that anytype and ... are not included
+        /// in the params slice, since they are simple identifiers and
+        /// not sub-expressions.
+        pub const Iterator = struct {
+            tree: *const Tree,
+            fn_proto: *const FnProto,
+            param_i: usize,
+            tok_i: TokenIndex,
+            tok_flag: bool,
+
+            pub fn next(it: *Iterator) ?Param {
+                const token_tags = it.tree.tokens.items(.tag);
+                while (true) {
+                    var first_doc_comment: ?TokenIndex = null;
+                    var comptime_noalias: ?TokenIndex = null;
+                    var name_token: ?TokenIndex = null;
+                    if (!it.tok_flag) {
+                        if (it.param_i >= it.fn_proto.ast.params.len) {
+                            return null;
+                        }
+                        const param_type = it.fn_proto.ast.params[it.param_i];
+                        var tok_i = tree.firstToken(param_type) - 1;
+                        while (true) : (tok_i -= 1) switch (token_tags[tok_i]) {
+                            .colon => continue,
+                            .identifier => name_token = tok_i,
+                            .doc_comment => first_doc_comment = tok_i,
+                            .keyword_comptime, .keyword_noalias => comptime_noalias = tok_i,
+                            else => break,
+                        };
+                        it.param_i += 1;
+                        it.tok_i = tree.lastToken(param_type) + 1;
+                        it.tok_flag = true;
+                        return Param{
+                            .first_doc_comment = first_doc_comment,
+                            .comptime_noalias = comptime_noalias,
+                            .name_token = name_token,
+                            .anytype_ellipsis3 = null,
+                            .type_expr = param_type,
+                        };
+                    }
+                    // Look for anytype and ... params afterwards.
+                    if (token_tags[it.tok_i] == .comma) {
+                        it.tok_i += 1;
+                    } else {
+                        return null;
+                    }
+                    if (token_tags[it.tok_i] == .doc_comment) {
+                        first_doc_comment = it.tok_i;
+                        while (token_tags[it.tok_i] == .doc_comment) {
+                            it.tok_i += 1;
+                        }
+                    }
+                    switch (token_tags[it.tok_i]) {
+                        .ellipsis3 => {
+                            it.tok_flag = false; // Next iteration should return null.
+                            return Param{
+                                .first_doc_comment = first_doc_comment,
+                                .comptime_noalias = null,
+                                .name_token = null,
+                                .anytype_ellipsis3 = it.tok_i,
+                                .type_expr = 0,
+                            };
+                        },
+                        .keyword_noalias, .keyword_comptime => {
+                            comptime_noalias = it.tok_i;
+                            it.tok_i += 1;
+                        },
+                        else => {},
+                    }
+                    if (token_tags[it.tok_i] == .identifier and
+                        token_tags[it.tok_i + 1] == .colon)
+                    {
+                        name_token = it.tok_i;
+                        it.tok_i += 2;
+                    }
+                    if (token_tags[it.tok_i] == .keyword_anytype) {
+                        it.tok_i += 1;
+                        return Param{
+                            .first_doc_comment = first_doc_comment,
+                            .comptime_noalias = comptime_noalias,
+                            .name_token = name_token,
+                            .anytype_ellipsis3 = it.tok_i - 1,
+                            .type_expr = param_type,
+                        };
+                    }
+                    it.tok_flag = false;
+                }
+            }
+        };
+
+        pub fn iterate(fn_proto: FnProto, tree: Tree) Iterator {
+            return .{
+                .tree = &tree,
+                .fn_proto = &fn_proto,
+                .param_i = 0,
+                .tok_i = undefined,
+                .tok_flag = false,
+            };
+        }
     };
 
     pub const StructInit = struct {
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index aad9ad3ed6..9046feda85 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -4223,7 +4223,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
         return error.ParseError;
     }
 
-    const formatted = try std.zig.render(allocator, tree);
+    const formatted = try tree.render(allocator);
     anything_changed.* = !mem.eql(u8, formatted, source);
     return formatted;
 }
diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig
index 8f98acc20c..1410ef5c13 100644
--- a/lib/std/zig/render.zig
+++ b/lib/std/zig/render.zig
@@ -13,28 +13,24 @@ const Token = std.zig.Token;
 const indent_delta = 4;
 const asm_indent_delta = 2;
 
-pub const Error = error{
-    /// Ran out of memory allocating call stack frames to complete rendering, or
-    /// ran out of memory allocating space in the output buffer.
-    OutOfMemory,
-};
+pub const Error = ast.Tree.RenderError;
 
 const Writer = std.ArrayList(u8).Writer;
 const Ais = std.io.AutoIndentingStream(Writer);
 
-/// `gpa` is used for allocating the resulting formatted source code, as well as
-/// for allocating extra stack memory if needed, because this function utilizes recursion.
-/// Note: that's not actually true yet, see https://github.com/ziglang/zig/issues/1006.
-/// Caller owns the returned slice of bytes, allocated with `gpa`.
-pub fn render(gpa: *mem.Allocator, tree: ast.Tree) Error![]u8 {
+pub fn renderTree(buffer: *std.ArrayList(u8), tree: ast.Tree) Error!void {
     assert(tree.errors.len == 0); // Cannot render an invalid tree.
+    var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, buffer.writer());
+    const ais = &auto_indenting_stream;
 
-    var buffer = std.ArrayList(u8).init(gpa);
-    defer buffer.deinit();
+    // Render all the line comments at the beginning of the file.
+    const src_start: usize = if (mem.startsWith(u8, tree.source, "\xEF\xBB\xBF")) 3 else 0;
+    const comment_end_loc: usize = tree.tokens.items(.start)[0];
+    _ = try renderCommentsAndNewlines(ais, tree, src_start, comment_end_loc);
 
-    var auto_indenting_stream = std.io.autoIndentingStream(indent_delta, buffer.writer());
-    try renderRoot(&auto_indenting_stream, tree);
-    return buffer.toOwnedSlice();
+    for (tree.rootDecls()) |decl| {
+        try renderMember(ais, tree, decl, .newline);
+    }
 }
 
 /// Assumes that start is the first byte past the previous token and
@@ -75,21 +71,6 @@ fn renderCommentsAndNewlines(ais: *Ais, tree: ast.Tree, start: usize, end: usize
     return index != start;
 }
 
-fn renderRoot(ais: *Ais, tree: ast.Tree) Error!void {
-    // Render all the line comments at the beginning of the file.
-    const src_start: usize = if (mem.startsWith(u8, tree.source, "\xEF\xBB\xBF")) 3 else 0;
-    const comment_end_loc: usize = tree.tokens.items(.start)[0];
-    _ = try renderCommentsAndNewlines(ais, tree, src_start, comment_end_loc);
-
-    // Root is always index 0.
-    const nodes_data = tree.nodes.items(.data);
-    const root_decls = tree.extra_data[nodes_data[0].lhs..nodes_data[0].rhs];
-
-    for (root_decls) |decl| {
-        try renderMember(ais, tree, decl, .newline);
-    }
-}
-
 fn renderMember(ais: *Ais, tree: ast.Tree, decl: ast.Node.Index, space: Space) Error!void {
     const token_tags = tree.tokens.items(.tag);
     const main_tokens = tree.nodes.items(.main_token);
@@ -1944,17 +1925,7 @@ fn renderToken(ais: *Ais, tree: ast.Tree, token_index: ast.TokenIndex, space: Sp
     const token_starts = tree.tokens.items(.start);
 
     const token_start = token_starts[token_index];
-    const token_tag = token_tags[token_index];
-    const lexeme = token_tag.lexeme() orelse lexeme: {
-        var tokenizer: std.zig.Tokenizer = .{
-            .buffer = tree.source,
-            .index = token_start,
-            .pending_invalid_token = null,
-        };
-        const token = tokenizer.next();
-        assert(token.tag == token_tag);
-        break :lexeme tree.source[token.loc.start..token.loc.end];
-    };
+    const lexeme = tree.tokenSlice(token_index);
     try ais.writer().writeAll(lexeme);
 
     switch (space) {
diff --git a/src/Compilation.zig b/src/Compilation.zig
index c7bb260aa7..89c047065c 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -921,7 +921,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
                         // TODO this is duped so it can be freed in Container.deinit
                         .sub_file_path = try gpa.dupe(u8, root_pkg.root_src_path),
                         .source = .{ .unloaded = {} },
-                        .contents = .{ .not_available = {} },
+                        .tree = undefined,
                         .status = .never_loaded,
                         .pkg = root_pkg,
                         .root_container = .{
@@ -1882,7 +1882,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"});
         const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path);
         var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{};
-        const tree = translate_c.translate(
+        var tree = translate_c.translate(
             comp.gpa,
             new_argv.ptr,
             new_argv.ptr + new_argv.len,
@@ -1901,7 +1901,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
                 };
             },
         };
-        defer tree.deinit();
+        defer tree.deinit(comp.gpa);
 
         if (comp.verbose_cimport) {
             log.info("C import .d file: {s}", .{out_dep_path});
@@ -1919,9 +1919,10 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{});
         defer out_zig_file.close();
 
-        var bos = std.io.bufferedWriter(out_zig_file.writer());
-        _ = try std.zig.render(comp.gpa, bos.writer(), tree);
-        try bos.flush();
+        const formatted = try tree.render(comp.gpa);
+        defer comp.gpa.free(formatted);
+
+        try out_zig_file.writeAll(formatted);
 
         man.writeManifest() catch |err| {
             log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)});
diff --git a/src/Module.zig b/src/Module.zig
index fa9722814e..bd47332d15 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -244,9 +244,9 @@ pub const Decl = struct {
     }
 
     pub fn src(self: Decl) usize {
-        const tree = self.container.file_scope.contents.tree;
-        const decl_node = tree.root_node.decls()[self.src_index];
-        return tree.token_locs[decl_node.firstToken()].start;
+        const tree = &self.container.file_scope.tree;
+        const decl_node = tree.rootDecls()[self.src_index];
+        return tree.tokens.items(.start)[tree.firstToken(decl_node)];
     }
 
     pub fn fullyQualifiedNameHash(self: Decl) Scope.NameHash {
@@ -536,6 +536,12 @@ pub const Scope = struct {
     pub const File = struct {
         pub const base_tag: Tag = .file;
         base: Scope = Scope{ .tag = base_tag },
+        status: enum {
+            never_loaded,
+            unloaded_success,
+            unloaded_parse_failure,
+            loaded_success,
+        },
 
         /// Relative to the owning package's root_src_dir.
         /// Reference to external memory, not owned by File.
@@ -544,16 +550,8 @@ pub const Scope = struct {
             unloaded: void,
             bytes: [:0]const u8,
         },
-        contents: union {
-            not_available: void,
-            tree: *ast.Tree,
-        },
-        status: enum {
-            never_loaded,
-            unloaded_success,
-            unloaded_parse_failure,
-            loaded_success,
-        },
+        /// Whether this is populated or not depends on `status`.
+        tree: ast.Tree,
         /// Package that this file is a part of, managed externally.
         pkg: *Package,
 
@@ -567,7 +565,7 @@ pub const Scope = struct {
                 => {},
 
                 .loaded_success => {
-                    self.contents.tree.deinit();
+                    self.tree.deinit(gpa);
                     self.status = .unloaded_success;
                 },
             }
@@ -905,7 +903,7 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
         .unreferenced => false,
     };
 
-    const type_changed = mod.astGenAndAnalyzeDecl(decl) catch |err| switch (err) {
+    const type_changed = mod.astgenAndSemaDecl(decl) catch |err| switch (err) {
         error.OutOfMemory => return error.OutOfMemory,
         error.AnalysisFail => return error.AnalysisFail,
         else => {
@@ -947,129 +945,69 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
     }
 }
 
-fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
+/// Returns `true` if the Decl type changed.
+/// Returns `true` if this is the first time analyzing the Decl.
+/// Returns `false` otherwise.
+fn astgenAndSemaDecl(mod: *Module, decl: *Decl) !bool {
     const tracy = trace(@src());
     defer tracy.end();
 
-    const tree = try self.getAstTree(decl.container.file_scope);
-    const ast_node = tree.root_node.decls()[decl.src_index];
-    switch (ast_node.tag) {
-        .FnProto => {
-            const fn_proto = ast_node.castTag(.FnProto).?;
+    const tree = try mod.getAstTree(decl.container.file_scope);
+    const node_tags = tree.nodes.items(.tag);
+    const node_datas = tree.nodes.items(.data);
+    const decl_node = tree.rootDecls()[decl.src_index];
+    switch (node_tags[decl_node]) {
+        .fn_decl => {
+            const fn_proto = node_datas[decl_node].lhs;
+            const body = node_datas[decl_node].rhs;
+            switch (node_tags[fn_proto]) {
+                .fn_proto_simple => {
+                    var params: [1]ast.Node.Index = undefined;
+                    return mod.astgenAndSemaFn(decl, tree, body, tree.fnProtoSimple(&params, fn_proto));
+                },
+                .fn_proto_multi => return mod.astgenAndSemaFn(decl, tree, body, tree.fnProtoMulti(fn_proto)),
+                .fn_proto_one => {
+                    var params: [1]ast.Node.Index = undefined;
+                    return mod.astgenAndSemaFn(decl, tree, body, tree.fnProtoOne(&params, fn_proto));
+                },
+                .fn_proto => return mod.astgenAndSemaFn(decl, tree, body, tree.fnProto(fn_proto)),
+                else => unreachable,
+            }
+        },
+        .fn_proto_simple => {
+            var params: [1]ast.Node.Index = undefined;
+            return mod.astgenAndSemaFn(decl, tree, null, tree.fnProtoSimple(&params, decl_node));
+        },
+        .fn_proto_multi => return mod.astgenAndSemaFn(decl, tree, null, tree.fnProtoMulti(decl_node)),
+        .fn_proto_one => {
+            var params: [1]ast.Node.Index = undefined;
+            return mod.astgenAndSemaFn(decl, tree, null, tree.fnProtoOne(&params, decl_node));
+        },
+        .fn_proto => return mod.astgenAndSemaFn(decl, tree, null, tree.fnProto(decl_node)),
 
+        .global_var_decl => return mod.astgenAndSemaVarDecl(decl, tree, tree.globalVarDecl(decl_node)),
+        .local_var_decl => return mod.astgenAndSemaVarDecl(decl, tree, tree.localVarDecl(decl_node)),
+        .simple_var_decl => return mod.astgenAndSemaVarDecl(decl, tree, tree.simpleVarDecl(decl_node)),
+        .aligned_var_decl => return mod.astgenAndSemaVarDecl(decl, tree, tree.alignedVarDecl(decl_node)),
+
+        .@"comptime" => {
             decl.analysis = .in_progress;
 
-            // This arena allocator's memory is discarded at the end of this function. It is used
-            // to determine the type of the function, and hence the type of the decl, which is needed
-            // to complete the Decl analysis.
-            var fn_type_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
-            defer fn_type_scope_arena.deinit();
-            var fn_type_scope: Scope.GenZIR = .{
+            // A comptime decl does not store any value so we can just deinit this arena after analysis is done.
+            var analysis_arena = std.heap.ArenaAllocator.init(self.gpa);
+            defer analysis_arena.deinit();
+            var gen_scope: Scope.GenZIR = .{
                 .decl = decl,
-                .arena = &fn_type_scope_arena.allocator,
+                .arena = &analysis_arena.allocator,
                 .parent = &decl.container.base,
             };
-            defer fn_type_scope.instructions.deinit(self.gpa);
-
-            decl.is_pub = fn_proto.getVisibToken() != null;
-
-            const param_decls = fn_proto.params();
-            const param_types = try fn_type_scope.arena.alloc(*zir.Inst, param_decls.len);
-
-            const fn_src = tree.token_locs[fn_proto.fn_token].start;
-            const type_type = try astgen.addZIRInstConst(self, &fn_type_scope.base, fn_src, .{
-                .ty = Type.initTag(.type),
-                .val = Value.initTag(.type_type),
-            });
-            const type_type_rl: astgen.ResultLoc = .{ .ty = type_type };
-            for (param_decls) |param_decl, i| {
-                const param_type_node = switch (param_decl.param_type) {
-                    .any_type => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement anytype parameter", .{}),
-                    .type_expr => |node| node,
-                };
-                param_types[i] = try astgen.expr(self, &fn_type_scope.base, type_type_rl, param_type_node);
-            }
-            if (fn_proto.getVarArgsToken()) |var_args_token| {
-                return self.failTok(&fn_type_scope.base, var_args_token, "TODO implement var args", .{});
-            }
-            if (fn_proto.getLibName()) |lib_name| blk: {
-                const lib_name_str = mem.trim(u8, tree.tokenSlice(lib_name.firstToken()), "\""); // TODO: call identifierTokenString
-                log.debug("extern fn symbol expected in lib '{s}'", .{lib_name_str});
-                const target = self.comp.getTarget();
-                if (target_util.is_libc_lib_name(target, lib_name_str)) {
-                    if (!self.comp.bin_file.options.link_libc) {
-                        return self.failNode(
-                            &fn_type_scope.base,
-                            lib_name,
-                            "dependency on libc must be explicitly specified in the build command",
-                            .{},
-                        );
-                    }
-                    break :blk;
-                }
-                if (target_util.is_libcpp_lib_name(target, lib_name_str)) {
-                    if (!self.comp.bin_file.options.link_libcpp) {
-                        return self.failNode(
-                            &fn_type_scope.base,
-                            lib_name,
-                            "dependency on libc++ must be explicitly specified in the build command",
-                            .{},
-                        );
-                    }
-                    break :blk;
-                }
-                if (!target.isWasm() and !self.comp.bin_file.options.pic) {
-                    return self.failNode(
-                        &fn_type_scope.base,
-                        lib_name,
-                        "dependency on dynamic library '{s}' requires enabling Position Independent Code. Fixed by `-l{s}` or `-fPIC`.",
-                        .{ lib_name, lib_name },
-                    );
-                }
-                self.comp.stage1AddLinkLib(lib_name_str) catch |err| {
-                    return self.failNode(
-                        &fn_type_scope.base,
-                        lib_name,
-                        "unable to add link lib '{s}': {s}",
-                        .{ lib_name, @errorName(err) },
-                    );
-                };
-            }
-            if (fn_proto.getAlignExpr()) |align_expr| {
-                return self.failNode(&fn_type_scope.base, align_expr, "TODO implement function align expression", .{});
-            }
-            if (fn_proto.getSectionExpr()) |sect_expr| {
-                return self.failNode(&fn_type_scope.base, sect_expr, "TODO implement function section expression", .{});
-            }
-            if (fn_proto.getCallconvExpr()) |callconv_expr| {
-                return self.failNode(
-                    &fn_type_scope.base,
-                    callconv_expr,
-                    "TODO implement function calling convention expression",
-                    .{},
-                );
-            }
-            const return_type_expr = switch (fn_proto.return_type) {
-                .Explicit => |node| node,
-                .InferErrorSet => |node| return self.failNode(&fn_type_scope.base, node, "TODO implement inferred error sets", .{}),
-                .Invalid => |tok| return self.failTok(&fn_type_scope.base, tok, "unable to parse return type", .{}),
-            };
-
-            const return_type_inst = try astgen.expr(self, &fn_type_scope.base, type_type_rl, return_type_expr);
-            const fn_type_inst = try astgen.addZIRInst(self, &fn_type_scope.base, fn_src, zir.Inst.FnType, .{
-                .return_type = return_type_inst,
-                .param_types = param_types,
-            }, .{});
+            defer gen_scope.instructions.deinit(self.gpa);
 
+            _ = try astgen.comptimeExpr(self, &gen_scope.base, .none, comptime_decl.expr);
             if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
-                zir.dumpZir(self.gpa, "fn_type", decl.name, fn_type_scope.instructions.items) catch {};
+                zir.dumpZir(self.gpa, "comptime_block", decl.name, gen_scope.instructions.items) catch {};
             }
 
-            // We need the memory for the Type to go into the arena for the Decl
-            var decl_arena = std.heap.ArenaAllocator.init(self.gpa);
-            errdefer decl_arena.deinit();
-            const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State);
-
             var inst_table = Scope.Block.InstTable.init(self.gpa);
             defer inst_table.deinit();
 
@@ -1082,426 +1020,561 @@ fn astGenAndAnalyzeDecl(self: *Module, decl: *Decl) !bool {
                 .owner_decl = decl,
                 .src_decl = decl,
                 .instructions = .{},
-                .arena = &decl_arena.allocator,
+                .arena = &analysis_arena.allocator,
                 .inlining = null,
-                .is_comptime = false,
+                .is_comptime = true,
                 .branch_quota = &branch_quota,
             };
             defer block_scope.instructions.deinit(self.gpa);
 
-            const fn_type = try zir_sema.analyzeBodyValueAsType(self, &block_scope, fn_type_inst, .{
-                .instructions = fn_type_scope.instructions.items,
+            _ = try zir_sema.analyzeBody(self, &block_scope, .{
+                .instructions = gen_scope.instructions.items,
             });
-            const body_node = fn_proto.getBodyNode() orelse {
-                // Extern function.
-                var type_changed = true;
-                if (decl.typedValueManaged()) |tvm| {
-                    type_changed = !tvm.typed_value.ty.eql(fn_type);
 
-                    tvm.deinit(self.gpa);
-                }
-                const fn_val = try Value.Tag.extern_fn.create(&decl_arena.allocator, decl);
+            decl.analysis = .complete;
+            decl.generation = self.generation;
+            return true;
+        },
+        .UsingNamespace => @panic("TODO usingnamespace decl"),
+        else => unreachable,
+    }
+}
 
-                decl_arena_state.* = decl_arena.state;
-                decl.typed_value = .{
-                    .most_recent = .{
-                        .typed_value = .{ .ty = fn_type, .val = fn_val },
-                        .arena = decl_arena_state,
-                    },
-                };
-                decl.analysis = .complete;
-                decl.generation = self.generation;
+fn astgenAndSemaFn(
+    mod: *Module,
+    decl: *Decl,
+    tree: ast.Tree,
+    body_node: ast.Node.Index,
+    fn_proto: ast.full.FnProto,
+) !bool {
+    const tracy = trace(@src());
+    defer tracy.end();
 
-                try self.comp.bin_file.allocateDeclIndexes(decl);
-                try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
+    decl.analysis = .in_progress;
 
-                if (type_changed and self.emit_h != null) {
-                    try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
-                }
+    const token_starts = tree.tokens.items(.start);
 
-                return type_changed;
-            };
+    // This arena allocator's memory is discarded at the end of this function. It is used
+    // to determine the type of the function, and hence the type of the decl, which is needed
+    // to complete the Decl analysis.
+    var fn_type_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
+    defer fn_type_scope_arena.deinit();
+    var fn_type_scope: Scope.GenZIR = .{
+        .decl = decl,
+        .arena = &fn_type_scope_arena.allocator,
+        .parent = &decl.container.base,
+    };
+    defer fn_type_scope.instructions.deinit(self.gpa);
 
-            const new_func = try decl_arena.allocator.create(Fn);
-            const fn_payload = try decl_arena.allocator.create(Value.Payload.Function);
+    decl.is_pub = fn_proto.visib_token != null;
 
-            const fn_zir: zir.Body = blk: {
-                // We put the ZIR inside the Decl arena.
-                var gen_scope: Scope.GenZIR = .{
-                    .decl = decl,
-                    .arena = &decl_arena.allocator,
-                    .parent = &decl.container.base,
-                };
-                defer gen_scope.instructions.deinit(self.gpa);
-
-                // We need an instruction for each parameter, and they must be first in the body.
-                try gen_scope.instructions.resize(self.gpa, fn_proto.params_len);
-                var params_scope = &gen_scope.base;
-                for (fn_proto.params()) |param, i| {
-                    const name_token = param.name_token.?;
-                    const src = tree.token_locs[name_token].start;
-                    const param_name = try self.identifierTokenString(&gen_scope.base, name_token);
-                    const arg = try decl_arena.allocator.create(zir.Inst.Arg);
-                    arg.* = .{
-                        .base = .{
-                            .tag = .arg,
-                            .src = src,
-                        },
-                        .positionals = .{
-                            .name = param_name,
-                        },
-                        .kw_args = .{},
-                    };
-                    gen_scope.instructions.items[i] = &arg.base;
-                    const sub_scope = try decl_arena.allocator.create(Scope.LocalVal);
-                    sub_scope.* = .{
-                        .parent = params_scope,
-                        .gen_zir = &gen_scope,
-                        .name = param_name,
-                        .inst = &arg.base,
-                    };
-                    params_scope = &sub_scope.base;
+    // The AST params array does not contain anytype and ... parameters.
+    // We must iterate to count how many param types to allocate.
+    const param_count = blk: {
+        var count: usize = 0;
+        var it = fn_proto.iterate(tree);
+        while (it.next()) |_| {
+            count += 1;
+        }
+        break :blk count;
+    };
+    const param_types = try fn_type_scope.arena.alloc(*zir.Inst, param_count);
+    const fn_src = token_starts[fn_proto.ast.fn_token];
+    const type_type = try astgen.addZIRInstConst(self, &fn_type_scope.base, fn_src, .{
+        .ty = Type.initTag(.type),
+        .val = Value.initTag(.type_type),
+    });
+    const type_type_rl: astgen.ResultLoc = .{ .ty = type_type };
+
+    {
+        var param_type_i: usize = 0;
+        var it = fn_proto.iterate(tree);
+        while (it.next()) |param| : (param_type_i += 1) {
+            if (param.anytype_ellipsis3) |token| {
+                switch (token_tags[token]) {
+                    .keyword_anytype => return self.failTok(
+                        &fn_type_scope.base,
+                        tok_i,
+                        "TODO implement anytype parameter",
+                        .{},
+                    ),
+                    .ellipsis3 => return self.failTok(
+                        &fn_type_scope.base,
+                        token,
+                        "TODO implement var args",
+                        .{},
+                    ),
+                    else => unreachable,
                 }
+            }
+            const param_type_node = param.type_expr;
+            assert(param_type_node != 0);
+            param_types[param_type_i] =
+                try astgen.expr(self, &fn_type_scope.base, type_type_rl, param_type_node);
+        }
+        assert(param_type_i == param_count);
+    }
+    if (fn_proto.lib_name) |lib_name| blk: {
+        // TODO call std.zig.parseStringLiteral
+        const lib_name_str = mem.trim(u8, tree.tokenSlice(lib_name), "\"");
+        log.debug("extern fn symbol expected in lib '{s}'", .{lib_name_str});
+        const target = self.comp.getTarget();
+        if (target_util.is_libc_lib_name(target, lib_name_str)) {
+            if (!self.comp.bin_file.options.link_libc) {
+                return self.failTok(
+                    &fn_type_scope.base,
+                    lib_name,
+                    "dependency on libc must be explicitly specified in the build command",
+                    .{},
+                );
+            }
+            break :blk;
+        }
+        if (target_util.is_libcpp_lib_name(target, lib_name_str)) {
+            if (!self.comp.bin_file.options.link_libcpp) {
+                return self.failTok(
+                    &fn_type_scope.base,
+                    lib_name,
+                    "dependency on libc++ must be explicitly specified in the build command",
+                    .{},
+                );
+            }
+            break :blk;
+        }
+        if (!target.isWasm() and !self.comp.bin_file.options.pic) {
+            return self.failTok(
+                &fn_type_scope.base,
+                lib_name,
+                "dependency on dynamic library '{s}' requires enabling Position Independent Code. Fixed by `-l{s}` or `-fPIC`.",
+                .{ lib_name, lib_name },
+            );
+        }
+        self.comp.stage1AddLinkLib(lib_name_str) catch |err| {
+            return self.failTok(
+                &fn_type_scope.base,
+                lib_name,
+                "unable to add link lib '{s}': {s}",
+                .{ lib_name, @errorName(err) },
+            );
+        };
+    }
+    if (fn_proto.ast.align_expr) |align_expr| {
+        return self.failNode(&fn_type_scope.base, align_expr, "TODO implement function align expression", .{});
+    }
+    if (fn_proto.ast.section_expr) |sect_expr| {
+        return self.failNode(&fn_type_scope.base, sect_expr, "TODO implement function section expression", .{});
+    }
+    if (fn_proto.ast.callconv_expr) |callconv_expr| {
+        return self.failNode(
+            &fn_type_scope.base,
+            callconv_expr,
+            "TODO implement function calling convention expression",
+            .{},
+        );
+    }
+    const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1;
+    if (token_tags[maybe_bang] == .bang) {
+        return self.failTok(&fn_type_scope.base, maybe_bang, "TODO implement inferred error sets", .{});
+    }
+    const return_type_inst = try astgen.expr(
+        self,
+        &fn_type_scope.base,
+        type_type_rl,
+        fn_proto.ast.return_type,
+    );
+    const fn_type_inst = try astgen.addZIRInst(self, &fn_type_scope.base, fn_src, zir.Inst.FnType, .{
+        .return_type = return_type_inst,
+        .param_types = param_types,
+    }, .{});
 
-                const body_block = body_node.cast(ast.Node.Block).?;
+    if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
+        zir.dumpZir(self.gpa, "fn_type", decl.name, fn_type_scope.instructions.items) catch {};
+    }
 
-                try astgen.blockExpr(self, params_scope, body_block);
+    // We need the memory for the Type to go into the arena for the Decl
+    var decl_arena = std.heap.ArenaAllocator.init(self.gpa);
+    errdefer decl_arena.deinit();
+    const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State);
 
-                if (gen_scope.instructions.items.len == 0 or
-                    !gen_scope.instructions.items[gen_scope.instructions.items.len - 1].tag.isNoReturn())
-                {
-                    const src = tree.token_locs[body_block.rbrace].start;
-                    _ = try astgen.addZIRNoOp(self, &gen_scope.base, src, .returnvoid);
-                }
+    var inst_table = Scope.Block.InstTable.init(self.gpa);
+    defer inst_table.deinit();
 
-                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
-                    zir.dumpZir(self.gpa, "fn_body", decl.name, gen_scope.instructions.items) catch {};
-                }
+    var branch_quota: u32 = default_eval_branch_quota;
 
-                break :blk .{
-                    .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items),
-                };
-            };
+    var block_scope: Scope.Block = .{
+        .parent = null,
+        .inst_table = &inst_table,
+        .func = null,
+        .owner_decl = decl,
+        .src_decl = decl,
+        .instructions = .{},
+        .arena = &decl_arena.allocator,
+        .inlining = null,
+        .is_comptime = false,
+        .branch_quota = &branch_quota,
+    };
+    defer block_scope.instructions.deinit(self.gpa);
 
-            const is_inline = blk: {
-                if (fn_proto.getExternExportInlineToken()) |maybe_inline_token| {
-                    if (tree.token_ids[maybe_inline_token] == .Keyword_inline) {
-                        break :blk true;
-                    }
-                }
-                break :blk false;
-            };
-            const anal_state = ([2]Fn.Analysis{ .queued, .inline_only })[@boolToInt(is_inline)];
+    const fn_type = try zir_sema.analyzeBodyValueAsType(self, &block_scope, fn_type_inst, .{
+        .instructions = fn_type_scope.instructions.items,
+    });
+    if (body_node == 0) {
+        // Extern function.
+        var type_changed = true;
+        if (decl.typedValueManaged()) |tvm| {
+            type_changed = !tvm.typed_value.ty.eql(fn_type);
 
-            new_func.* = .{
-                .state = anal_state,
-                .zir = fn_zir,
-                .body = undefined,
-                .owner_decl = decl,
-            };
-            fn_payload.* = .{
-                .base = .{ .tag = .function },
-                .data = new_func,
-            };
+            tvm.deinit(self.gpa);
+        }
+        const fn_val = try Value.Tag.extern_fn.create(&decl_arena.allocator, decl);
 
-            var prev_type_has_bits = false;
-            var prev_is_inline = false;
-            var type_changed = true;
+        decl_arena_state.* = decl_arena.state;
+        decl.typed_value = .{
+            .most_recent = .{
+                .typed_value = .{ .ty = fn_type, .val = fn_val },
+                .arena = decl_arena_state,
+            },
+        };
+        decl.analysis = .complete;
+        decl.generation = self.generation;
 
-            if (decl.typedValueManaged()) |tvm| {
-                prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits();
-                type_changed = !tvm.typed_value.ty.eql(fn_type);
-                if (tvm.typed_value.val.castTag(.function)) |payload| {
-                    const prev_func = payload.data;
-                    prev_is_inline = prev_func.state == .inline_only;
-                }
+        try self.comp.bin_file.allocateDeclIndexes(decl);
+        try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
 
-                tvm.deinit(self.gpa);
-            }
+        if (type_changed and self.emit_h != null) {
+            try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
+        }
 
-            decl_arena_state.* = decl_arena.state;
-            decl.typed_value = .{
-                .most_recent = .{
-                    .typed_value = .{
-                        .ty = fn_type,
-                        .val = Value.initPayload(&fn_payload.base),
-                    },
-                    .arena = decl_arena_state,
+        return type_changed;
+    }
+
+    const new_func = try decl_arena.allocator.create(Fn);
+    const fn_payload = try decl_arena.allocator.create(Value.Payload.Function);
+
+    const fn_zir: zir.Body = blk: {
+        // We put the ZIR inside the Decl arena.
+        var gen_scope: Scope.GenZIR = .{
+            .decl = decl,
+            .arena = &decl_arena.allocator,
+            .parent = &decl.container.base,
+        };
+        defer gen_scope.instructions.deinit(self.gpa);
+
+        // We need an instruction for each parameter, and they must be first in the body.
+        try gen_scope.instructions.resize(self.gpa, param_count);
+        var params_scope = &gen_scope.base;
+        var i: usize = 0;
+        var it = fn_proto.iterate(tree);
+        while (it.next()) |param| : (i += 1) {
+            const name_token = param.name_token.?;
+            const src = token_starts[name_token];
+            const param_name = try self.identifierTokenString(&gen_scope.base, name_token);
+            const arg = try decl_arena.allocator.create(zir.Inst.NoOp);
+            arg.* = .{
+                .base = .{
+                    .tag = .arg,
+                    .src = src,
                 },
+                .positionals = .{},
+                .kw_args = .{},
             };
-            decl.analysis = .complete;
-            decl.generation = self.generation;
+            gen_scope.instructions.items[i] = &arg.base;
+            const sub_scope = try decl_arena.allocator.create(Scope.LocalVal);
+            sub_scope.* = .{
+                .parent = params_scope,
+                .gen_zir = &gen_scope,
+                .name = param_name,
+                .inst = &arg.base,
+            };
+            params_scope = &sub_scope.base;
+        }
 
-            if (!is_inline and fn_type.hasCodeGenBits()) {
-                // We don't fully codegen the decl until later, but we do need to reserve a global
-                // offset table index for it. This allows us to codegen decls out of dependency order,
-                // increasing how many computations can be done in parallel.
-                try self.comp.bin_file.allocateDeclIndexes(decl);
-                try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
-                if (type_changed and self.emit_h != null) {
-                    try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
-                }
-            } else if (!prev_is_inline and prev_type_has_bits) {
-                self.comp.bin_file.freeDecl(decl);
-            }
+        try astgen.blockExpr(self, params_scope, body_node);
 
-            if (fn_proto.getExternExportInlineToken()) |maybe_export_token| {
-                if (tree.token_ids[maybe_export_token] == .Keyword_export) {
-                    if (is_inline) {
-                        return self.failTok(
-                            &block_scope.base,
-                            maybe_export_token,
-                            "export of inline function",
-                            .{},
-                        );
-                    }
-                    const export_src = tree.token_locs[maybe_export_token].start;
-                    const name_loc = tree.token_locs[fn_proto.getNameToken().?];
-                    const name = tree.tokenSliceLoc(name_loc);
-                    // The scope needs to have the decl in it.
-                    try self.analyzeExport(&block_scope.base, export_src, name, decl);
-                }
-            }
-            return type_changed or is_inline != prev_is_inline;
-        },
-        .VarDecl => {
-            const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", ast_node);
+        if (gen_scope.instructions.items.len == 0 or
+            !gen_scope.instructions.items[gen_scope.instructions.items.len - 1].tag.isNoReturn())
+        {
+            const src = token_starts[tree.lastToken(body_node)];
+            _ = try astgen.addZIRNoOp(self, &gen_scope.base, src, .returnvoid);
+        }
 
-            decl.analysis = .in_progress;
+        if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
+            zir.dumpZir(self.gpa, "fn_body", decl.name, gen_scope.instructions.items) catch {};
+        }
 
-            // We need the memory for the Type to go into the arena for the Decl
-            var decl_arena = std.heap.ArenaAllocator.init(self.gpa);
-            errdefer decl_arena.deinit();
-            const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State);
+        break :blk .{
+            .instructions = try gen_scope.arena.dupe(*zir.Inst, gen_scope.instructions.items),
+        };
+    };
 
-            var decl_inst_table = Scope.Block.InstTable.init(self.gpa);
-            defer decl_inst_table.deinit();
+    const is_inline = fn_type.fnCallingConvention() == .Inline;
+    const anal_state: Fn.Analysis = if (is_inline) .inline_only else .queued;
 
-            var branch_quota: u32 = default_eval_branch_quota;
+    new_func.* = .{
+        .state = anal_state,
+        .zir = fn_zir,
+        .body = undefined,
+        .owner_decl = decl,
+    };
+    fn_payload.* = .{
+        .base = .{ .tag = .function },
+        .data = new_func,
+    };
 
-            var block_scope: Scope.Block = .{
-                .parent = null,
-                .inst_table = &decl_inst_table,
-                .func = null,
-                .owner_decl = decl,
-                .src_decl = decl,
-                .instructions = .{},
-                .arena = &decl_arena.allocator,
-                .inlining = null,
-                .is_comptime = true,
-                .branch_quota = &branch_quota,
-            };
-            defer block_scope.instructions.deinit(self.gpa);
+    var prev_type_has_bits = false;
+    var prev_is_inline = false;
+    var type_changed = true;
 
-            decl.is_pub = var_decl.getVisibToken() != null;
-            const is_extern = blk: {
-                const maybe_extern_token = var_decl.getExternExportToken() orelse
-                    break :blk false;
-                if (tree.token_ids[maybe_extern_token] != .Keyword_extern) break :blk false;
-                if (var_decl.getInitNode()) |some| {
-                    return self.failNode(&block_scope.base, some, "extern variables have no initializers", .{});
-                }
-                break :blk true;
-            };
-            if (var_decl.getLibName()) |lib_name| {
-                assert(is_extern);
-                return self.failNode(&block_scope.base, lib_name, "TODO implement function library name", .{});
-            }
-            const is_mutable = tree.token_ids[var_decl.mut_token] == .Keyword_var;
-            const is_threadlocal = if (var_decl.getThreadLocalToken()) |some| blk: {
-                if (!is_mutable) {
-                    return self.failTok(&block_scope.base, some, "threadlocal variable cannot be constant", .{});
-                }
-                break :blk true;
-            } else false;
-            assert(var_decl.getComptimeToken() == null);
-            if (var_decl.getAlignNode()) |align_expr| {
-                return self.failNode(&block_scope.base, align_expr, "TODO implement function align expression", .{});
-            }
-            if (var_decl.getSectionNode()) |sect_expr| {
-                return self.failNode(&block_scope.base, sect_expr, "TODO implement function section expression", .{});
-            }
+    if (decl.typedValueManaged()) |tvm| {
+        prev_type_has_bits = tvm.typed_value.ty.hasCodeGenBits();
+        type_changed = !tvm.typed_value.ty.eql(fn_type);
+        if (tvm.typed_value.val.castTag(.function)) |payload| {
+            const prev_func = payload.data;
+            prev_is_inline = prev_func.state == .inline_only;
+        }
 
-            const var_info: struct { ty: Type, val: ?Value } = if (var_decl.getInitNode()) |init_node| vi: {
-                var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
-                defer gen_scope_arena.deinit();
-                var gen_scope: Scope.GenZIR = .{
-                    .decl = decl,
-                    .arena = &gen_scope_arena.allocator,
-                    .parent = &decl.container.base,
-                };
-                defer gen_scope.instructions.deinit(self.gpa);
+        tvm.deinit(self.gpa);
+    }
 
-                const init_result_loc: astgen.ResultLoc = if (var_decl.getTypeNode()) |type_node| rl: {
-                    const src = tree.token_locs[type_node.firstToken()].start;
-                    const type_type = try astgen.addZIRInstConst(self, &gen_scope.base, src, .{
-                        .ty = Type.initTag(.type),
-                        .val = Value.initTag(.type_type),
-                    });
-                    const var_type = try astgen.expr(self, &gen_scope.base, .{ .ty = type_type }, type_node);
-                    break :rl .{ .ty = var_type };
-                } else .none;
+    decl_arena_state.* = decl_arena.state;
+    decl.typed_value = .{
+        .most_recent = .{
+            .typed_value = .{
+                .ty = fn_type,
+                .val = Value.initPayload(&fn_payload.base),
+            },
+            .arena = decl_arena_state,
+        },
+    };
+    decl.analysis = .complete;
+    decl.generation = self.generation;
+
+    if (!is_inline and fn_type.hasCodeGenBits()) {
+        // We don't fully codegen the decl until later, but we do need to reserve a global
+        // offset table index for it. This allows us to codegen decls out of dependency order,
+        // increasing how many computations can be done in parallel.
+        try self.comp.bin_file.allocateDeclIndexes(decl);
+        try self.comp.work_queue.writeItem(.{ .codegen_decl = decl });
+        if (type_changed and self.emit_h != null) {
+            try self.comp.work_queue.writeItem(.{ .emit_h_decl = decl });
+        }
+    } else if (!prev_is_inline and prev_type_has_bits) {
+        self.comp.bin_file.freeDecl(decl);
+    }
 
-                const init_inst = try astgen.comptimeExpr(self, &gen_scope.base, init_result_loc, init_node);
-                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
-                    zir.dumpZir(self.gpa, "var_init", decl.name, gen_scope.instructions.items) catch {};
-                }
+    if (fn_proto.extern_export_token) |maybe_export_token| {
+        if (token_tags[maybe_export_token] == .Keyword_export) {
+            if (is_inline) {
+                return self.failTok(
+                    &block_scope.base,
+                    maybe_export_token,
+                    "export of inline function",
+                    .{},
+                );
+            }
+            const export_src = token_starts[maybe_export_token];
+            const name = tree.tokenSlice(fn_proto.name_token.?); // TODO identifierTokenString
+            // The scope needs to have the decl in it.
+            try self.analyzeExport(&block_scope.base, export_src, name, decl);
+        }
+    }
+    return type_changed or is_inline != prev_is_inline;
+}
 
-                var var_inst_table = Scope.Block.InstTable.init(self.gpa);
-                defer var_inst_table.deinit();
-
-                var branch_quota_vi: u32 = default_eval_branch_quota;
-                var inner_block: Scope.Block = .{
-                    .parent = null,
-                    .inst_table = &var_inst_table,
-                    .func = null,
-                    .owner_decl = decl,
-                    .src_decl = decl,
-                    .instructions = .{},
-                    .arena = &gen_scope_arena.allocator,
-                    .inlining = null,
-                    .is_comptime = true,
-                    .branch_quota = &branch_quota_vi,
-                };
-                defer inner_block.instructions.deinit(self.gpa);
-                try zir_sema.analyzeBody(self, &inner_block, .{
-                    .instructions = gen_scope.instructions.items,
-                });
+fn astgenAndSemaVarDecl(
+    mod: *Module,
+    decl: *Decl,
+    tree: ast.Tree,
+    var_decl: ast.full.VarDecl,
+) !bool {
+    const tracy = trace(@src());
+    defer tracy.end();
 
-                // The result location guarantees the type coercion.
-                const analyzed_init_inst = var_inst_table.get(init_inst).?;
-                // The is_comptime in the Scope.Block guarantees the result is comptime-known.
-                const val = analyzed_init_inst.value().?;
+    decl.analysis = .in_progress;
 
-                const ty = try analyzed_init_inst.ty.copy(block_scope.arena);
-                break :vi .{
-                    .ty = ty,
-                    .val = try val.copy(block_scope.arena),
-                };
-            } else if (!is_extern) {
-                return self.failTok(&block_scope.base, var_decl.firstToken(), "variables must be initialized", .{});
-            } else if (var_decl.getTypeNode()) |type_node| vi: {
-                // Temporary arena for the zir instructions.
-                var type_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
-                defer type_scope_arena.deinit();
-                var type_scope: Scope.GenZIR = .{
-                    .decl = decl,
-                    .arena = &type_scope_arena.allocator,
-                    .parent = &decl.container.base,
-                };
-                defer type_scope.instructions.deinit(self.gpa);
+    const token_starts = tree.tokens.items(.start);
 
-                const var_type = try astgen.typeExpr(self, &type_scope.base, type_node);
-                if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
-                    zir.dumpZir(self.gpa, "var_type", decl.name, type_scope.instructions.items) catch {};
-                }
+    // We need the memory for the Type to go into the arena for the Decl
+    var decl_arena = std.heap.ArenaAllocator.init(self.gpa);
+    errdefer decl_arena.deinit();
+    const decl_arena_state = try decl_arena.allocator.create(std.heap.ArenaAllocator.State);
 
-                const ty = try zir_sema.analyzeBodyValueAsType(self, &block_scope, var_type, .{
-                    .instructions = type_scope.instructions.items,
-                });
-                break :vi .{
-                    .ty = ty,
-                    .val = null,
-                };
-            } else {
-                return self.failTok(&block_scope.base, var_decl.firstToken(), "unable to infer variable type", .{});
-            };
+    var decl_inst_table = Scope.Block.InstTable.init(self.gpa);
+    defer decl_inst_table.deinit();
 
-            if (is_mutable and !var_info.ty.isValidVarType(is_extern)) {
-                return self.failTok(&block_scope.base, var_decl.firstToken(), "variable of type '{}' must be const", .{var_info.ty});
-            }
+    var branch_quota: u32 = default_eval_branch_quota;
 
-            var type_changed = true;
-            if (decl.typedValueManaged()) |tvm| {
-                type_changed = !tvm.typed_value.ty.eql(var_info.ty);
+    var block_scope: Scope.Block = .{
+        .parent = null,
+        .inst_table = &decl_inst_table,
+        .func = null,
+        .owner_decl = decl,
+        .src_decl = decl,
+        .instructions = .{},
+        .arena = &decl_arena.allocator,
+        .inlining = null,
+        .is_comptime = true,
+        .branch_quota = &branch_quota,
+    };
+    defer block_scope.instructions.deinit(self.gpa);
+
+    decl.is_pub = var_decl.getVisibToken() != null;
+    const is_extern = blk: {
+        const maybe_extern_token = var_decl.getExternExportToken() orelse
+            break :blk false;
+        if (tree.token_ids[maybe_extern_token] != .Keyword_extern) break :blk false;
+        if (var_decl.getInitNode()) |some| {
+            return self.failNode(&block_scope.base, some, "extern variables have no initializers", .{});
+        }
+        break :blk true;
+    };
+    if (var_decl.getLibName()) |lib_name| {
+        assert(is_extern);
+        return self.failNode(&block_scope.base, lib_name, "TODO implement function library name", .{});
+    }
+    const is_mutable = tree.token_ids[var_decl.mut_token] == .Keyword_var;
+    const is_threadlocal = if (var_decl.getThreadLocalToken()) |some| blk: {
+        if (!is_mutable) {
+            return self.failTok(&block_scope.base, some, "threadlocal variable cannot be constant", .{});
+        }
+        break :blk true;
+    } else false;
+    assert(var_decl.getComptimeToken() == null);
+    if (var_decl.getAlignNode()) |align_expr| {
+        return self.failNode(&block_scope.base, align_expr, "TODO implement function align expression", .{});
+    }
+    if (var_decl.getSectionNode()) |sect_expr| {
+        return self.failNode(&block_scope.base, sect_expr, "TODO implement function section expression", .{});
+    }
+
+    const var_info: struct { ty: Type, val: ?Value } = if (var_decl.getInitNode()) |init_node| vi: {
+        var gen_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
+        defer gen_scope_arena.deinit();
+        var gen_scope: Scope.GenZIR = .{
+            .decl = decl,
+            .arena = &gen_scope_arena.allocator,
+            .parent = &decl.container.base,
+        };
+        defer gen_scope.instructions.deinit(self.gpa);
 
-                tvm.deinit(self.gpa);
-            }
+        const init_result_loc: astgen.ResultLoc = if (var_decl.getTypeNode()) |type_node| rl: {
+            const src = token_starts[type_node.firstToken()];
+            const type_type = try astgen.addZIRInstConst(self, &gen_scope.base, src, .{
+                .ty = Type.initTag(.type),
+                .val = Value.initTag(.type_type),
+            });
+            const var_type = try astgen.expr(self, &gen_scope.base, .{ .ty = type_type }, type_node);
+            break :rl .{ .ty = var_type };
+        } else .none;
 
-            const new_variable = try decl_arena.allocator.create(Var);
-            new_variable.* = .{
-                .owner_decl = decl,
-                .init = var_info.val orelse undefined,
-                .is_extern = is_extern,
-                .is_mutable = is_mutable,
-                .is_threadlocal = is_threadlocal,
-            };
-            const var_val = try Value.Tag.variable.create(&decl_arena.allocator, new_variable);
-
-            decl_arena_state.* = decl_arena.state;
-            decl.typed_value = .{
-                .most_recent = .{
-                    .typed_value = .{
-                        .ty = var_info.ty,
-                        .val = var_val,
-                    },
-                    .arena = decl_arena_state,
-                },
-            };
-            decl.analysis = .complete;
-            decl.generation = self.generation;
+        const init_inst = try astgen.comptimeExpr(self, &gen_scope.base, init_result_loc, init_node);
+        if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
+            zir.dumpZir(self.gpa, "var_init", decl.name, gen_scope.instructions.items) catch {};
+        }
 
-            if (var_decl.getExternExportToken()) |maybe_export_token| {
-                if (tree.token_ids[maybe_export_token] == .Keyword_export) {
-                    const export_src = tree.token_locs[maybe_export_token].start;
-                    const name_loc = tree.token_locs[var_decl.name_token];
-                    const name = tree.tokenSliceLoc(name_loc);
-                    // The scope needs to have the decl in it.
-                    try self.analyzeExport(&block_scope.base, export_src, name, decl);
-                }
-            }
-            return type_changed;
-        },
-        .Comptime => {
-            const comptime_decl = @fieldParentPtr(ast.Node.Comptime, "base", ast_node);
+        var var_inst_table = Scope.Block.InstTable.init(self.gpa);
+        defer var_inst_table.deinit();
+
+        var branch_quota_vi: u32 = default_eval_branch_quota;
+        var inner_block: Scope.Block = .{
+            .parent = null,
+            .inst_table = &var_inst_table,
+            .func = null,
+            .owner_decl = decl,
+            .src_decl = decl,
+            .instructions = .{},
+            .arena = &gen_scope_arena.allocator,
+            .inlining = null,
+            .is_comptime = true,
+            .branch_quota = &branch_quota_vi,
+        };
+        defer inner_block.instructions.deinit(self.gpa);
+        try zir_sema.analyzeBody(self, &inner_block, .{
+            .instructions = gen_scope.instructions.items,
+        });
 
-            decl.analysis = .in_progress;
+        // The result location guarantees the type coercion.
+        const analyzed_init_inst = var_inst_table.get(init_inst).?;
+        // The is_comptime in the Scope.Block guarantees the result is comptime-known.
+        const val = analyzed_init_inst.value().?;
 
-            // A comptime decl does not store any value so we can just deinit this arena after analysis is done.
-            var analysis_arena = std.heap.ArenaAllocator.init(self.gpa);
-            defer analysis_arena.deinit();
-            var gen_scope: Scope.GenZIR = .{
-                .decl = decl,
-                .arena = &analysis_arena.allocator,
-                .parent = &decl.container.base,
-            };
-            defer gen_scope.instructions.deinit(self.gpa);
+        const ty = try analyzed_init_inst.ty.copy(block_scope.arena);
+        break :vi .{
+            .ty = ty,
+            .val = try val.copy(block_scope.arena),
+        };
+    } else if (!is_extern) {
+        return self.failTok(&block_scope.base, var_decl.firstToken(), "variables must be initialized", .{});
+    } else if (var_decl.getTypeNode()) |type_node| vi: {
+        // Temporary arena for the zir instructions.
+        var type_scope_arena = std.heap.ArenaAllocator.init(self.gpa);
+        defer type_scope_arena.deinit();
+        var type_scope: Scope.GenZIR = .{
+            .decl = decl,
+            .arena = &type_scope_arena.allocator,
+            .parent = &decl.container.base,
+        };
+        defer type_scope.instructions.deinit(self.gpa);
 
-            _ = try astgen.comptimeExpr(self, &gen_scope.base, .none, comptime_decl.expr);
-            if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
-                zir.dumpZir(self.gpa, "comptime_block", decl.name, gen_scope.instructions.items) catch {};
-            }
+        const var_type = try astgen.typeExpr(self, &type_scope.base, type_node);
+        if (std.builtin.mode == .Debug and self.comp.verbose_ir) {
+            zir.dumpZir(self.gpa, "var_type", decl.name, type_scope.instructions.items) catch {};
+        }
 
-            var inst_table = Scope.Block.InstTable.init(self.gpa);
-            defer inst_table.deinit();
+        const ty = try zir_sema.analyzeBodyValueAsType(self, &block_scope, var_type, .{
+            .instructions = type_scope.instructions.items,
+        });
+        break :vi .{
+            .ty = ty,
+            .val = null,
+        };
+    } else {
+        return self.failTok(&block_scope.base, var_decl.firstToken(), "unable to infer variable type", .{});
+    };
 
-            var branch_quota: u32 = default_eval_branch_quota;
+    if (is_mutable and !var_info.ty.isValidVarType(is_extern)) {
+        return self.failTok(&block_scope.base, var_decl.firstToken(), "variable of type '{}' must be const", .{var_info.ty});
+    }
 
-            var block_scope: Scope.Block = .{
-                .parent = null,
-                .inst_table = &inst_table,
-                .func = null,
-                .owner_decl = decl,
-                .src_decl = decl,
-                .instructions = .{},
-                .arena = &analysis_arena.allocator,
-                .inlining = null,
-                .is_comptime = true,
-                .branch_quota = &branch_quota,
-            };
-            defer block_scope.instructions.deinit(self.gpa);
+    var type_changed = true;
+    if (decl.typedValueManaged()) |tvm| {
+        type_changed = !tvm.typed_value.ty.eql(var_info.ty);
 
-            _ = try zir_sema.analyzeBody(self, &block_scope, .{
-                .instructions = gen_scope.instructions.items,
-            });
+        tvm.deinit(self.gpa);
+    }
 
-            decl.analysis = .complete;
-            decl.generation = self.generation;
-            return true;
+    const new_variable = try decl_arena.allocator.create(Var);
+    new_variable.* = .{
+        .owner_decl = decl,
+        .init = var_info.val orelse undefined,
+        .is_extern = is_extern,
+        .is_mutable = is_mutable,
+        .is_threadlocal = is_threadlocal,
+    };
+    const var_val = try Value.Tag.variable.create(&decl_arena.allocator, new_variable);
+
+    decl_arena_state.* = decl_arena.state;
+    decl.typed_value = .{
+        .most_recent = .{
+            .typed_value = .{
+                .ty = var_info.ty,
+                .val = var_val,
+            },
+            .arena = decl_arena_state,
         },
-        .Use => @panic("TODO usingnamespace decl"),
-        else => unreachable,
+    };
+    decl.analysis = .complete;
+    decl.generation = self.generation;
+
+    if (var_decl.getExternExportToken()) |maybe_export_token| {
+        if (tree.token_ids[maybe_export_token] == .Keyword_export) {
+            const export_src = token_starts[maybe_export_token];
+            const name = tree.tokenSlice(var_decl.name_token); // TODO identifierTokenString
+            // The scope needs to have the decl in it.
+            try self.analyzeExport(&block_scope.base, export_src, name, decl);
+        }
     }
+    return type_changed;
 }
 
 fn declareDeclDependency(self: *Module, depender: *Decl, dependee: *Decl) !void {
@@ -1512,7 +1585,7 @@ fn declareDeclDependency(self: *Module, depender: *Decl, dependee: *Decl) !void
     dependee.dependants.putAssumeCapacity(depender, {});
 }
 
-pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
+pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*const ast.Tree {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -1523,8 +1596,10 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
             const source = try root_scope.getSource(self);
 
             var keep_tree = false;
-            const tree = try std.zig.parse(self.gpa, source);
-            defer if (!keep_tree) tree.deinit();
+            root_scope.tree = try std.zig.parse(self.gpa, source);
+            defer if (!keep_tree) root_scope.tree.deinit(self.gpa);
+
+            const tree = &root_scope.tree;
 
             if (tree.errors.len != 0) {
                 const parse_err = tree.errors[0];
@@ -1532,12 +1607,12 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
                 var msg = std.ArrayList(u8).init(self.gpa);
                 defer msg.deinit();
 
-                try parse_err.render(tree.token_ids, msg.writer());
+                try tree.renderError(parse_err, msg.writer());
                 const err_msg = try self.gpa.create(ErrorMsg);
                 err_msg.* = .{
                     .src_loc = .{
                         .file_scope = root_scope,
-                        .byte_offset = tree.token_locs[parse_err.loc()].start,
+                        .byte_offset = tree.tokens.items(.start)[parse_err.loc()],
                     },
                     .msg = msg.toOwnedSlice(),
                 };
@@ -1548,7 +1623,6 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
             }
 
             root_scope.status = .loaded_success;
-            root_scope.contents = .{ .tree = tree };
             keep_tree = true;
 
             return tree;
@@ -1556,144 +1630,336 @@ pub fn getAstTree(self: *Module, root_scope: *Scope.File) !*ast.Tree {
 
         .unloaded_parse_failure => return error.AnalysisFail,
 
-        .loaded_success => return root_scope.contents.tree,
+        .loaded_success => return &root_scope.tree,
     }
 }
 
-pub fn analyzeContainer(self: *Module, container_scope: *Scope.Container) !void {
+pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
     // We may be analyzing it for the first time, or this may be
     // an incremental update. This code handles both cases.
-    const tree = try self.getAstTree(container_scope.file_scope);
-    const decls = tree.root_node.decls();
+    const tree = try mod.getAstTree(container_scope.file_scope);
+    const node_tags = tree.nodes.items(.tag);
+    const node_datas = tree.nodes.items(.data);
+    const decls = tree.rootDecls();
 
-    try self.comp.work_queue.ensureUnusedCapacity(decls.len);
-    try container_scope.decls.ensureCapacity(self.gpa, decls.len);
+    try mod.comp.work_queue.ensureUnusedCapacity(decls.len);
+    try container_scope.decls.ensureCapacity(mod.gpa, decls.len);
 
     // Keep track of the decls that we expect to see in this file so that
     // we know which ones have been deleted.
-    var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(self.gpa);
+    var deleted_decls = std.AutoArrayHashMap(*Decl, void).init(mod.gpa);
     defer deleted_decls.deinit();
     try deleted_decls.ensureCapacity(container_scope.decls.items().len);
     for (container_scope.decls.items()) |entry| {
         deleted_decls.putAssumeCapacityNoClobber(entry.key, {});
     }
 
-    for (decls) |src_decl, decl_i| {
-        if (src_decl.cast(ast.Node.FnProto)) |fn_proto| {
-            // We will create a Decl for it regardless of analysis status.
-            const name_tok = fn_proto.getNameToken() orelse {
-                @panic("TODO missing function name");
-            };
-
-            const name_loc = tree.token_locs[name_tok];
-            const name = tree.tokenSliceLoc(name_loc);
-            const name_hash = container_scope.fullyQualifiedNameHash(name);
-            const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl));
-            if (self.decl_table.get(name_hash)) |decl| {
-                // Update the AST Node index of the decl, even if its contents are unchanged, it may
-                // have been re-ordered.
-                decl.src_index = decl_i;
-                if (deleted_decls.swapRemove(decl) == null) {
-                    decl.analysis = .sema_failure;
-                    const msg = try ErrorMsg.create(self.gpa, .{
-                        .file_scope = container_scope.file_scope,
-                        .byte_offset = tree.token_locs[name_tok].start,
-                    }, "redefinition of '{s}'", .{decl.name});
-                    errdefer msg.destroy(self.gpa);
-                    try self.failed_decls.putNoClobber(self.gpa, decl, msg);
-                } else {
-                    if (!srcHashEql(decl.contents_hash, contents_hash)) {
-                        try self.markOutdatedDecl(decl);
-                        decl.contents_hash = contents_hash;
-                    } else switch (self.comp.bin_file.tag) {
-                        .coff => {
-                            // TODO Implement for COFF
-                        },
-                        .elf => if (decl.fn_link.elf.len != 0) {
-                            // TODO Look into detecting when this would be unnecessary by storing enough state
-                            // in `Decl` to notice that the line number did not change.
-                            self.comp.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl });
-                        },
-                        .macho => if (decl.fn_link.macho.len != 0) {
-                            // TODO Look into detecting when this would be unnecessary by storing enough state
-                            // in `Decl` to notice that the line number did not change.
-                            self.comp.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl });
-                        },
-                        .c, .wasm => {},
-                    }
-                }
-            } else {
-                const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
-                container_scope.decls.putAssumeCapacity(new_decl, {});
-                if (fn_proto.getExternExportInlineToken()) |maybe_export_token| {
-                    if (tree.token_ids[maybe_export_token] == .Keyword_export) {
-                        self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
-                    }
-                }
-            }
-        } else if (src_decl.castTag(.VarDecl)) |var_decl| {
-            const name_loc = tree.token_locs[var_decl.name_token];
-            const name = tree.tokenSliceLoc(name_loc);
-            const name_hash = container_scope.fullyQualifiedNameHash(name);
-            const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl));
-            if (self.decl_table.get(name_hash)) |decl| {
-                // Update the AST Node index of the decl, even if its contents are unchanged, it may
-                // have been re-ordered.
-                decl.src_index = decl_i;
-                if (deleted_decls.swapRemove(decl) == null) {
-                    decl.analysis = .sema_failure;
-                    const err_msg = try ErrorMsg.create(self.gpa, .{
-                        .file_scope = container_scope.file_scope,
-                        .byte_offset = name_loc.start,
-                    }, "redefinition of '{s}'", .{decl.name});
-                    errdefer err_msg.destroy(self.gpa);
-                    try self.failed_decls.putNoClobber(self.gpa, decl, err_msg);
-                } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
-                    try self.markOutdatedDecl(decl);
-                    decl.contents_hash = contents_hash;
-                }
-            } else {
-                const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
-                container_scope.decls.putAssumeCapacity(new_decl, {});
-                if (var_decl.getExternExportToken()) |maybe_export_token| {
-                    if (tree.token_ids[maybe_export_token] == .Keyword_export) {
-                        self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
-                    }
-                }
+    for (decls) |decl_node, decl_i| switch (node_tags[decl_node]) {
+        .fn_decl => {
+            const fn_proto = node_datas[decl_node].lhs;
+            const body = node_datas[decl_node].rhs;
+            switch (node_tags[fn_proto]) {
+                .fn_proto_simple => {
+                    var params: [1]ast.Node.Index = undefined;
+                    try mod.semaContainerFn(
+                        container_scope,
+                        &deleted_decls,
+                        decl_node,
+                        decl_i,
+                        tree.*,
+                        body,
+                        tree.fnProtoSimple(&params, fn_proto),
+                    );
+                },
+                .fn_proto_multi => try mod.semaContainerFn(
+                    container_scope,
+                    &deleted_decls,
+                    decl_node,
+                    decl_i,
+                    tree.*,
+                    body,
+                    tree.fnProtoMulti(fn_proto),
+                ),
+                .fn_proto_one => {
+                    var params: [1]ast.Node.Index = undefined;
+                    try mod.semaContainerFn(
+                        container_scope,
+                        &deleted_decls,
+                        decl_node,
+                        decl_i,
+                        tree.*,
+                        body,
+                        tree.fnProtoOne(&params, fn_proto),
+                    );
+                },
+                .fn_proto => try mod.semaContainerFn(
+                    container_scope,
+                    &deleted_decls,
+                    decl_node,
+                    decl_i,
+                    tree.*,
+                    body,
+                    tree.fnProto(fn_proto),
+                ),
+                else => unreachable,
             }
-        } else if (src_decl.castTag(.Comptime)) |comptime_node| {
-            const name_index = self.getNextAnonNameIndex();
-            const name = try std.fmt.allocPrint(self.gpa, "__comptime_{d}", .{name_index});
-            defer self.gpa.free(name);
+        },
+        .fn_proto_simple => {
+            var params: [1]ast.Node.Index = undefined;
+            try mod.semaContainerFn(
+                container_scope,
+                &deleted_decls,
+                decl_node,
+                decl_i,
+                tree.*,
+                null,
+                tree.fnProtoSimple(&params, decl_node),
+            );
+        },
+        .fn_proto_multi => try mod.semaContainerFn(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            null,
+            tree.fnProtoMulti(decl_node),
+        ),
+        .fn_proto_one => {
+            var params: [1]ast.Node.Index = undefined;
+            try mod.semaContainerFn(
+                container_scope,
+                &deleted_decls,
+                decl_node,
+                decl_i,
+                tree.*,
+                null,
+                tree.fnProtoOne(&params, decl_node),
+            );
+        },
+        .fn_proto => try mod.semaContainerFn(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            null,
+            tree.fnProto(decl_node),
+        ),
+
+        .global_var_decl => try mod.semaContainerVar(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.globalVarDecl(decl_node),
+        ),
+        .local_var_decl => try mod.semaContainerVar(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.localVarDecl(decl_node),
+        ),
+        .simple_var_decl => try mod.semaContainerVar(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.simpleVarDecl(decl_node),
+        ),
+        .aligned_var_decl => try mod.semaContainerVar(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.alignedVarDecl(decl_node),
+        ),
+
+        .@"comptime" => {
+            const name_index = mod.getNextAnonNameIndex();
+            const name = try std.fmt.allocPrint(mod.gpa, "__comptime_{d}", .{name_index});
+            defer mod.gpa.free(name);
 
             const name_hash = container_scope.fullyQualifiedNameHash(name);
-            const contents_hash = std.zig.hashSrc(tree.getNodeSource(src_decl));
+            const contents_hash = std.zig.hashSrc(tree.getNodeSource(decl_node));
 
-            const new_decl = try self.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
+            const new_decl = try mod.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
             container_scope.decls.putAssumeCapacity(new_decl, {});
-            self.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
-        } else if (src_decl.castTag(.ContainerField)) |container_field| {
-            log.err("TODO: analyze container field", .{});
-        } else if (src_decl.castTag(.TestDecl)) |test_decl| {
+            mod.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
+        },
+
+        .container_field_init => try mod.semaContainerField(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.containerFieldInit(decl),
+        ),
+        .container_field_align => try mod.semaContainerField(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.containerFieldAlign(decl),
+        ),
+        .container_field => try mod.semaContainerField(
+            container_scope,
+            &deleted_decls,
+            decl_node,
+            decl_i,
+            tree.*,
+            tree.containerField(decl),
+        ),
+
+        .test_decl => {
             log.err("TODO: analyze test decl", .{});
-        } else if (src_decl.castTag(.Use)) |use_decl| {
+        },
+        .@"usingnamespace" => {
             log.err("TODO: analyze usingnamespace decl", .{});
-        } else {
-            unreachable;
-        }
-    }
+        },
+        else => unreachable,
+    };
     // Handle explicitly deleted decls from the source code. Not to be confused
     // with when we delete decls because they are no longer referenced.
     for (deleted_decls.items()) |entry| {
         log.debug("noticed '{s}' deleted from source\n", .{entry.key.name});
-        try self.deleteDecl(entry.key);
+        try mod.deleteDecl(entry.key);
+    }
+}
+
+fn semaContainerFn(
+    mod: *Module,
+    container_scope: *Scope.Container,
+    deleted_decls: *std.AutoArrayHashMap(*Decl, void),
+    decl_node: ast.Node.Index,
+    decl_i: usize,
+    tree: ast.Tree,
+    body_node: ast.Node.Index,
+    fn_proto: ast.full.FnProto,
+) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const token_starts = tree.tokens.items(.start);
+    const token_tags = tree.tokens.items(.tag);
+
+    // We will create a Decl for it regardless of analysis status.
+    const name_tok = fn_proto.name_token orelse {
+        @panic("TODO missing function name");
+    };
+    const name = tree.tokenSlice(name_tok); // TODO use identifierTokenString
+    const name_hash = container_scope.fullyQualifiedNameHash(name);
+    const contents_hash = std.zig.hashSrc(tree.getNodeSource(decl_node));
+    if (mod.decl_table.get(name_hash)) |decl| {
+        // Update the AST Node index of the decl, even if its contents are unchanged, it may
+        // have been re-ordered.
+        decl.src_index = decl_i;
+        if (deleted_decls.swapRemove(decl) == null) {
+            decl.analysis = .sema_failure;
+            const msg = try ErrorMsg.create(mod.gpa, .{
+                .file_scope = container_scope.file_scope,
+                .byte_offset = token_starts[name_tok],
+            }, "redefinition of '{s}'", .{decl.name});
+            errdefer msg.destroy(mod.gpa);
+            try mod.failed_decls.putNoClobber(mod.gpa, decl, msg);
+        } else {
+            if (!srcHashEql(decl.contents_hash, contents_hash)) {
+                try mod.markOutdatedDecl(decl);
+                decl.contents_hash = contents_hash;
+            } else switch (mod.comp.bin_file.tag) {
+                .coff => {
+                    // TODO Implement for COFF
+                },
+                .elf => if (decl.fn_link.elf.len != 0) {
+                    // TODO Look into detecting when this would be unnecessary by storing enough state
+                    // in `Decl` to notice that the line number did not change.
+                    mod.comp.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl });
+                },
+                .macho => if (decl.fn_link.macho.len != 0) {
+                    // TODO Look into detecting when this would be unnecessary by storing enough state
+                    // in `Decl` to notice that the line number did not change.
+                    mod.comp.work_queue.writeItemAssumeCapacity(.{ .update_line_number = decl });
+                },
+                .c, .wasm => {},
+            }
+        }
+    } else {
+        const new_decl = try mod.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
+        container_scope.decls.putAssumeCapacity(new_decl, {});
+        if (fn_proto.getExternExportInlineToken()) |maybe_export_token| {
+            if (tree.token_ids[maybe_export_token] == .Keyword_export) {
+                mod.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
+            }
+        }
+    }
+}
+
+fn semaContainerVar(
+    mod: *Module,
+    container_scope: *Scope.Container,
+    deleted_decls: *std.AutoArrayHashMap(*Decl, void),
+    decl_node: ast.Node.Index,
+    decl_i: usize,
+    tree: ast.Tree,
+    var_decl: ast.full.VarDecl,
+) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const token_starts = tree.tokens.items(.start);
+
+    const name_src = token_starts[var_decl.name_token];
+    const name = tree.tokenSlice(var_decl.name_token); // TODO identifierTokenString
+    const name_hash = container_scope.fullyQualifiedNameHash(name);
+    const contents_hash = std.zig.hashSrc(tree.getNodeSource(decl_node));
+    if (mod.decl_table.get(name_hash)) |decl| {
+        // Update the AST Node index of the decl, even if its contents are unchanged, it may
+        // have been re-ordered.
+        decl.src_index = decl_i;
+        if (deleted_decls.swapRemove(decl) == null) {
+            decl.analysis = .sema_failure;
+            const err_msg = try ErrorMsg.create(mod.gpa, .{
+                .file_scope = container_scope.file_scope,
+                .byte_offset = name_src,
+            }, "redefinition of '{s}'", .{decl.name});
+            errdefer err_msg.destroy(mod.gpa);
+            try mod.failed_decls.putNoClobber(mod.gpa, decl, err_msg);
+        } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
+            try mod.markOutdatedDecl(decl);
+            decl.contents_hash = contents_hash;
+        }
+    } else {
+        const new_decl = try mod.createNewDecl(&container_scope.base, name, decl_i, name_hash, contents_hash);
+        container_scope.decls.putAssumeCapacity(new_decl, {});
+        if (var_decl.getExternExportToken()) |maybe_export_token| {
+            if (tree.token_ids[maybe_export_token] == .Keyword_export) {
+                mod.comp.work_queue.writeItemAssumeCapacity(.{ .analyze_decl = new_decl });
+            }
+        }
     }
 }
 
+fn semaContainerField() void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    log.err("TODO: analyze container field", .{});
+}
+
 pub fn deleteDecl(self: *Module, decl: *Decl) !void {
+    const tracy = trace(@src());
+    defer tracy.end();
+
     try self.deletion_set.ensureCapacity(self.gpa, self.deletion_set.items.len + decl.dependencies.items().len);
 
     // Remove from the namespace it resides in. In the case of an anonymous Decl it will
@@ -2338,15 +2604,16 @@ pub fn createContainerDecl(
 fn getAnonTypeName(self: *Module, scope: *Scope, base_token: std.zig.ast.TokenIndex) ![]u8 {
     // TODO add namespaces, generic function signatrues
     const tree = scope.tree();
-    const base_name = switch (tree.token_ids[base_token]) {
-        .Keyword_struct => "struct",
-        .Keyword_enum => "enum",
-        .Keyword_union => "union",
-        .Keyword_opaque => "opaque",
+    const token_tags = tree.tokens.items(.tag);
+    const base_name = switch (token_tags[base_token]) {
+        .keyword_struct => "struct",
+        .keyword_enum => "enum",
+        .keyword_union => "union",
+        .keyword_opaque => "opaque",
         else => unreachable,
     };
-    const loc = tree.tokenLocationLoc(0, tree.token_locs[base_token]);
-    return std.fmt.allocPrint(self.gpa, "{}:{}:{}", .{ base_name, loc.line, loc.column });
+    const loc = tree.tokenLocation(0, base_token);
+    return std.fmt.allocPrint(self.gpa, "{s}:{d}:{d}", .{ base_name, loc.line, loc.column });
 }
 
 fn getNextAnonNameIndex(self: *Module) usize {
@@ -3092,7 +3359,7 @@ pub fn failTok(
     comptime format: []const u8,
     args: anytype,
 ) InnerError {
-    const src = scope.tree().token_locs[token_index].start;
+    const src = scope.tree().tokens.items(.start)[token_index];
     return self.fail(scope, src, format, args);
 }
 
@@ -3103,7 +3370,7 @@ pub fn failNode(
     comptime format: []const u8,
     args: anytype,
 ) InnerError {
-    const src = scope.tree().token_locs[ast_node.firstToken()].start;
+    const src = scope.tree().tokens.items(.start)[ast_node.firstToken()];
     return self.fail(scope, src, format, args);
 }
 
@@ -3537,6 +3804,7 @@ pub fn validateVarType(mod: *Module, scope: *Scope, src: usize, ty: Type) !void
 /// Identifier token -> String (allocated in scope.arena())
 pub fn identifierTokenString(mod: *Module, scope: *Scope, token: ast.TokenIndex) InnerError![]const u8 {
     const tree = scope.tree();
+    const token_starts = tree.tokens.items(.start);
 
     const ident_name = tree.tokenSlice(token);
     if (mem.startsWith(u8, ident_name, "@")) {
@@ -3545,7 +3813,7 @@ pub fn identifierTokenString(mod: *Module, scope: *Scope, token: ast.TokenIndex)
         return std.zig.parseStringLiteral(scope.arena(), raw_string, &bad_index) catch |err| switch (err) {
             error.InvalidCharacter => {
                 const bad_byte = raw_string[bad_index];
-                const src = tree.token_locs[token].start;
+                const src = token_starts[token];
                 return mod.fail(scope, src + 1 + bad_index, "invalid string literal character: '{c}'\n", .{bad_byte});
             },
             else => |e| return e,
diff --git a/src/main.zig b/src/main.zig
index 26e222016a..2ae87fa56c 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -2153,7 +2153,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
         const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"});
         const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path);
         var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{};
-        const tree = translate_c.translate(
+        var tree = translate_c.translate(
             comp.gpa,
             new_argv.ptr,
             new_argv.ptr + new_argv.len,
@@ -2174,7 +2174,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
                 process.exit(1);
             },
         };
-        defer tree.deinit();
+        defer tree.deinit(comp.gpa);
 
         if (out_dep_path) |dep_file_path| {
             const dep_basename = std.fs.path.basename(dep_file_path);
@@ -2188,16 +2188,21 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator, enable_cache: bool) !voi
 
         const digest = man.final();
         const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest });
+
         var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
         defer o_dir.close();
+
         var zig_file = try o_dir.createFile(translated_zig_basename, .{});
         defer zig_file.close();
 
-        var bw = io.bufferedWriter(zig_file.writer());
-        _ = try std.zig.render(comp.gpa, bw.writer(), tree);
-        try bw.flush();
+        const formatted = try tree.render(comp.gpa);
+        defer comp.gpa.free(formatted);
 
-        man.writeManifest() catch |err| warn("failed to write cache manifest: {s}", .{@errorName(err)});
+        try zig_file.writeAll(formatted);
+
+        man.writeManifest() catch |err| warn("failed to write cache manifest: {s}", .{
+            @errorName(err),
+        });
 
         break :digest digest;
     };
@@ -2684,10 +2689,10 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
         const source_code = try stdin.readAllAlloc(gpa, max_src_size);
         defer gpa.free(source_code);
 
-        const tree = std.zig.parse(gpa, source_code) catch |err| {
+        var tree = std.zig.parse(gpa, source_code) catch |err| {
             fatal("error parsing stdin: {s}", .{err});
         };
-        defer tree.deinit();
+        defer tree.deinit(gpa);
 
         for (tree.errors) |parse_error| {
             try printErrMsgToFile(gpa, parse_error, tree, "<stdin>", stderr_file, color);
@@ -2695,16 +2700,15 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
         if (tree.errors.len != 0) {
             process.exit(1);
         }
+        const formatted = try tree.render(gpa);
+        defer gpa.free(formatted);
+
         if (check_flag) {
-            const anything_changed = try std.zig.render(gpa, io.null_writer, tree);
-            const code = if (anything_changed) @as(u8, 1) else @as(u8, 0);
+            const code: u8 = @boolToInt(mem.eql(u8, formatted, source_code));
             process.exit(code);
         }
 
-        var bw = io.bufferedWriter(io.getStdOut().writer());
-        _ = try std.zig.render(gpa, bw.writer(), tree);
-        try bw.flush();
-        return;
+        return io.getStdOut().writeAll(formatted);
     }
 
     if (input_files.items.len == 0) {
@@ -2841,8 +2845,8 @@ fn fmtPathFile(
     // Add to set after no longer possible to get error.IsDir.
     if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;
 
-    const tree = try std.zig.parse(fmt.gpa, source_code);
-    defer tree.deinit();
+    var tree = try std.zig.parse(fmt.gpa, source_code);
+    defer tree.deinit(fmt.gpa);
 
     for (tree.errors) |parse_error| {
         try printErrMsgToFile(fmt.gpa, parse_error, tree, file_path, std.io.getStdErr(), fmt.color);
@@ -2852,22 +2856,20 @@ fn fmtPathFile(
         return;
     }
 
+    // As a heuristic, we make enough capacity for the same as the input source.
+    fmt.out_buffer.shrinkRetainingCapacity(0);
+    try fmt.out_buffer.ensureCapacity(source_code.len);
+
+    try tree.renderToArrayList(&fmt.out_buffer);
+    const anything_changed = mem.eql(u8, fmt.out_buffer.items, source_code);
+    if (!anything_changed)
+        return;
+
     if (check_mode) {
-        const anything_changed = try std.zig.render(fmt.gpa, io.null_writer, tree);
-        if (anything_changed) {
-            const stdout = io.getStdOut().writer();
-            try stdout.print("{s}\n", .{file_path});
-            fmt.any_error = true;
-        }
+        const stdout = io.getStdOut().writer();
+        try stdout.print("{s}\n", .{file_path});
+        fmt.any_error = true;
     } else {
-        // As a heuristic, we make enough capacity for the same as the input source.
-        try fmt.out_buffer.ensureCapacity(source_code.len);
-        fmt.out_buffer.items.len = 0;
-        const writer = fmt.out_buffer.writer();
-        const anything_changed = try std.zig.render(fmt.gpa, writer, tree);
-        if (!anything_changed)
-            return; // Good thing we didn't waste any file system access on this.
-
         var af = try dir.atomicFile(sub_path, .{ .mode = stat.mode });
         defer af.deinit();
 
@@ -2881,7 +2883,7 @@ fn fmtPathFile(
 fn printErrMsgToFile(
     gpa: *mem.Allocator,
     parse_error: ast.Error,
-    tree: *ast.Tree,
+    tree: ast.Tree,
     path: []const u8,
     file: fs.File,
     color: Color,
@@ -2892,18 +2894,16 @@ fn printErrMsgToFile(
         .off => false,
     };
     const lok_token = parse_error.loc();
-    const span_first = lok_token;
-    const span_last = lok_token;
 
-    const first_token = tree.token_locs[span_first];
-    const last_token = tree.token_locs[span_last];
-    const start_loc = tree.tokenLocationLoc(0, first_token);
-    const end_loc = tree.tokenLocationLoc(first_token.end, last_token);
+    const token_starts = tree.tokens.items(.start);
+    const token_tags = tree.tokens.items(.tag);
+    const first_token_start = token_starts[lok_token];
+    const start_loc = tree.tokenLocation(0, lok_token);
 
     var text_buf = std.ArrayList(u8).init(gpa);
     defer text_buf.deinit();
     const writer = text_buf.writer();
-    try parse_error.render(tree.token_ids, writer);
+    try tree.renderError(parse_error, writer);
     const text = text_buf.items;
 
     const stream = file.writer();
@@ -2920,8 +2920,12 @@ fn printErrMsgToFile(
     }
     try stream.writeByte('\n');
     try stream.writeByteNTimes(' ', start_loc.column);
-    try stream.writeByteNTimes('~', last_token.end - first_token.start);
-    try stream.writeByte('\n');
+    if (token_tags[lok_token].lexeme()) |lexeme| {
+        try stream.writeByteNTimes('~', lexeme.len);
+        try stream.writeByte('\n');
+    } else {
+        try stream.writeAll("^\n");
+    }
 }
 
 pub const info_zen =
diff --git a/src/translate_c.zig b/src/translate_c.zig
index bca9ff3a20..c37355298a 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -375,7 +375,7 @@ pub fn translate(
     args_end: [*]?[*]const u8,
     errors: *[]ClangErrMsg,
     resources_path: [*:0]const u8,
-) !*ast.Tree {
+) !ast.Tree {
     const ast_unit = clang.LoadFromCommandLine(
         args_begin,
         args_end,
@@ -396,6 +396,14 @@ pub fn translate(
     var arena = std.heap.ArenaAllocator.init(gpa);
     errdefer arena.deinit();
 
+    if (true) {
+        var x = false;
+        if (x) {
+            return error.OutOfMemory;
+        }
+        @panic("TODO update translate-c");
+    }
+
     var context = Context{
         .gpa = gpa,
         .arena = &arena.allocator,
diff --git a/src/zir.zig b/src/zir.zig
index 9e5830e79a..5b683c7fbf 100644
--- a/src/zir.zig
+++ b/src/zir.zig
@@ -357,6 +357,7 @@ pub const Inst = struct {
                 .ret_type,
                 .unreach_nocheck,
                 .@"unreachable",
+                .arg,
                 => NoOp,
 
                 .alloc,
@@ -449,7 +450,6 @@ pub const Inst = struct {
                 .block_comptime_flat,
                 => Block,
 
-                .arg => Arg,
                 .array_type_sentinel => ArrayTypeSentinel,
                 .@"break" => Break,
                 .breakvoid => BreakVoid,
@@ -685,16 +685,6 @@ pub const Inst = struct {
         kw_args: struct {},
     };
 
-    pub const Arg = struct {
-        pub const base_tag = Tag.arg;
-        base: Inst,
-
-        positionals: struct {
-            name: []const u8,
-        },
-        kw_args: struct {},
-    };
-
     pub const Block = struct {
         pub const base_tag = Tag.block;
         base: Inst,
@@ -1608,6 +1598,7 @@ const DumpTzir = struct {
                 .unreach,
                 .breakpoint,
                 .dbg_stmt,
+                .arg,
                 => {},
 
                 .ref,
@@ -1652,8 +1643,6 @@ const DumpTzir = struct {
                     try dtz.findConst(bin_op.rhs);
                 },
 
-                .arg => {},
-
                 .br => {
                     const br = inst.castTag(.br).?;
                     try dtz.findConst(&br.block.base);
-- 
cgit v1.2.3


From c0540967e974b59bf85f71be9f085b5852d20ef6 Mon Sep 17 00:00:00 2001
From: Veikka Tuominen <git@vexu.eu>
Date: Sun, 14 Feb 2021 18:06:15 +0200
Subject: translate-c: render array stuff

---
 src/Compilation.zig     |   2 +-
 src/translate_c.zig     |  64 +++++++-------
 src/translate_c/ast.zig | 220 ++++++++++++++++++++++++++++++++++++++++++------
 3 files changed, 227 insertions(+), 59 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index 89c047065c..b3ee73f03f 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1334,7 +1334,7 @@ pub fn update(self: *Compilation) !void {
         self.c_object_work_queue.writeItemAssumeCapacity(entry.key);
     }
 
-    const use_stage1 = build_options.is_stage1 and self.bin_file.options.use_llvm;
+    const use_stage1 = build_options.omit_stage2 or build_options.is_stage1 and self.bin_file.options.use_llvm;
     if (!use_stage1) {
         if (self.bin_file.options.module) |module| {
             module.compile_log_text.shrinkAndFree(module.gpa, 0);
diff --git a/src/translate_c.zig b/src/translate_c.zig
index 894502ecc5..ce8fea7a71 100644
--- a/src/translate_c.zig
+++ b/src/translate_c.zig
@@ -1689,7 +1689,10 @@ fn transStringLiteralAsArray(
         init_list[i] = try transCreateNodeNumber(c, 0, .int);
     }
 
-    return Tag.array_init.create(c.arena, init_list);
+    return Tag.array_init.create(c.arena, .{
+        .cond = arr_type,
+        .cases = init_list,
+    });
 }
 
 fn cIsEnum(qt: clang.QualType) bool {
@@ -1880,6 +1883,7 @@ fn transInitListExprArray(
 ) TransError!Node {
     const arr_type = ty.getAsArrayTypeUnsafe();
     const child_qt = arr_type.getElementType();
+    const child_type = try transQualType(c, child_qt, loc);
     const init_count = expr.getNumInits();
     assert(@ptrCast(*const clang.Type, arr_type).isConstantArrayType());
     const const_arr_ty = @ptrCast(*const clang.ConstantArrayType, arr_type);
@@ -1888,18 +1892,20 @@ fn transInitListExprArray(
     const leftover_count = all_count - init_count;
 
     if (all_count == 0) {
-        return Tag.empty_array.create(c.arena, try transQualType(c, child_qt, loc));
+        return Tag.empty_array.create(c.arena, child_type);
     }
 
-    const ty_node = try transType(c, ty, loc);
     const init_node = if (init_count != 0) blk: {
         const init_list = try c.arena.alloc(Node, init_count);
 
         for (init_list) |*init, i| {
             const elem_expr = expr.getInit(@intCast(c_uint, i));
-            init.* = try transExpr(c, scope, elem_expr, .used);
+            init.* = try transExprCoercing(c, scope, elem_expr, .used);
         }
-        const init_node = try Tag.array_init.create(c.arena, init_list);
+        const init_node = try Tag.array_init.create(c.arena, .{
+            .cond = try Tag.array_type.create(c.arena, .{ .len = init_count, .elem_type = child_type }),
+            .cases = init_list,
+        });
         if (leftover_count == 0) {
             return init_node;
         }
@@ -1908,8 +1914,8 @@ fn transInitListExprArray(
 
     const filler_val_expr = expr.getArrayFiller();
     const filler_node = try Tag.array_filler.create(c.arena, .{
-        .type = ty_node,
-        .filler = try transExpr(c, scope, filler_val_expr, .used),
+        .type = child_type,
+        .filler = try transExprCoercing(c, scope, filler_val_expr, .used),
         .count = leftover_count,
     });
 
@@ -2422,9 +2428,7 @@ fn transMemberExpr(c: *Context, scope: *Scope, stmt: *const clang.MemberExpr, re
         const decl = @ptrCast(*const clang.NamedDecl, member_decl);
         break :blk try c.str(decl.getName_bytes_begin());
     };
-    const ident = try Tag.identifier.create(c.arena, name);
-
-    const node = try Tag.field_access.create(c.arena, .{ .lhs = container_node, .rhs = ident });
+    const node = try Tag.field_access.create(c.arena, .{ .lhs = container_node, .field_name = name });
     return maybeSuppressResult(c, scope, result_used, node);
 }
 
@@ -2698,14 +2702,14 @@ fn transCreatePreCrement(
     defer block_scope.deinit();
     const ref = try block_scope.makeMangledName(c, "ref");
 
-    const expr = try transExpr(c, scope, op_expr, .used);
+    const expr = try transExpr(c, &block_scope.base, op_expr, .used);
     const addr_of = try Tag.address_of.create(c.arena, expr);
     const ref_decl = try Tag.var_simple.create(c.arena, .{ .name = ref, .init = addr_of });
     try block_scope.statements.append(ref_decl);
 
     const lhs_node = try Tag.identifier.create(c.arena, ref);
     const ref_node = try Tag.deref.create(c.arena, lhs_node);
-    const node = try transCreateNodeInfixOp(c, scope, op, ref_node, Tag.one_literal.init(), .used);
+    const node = try transCreateNodeInfixOp(c, &block_scope.base, op, ref_node, Tag.one_literal.init(), .used);
     try block_scope.statements.append(node);
 
     const break_node = try Tag.break_val.create(c.arena, .{
@@ -2745,7 +2749,7 @@ fn transCreatePostCrement(
     defer block_scope.deinit();
     const ref = try block_scope.makeMangledName(c, "ref");
 
-    const expr = try transExpr(c, scope, op_expr, .used);
+    const expr = try transExpr(c, &block_scope.base, op_expr, .used);
     const addr_of = try Tag.address_of.create(c.arena, expr);
     const ref_decl = try Tag.var_simple.create(c.arena, .{ .name = ref, .init = addr_of });
     try block_scope.statements.append(ref_decl);
@@ -2757,7 +2761,7 @@ fn transCreatePostCrement(
     const tmp_decl = try Tag.var_simple.create(c.arena, .{ .name = tmp, .init = ref_node });
     try block_scope.statements.append(tmp_decl);
 
-    const node = try transCreateNodeInfixOp(c, scope, op, ref_node, Tag.one_literal.init(), .used);
+    const node = try transCreateNodeInfixOp(c, &block_scope.base, op, ref_node, Tag.one_literal.init(), .used);
     try block_scope.statements.append(node);
 
     const break_node = try Tag.break_val.create(c.arena, .{
@@ -2864,7 +2868,7 @@ fn transCreateCompoundAssign(
     defer block_scope.deinit();
     const ref = try block_scope.makeMangledName(c, "ref");
 
-    const expr = try transExpr(c, scope, lhs, .used);
+    const expr = try transExpr(c, &block_scope.base, lhs, .used);
     const addr_of = try Tag.address_of.create(c.arena, expr);
     const ref_decl = try Tag.var_simple.create(c.arena, .{ .name = ref, .init = addr_of });
     try block_scope.statements.append(ref_decl);
@@ -2873,16 +2877,16 @@ fn transCreateCompoundAssign(
     const ref_node = try Tag.deref.create(c.arena, lhs_node);
 
     if ((is_mod or is_div) and is_signed) {
-        const rhs_node = try transExpr(c, scope, rhs, .used);
+        const rhs_node = try transExpr(c, &block_scope.base, rhs, .used);
         const builtin = if (is_mod)
             try Tag.rem.create(c.arena, .{ .lhs = lhs_node, .rhs = rhs_node })
         else
             try Tag.div_trunc.create(c.arena, .{ .lhs = lhs_node, .rhs = rhs_node });
 
-        const assign = try transCreateNodeInfixOp(c, scope, .assign, lhs_node, builtin, .used);
+        const assign = try transCreateNodeInfixOp(c, &block_scope.base, .assign, lhs_node, builtin, .used);
         try block_scope.statements.append(assign);
     } else {
-        var rhs_node = try transExpr(c, scope, rhs, .used);
+        var rhs_node = try transExpr(c, &block_scope.base, rhs, .used);
 
         if (is_shift or requires_int_cast) {
             // @intCast(rhs)
@@ -2894,7 +2898,7 @@ fn transCreateCompoundAssign(
             rhs_node = try Tag.int_cast.create(c.arena, .{ .lhs = cast_to_type, .rhs = rhs_node });
         }
 
-        const assign = try transCreateNodeInfixOp(c, scope, op, ref_node, rhs_node, .used);
+        const assign = try transCreateNodeInfixOp(c, &block_scope.base, op, ref_node, rhs_node, .used);
         try block_scope.statements.append(assign);
     }
 
@@ -3395,7 +3399,7 @@ fn transCreateNodeAssign(
     defer block_scope.deinit();
 
     const tmp = try block_scope.makeMangledName(c, "tmp");
-    const rhs_node = try transExpr(c, scope, rhs, .used);
+    const rhs_node = try transExpr(c, &block_scope.base, rhs, .used);
     const tmp_decl = try Tag.var_simple.create(c.arena, .{ .name = tmp, .init = rhs_node });
     try block_scope.statements.append(tmp_decl);
 
@@ -4756,8 +4760,7 @@ fn parseCPostfixExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!Node {
                     return error.ParseError;
                 }
 
-                const ident = try Tag.identifier.create(c.arena, m.slice());
-                node = try Tag.field_access.create(c.arena, .{ .lhs = node, .rhs = ident });
+                node = try Tag.field_access.create(c.arena, .{ .lhs = node, .field_name = m.slice() });
             },
             .Arrow => {
                 if (m.next().? != .Identifier) {
@@ -4766,8 +4769,7 @@ fn parseCPostfixExpr(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!Node {
                 }
 
                 const deref = try Tag.deref.create(c.arena, node);
-                const ident = try Tag.identifier.create(c.arena, m.slice());
-                node = try Tag.field_access.create(c.arena, .{ .lhs = deref, .rhs = ident });
+                node = try Tag.field_access.create(c.arena, .{ .lhs = deref, .field_name = m.slice() });
             },
             .LBracket => {
                 const index = try macroBoolToInt(c, try parseCExpr(c, m, scope));
@@ -4914,13 +4916,12 @@ fn getContainer(c: *Context, node: Node) ?Node {
         },
 
         .field_access => {
-            const infix = node.castTag(.field_access).?;
+            const field_access = node.castTag(.field_access).?;
 
-            if (getContainerTypeOf(c, infix.data.lhs)) |ty_node| {
+            if (getContainerTypeOf(c, field_access.data.lhs)) |ty_node| {
                 if (ty_node.castTag(.@"struct") orelse ty_node.castTag(.@"union")) |container| {
                     for (container.data.fields) |field| {
-                        const ident = infix.data.rhs.castTag(.identifier).?;
-                        if (mem.eql(u8, field.name, ident.data)) {
+                        if (mem.eql(u8, field.name, field_access.data.field_name)) {
                             return getContainer(c, field.type);
                         }
                     }
@@ -4940,12 +4941,11 @@ fn getContainerTypeOf(c: *Context, ref: Node) ?Node {
                 return getContainer(c, var_decl.data.type);
             }
         }
-    } else if (ref.castTag(.field_access)) |infix| {
-        if (getContainerTypeOf(c, infix.data.lhs)) |ty_node| {
+    } else if (ref.castTag(.field_access)) |field_access| {
+        if (getContainerTypeOf(c, field_access.data.lhs)) |ty_node| {
             if (ty_node.castTag(.@"struct") orelse ty_node.castTag(.@"union")) |container| {
                 for (container.data.fields) |field| {
-                    const ident = infix.data.rhs.castTag(.identifier).?;
-                    if (mem.eql(u8, field.name, ident.data)) {
+                    if (mem.eql(u8, field.name, field_access.data.field_name)) {
                         return getContainer(c, field.type);
                     }
                 }
diff --git a/src/translate_c/ast.zig b/src/translate_c/ast.zig
index d108a2852e..014236e002 100644
--- a/src/translate_c/ast.zig
+++ b/src/translate_c/ast.zig
@@ -291,7 +291,6 @@ pub const Node = extern union {
                 .array_cat,
                 .ellipsis3,
                 .switch_prong,
-                .field_access,
                 .assign,
                 .align_cast,
                 .array_access,
@@ -309,7 +308,7 @@ pub const Node = extern union {
                 => Payload.Value,
                 .@"if" => Payload.If,
                 .@"while" => Payload.While,
-                .@"switch" => Payload.Switch,
+                .@"switch", .array_init => Payload.Switch,
                 .@"break" => Payload.Break,
                 .break_val => Payload.BreakVal,
                 .call => Payload.Call,
@@ -317,7 +316,7 @@ pub const Node = extern union {
                 .func => Payload.Func,
                 .@"enum" => Payload.Enum,
                 .@"struct", .@"union" => Payload.Record,
-                .array_init, .tuple => Payload.ArrayInit,
+                .tuple => Payload.TupleInit,
                 .container_init => Payload.ContainerInit,
                 .std_meta_cast => Payload.Infix,
                 .block => Payload.Block,
@@ -329,6 +328,7 @@ pub const Node = extern union {
                 .enum_redecl => Payload.EnumRedecl,
                 .array_filler => Payload.ArrayFiller,
                 .pub_inline_fn => Payload.PubInlineFn,
+                .field_access => Payload.FieldAccess,
             };
         }
 
@@ -513,7 +513,7 @@ pub const Payload = struct {
         };
     };
 
-    pub const ArrayInit = struct {
+    pub const TupleInit = struct {
         base: Payload,
         data: []Node,
     };
@@ -601,6 +601,14 @@ pub const Payload = struct {
             body: Node,
         },
     };
+
+    pub const FieldAccess = struct {
+        base: Payload,
+        data: struct {
+            lhs: Node,
+            field_name: []const u8,
+        },
+    };
 };
 
 /// Converts the nodes into a Zig ast.
@@ -995,6 +1003,32 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
             });
         },
         .var_decl => return renderVar(c, node),
+        .arg_redecl, .alias => {
+            const payload = @fieldParentPtr(Payload.ArgRedecl, "base", node.ptr_otherwise).data;
+            if (node.tag() == .alias) _ = try c.addToken(.keyword_pub, "pub");
+            const mut_tok = if (node.tag() == .alias)
+                try c.addToken(.keyword_const, "const")
+            else
+                try c.addToken(.keyword_var, "var");
+            _ = try c.addIdentifier(payload.actual);
+            _ = try c.addToken(.equal, "=");
+
+            const init = try c.addNode(.{
+                .tag = .identifier,
+                .main_token = try c.addIdentifier(payload.mangled),
+                .data = undefined,
+            });
+            _ = try c.addToken(.semicolon, ";");
+
+            return c.addNode(.{
+                .tag = .simple_var_decl,
+                .main_token = mut_tok,
+                .data = .{
+                    .lhs = 0,
+                    .rhs = init,
+                },
+            });
+        },
         .int_cast => {
             const payload = node.castTag(.int_cast).?.data;
             return renderBuiltinCall(c, "@intCast", &.{ payload.lhs, payload.rhs });
@@ -1339,7 +1373,7 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
             _ = try c.addToken(.l_paren, "(");
             const cond = try c.addNode(.{
                 .tag = .bool_not,
-                .main_token = try c.addToken(.bang,  "!"),
+                .main_token = try c.addToken(.bang, "!"),
                 .data = .{
                     .lhs = try renderNodeGrouped(c, payload),
                     .rhs = undefined,
@@ -1430,7 +1464,77 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
                 },
             });
         },
-        else => return c.addNode(.{
+        .array_access => {
+            const payload = node.castTag(.array_access).?.data;
+            const lhs = try renderNode(c, payload.lhs);
+            const l_bracket = try c.addToken(.l_bracket, "[");
+            const index_expr = try renderNode(c, payload.rhs);
+            _ = try c.addToken(.r_bracket, "]");
+            return c.addNode(.{
+                .tag = .array_access,
+                .main_token = l_bracket,
+                .data = .{
+                    .lhs = lhs,
+                    .rhs = index_expr,
+                },
+            });
+        },
+        .array_type => {
+            const payload = node.castTag(.array_type).?.data;
+            return renderArrayType(c, payload.len, payload.elem_type);
+        },
+        .array_filler => {
+            const payload = node.castTag(.array_filler).?.data;
+
+            const type_expr = try renderArrayType(c, 1, payload.type);
+            const l_brace = try c.addToken(.l_brace, "{");
+            const val = try renderNode(c, payload.filler);
+            _ = try c.addToken(.r_brace, "}");
+
+            const init = try c.addNode(.{
+                .tag = .array_init_one,
+                .main_token = l_brace,
+                .data = .{
+                    .lhs = type_expr,
+                    .rhs = val,
+                },
+            });
+            return c.addNode(.{
+                .tag = .array_cat,
+                .main_token = try c.addToken(.asterisk_asterisk, "**"),
+                .data = .{
+                    .lhs = init,
+                    .rhs = try c.addNode(.{
+                        .tag = .integer_literal,
+                        .main_token = try c.addTokenFmt(.integer_literal, "{d}", .{payload.count}),
+                        .data = .{ .lhs = undefined, .rhs = undefined },
+                    }),
+                },
+            });
+        },
+        .empty_array => {
+            const payload = node.castTag(.empty_array).?.data;
+
+            const type_expr = try renderArrayType(c, 0, payload);
+            return renderArrayInit(c, 0, &.{});
+        },
+        .array_init => {
+            const payload = node.castTag(.array_init).?.data;
+            const type_expr = try renderNode(c, payload.cond);
+            return renderArrayInit(c, type_expr, payload.cases);
+        },
+        .field_access => {
+            const payload = node.castTag(.field_access).?.data;
+            const lhs = try renderNode(c, payload.lhs);
+            return renderFieldAccess(c, lhs, payload.field_name);
+        },
+        .tuple,
+        .@"enum",
+        .@"struct",
+        .@"union",
+        .container_init,
+        .enum_redecl,
+        => return c.addNode(.{
             .tag = .identifier,
             .main_token = try c.addTokenFmt(.identifier, "@\"TODO {}\"", .{node.tag()}),
             .data = .{
@@ -1441,10 +1545,88 @@ fn renderNode(c: *Context, node: Node) Allocator.Error!NodeIndex {
     }
 }
 
+fn renderFieldAccess(c: *Context, lhs: NodeIndex, field_name: []const u8) !NodeIndex {
+    return c.addNode(.{
+        .tag = .field_access,
+        .main_token = try c.addToken(.period, "."),
+        .data = .{
+            .lhs = lhs,
+            .rhs = try c.addIdentifier(field_name),
+        },
+    });
+}
+
+fn renderArrayInit(c: *Context, lhs: NodeIndex, inits: []const Node) !NodeIndex {
+    const l_brace = try c.addToken(.l_brace, "{");
+    const res = switch (inits.len) {
+        0 => try c.addNode(.{
+            .tag = .struct_init_one,
+            .main_token = l_brace,
+            .data = .{
+                .lhs = lhs,
+                .rhs = 0,
+            },
+        }),
+        1 => blk: {
+            const init = try renderNode(c, inits[0]);
+            break :blk try c.addNode(.{
+                .tag = .array_init_one,
+                .main_token = l_brace,
+                .data = .{
+                    .lhs = lhs,
+                    .rhs = init,
+                },
+            });
+        },
+        else => blk: {
+            var rendered = try c.gpa.alloc(NodeIndex, inits.len);
+            defer c.gpa.free(rendered);
+
+            for (inits) |init, i| {
+                if (i != 0) _ = try c.addToken(.comma, ",");
+                rendered[i] = try renderNode(c, init);
+            }
+            const span = try c.listToSpan(rendered);
+            break :blk try c.addNode(.{
+                .tag = .array_init,
+                .main_token = l_brace,
+                .data = .{
+                    .lhs = lhs,
+                    .rhs = try c.addExtra(NodeSubRange{
+                        .start = span.start,
+                        .end = span.end,
+                    }),
+                },
+            });
+        },
+    };
+    _ = try c.addToken(.r_brace, "}");
+    return res;
+}
+
+fn renderArrayType(c: *Context, len: usize, elem_type: Node) !NodeIndex {
+    const l_bracket = try c.addToken(.l_bracket, "[");
+    const len_expr = try c.addNode(.{
+        .tag = .integer_literal,
+        .main_token = try c.addTokenFmt(.integer_literal, "{d}", .{len}),
+        .data = .{ .lhs = undefined, .rhs = undefined },
+    });
+    _ = try c.addToken(.r_bracket, "]");
+    const elem_type_expr = try renderNode(c, elem_type);
+    return c.addNode(.{
+        .tag = .array_type,
+        .main_token = l_bracket,
+        .data = .{
+            .lhs = len_expr,
+            .rhs = elem_type_expr,
+        },
+    });
+}
+
 fn addSemicolonIfNeeded(c: *Context, node: Node) !void {
     switch (node.tag()) {
         .warning => unreachable,
-        .var_decl, .var_simple, .block, .empty_block, .@"switch" => {},
+        .var_decl, .var_simple, .arg_redecl, .alias, .enum_redecl, .block, .empty_block, .@"switch" => {},
         .while_true => {
             const payload = node.castTag(.while_true).?.data;
             return addSemicolonIfNeeded(c, payload);
@@ -1517,6 +1699,7 @@ fn renderNodeGrouped(c: *Context, node: Node) !NodeIndex {
         .negate,
         .negate_wrap,
         .bit_not,
+        .func,
         => {
             // no grouping needed
             return renderNode(c, node);
@@ -1572,7 +1755,6 @@ fn renderNodeGrouped(c: *Context, node: Node) !NodeIndex {
         .switch_prong,
         .warning,
         .var_decl,
-        .func,
         .fail_decl,
         .arg_redecl,
         .alias,
@@ -1658,22 +1840,8 @@ fn renderStdImport(c: *Context, first: []const u8, second: []const u8) !NodeInde
     });
 
     var access_chain = import_node;
-    access_chain = try c.addNode(.{
-        .tag = .field_access,
-        .main_token = try c.addToken(.period, "."),
-        .data = .{
-            .lhs = access_chain,
-            .rhs = try c.addIdentifier(first),
-        },
-    });
-    access_chain = try c.addNode(.{
-        .tag = .field_access,
-        .main_token = try c.addToken(.period, "."),
-        .data = .{
-            .lhs = access_chain,
-            .rhs = try c.addIdentifier(second),
-        },
-    });
+    access_chain = try renderFieldAccess(c, access_chain, first);
+    access_chain = try renderFieldAccess(c, access_chain, second);
     return access_chain;
 }
 
@@ -1974,10 +2142,10 @@ fn renderFunc(c: *Context, node: Node) !NodeIndex {
 
     const body = if (payload.body) |some|
         try renderNode(c, some)
-    else blk: {
+    else if (payload.is_extern) blk: {
         _ = try c.addToken(.semicolon, ";");
         break :blk 0;
-    };
+    } else return fn_proto;
 
     return c.addNode(.{
         .tag = .fn_decl,
-- 
cgit v1.2.3


From 70761d7c52cd3634e086b0914c9520ef4dc01eee Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Fri, 19 Feb 2021 20:27:06 -0700
Subject: stage2: remove incorrect newlines from log statements

---
 src/Compilation.zig |  4 ++--
 src/Module.zig      | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index b3ee73f03f..227355de93 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1935,7 +1935,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         "o", &digest, cimport_zig_basename,
     });
     if (comp.verbose_cimport) {
-        log.info("C import output: {s}\n", .{out_zig_path});
+        log.info("C import output: {s}", .{out_zig_path});
     }
     return CImportResult{
         .out_zig_path = out_zig_path,
@@ -2999,7 +2999,7 @@ pub fn updateSubCompilation(sub_compilation: *Compilation) !void {
         for (errors.list) |full_err_msg| {
             switch (full_err_msg) {
                 .src => |src| {
-                    log.err("{s}:{d}:{d}: {s}\n", .{
+                    log.err("{s}:{d}:{d}: {s}", .{
                         src.src_path,
                         src.line + 1,
                         src.column + 1,
diff --git a/src/Module.zig b/src/Module.zig
index 8f2ac3721e..9a918321c4 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -924,7 +924,7 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
         .complete => return,
 
         .outdated => blk: {
-            log.debug("re-analyzing {s}\n", .{decl.name});
+            log.debug("re-analyzing {s}", .{decl.name});
 
             // The exports this Decl performs will be re-discovered, so we remove them here
             // prior to re-analysis.
@@ -1943,7 +1943,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
     // Handle explicitly deleted decls from the source code. Not to be confused
     // with when we delete decls because they are no longer referenced.
     for (deleted_decls.items()) |entry| {
-        log.debug("noticed '{s}' deleted from source\n", .{entry.key.name});
+        log.debug("noticed '{s}' deleted from source", .{entry.key.name});
         try mod.deleteDecl(entry.key);
     }
 }
@@ -2087,7 +2087,7 @@ pub fn deleteDecl(self: *Module, decl: *Decl) !void {
     // not be present in the set, and this does nothing.
     decl.container.removeDecl(decl);
 
-    log.debug("deleting decl '{s}'\n", .{decl.name});
+    log.debug("deleting decl '{s}'", .{decl.name});
     const name_hash = decl.fullyQualifiedNameHash();
     self.decl_table.removeAssertDiscard(name_hash);
     // Remove itself from its dependencies, because we are about to destroy the decl pointer.
@@ -2189,18 +2189,18 @@ pub fn analyzeFnBody(self: *Module, decl: *Decl, func: *Fn) !void {
     defer inner_block.instructions.deinit(self.gpa);
 
     func.state = .in_progress;
-    log.debug("set {s} to in_progress\n", .{decl.name});
+    log.debug("set {s} to in_progress", .{decl.name});
 
     try zir_sema.analyzeBody(self, &inner_block, func.zir);
 
     const instructions = try arena.allocator.dupe(*Inst, inner_block.instructions.items);
     func.state = .success;
     func.body = .{ .instructions = instructions };
-    log.debug("set {s} to success\n", .{decl.name});
+    log.debug("set {s} to success", .{decl.name});
 }
 
 fn markOutdatedDecl(self: *Module, decl: *Decl) !void {
-    log.debug("mark {s} outdated\n", .{decl.name});
+    log.debug("mark {s} outdated", .{decl.name});
     try self.comp.work_queue.writeItem(.{ .analyze_decl = decl });
     if (self.failed_decls.swapRemove(decl)) |entry| {
         entry.value.destroy(self.gpa);
-- 
cgit v1.2.3


From 153cd4da0c723fe949bbf9eaf9d780500320bffc Mon Sep 17 00:00:00 2001
From: Michael Dusan <michael.dusan@gmail.com>
Date: Fri, 19 Feb 2021 15:52:34 -0500
Subject: macos: fix cond to enable ZIG_SYSTEM_LINKER_HACK

closes #8037
---
 src/Compilation.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/Compilation.zig')

diff --git a/src/Compilation.zig b/src/Compilation.zig
index c7bb260aa7..ae3385b2dc 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -645,7 +645,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
         };
 
         const darwin_options: DarwinOptions = if (build_options.have_llvm and comptime std.Target.current.isDarwin()) outer: {
-            const opts: DarwinOptions = if (use_lld and options.is_native_os and options.target.isDarwin()) inner: {
+            const opts: DarwinOptions = if (use_lld and std.builtin.os.tag == .macos and options.target.isDarwin()) inner: {
                 // TODO Revisit this targeting versions lower than macOS 11 when LLVM 12 is out.
                 // See https://github.com/ziglang/zig/issues/6996
                 const at_least_big_sur = options.target.os.getVersionRange().semver.min.major >= 11;
-- 
cgit v1.2.3


From 1bd434fd18e0cb769ca46849dac056a562ce7ce3 Mon Sep 17 00:00:00 2001
From: jacob gw <jacoblevgw@gmail.com>
Date: Mon, 1 Feb 2021 09:18:52 -0500
Subject: std.Progress: improve support for "dumb" terminals

---
 lib/std/Progress.zig | 18 +++++++++++++++++-
 src/Compilation.zig  |  4 +++-
 src/stage1.zig       |  4 +++-
 3 files changed, 23 insertions(+), 3 deletions(-)

(limited to 'src/Compilation.zig')

diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig
index 131aff2d2b..4afd191b93 100644
--- a/lib/std/Progress.zig
+++ b/lib/std/Progress.zig
@@ -25,6 +25,13 @@ terminal: ?std.fs.File = undefined,
 /// Whether the terminal supports ANSI escape codes.
 supports_ansi_escape_codes: bool = false,
 
+/// If the terminal is "dumb", don't print output.
+/// This can be useful if you don't want to print all
+/// the stages of code generation if there are a lot.
+/// You should not use it if the user should see output
+/// for example showing the user what tests run.
+dont_print_on_dumb: bool = false,
+
 root: Node = undefined,
 
 /// Keeps track of how much time has passed since the beginning.
@@ -141,6 +148,9 @@ pub fn start(self: *Progress, name: []const u8, estimated_total_items: usize) !*
         self.supports_ansi_escape_codes = true;
     } else if (std.builtin.os.tag == .windows and stderr.isTty()) {
         self.terminal = stderr;
+    } else if (std.builtin.os.tag != .windows) {
+        // we are in a "dumb" terminal like in acme or writing to a file
+        self.terminal = stderr;
     }
     self.root = Node{
         .context = self,
@@ -178,6 +188,8 @@ pub fn refresh(self: *Progress) void {
 }
 
 fn refreshWithHeldLock(self: *Progress) void {
+    const is_dumb = !self.supports_ansi_escape_codes and !(std.builtin.os.tag == .windows);
+    if (is_dumb and self.dont_print_on_dumb) return;
     const file = self.terminal orelse return;
 
     const prev_columns_written = self.columns_written;
@@ -226,7 +238,11 @@ fn refreshWithHeldLock(self: *Progress) void {
 
             if (windows.kernel32.SetConsoleCursorPosition(file.handle, cursor_pos) != windows.TRUE)
                 unreachable;
-        } else unreachable;
+        } else {
+            // we are in a "dumb" terminal like in acme or writing to a file
+            self.output_buffer[end] = '\n';
+            end += 1;
+        }
 
         self.columns_written = 0;
     }
diff --git a/src/Compilation.zig b/src/Compilation.zig
index ae3385b2dc..180d49a196 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1538,7 +1538,9 @@ pub fn getCompileLogOutput(self: *Compilation) []const u8 {
 }
 
 pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemory }!void {
-    var progress: std.Progress = .{};
+    // If the terminal is dumb, we dont want to show the user all the
+    // output.
+    var progress: std.Progress = .{ .dont_print_on_dumb = true };
     var main_progress_node = try progress.start("", 0);
     defer main_progress_node.end();
     if (self.color == .off) progress.terminal = null;
diff --git a/src/stage1.zig b/src/stage1.zig
index 8ab3b1d94d..1b7eadd1a8 100644
--- a/src/stage1.zig
+++ b/src/stage1.zig
@@ -278,7 +278,9 @@ export fn stage2_attach_segfault_handler() void {
 // ABI warning
 export fn stage2_progress_create() *std.Progress {
     const ptr = std.heap.c_allocator.create(std.Progress) catch @panic("out of memory");
-    ptr.* = std.Progress{};
+    // If the terminal is dumb, we dont want to show the user all the
+    // output.
+    ptr.* = std.Progress{ .dont_print_on_dumb = true };
     return ptr;
 }
 
-- 
cgit v1.2.3