aboutsummaryrefslogtreecommitdiff
path: root/lib/tsan/tsan_sync.cpp
diff options
context:
space:
mode:
authorAlex Rønne Petersen <alex@alexrp.com>2025-04-10 19:19:09 +0200
committerAlex Rønne Petersen <alex@alexrp.com>2025-04-11 02:08:18 +0200
commitee0ff134e9f82bf87751a5174c27b191c04e16c0 (patch)
treeb06060c8a3a26ade0be2139b7d1fa2b0d285d13f /lib/tsan/tsan_sync.cpp
parent71a237e764717a69495af531ea0d4f123bec7294 (diff)
downloadzig-ee0ff134e9f82bf87751a5174c27b191c04e16c0.tar.gz
zig-ee0ff134e9f82bf87751a5174c27b191c04e16c0.zip
tsan: Rename lib/tsan to lib/libtsan.
For consistency with other vendored C/C++ libraries.
Diffstat (limited to 'lib/tsan/tsan_sync.cpp')
-rw-r--r--lib/tsan/tsan_sync.cpp289
1 files changed, 0 insertions, 289 deletions
diff --git a/lib/tsan/tsan_sync.cpp b/lib/tsan/tsan_sync.cpp
deleted file mode 100644
index 09d41780d1..0000000000
--- a/lib/tsan/tsan_sync.cpp
+++ /dev/null
@@ -1,289 +0,0 @@
-//===-- tsan_sync.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_placement_new.h"
-#include "tsan_sync.h"
-#include "tsan_rtl.h"
-#include "tsan_mman.h"
-
-namespace __tsan {
-
-void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
-
-SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }
-
-void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
- Reset();
- this->addr = addr;
- next = 0;
- if (save_stack && !SANITIZER_GO) // Go does not use them
- creation_stack_id = CurrentStackId(thr, pc);
- if (common_flags()->detect_deadlocks)
- DDMutexInit(thr, pc, this);
-}
-
-void SyncVar::Reset() {
- CHECK(!ctx->resetting);
- creation_stack_id = kInvalidStackID;
- owner_tid = kInvalidTid;
- last_lock.Reset();
- recursion = 0;
- atomic_store_relaxed(&flags, 0);
- Free(clock);
- Free(read_clock);
-}
-
-MetaMap::MetaMap()
- : block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}
-
-void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
- u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
- MBlock *b = block_alloc_.Map(idx);
- b->siz = sz;
- b->tag = 0;
- b->tid = thr->tid;
- b->stk = CurrentStackId(thr, pc);
- u32 *meta = MemToMeta(p);
- DCHECK_EQ(*meta, 0);
- *meta = idx | kFlagBlock;
-}
-
-uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
- MBlock* b = GetBlock(p);
- if (b == 0)
- return 0;
- uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
- FreeRange(proc, p, sz, reset);
- return sz;
-}
-
-bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
- bool has_something = false;
- u32 *meta = MemToMeta(p);
- u32 *end = MemToMeta(p + sz);
- if (end == meta)
- end++;
- for (; meta < end; meta++) {
- u32 idx = *meta;
- if (idx == 0) {
- // Note: don't write to meta in this case -- the block can be huge.
- continue;
- }
- *meta = 0;
- has_something = true;
- while (idx != 0) {
- if (idx & kFlagBlock) {
- block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask);
- break;
- } else if (idx & kFlagSync) {
- DCHECK(idx & kFlagSync);
- SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
- u32 next = s->next;
- if (reset)
- s->Reset();
- sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
- idx = next;
- } else {
- CHECK(0);
- }
- }
- }
- return has_something;
-}
-
-// ResetRange removes all meta objects from the range.
-// It is called for large mmap-ed regions. The function is best-effort wrt
-// freeing of meta objects, because we don't want to page in the whole range
-// which can be huge. The function probes pages one-by-one until it finds a page
-// without meta objects, at this point it stops freeing meta objects. Because
-// thread stacks grow top-down, we do the same starting from end as well.
-void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
- if (SANITIZER_GO) {
- // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
- // so we do the optimization only for C/C++.
- FreeRange(proc, p, sz, reset);
- return;
- }
- const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
- const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
- if (sz <= 4 * kPageSize) {
- // If the range is small, just do the normal free procedure.
- FreeRange(proc, p, sz, reset);
- return;
- }
- // First, round both ends of the range to page size.
- uptr diff = RoundUp(p, kPageSize) - p;
- if (diff != 0) {
- FreeRange(proc, p, diff, reset);
- p += diff;
- sz -= diff;
- }
- diff = p + sz - RoundDown(p + sz, kPageSize);
- if (diff != 0) {
- FreeRange(proc, p + sz - diff, diff, reset);
- sz -= diff;
- }
- // Now we must have a non-empty page-aligned range.
- CHECK_GT(sz, 0);
- CHECK_EQ(p, RoundUp(p, kPageSize));
- CHECK_EQ(sz, RoundUp(sz, kPageSize));
- const uptr p0 = p;
- const uptr sz0 = sz;
- // Probe start of the range.
- for (uptr checked = 0; sz > 0; checked += kPageSize) {
- bool has_something = FreeRange(proc, p, kPageSize, reset);
- p += kPageSize;
- sz -= kPageSize;
- if (!has_something && checked > (128 << 10))
- break;
- }
- // Probe end of the range.
- for (uptr checked = 0; sz > 0; checked += kPageSize) {
- bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
- sz -= kPageSize;
- // Stacks grow down, so sync object are most likely at the end of the region
- // (if it is a stack). The very end of the stack is TLS and tsan increases
- // TLS by at least 256K, so check at least 512K.
- if (!has_something && checked > (512 << 10))
- break;
- }
- // Finally, page out the whole range (including the parts that we've just
- // freed). Note: we can't simply madvise, because we need to leave a zeroed
- // range (otherwise __tsan_java_move can crash if it encounters a left-over
- // meta objects in java heap).
- uptr metap = (uptr)MemToMeta(p0);
- uptr metasz = sz0 / kMetaRatio;
- UnmapOrDie((void*)metap, metasz);
- if (!MmapFixedSuperNoReserve(metap, metasz))
- Die();
-}
-
-void MetaMap::ResetClocks() {
- // This can be called from the background thread
- // which does not have proc/cache.
- // The cache is too large for stack.
- static InternalAllocatorCache cache;
- internal_memset(&cache, 0, sizeof(cache));
- internal_allocator()->InitCache(&cache);
- sync_alloc_.ForEach([&](SyncVar *s) {
- if (s->clock) {
- InternalFree(s->clock, &cache);
- s->clock = nullptr;
- }
- if (s->read_clock) {
- InternalFree(s->read_clock, &cache);
- s->read_clock = nullptr;
- }
- s->last_lock.Reset();
- });
- internal_allocator()->DestroyCache(&cache);
-}
-
-MBlock* MetaMap::GetBlock(uptr p) {
- u32 *meta = MemToMeta(p);
- u32 idx = *meta;
- for (;;) {
- if (idx == 0)
- return 0;
- if (idx & kFlagBlock)
- return block_alloc_.Map(idx & ~kFlagMask);
- DCHECK(idx & kFlagSync);
- SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
- idx = s->next;
- }
-}
-
-SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
- bool save_stack) {
- DCHECK(!create || thr->slot_locked);
- u32 *meta = MemToMeta(addr);
- u32 idx0 = *meta;
- u32 myidx = 0;
- SyncVar *mys = nullptr;
- for (;;) {
- for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
- DCHECK(idx & kFlagSync);
- SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
- if (LIKELY(s->addr == addr)) {
- if (UNLIKELY(myidx != 0)) {
- mys->Reset();
- sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
- }
- return s;
- }
- idx = s->next;
- }
- if (!create)
- return nullptr;
- if (UNLIKELY(*meta != idx0)) {
- idx0 = *meta;
- continue;
- }
-
- if (LIKELY(myidx == 0)) {
- myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
- mys = sync_alloc_.Map(myidx);
- mys->Init(thr, pc, addr, save_stack);
- }
- mys->next = idx0;
- if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
- myidx | kFlagSync, memory_order_release)) {
- return mys;
- }
- }
-}
-
-void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
- // src and dst can overlap,
- // there are no concurrent accesses to the regions (e.g. stop-the-world).
- CHECK_NE(src, dst);
- CHECK_NE(sz, 0);
- uptr diff = dst - src;
- u32 *src_meta = MemToMeta(src);
- u32 *dst_meta = MemToMeta(dst);
- u32 *src_meta_end = MemToMeta(src + sz);
- uptr inc = 1;
- if (dst > src) {
- src_meta = MemToMeta(src + sz) - 1;
- dst_meta = MemToMeta(dst + sz) - 1;
- src_meta_end = MemToMeta(src) - 1;
- inc = -1;
- }
- for (; src_meta != src_meta_end; src_meta += inc, dst_meta += inc) {
- CHECK_EQ(*dst_meta, 0);
- u32 idx = *src_meta;
- *src_meta = 0;
- *dst_meta = idx;
- // Patch the addresses in sync objects.
- while (idx != 0) {
- if (idx & kFlagBlock)
- break;
- CHECK(idx & kFlagSync);
- SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
- s->addr += diff;
- idx = s->next;
- }
- }
-}
-
-void MetaMap::OnProcIdle(Processor *proc) {
- block_alloc_.FlushCache(&proc->block_cache);
- sync_alloc_.FlushCache(&proc->sync_cache);
-}
-
-MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
- MemoryStats stats;
- stats.mem_block = block_alloc_.AllocatedMemory();
- stats.sync_obj = sync_alloc_.AllocatedMemory();
- return stats;
-}
-
-} // namespace __tsan