HAXY

▀ ▄▄▄ ▀ ▀▀ ▄▀ ▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▄▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀ ▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▄▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▄▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀ ▄▄ ▀▀▀ ▀▀▀▀▀▀▀▀▀ ▀ ▀▀▀▀ ▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▄▀▀ ▀▀▀▀▀▀▀▀▀▀▄▀▄ ▄▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▄▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀

╭ tsan_rtl_access.cpp ──────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯

╭ find file ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╮ ╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │ │ │http│ ssh │ │ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─╯ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯

1 //===-- tsan_rtl_access.cpp -----------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Definitions of memory access and function entry/exit entry points. 12 //===----------------------------------------------------------------------===// 13 14 #include "tsan_rtl.h" 15 16 namespace __tsan { 17 18 ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc, 19 uptr addr, uptr size, 20 AccessType typ) { 21 DCHECK(size == 1 || size == 2 || size == 4 || size == 8); 22 if (!kCollectHistory) 23 return true; 24 EventAccess* ev; 25 if (UNLIKELY(!TraceAcquire(thr, &ev))) 26 return false; 27 u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3; 28 uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1)); 29 thr->trace_prev_pc = pc; 30 if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) { 31 ev->is_access = 1; 32 ev->is_read = !!(typ & kAccessRead); 33 ev->is_atomic = !!(typ & kAccessAtomic); 34 ev->size_log = size_log; 35 ev->pc_delta = pc_delta; 36 DCHECK_EQ(ev->pc_delta, pc_delta); 37 ev->addr = CompressAddr(addr); 38 TraceRelease(thr, ev); 39 return true; 40 } 41 auto* evex = reinterpret_cast<EventAccessExt*>(ev); 42 evex->is_access = 0; 43 evex->is_func = 0; 44 evex->type = EventType::kAccessExt; 45 evex->is_read = !!(typ & kAccessRead); 46 evex->is_atomic = !!(typ & kAccessAtomic); 47 evex->size_log = size_log; 48 // Note: this is important, see comment in EventAccessExt. 49 evex->_ = 0; 50 evex->addr = CompressAddr(addr); 51 evex->pc = pc; 52 TraceRelease(thr, evex); 53 return true; 54 } 55 56 ALWAYS_INLINE 57 bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size, 58 AccessType typ) { 59 if (!kCollectHistory) 60 return true; 61 EventAccessRange* ev; 62 if (UNLIKELY(!TraceAcquire(thr, &ev))) 63 return false; 64 thr->trace_prev_pc = pc; 65 ev->is_access = 0; 66 ev->is_func = 0; 67 ev->type = EventType::kAccessRange; 68 ev->is_read = !!(typ & kAccessRead); 69 ev->is_free = !!(typ & kAccessFree); 70 ev->size_lo = size; 71 ev->pc = CompressAddr(pc); 72 ev->addr = CompressAddr(addr); 73 ev->size_hi = size >> EventAccessRange::kSizeLoBits; 74 TraceRelease(thr, ev); 75 return true; 76 } 77 78 void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size, 79 AccessType typ) { 80 if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ))) 81 return; 82 TraceSwitchPart(thr); 83 UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ); 84 DCHECK(res); 85 } 86 87 void TraceFunc(ThreadState* thr, uptr pc) { 88 if (LIKELY(TryTraceFunc(thr, pc))) 89 return; 90 TraceSwitchPart(thr); 91 UNUSED bool res = TryTraceFunc(thr, pc); 92 DCHECK(res); 93 } 94 95 NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) { 96 TraceSwitchPart(thr); 97 FuncEntry(thr, pc); 98 } 99 100 NOINLINE void TraceRestartFuncExit(ThreadState* thr) { 101 TraceSwitchPart(thr); 102 FuncExit(thr); 103 } 104 105 void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr, 106 StackID stk) { 107 DCHECK(type == EventType::kLock || type == EventType::kRLock); 108 if (!kCollectHistory) 109 return; 110 EventLock ev; 111 ev.is_access = 0; 112 ev.is_func = 0; 113 ev.type = type; 114 ev.pc = CompressAddr(pc); 115 ev.stack_lo = stk; 116 ev.stack_hi = stk >> EventLock::kStackIDLoBits; 117 ev._ = 0; 118 ev.addr = CompressAddr(addr); 119 TraceEvent(thr, ev); 120 } 121 122 void TraceMutexUnlock(ThreadState* thr, uptr addr) { 123 if (!kCollectHistory) 124 return; 125 EventUnlock ev; 126 ev.is_access = 0; 127 ev.is_func = 0; 128 ev.type = EventType::kUnlock; 129 ev._ = 0; 130 ev.addr = CompressAddr(addr); 131 TraceEvent(thr, ev); 132 } 133 134 void TraceTime(ThreadState* thr) { 135 if (!kCollectHistory) 136 return; 137 FastState fast_state = thr->fast_state; 138 EventTime ev; 139 ev.is_access = 0; 140 ev.is_func = 0; 141 ev.type = EventType::kTime; 142 ev.sid = static_cast<u64>(fast_state.sid()); 143 ev.epoch = static_cast<u64>(fast_state.epoch()); 144 ev._ = 0; 145 TraceEvent(thr, ev); 146 } 147 148 NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, 149 Shadow old, 150 AccessType typ) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { 151 // For the free shadow markers the first element (that contains kFreeSid) 152 // triggers the race, but the second element contains info about the freeing 153 // thread, take it. 154 if (old.sid() == kFreeSid) 155 old = Shadow(LoadShadow(&shadow_mem[1])); 156 // This prevents trapping on this address in future. 157 for (uptr i = 0; i < kShadowCnt; i++) 158 StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty); 159 // See the comment in MemoryRangeFreed as to why the slot is locked 160 // for free memory accesses. ReportRace must not be called with 161 // the slot locked because of the fork. But MemoryRangeFreed is not 162 // called during fork because fork sets ignore_reads_and_writes, 163 // so simply unlocking the slot should be fine. 164 if (typ & kAccessSlotLocked) 165 SlotUnlock(thr); 166 ReportRace(thr, shadow_mem, cur, Shadow(old), typ); 167 if (typ & kAccessSlotLocked) 168 SlotLock(thr); 169 } 170 171 #if !TSAN_VECTORIZE 172 ALWAYS_INLINE 173 bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1, 174 AccessType typ) { 175 for (uptr i = 0; i < kShadowCnt; i++) { 176 auto old = LoadShadow(&s[i]); 177 if (!(typ & kAccessRead)) { 178 if (old == cur.raw()) 179 return true; 180 continue; 181 } 182 auto masked = static_cast<RawShadow>(static_cast<u32>(old) | 183 static_cast<u32>(Shadow::kRodata)); 184 if (masked == cur.raw()) 185 return true; 186 if (!(typ & kAccessNoRodata) && !SANITIZER_GO) { 187 if (old == Shadow::kRodata) 188 return true; 189 } 190 } 191 return false; 192 } 193 194 ALWAYS_INLINE 195 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, 196 int unused0, int unused1, AccessType typ) { 197 bool stored = false; 198 for (uptr idx = 0; idx < kShadowCnt; idx++) { 199 RawShadow* sp = &shadow_mem[idx]; 200 Shadow old(LoadShadow(sp)); 201 if (LIKELY(old.raw() == Shadow::kEmpty)) { 202 if (!(typ & kAccessCheckOnly) && !stored) 203 StoreShadow(sp, cur.raw()); 204 return false; 205 } 206 if (LIKELY(!(cur.access() & old.access()))) 207 continue; 208 if (LIKELY(cur.sid() == old.sid())) { 209 if (!(typ & kAccessCheckOnly) && 210 LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) { 211 StoreShadow(sp, cur.raw()); 212 stored = true; 213 } 214 continue; 215 } 216 if (LIKELY(old.IsBothReadsOrAtomic(typ))) 217 continue; 218 if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch())) 219 continue; 220 DoReportRace(thr, shadow_mem, cur, old, typ); 221 return true; 222 } 223 // We did not find any races and had already stored 224 // the current access info, so we are done. 225 if (LIKELY(stored)) 226 return false; 227 // Choose a random candidate slot and replace it. 228 uptr index = 229 atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt; 230 StoreShadow(&shadow_mem[index], cur.raw()); 231 return false; 232 } 233 234 # define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0 235 236 #else /* !TSAN_VECTORIZE */ 237 238 ALWAYS_INLINE 239 bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow, 240 m128 access, AccessType typ) { 241 // Note: we could check if there is a larger access of the same type, 242 // e.g. we just allocated/memset-ed a block (so it contains 8 byte writes) 243 // and now do smaller reads/writes, these can also be considered as "same 244 // access". However, it will make the check more expensive, so it's unclear 245 // if it's worth it. But this would conserve trace space, so it's useful 246 // besides potential speed up. 247 if (!(typ & kAccessRead)) { 248 const m128 same = _mm_cmpeq_epi32(shadow, access); 249 return _mm_movemask_epi8(same); 250 } 251 // For reads we need to reset read bit in the shadow, 252 // because we need to match read with both reads and writes. 253 // Shadow::kRodata has only read bit set, so it does what we want. 254 // We also abuse it for rodata check to save few cycles 255 // since we already loaded Shadow::kRodata into a register. 256 // Reads from rodata can't race. 257 // Measurements show that they can be 10-20% of all memory accesses. 258 // Shadow::kRodata has epoch 0 which cannot appear in shadow normally 259 // (thread epochs start from 1). So the same read bit mask 260 // serves as rodata indicator. 261 const m128 read_mask = _mm_set1_epi32(static_cast<u32>(Shadow::kRodata)); 262 const m128 masked_shadow = _mm_or_si128(shadow, read_mask); 263 m128 same = _mm_cmpeq_epi32(masked_shadow, access); 264 // Range memory accesses check Shadow::kRodata before calling this, 265 // Shadow::kRodatas is not possible for free memory access 266 // and Go does not use Shadow::kRodata. 267 if (!(typ & kAccessNoRodata) && !SANITIZER_GO) { 268 const m128 ro = _mm_cmpeq_epi32(shadow, read_mask); 269 same = _mm_or_si128(ro, same); 270 } 271 return _mm_movemask_epi8(same); 272 } 273 274 NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, 275 u32 race_mask, m128 shadow, AccessType typ) { 276 // race_mask points which of the shadow elements raced with the current 277 // access. Extract that element. 278 CHECK_NE(race_mask, 0); 279 u32 old; 280 // Note: _mm_extract_epi32 index must be a constant value. 281 switch (__builtin_ffs(race_mask) / 4) { 282 case 0: 283 old = _mm_extract_epi32(shadow, 0); 284 break; 285 case 1: 286 old = _mm_extract_epi32(shadow, 1); 287 break; 288 case 2: 289 old = _mm_extract_epi32(shadow, 2); 290 break; 291 case 3: 292 old = _mm_extract_epi32(shadow, 3); 293 break; 294 } 295 Shadow prev(static_cast<RawShadow>(old)); 296 // For the free shadow markers the first element (that contains kFreeSid) 297 // triggers the race, but the second element contains info about the freeing 298 // thread, take it. 299 if (prev.sid() == kFreeSid) 300 prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1))); 301 DoReportRace(thr, shadow_mem, cur, prev, typ); 302 } 303 304 ALWAYS_INLINE 305 bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, 306 m128 shadow, m128 access, AccessType typ) { 307 // Note: empty/zero slots don't intersect with any access. 308 const m128 zero = _mm_setzero_si128(); 309 const m128 mask_access = _mm_set1_epi32(0x000000ff); 310 const m128 mask_sid = _mm_set1_epi32(0x0000ff00); 311 const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000); 312 const m128 access_and = _mm_and_si128(access, shadow); 313 const m128 access_xor = _mm_xor_si128(access, shadow); 314 const m128 intersect = _mm_and_si128(access_and, mask_access); 315 const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero); 316 const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid); 317 const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero); 318 const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic); 319 const m128 no_race = 320 _mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic); 321 const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero)); 322 if (UNLIKELY(race_mask)) 323 goto SHARED; 324 325 STORE : { 326 if (typ & kAccessCheckOnly) 327 return false; 328 // We could also replace different sid's if access is the same, 329 // rw weaker and happens before. However, just checking access below 330 // is not enough because we also need to check that !both_read_or_atomic 331 // (reads from different sids can be concurrent). 332 // Theoretically we could replace smaller accesses with larger accesses, 333 // but it's unclear if it's worth doing. 334 const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff); 335 const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid); 336 const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero); 337 const m128 access_read_atomic = 338 _mm_set1_epi32((typ & (kAccessRead | kAccessAtomic)) << 30); 339 const m128 rw_weaker = 340 _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow); 341 const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker); 342 const int rewrite_mask = _mm_movemask_epi8(rewrite); 343 int index = __builtin_ffs(rewrite_mask); 344 if (UNLIKELY(index == 0)) { 345 const m128 empty = _mm_cmpeq_epi32(shadow, zero); 346 const int empty_mask = _mm_movemask_epi8(empty); 347 index = __builtin_ffs(empty_mask); 348 if (UNLIKELY(index == 0)) 349 index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16; 350 } 351 StoreShadow(&shadow_mem[index / 4], cur.raw()); 352 // We could zero other slots determined by rewrite_mask. 353 // That would help other threads to evict better slots, 354 // but it's unclear if it's worth it. 355 return false; 356 } 357 358 SHARED: 359 m128 thread_epochs = _mm_set1_epi32(0x7fffffff); 360 // Need to unwind this because _mm_extract_epi8/_mm_insert_epi32 361 // indexes must be constants. 362 # define LOAD_EPOCH(idx) \ 363 if (LIKELY(race_mask & (1 << (idx * 4)))) { \ 364 u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \ 365 u16 epoch = static_cast<u16>(thr->clock.Get(static_cast<Sid>(sid))); \ 366 thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \ 367 } 368 LOAD_EPOCH(0); 369 LOAD_EPOCH(1); 370 LOAD_EPOCH(2); 371 LOAD_EPOCH(3); 372 # undef LOAD_EPOCH 373 const m128 mask_epoch = _mm_set1_epi32(0x3fff0000); 374 const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch); 375 const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs); 376 const int concurrent_mask = _mm_movemask_epi8(concurrent); 377 if (LIKELY(concurrent_mask == 0)) 378 goto STORE; 379 380 DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ); 381 return true; 382 } 383 384 # define LOAD_CURRENT_SHADOW(cur, shadow_mem) \ 385 const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \ 386 const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem)) 387 #endif 388 389 char* DumpShadow(char* buf, RawShadow raw) { 390 if (raw == Shadow::kEmpty) { 391 internal_snprintf(buf, 64, "0"); 392 return buf; 393 } 394 Shadow s(raw); 395 AccessType typ; 396 s.GetAccess(nullptr, nullptr, &typ); 397 internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}", 398 static_cast<u32>(s.sid()), static_cast<u32>(s.epoch()), 399 s.access(), static_cast<u32>(typ)); 400 return buf; 401 } 402 403 // TryTrace* and TraceRestart* functions allow to turn memory access and func 404 // entry/exit callbacks into leaf functions with all associated performance 405 // benefits. These hottest callbacks do only 2 slow path calls: report a race 406 // and trace part switching. Race reporting is easy to turn into a tail call, we 407 // just always return from the runtime after reporting a race. But trace part 408 // switching is harder because it needs to be in the middle of callbacks. To 409 // turn it into a tail call we immidiately return after TraceRestart* functions, 410 // but TraceRestart* functions themselves recurse into the callback after 411 // switching trace part. As the result the hottest callbacks contain only tail 412 // calls, which effectively makes them leaf functions (can use all registers, 413 // no frame setup, etc). 414 NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr, 415 uptr size, AccessType typ) { 416 TraceSwitchPart(thr); 417 MemoryAccess(thr, pc, addr, size, typ); 418 } 419 420 ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr, 421 uptr size, AccessType typ) { 422 #if SANITIZER_APPLE && !SANITIZER_GO 423 // Swift symbolizer can be intercepted and deadlock without this 424 if (thr->in_symbolizer) 425 return; 426 #endif 427 RawShadow* shadow_mem = MemToShadow(addr); 428 UNUSED char memBuf[4][64]; 429 DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid, 430 static_cast<int>(thr->fast_state.sid()), 431 static_cast<int>(thr->fast_state.epoch()), (void*)addr, size, 432 static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]), 433 DumpShadow(memBuf[1], shadow_mem[1]), 434 DumpShadow(memBuf[2], shadow_mem[2]), 435 DumpShadow(memBuf[3], shadow_mem[3])); 436 437 FastState fast_state = thr->fast_state; 438 Shadow cur(fast_state, addr, size, typ); 439 440 LOAD_CURRENT_SHADOW(cur, shadow_mem); 441 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 442 return; 443 if (UNLIKELY(fast_state.GetIgnoreBit())) 444 return; 445 if (!TryTraceMemoryAccess(thr, pc, addr, size, typ)) 446 return TraceRestartMemoryAccess(thr, pc, addr, size, typ); 447 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); 448 } 449 450 void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ); 451 452 NOINLINE 453 void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr, 454 AccessType typ) { 455 TraceSwitchPart(thr); 456 MemoryAccess16(thr, pc, addr, typ); 457 } 458 459 ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, 460 AccessType typ) { 461 const uptr size = 16; 462 FastState fast_state = thr->fast_state; 463 if (UNLIKELY(fast_state.GetIgnoreBit())) 464 return; 465 Shadow cur(fast_state, 0, 8, typ); 466 RawShadow* shadow_mem = MemToShadow(addr); 467 bool traced = false; 468 { 469 LOAD_CURRENT_SHADOW(cur, shadow_mem); 470 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 471 goto SECOND; 472 if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) 473 return RestartMemoryAccess16(thr, pc, addr, typ); 474 traced = true; 475 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) 476 return; 477 } 478 SECOND: 479 shadow_mem += kShadowCnt; 480 LOAD_CURRENT_SHADOW(cur, shadow_mem); 481 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 482 return; 483 if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) 484 return RestartMemoryAccess16(thr, pc, addr, typ); 485 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); 486 } 487 488 NOINLINE 489 void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr, 490 uptr size, AccessType typ) { 491 TraceSwitchPart(thr); 492 UnalignedMemoryAccess(thr, pc, addr, size, typ); 493 } 494 495 ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc, 496 uptr addr, uptr size, 497 AccessType typ) { 498 DCHECK_LE(size, 8); 499 FastState fast_state = thr->fast_state; 500 if (UNLIKELY(fast_state.GetIgnoreBit())) 501 return; 502 RawShadow* shadow_mem = MemToShadow(addr); 503 bool traced = false; 504 uptr size1 = Min<uptr>(size, RoundUp(addr + 1, kShadowCell) - addr); 505 { 506 Shadow cur(fast_state, addr, size1, typ); 507 LOAD_CURRENT_SHADOW(cur, shadow_mem); 508 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 509 goto SECOND; 510 if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) 511 return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ); 512 traced = true; 513 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) 514 return; 515 } 516 SECOND: 517 uptr size2 = size - size1; 518 if (LIKELY(size2 == 0)) 519 return; 520 shadow_mem += kShadowCnt; 521 Shadow cur(fast_state, 0, size2, typ); 522 LOAD_CURRENT_SHADOW(cur, shadow_mem); 523 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 524 return; 525 if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) 526 return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ); 527 CheckRaces(thr, shadow_mem, cur, shadow, access, typ); 528 } 529 530 void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) { 531 DCHECK_LT(p, end); 532 DCHECK(IsShadowMem(p)); 533 DCHECK(IsShadowMem(end - 1)); 534 UNUSED const uptr kAlign = kShadowCnt * kShadowSize; 535 DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0); 536 DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0); 537 #if !TSAN_VECTORIZE 538 for (; p < end; p += kShadowCnt) { 539 p[0] = v; 540 for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty; 541 } 542 #else 543 m128 vv = _mm_setr_epi32( 544 static_cast<u32>(v), static_cast<u32>(Shadow::kEmpty), 545 static_cast<u32>(Shadow::kEmpty), static_cast<u32>(Shadow::kEmpty)); 546 m128* vp = reinterpret_cast<m128*>(p); 547 m128* vend = reinterpret_cast<m128*>(end); 548 for (; vp < vend; vp++) _mm_store_si128(vp, vv); 549 #endif 550 } 551 552 static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) { 553 if (size == 0) 554 return; 555 DCHECK_EQ(addr % kShadowCell, 0); 556 DCHECK_EQ(size % kShadowCell, 0); 557 // If a user passes some insane arguments (memset(0)), 558 // let it just crash as usual. 559 if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) 560 return; 561 RawShadow* begin = MemToShadow(addr); 562 RawShadow* end = begin + size / kShadowCell * kShadowCnt; 563 // Don't want to touch lots of shadow memory. 564 // If a program maps 10MB stack, there is no need reset the whole range. 565 // UnmapOrDie/MmapFixedNoReserve does not work on Windows. 566 if (SANITIZER_WINDOWS || 567 size <= common_flags()->clear_shadow_mmap_threshold) { 568 ShadowSet(begin, end, val); 569 return; 570 } 571 // The region is big, reset only beginning and end. 572 const uptr kPageSize = GetPageSizeCached(); 573 // Set at least first kPageSize/2 to page boundary. 574 RawShadow* mid1 = 575 Min(end, reinterpret_cast<RawShadow*>(RoundUp( 576 reinterpret_cast<uptr>(begin) + kPageSize / 2, kPageSize))); 577 // begin must < mid1 578 ShadowSet(begin, mid1, val); 579 // Reset middle part. 580 RawShadow* mid2 = RoundDown(end, kPageSize); 581 if (mid2 > mid1) { 582 if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1)) 583 Die(); 584 } 585 // Set the ending. 586 if (mid2 < end) 587 ShadowSet(mid2, end, val); 588 else 589 DCHECK_EQ(mid2, end); 590 } 591 592 void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) { 593 uptr addr1 = RoundDown(addr, kShadowCell); 594 uptr size1 = RoundUp(size + addr - addr1, kShadowCell); 595 MemoryRangeSet(addr1, size1, Shadow::kEmpty); 596 } 597 598 void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) { 599 // Callers must lock the slot to ensure synchronization with the reset. 600 // The problem with "freed" memory is that it's not "monotonic" 601 // with respect to bug detection: freed memory is bad to access, 602 // but then if the heap block is reallocated later, it's good to access. 603 // As the result a garbage "freed" shadow can lead to a false positive 604 // if it happens to match a real free in the thread trace, 605 // but the heap block was reallocated before the current memory access, 606 // so it's still good to access. It's not the case with data races. 607 DCHECK(thr->slot_locked); 608 DCHECK_EQ(addr % kShadowCell, 0); 609 size = RoundUp(size, kShadowCell); 610 // Processing more than 1k (2k of shadow) is expensive, 611 // can cause excessive memory consumption (user does not necessary touch 612 // the whole range) and most likely unnecessary. 613 size = Min<uptr>(size, 1024); 614 const AccessType typ = kAccessWrite | kAccessFree | kAccessSlotLocked | 615 kAccessCheckOnly | kAccessNoRodata; 616 TraceMemoryAccessRange(thr, pc, addr, size, typ); 617 RawShadow* shadow_mem = MemToShadow(addr); 618 Shadow cur(thr->fast_state, 0, kShadowCell, typ); 619 #if TSAN_VECTORIZE 620 const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw())); 621 const m128 freed = _mm_setr_epi32( 622 static_cast<u32>(Shadow::FreedMarker()), 623 static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0); 624 for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { 625 const m128 shadow = _mm_load_si128((m128*)shadow_mem); 626 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) 627 return; 628 _mm_store_si128((m128*)shadow_mem, freed); 629 } 630 #else 631 for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) { 632 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ))) 633 return; 634 StoreShadow(&shadow_mem[0], Shadow::FreedMarker()); 635 StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch())); 636 StoreShadow(&shadow_mem[2], Shadow::kEmpty); 637 StoreShadow(&shadow_mem[3], Shadow::kEmpty); 638 } 639 #endif 640 } 641 642 void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) { 643 DCHECK_EQ(addr % kShadowCell, 0); 644 size = RoundUp(size, kShadowCell); 645 TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite); 646 Shadow cur(thr->fast_state, 0, 8, kAccessWrite); 647 MemoryRangeSet(addr, size, cur.raw()); 648 } 649 650 void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr, 651 uptr size) { 652 if (thr->ignore_reads_and_writes == 0) 653 MemoryRangeImitateWrite(thr, pc, addr, size); 654 else 655 MemoryResetRange(thr, pc, addr, size); 656 } 657 658 ALWAYS_INLINE 659 bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, 660 AccessType typ) { 661 LOAD_CURRENT_SHADOW(cur, shadow_mem); 662 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) 663 return false; 664 return CheckRaces(thr, shadow_mem, cur, shadow, access, typ); 665 } 666 667 template <bool is_read> 668 NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, 669 uptr size) { 670 TraceSwitchPart(thr); 671 MemoryAccessRangeT<is_read>(thr, pc, addr, size); 672 } 673 674 template <bool is_read> 675 void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) { 676 const AccessType typ = 677 (is_read ? kAccessRead : kAccessWrite) | kAccessNoRodata; 678 RawShadow* shadow_mem = MemToShadow(addr); 679 DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid, 680 (void*)pc, (void*)addr, (int)size, is_read); 681 DCHECK_NE(size, 0); 682 #if SANITIZER_DEBUG 683 if (!IsAppMem(addr)) { 684 Printf("Access to non app mem start: %p\n", (void*)addr); 685 DCHECK(IsAppMem(addr)); 686 } 687 if (!IsAppMem(addr + size - 1)) { 688 Printf("Access to non app mem end: %p\n", (void*)(addr + size - 1)); 689 DCHECK(IsAppMem(addr + size - 1)); 690 } 691 if (!IsShadowMem(shadow_mem)) { 692 Printf("Bad shadow start addr: %p (%p)\n", (void*)shadow_mem, (void*)addr); 693 DCHECK(IsShadowMem(shadow_mem)); 694 } 695 696 uptr rounded_size = 697 (RoundUpTo(addr + size, kShadowCell) - RoundDownTo(addr, kShadowCell)); 698 RawShadow* shadow_mem_end = 699 shadow_mem + rounded_size / kShadowCell * kShadowCnt; 700 if (!IsShadowMem(shadow_mem_end - 1)) { 701 Printf("Bad shadow end addr: %p (%p)\n", (void*)(shadow_mem_end - 1), 702 (void*)(addr + size - 1)); 703 Printf( 704 "Shadow start addr (ok): %p (%p); size: 0x%zx; rounded_size: 0x%zx; " 705 "kShadowMultiplier: %zx\n", 706 (void*)shadow_mem, (void*)addr, size, rounded_size, kShadowMultiplier); 707 DCHECK(IsShadowMem(shadow_mem_end - 1)); 708 } 709 #endif 710 711 // Access to .rodata section, no races here. 712 // Measurements show that it can be 10-20% of all memory accesses. 713 // Check here once to not check for every access separately. 714 // Note: we could (and should) do this only for the is_read case 715 // (writes shouldn't go to .rodata). But it happens in Chromium tests: 716 // https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19 717 // Details are unknown since it happens only on CI machines. 718 if (*shadow_mem == Shadow::kRodata) 719 return; 720 721 FastState fast_state = thr->fast_state; 722 if (UNLIKELY(fast_state.GetIgnoreBit())) 723 return; 724 725 if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) 726 return RestartMemoryAccessRange<is_read>(thr, pc, addr, size); 727 728 if (UNLIKELY(addr % kShadowCell)) { 729 // Handle unaligned beginning, if any. 730 uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr); 731 size -= size1; 732 Shadow cur(fast_state, addr, size1, typ); 733 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) 734 return; 735 shadow_mem += kShadowCnt; 736 } 737 // Handle middle part, if any. 738 Shadow cur(fast_state, 0, kShadowCell, typ); 739 for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) { 740 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) 741 return; 742 } 743 // Handle ending, if any. 744 if (UNLIKELY(size)) { 745 Shadow cur(fast_state, 0, size, typ); 746 if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ))) 747 return; 748 } 749 } 750 751 template void MemoryAccessRangeT<true>(ThreadState* thr, uptr pc, uptr addr, 752 uptr size); 753 template void MemoryAccessRangeT<false>(ThreadState* thr, uptr pc, uptr addr, 754 uptr size); 755 756 } // namespace __tsan 757 758 #if !SANITIZER_GO 759 // Must be included in this file to make sure everything is inlined. 760 # include "tsan_interface.inc" 761 #endif 762