1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
| //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
// Not intended for direct inclusion. Include sanitizer_atomic.h.
//
//===----------------------------------------------------------------------===//
#ifndef SANITIZER_ATOMIC_MSVC_H
#define SANITIZER_ATOMIC_MSVC_H
extern "C" void _ReadWriteBarrier();
#pragma intrinsic(_ReadWriteBarrier)
extern "C" void _mm_mfence();
#pragma intrinsic(_mm_mfence)
extern "C" void _mm_pause();
#pragma intrinsic(_mm_pause)
extern "C" char _InterlockedExchange8(char volatile *Addend, char Value);
#pragma intrinsic(_InterlockedExchange8)
extern "C" short _InterlockedExchange16(short volatile *Addend, short Value);
#pragma intrinsic(_InterlockedExchange16)
extern "C" long _InterlockedExchange(long volatile *Addend, long Value);
#pragma intrinsic(_InterlockedExchange)
extern "C" long _InterlockedExchangeAdd(long volatile *Addend, long Value);
#pragma intrinsic(_InterlockedExchangeAdd)
extern "C" char _InterlockedCompareExchange8(char volatile *Destination,
char Exchange, char Comparand);
#pragma intrinsic(_InterlockedCompareExchange8)
extern "C" short _InterlockedCompareExchange16(short volatile *Destination,
short Exchange, short Comparand);
#pragma intrinsic(_InterlockedCompareExchange16)
extern "C" long long _InterlockedCompareExchange64(
long long volatile *Destination, long long Exchange, long long Comparand);
#pragma intrinsic(_InterlockedCompareExchange64)
extern "C" void *_InterlockedCompareExchangePointer(
void *volatile *Destination,
void *Exchange, void *Comparand);
#pragma intrinsic(_InterlockedCompareExchangePointer)
extern "C" long __cdecl _InterlockedCompareExchange(long volatile *Destination,
long Exchange,
long Comparand);
#pragma intrinsic(_InterlockedCompareExchange)
#ifdef _WIN64
extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend,
long long Value);
#pragma intrinsic(_InterlockedExchangeAdd64)
#endif
namespace __sanitizer {
INLINE void atomic_signal_fence(memory_order) {
_ReadWriteBarrier();
}
INLINE void atomic_thread_fence(memory_order) {
_mm_mfence();
}
INLINE void proc_yield(int cnt) {
for (int i = 0; i < cnt; i++)
_mm_pause();
}
template<typename T>
INLINE typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
typename T::Type v;
// FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
if (mo == memory_order_relaxed) {
v = a->val_dont_use;
} else {
atomic_signal_fence(memory_order_seq_cst);
v = a->val_dont_use;
atomic_signal_fence(memory_order_seq_cst);
}
return v;
}
template<typename T>
INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
// FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
if (mo == memory_order_relaxed) {
a->val_dont_use = v;
} else {
atomic_signal_fence(memory_order_seq_cst);
a->val_dont_use = v;
atomic_signal_fence(memory_order_seq_cst);
}
if (mo == memory_order_seq_cst)
atomic_thread_fence(memory_order_seq_cst);
}
INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
(long)v);
}
INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
#ifdef _WIN64
return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
(long long)v);
#else
return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
(long)v);
#endif
}
INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
-(long)v);
}
INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
#ifdef _WIN64
return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
-(long long)v);
#else
return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
-(long)v);
#endif
}
INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
u8 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
}
INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
u16 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
}
INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
}
INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
u8 *cmp,
u8 xchgv,
memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
u8 cmpv = *cmp;
#ifdef _WIN64
u8 prev = (u8)_InterlockedCompareExchange8(
(volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
#else
u8 prev;
__asm {
mov al, cmpv
mov ecx, a
mov dl, xchgv
lock cmpxchg [ecx], dl
mov prev, al
}
#endif
if (prev == cmpv)
return true;
*cmp = prev;
return false;
}
INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
uptr *cmp,
uptr xchg,
memory_order mo) {
uptr cmpv = *cmp;
uptr prev = (uptr)_InterlockedCompareExchangePointer(
(void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
if (prev == cmpv)
return true;
*cmp = prev;
return false;
}
INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
u16 *cmp,
u16 xchg,
memory_order mo) {
u16 cmpv = *cmp;
u16 prev = (u16)_InterlockedCompareExchange16(
(volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
if (prev == cmpv)
return true;
*cmp = prev;
return false;
}
INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
u32 *cmp,
u32 xchg,
memory_order mo) {
u32 cmpv = *cmp;
u32 prev = (u32)_InterlockedCompareExchange(
(volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
if (prev == cmpv)
return true;
*cmp = prev;
return false;
}
INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
u64 *cmp,
u64 xchg,
memory_order mo) {
u64 cmpv = *cmp;
u64 prev = (u64)_InterlockedCompareExchange64(
(volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
if (prev == cmpv)
return true;
*cmp = prev;
return false;
}
template<typename T>
INLINE bool atomic_compare_exchange_weak(volatile T *a,
typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
return atomic_compare_exchange_strong(a, cmp, xchg, mo);
}
} // namespace __sanitizer
#endif // SANITIZER_ATOMIC_CLANG_H
|