Line data Source code
1 : /*
2 : * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a copy
5 : * of this software and associated documentation files (the "Software"), to deal
6 : * in the Software without restriction, including without limitation the rights
7 : * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 : * copies of the Software, and to permit persons to whom the Software is
9 : * furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice shall be included in
12 : * all copies or substantial portions of the Software.
13 : *
14 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 : * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 : * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 : * SOFTWARE.
21 : */
22 :
23 : /* char_fetch_compare_and_swap */
24 : #if defined(AO_HAVE_char_fetch_compare_and_swap) \
25 : && defined(AO_HAVE_nop_full) \
26 : && !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
27 : AO_INLINE unsigned/**/char
28 : AO_char_fetch_compare_and_swap_acquire(volatile unsigned/**/char *addr,
29 : unsigned/**/char old_val, unsigned/**/char new_val)
30 : {
31 : unsigned/**/char result = AO_char_fetch_compare_and_swap(addr, old_val, new_val);
32 : AO_nop_full();
33 : return result;
34 : }
35 : # define AO_HAVE_char_fetch_compare_and_swap_acquire
36 : #endif
37 : #if defined(AO_HAVE_char_fetch_compare_and_swap) \
38 : && defined(AO_HAVE_nop_full) \
39 : && !defined(AO_HAVE_char_fetch_compare_and_swap_release)
40 : # define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
41 : (AO_nop_full(), \
42 : AO_char_fetch_compare_and_swap(addr, old_val, new_val))
43 : # define AO_HAVE_char_fetch_compare_and_swap_release
44 : #endif
45 : #if defined(AO_HAVE_char_fetch_compare_and_swap_full)
46 : # if !defined(AO_HAVE_char_fetch_compare_and_swap_release)
47 : # define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
48 : AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
49 : # define AO_HAVE_char_fetch_compare_and_swap_release
50 : # endif
51 : # if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
52 : # define AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
53 : AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
54 : # define AO_HAVE_char_fetch_compare_and_swap_acquire
55 : # endif
56 : # if !defined(AO_HAVE_char_fetch_compare_and_swap_write)
57 : # define AO_char_fetch_compare_and_swap_write(addr, old_val, new_val) \
58 : AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
59 : # define AO_HAVE_char_fetch_compare_and_swap_write
60 : # endif
61 : # if !defined(AO_HAVE_char_fetch_compare_and_swap_read)
62 : # define AO_char_fetch_compare_and_swap_read(addr, old_val, new_val) \
63 : AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
64 : # define AO_HAVE_char_fetch_compare_and_swap_read
65 : # endif
66 : #endif /* AO_HAVE_char_fetch_compare_and_swap_full */
67 :
68 : #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
69 : && defined(AO_HAVE_char_fetch_compare_and_swap_release)
70 : # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
71 : AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
72 : # define AO_HAVE_char_fetch_compare_and_swap
73 : #endif
74 : #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
75 : && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
76 : # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
77 : AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
78 : # define AO_HAVE_char_fetch_compare_and_swap
79 : #endif
80 : #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
81 : && defined(AO_HAVE_char_fetch_compare_and_swap_write)
82 : # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
83 : AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
84 : # define AO_HAVE_char_fetch_compare_and_swap
85 : #endif
86 : #if !defined(AO_HAVE_char_fetch_compare_and_swap) \
87 : && defined(AO_HAVE_char_fetch_compare_and_swap_read)
88 : # define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
89 : AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
90 : # define AO_HAVE_char_fetch_compare_and_swap
91 : #endif
92 :
93 : #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
94 : && defined(AO_HAVE_nop_full) \
95 : && !defined(AO_HAVE_char_fetch_compare_and_swap_full)
96 : # define AO_char_fetch_compare_and_swap_full(addr, old_val, new_val) \
97 : (AO_nop_full(), \
98 : AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val))
99 : # define AO_HAVE_char_fetch_compare_and_swap_full
100 : #endif
101 :
102 : #if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
103 : && defined(AO_HAVE_char_fetch_compare_and_swap_write)
104 : # define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
105 : AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
106 : # define AO_HAVE_char_fetch_compare_and_swap_release_write
107 : #endif
108 : #if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
109 : && defined(AO_HAVE_char_fetch_compare_and_swap_release)
110 : # define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
111 : AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
112 : # define AO_HAVE_char_fetch_compare_and_swap_release_write
113 : #endif
114 : #if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
115 : && defined(AO_HAVE_char_fetch_compare_and_swap_read)
116 : # define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
117 : AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
118 : # define AO_HAVE_char_fetch_compare_and_swap_acquire_read
119 : #endif
120 : #if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
121 : && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
122 : # define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
123 : AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
124 : # define AO_HAVE_char_fetch_compare_and_swap_acquire_read
125 : #endif
126 :
127 : #ifdef AO_NO_DD_ORDERING
128 : # if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read)
129 : # define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
130 : AO_char_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
131 : # define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
132 : # endif
133 : #else
134 : # if defined(AO_HAVE_char_fetch_compare_and_swap)
135 : # define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
136 : AO_char_fetch_compare_and_swap(addr, old_val, new_val)
137 : # define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
138 : # endif
139 : #endif /* !AO_NO_DD_ORDERING */
140 :
141 : /* char_compare_and_swap */
142 : #if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
143 : && !defined(AO_HAVE_char_compare_and_swap_acquire)
144 : AO_INLINE int
145 : AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old,
146 : unsigned/**/char new_val)
147 : {
148 : int result = AO_char_compare_and_swap(addr, old, new_val);
149 : AO_nop_full();
150 : return result;
151 : }
152 : # define AO_HAVE_char_compare_and_swap_acquire
153 : #endif
154 : #if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
155 : && !defined(AO_HAVE_char_compare_and_swap_release)
156 : # define AO_char_compare_and_swap_release(addr, old, new_val) \
157 : (AO_nop_full(), AO_char_compare_and_swap(addr, old, new_val))
158 : # define AO_HAVE_char_compare_and_swap_release
159 : #endif
160 : #if defined(AO_HAVE_char_compare_and_swap_full)
161 : # if !defined(AO_HAVE_char_compare_and_swap_release)
162 : # define AO_char_compare_and_swap_release(addr, old, new_val) \
163 : AO_char_compare_and_swap_full(addr, old, new_val)
164 : # define AO_HAVE_char_compare_and_swap_release
165 : # endif
166 : # if !defined(AO_HAVE_char_compare_and_swap_acquire)
167 : # define AO_char_compare_and_swap_acquire(addr, old, new_val) \
168 : AO_char_compare_and_swap_full(addr, old, new_val)
169 : # define AO_HAVE_char_compare_and_swap_acquire
170 : # endif
171 : # if !defined(AO_HAVE_char_compare_and_swap_write)
172 : # define AO_char_compare_and_swap_write(addr, old, new_val) \
173 : AO_char_compare_and_swap_full(addr, old, new_val)
174 : # define AO_HAVE_char_compare_and_swap_write
175 : # endif
176 : # if !defined(AO_HAVE_char_compare_and_swap_read)
177 : # define AO_char_compare_and_swap_read(addr, old, new_val) \
178 : AO_char_compare_and_swap_full(addr, old, new_val)
179 : # define AO_HAVE_char_compare_and_swap_read
180 : # endif
181 : #endif /* AO_HAVE_char_compare_and_swap_full */
182 :
183 : #if !defined(AO_HAVE_char_compare_and_swap) \
184 : && defined(AO_HAVE_char_compare_and_swap_release)
185 : # define AO_char_compare_and_swap(addr, old, new_val) \
186 : AO_char_compare_and_swap_release(addr, old, new_val)
187 : # define AO_HAVE_char_compare_and_swap
188 : #endif
189 : #if !defined(AO_HAVE_char_compare_and_swap) \
190 : && defined(AO_HAVE_char_compare_and_swap_acquire)
191 : # define AO_char_compare_and_swap(addr, old, new_val) \
192 : AO_char_compare_and_swap_acquire(addr, old, new_val)
193 : # define AO_HAVE_char_compare_and_swap
194 : #endif
195 : #if !defined(AO_HAVE_char_compare_and_swap) \
196 : && defined(AO_HAVE_char_compare_and_swap_write)
197 : # define AO_char_compare_and_swap(addr, old, new_val) \
198 : AO_char_compare_and_swap_write(addr, old, new_val)
199 : # define AO_HAVE_char_compare_and_swap
200 : #endif
201 : #if !defined(AO_HAVE_char_compare_and_swap) \
202 : && defined(AO_HAVE_char_compare_and_swap_read)
203 : # define AO_char_compare_and_swap(addr, old, new_val) \
204 : AO_char_compare_and_swap_read(addr, old, new_val)
205 : # define AO_HAVE_char_compare_and_swap
206 : #endif
207 :
208 : #if defined(AO_HAVE_char_compare_and_swap_acquire) \
209 : && defined(AO_HAVE_nop_full) \
210 : && !defined(AO_HAVE_char_compare_and_swap_full)
211 : # define AO_char_compare_and_swap_full(addr, old, new_val) \
212 : (AO_nop_full(), \
213 : AO_char_compare_and_swap_acquire(addr, old, new_val))
214 : # define AO_HAVE_char_compare_and_swap_full
215 : #endif
216 :
217 : #if !defined(AO_HAVE_char_compare_and_swap_release_write) \
218 : && defined(AO_HAVE_char_compare_and_swap_write)
219 : # define AO_char_compare_and_swap_release_write(addr, old, new_val) \
220 : AO_char_compare_and_swap_write(addr, old, new_val)
221 : # define AO_HAVE_char_compare_and_swap_release_write
222 : #endif
223 : #if !defined(AO_HAVE_char_compare_and_swap_release_write) \
224 : && defined(AO_HAVE_char_compare_and_swap_release)
225 : # define AO_char_compare_and_swap_release_write(addr, old, new_val) \
226 : AO_char_compare_and_swap_release(addr, old, new_val)
227 : # define AO_HAVE_char_compare_and_swap_release_write
228 : #endif
229 : #if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
230 : && defined(AO_HAVE_char_compare_and_swap_read)
231 : # define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
232 : AO_char_compare_and_swap_read(addr, old, new_val)
233 : # define AO_HAVE_char_compare_and_swap_acquire_read
234 : #endif
235 : #if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
236 : && defined(AO_HAVE_char_compare_and_swap_acquire)
237 : # define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
238 : AO_char_compare_and_swap_acquire(addr, old, new_val)
239 : # define AO_HAVE_char_compare_and_swap_acquire_read
240 : #endif
241 :
242 : #ifdef AO_NO_DD_ORDERING
243 : # if defined(AO_HAVE_char_compare_and_swap_acquire_read)
244 : # define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
245 : AO_char_compare_and_swap_acquire_read(addr, old, new_val)
246 : # define AO_HAVE_char_compare_and_swap_dd_acquire_read
247 : # endif
248 : #else
249 : # if defined(AO_HAVE_char_compare_and_swap)
250 : # define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
251 : AO_char_compare_and_swap(addr, old, new_val)
252 : # define AO_HAVE_char_compare_and_swap_dd_acquire_read
253 : # endif
254 : #endif /* !AO_NO_DD_ORDERING */
255 :
256 : /* char_load */
257 : #if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire)
258 : # define AO_char_load_acquire(addr) AO_char_load_full(addr)
259 : # define AO_HAVE_char_load_acquire
260 : #endif
261 :
262 : #if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load)
263 : # define AO_char_load(addr) AO_char_load_acquire(addr)
264 : # define AO_HAVE_char_load
265 : #endif
266 :
267 : #if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read)
268 : # define AO_char_load_read(addr) AO_char_load_full(addr)
269 : # define AO_HAVE_char_load_read
270 : #endif
271 :
272 : #if !defined(AO_HAVE_char_load_acquire_read) \
273 : && defined(AO_HAVE_char_load_acquire)
274 : # define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr)
275 : # define AO_HAVE_char_load_acquire_read
276 : #endif
277 :
278 : #if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \
279 : && !defined(AO_HAVE_char_load_acquire)
280 : AO_INLINE unsigned/**/char
281 : AO_char_load_acquire(const volatile unsigned/**/char *addr)
282 : {
283 : unsigned/**/char result = AO_char_load(addr);
284 :
285 : /* Acquire barrier would be useless, since the load could be delayed */
286 : /* beyond it. */
287 : AO_nop_full();
288 : return result;
289 : }
290 : # define AO_HAVE_char_load_acquire
291 : #endif
292 :
293 : #if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \
294 : && !defined(AO_HAVE_char_load_read)
295 : AO_INLINE unsigned/**/char
296 : AO_char_load_read(const volatile unsigned/**/char *addr)
297 : {
298 : unsigned/**/char result = AO_char_load(addr);
299 :
300 : AO_nop_read();
301 : return result;
302 : }
303 : # define AO_HAVE_char_load_read
304 : #endif
305 :
306 : #if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \
307 : && !defined(AO_HAVE_char_load_full)
308 : # define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr))
309 : # define AO_HAVE_char_load_full
310 : #endif
311 :
312 : #if defined(AO_HAVE_char_compare_and_swap_read) \
313 : && !defined(AO_HAVE_char_load_read)
314 : # define AO_char_CAS_BASED_LOAD_READ
315 : AO_INLINE unsigned/**/char
316 : AO_char_load_read(const volatile unsigned/**/char *addr)
317 : {
318 : unsigned/**/char result;
319 :
320 : do {
321 : result = *(const unsigned/**/char *)addr;
322 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_read(
323 : (volatile unsigned/**/char *)addr,
324 : result, result)));
325 : return result;
326 : }
327 : # define AO_HAVE_char_load_read
328 : #endif
329 :
330 : #if !defined(AO_HAVE_char_load_acquire_read) \
331 : && defined(AO_HAVE_char_load_read)
332 : # define AO_char_load_acquire_read(addr) AO_char_load_read(addr)
333 : # define AO_HAVE_char_load_acquire_read
334 : #endif
335 :
336 : #if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) \
337 : && (!defined(AO_char_CAS_BASED_LOAD_READ) \
338 : || !defined(AO_HAVE_char_compare_and_swap))
339 : # define AO_char_load(addr) AO_char_load_acquire_read(addr)
340 : # define AO_HAVE_char_load
341 : #endif
342 :
343 : #if defined(AO_HAVE_char_compare_and_swap_full) \
344 : && !defined(AO_HAVE_char_load_full)
345 : AO_INLINE unsigned/**/char
346 : AO_char_load_full(const volatile unsigned/**/char *addr)
347 : {
348 : unsigned/**/char result;
349 :
350 : do {
351 : result = *(const unsigned/**/char *)addr;
352 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(
353 : (volatile unsigned/**/char *)addr,
354 : result, result)));
355 : return result;
356 : }
357 : # define AO_HAVE_char_load_full
358 : #endif
359 :
360 : #if defined(AO_HAVE_char_compare_and_swap_acquire) \
361 : && !defined(AO_HAVE_char_load_acquire)
362 : AO_INLINE unsigned/**/char
363 : AO_char_load_acquire(const volatile unsigned/**/char *addr)
364 : {
365 : unsigned/**/char result;
366 :
367 : do {
368 : result = *(const unsigned/**/char *)addr;
369 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(
370 : (volatile unsigned/**/char *)addr,
371 : result, result)));
372 : return result;
373 : }
374 : # define AO_HAVE_char_load_acquire
375 : #endif
376 :
377 : #if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_load)
378 : AO_INLINE unsigned/**/char
379 : AO_char_load(const volatile unsigned/**/char *addr)
380 : {
381 : unsigned/**/char result;
382 :
383 : do {
384 : result = *(const unsigned/**/char *)addr;
385 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(
386 : (volatile unsigned/**/char *)addr,
387 : result, result)));
388 : return result;
389 : }
390 : # define AO_HAVE_char_load
391 : #endif
392 :
393 : #ifdef AO_NO_DD_ORDERING
394 : # if defined(AO_HAVE_char_load_acquire_read)
395 : # define AO_char_load_dd_acquire_read(addr) \
396 : AO_char_load_acquire_read(addr)
397 : # define AO_HAVE_char_load_dd_acquire_read
398 : # endif
399 : #else
400 : # if defined(AO_HAVE_char_load)
401 : # define AO_char_load_dd_acquire_read(addr) AO_char_load(addr)
402 : # define AO_HAVE_char_load_dd_acquire_read
403 : # endif
404 : #endif /* !AO_NO_DD_ORDERING */
405 :
406 : /* char_store */
407 : #if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release)
408 : # define AO_char_store_release(addr, val) AO_char_store_full(addr, val)
409 : # define AO_HAVE_char_store_release
410 : #endif
411 :
412 : #if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store)
413 : # define AO_char_store(addr, val) AO_char_store_release(addr, val)
414 : # define AO_HAVE_char_store
415 : #endif
416 :
417 : #if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write)
418 : # define AO_char_store_write(addr, val) AO_char_store_full(addr, val)
419 : # define AO_HAVE_char_store_write
420 : #endif
421 :
422 : #if defined(AO_HAVE_char_store_release) \
423 : && !defined(AO_HAVE_char_store_release_write)
424 : # define AO_char_store_release_write(addr, val) \
425 : AO_char_store_release(addr, val)
426 : # define AO_HAVE_char_store_release_write
427 : #endif
428 :
429 : #if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store)
430 : # define AO_char_store(addr, val) AO_char_store_write(addr, val)
431 : # define AO_HAVE_char_store
432 : #endif
433 :
434 : #if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \
435 : && !defined(AO_HAVE_char_store_release)
436 : # define AO_char_store_release(addr, val) \
437 : (AO_nop_full(), AO_char_store(addr, val))
438 : # define AO_HAVE_char_store_release
439 : #endif
440 :
441 : #if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_write) \
442 : && !defined(AO_HAVE_char_store_write)
443 : # define AO_char_store_write(addr, val) \
444 : (AO_nop_write(), AO_char_store(addr, val))
445 : # define AO_HAVE_char_store_write
446 : #endif
447 :
448 : #if defined(AO_HAVE_char_compare_and_swap_write) \
449 : && !defined(AO_HAVE_char_store_write)
450 : AO_INLINE void
451 : AO_char_store_write(volatile unsigned/**/char *addr, unsigned/**/char new_val)
452 : {
453 : unsigned/**/char old_val;
454 :
455 : do {
456 : old_val = *(unsigned/**/char *)addr;
457 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_write(addr, old_val,
458 : new_val)));
459 : }
460 : # define AO_HAVE_char_store_write
461 : #endif
462 :
463 : #if defined(AO_HAVE_char_store_write) \
464 : && !defined(AO_HAVE_char_store_release_write)
465 : # define AO_char_store_release_write(addr, val) \
466 : AO_char_store_write(addr, val)
467 : # define AO_HAVE_char_store_release_write
468 : #endif
469 :
470 : #if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \
471 : && !defined(AO_HAVE_char_store_full)
472 : # define AO_char_store_full(addr, val) \
473 : (AO_char_store_release(addr, val), \
474 : AO_nop_full())
475 : # define AO_HAVE_char_store_full
476 : #endif
477 :
478 : #if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_store)
479 : AO_INLINE void
480 : AO_char_store(volatile unsigned/**/char *addr, unsigned/**/char new_val)
481 : {
482 : unsigned/**/char old_val;
483 :
484 : do {
485 : old_val = *(unsigned/**/char *)addr;
486 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr,
487 : old_val, new_val)));
488 : }
489 : # define AO_HAVE_char_store
490 : #endif
491 :
492 : #if defined(AO_HAVE_char_compare_and_swap_release) \
493 : && !defined(AO_HAVE_char_store_release)
494 : AO_INLINE void
495 : AO_char_store_release(volatile unsigned/**/char *addr, unsigned/**/char new_val)
496 : {
497 : unsigned/**/char old_val;
498 :
499 : do {
500 : old_val = *(unsigned/**/char *)addr;
501 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old_val,
502 : new_val)));
503 : }
504 : # define AO_HAVE_char_store_release
505 : #endif
506 :
507 : #if defined(AO_HAVE_char_compare_and_swap_full) \
508 : && !defined(AO_HAVE_char_store_full)
509 : AO_INLINE void
510 : AO_char_store_full(volatile unsigned/**/char *addr, unsigned/**/char new_val)
511 : {
512 : unsigned/**/char old_val;
513 :
514 : do {
515 : old_val = *(unsigned/**/char *)addr;
516 : } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old_val,
517 : new_val)));
518 : }
519 : # define AO_HAVE_char_store_full
520 : #endif
521 : /*
522 : * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
523 : *
524 : * Permission is hereby granted, free of charge, to any person obtaining a copy
525 : * of this software and associated documentation files (the "Software"), to deal
526 : * in the Software without restriction, including without limitation the rights
527 : * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
528 : * copies of the Software, and to permit persons to whom the Software is
529 : * furnished to do so, subject to the following conditions:
530 : *
531 : * The above copyright notice and this permission notice shall be included in
532 : * all copies or substantial portions of the Software.
533 : *
534 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
535 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
536 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
537 : * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
538 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
539 : * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
540 : * SOFTWARE.
541 : */
542 :
543 : /* short_fetch_compare_and_swap */
544 : #if defined(AO_HAVE_short_fetch_compare_and_swap) \
545 : && defined(AO_HAVE_nop_full) \
546 : && !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
547 : AO_INLINE unsigned/**/short
548 : AO_short_fetch_compare_and_swap_acquire(volatile unsigned/**/short *addr,
549 : unsigned/**/short old_val, unsigned/**/short new_val)
550 : {
551 : unsigned/**/short result = AO_short_fetch_compare_and_swap(addr, old_val, new_val);
552 : AO_nop_full();
553 : return result;
554 : }
555 : # define AO_HAVE_short_fetch_compare_and_swap_acquire
556 : #endif
557 : #if defined(AO_HAVE_short_fetch_compare_and_swap) \
558 : && defined(AO_HAVE_nop_full) \
559 : && !defined(AO_HAVE_short_fetch_compare_and_swap_release)
560 : # define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
561 : (AO_nop_full(), \
562 : AO_short_fetch_compare_and_swap(addr, old_val, new_val))
563 : # define AO_HAVE_short_fetch_compare_and_swap_release
564 : #endif
565 : #if defined(AO_HAVE_short_fetch_compare_and_swap_full)
566 : # if !defined(AO_HAVE_short_fetch_compare_and_swap_release)
567 : # define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
568 : AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
569 : # define AO_HAVE_short_fetch_compare_and_swap_release
570 : # endif
571 : # if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
572 : # define AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
573 : AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
574 : # define AO_HAVE_short_fetch_compare_and_swap_acquire
575 : # endif
576 : # if !defined(AO_HAVE_short_fetch_compare_and_swap_write)
577 : # define AO_short_fetch_compare_and_swap_write(addr, old_val, new_val) \
578 : AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
579 : # define AO_HAVE_short_fetch_compare_and_swap_write
580 : # endif
581 : # if !defined(AO_HAVE_short_fetch_compare_and_swap_read)
582 : # define AO_short_fetch_compare_and_swap_read(addr, old_val, new_val) \
583 : AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
584 : # define AO_HAVE_short_fetch_compare_and_swap_read
585 : # endif
586 : #endif /* AO_HAVE_short_fetch_compare_and_swap_full */
587 :
588 : #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
589 : && defined(AO_HAVE_short_fetch_compare_and_swap_release)
590 : # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
591 : AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
592 : # define AO_HAVE_short_fetch_compare_and_swap
593 : #endif
594 : #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
595 : && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
596 : # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
597 : AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
598 : # define AO_HAVE_short_fetch_compare_and_swap
599 : #endif
600 : #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
601 : && defined(AO_HAVE_short_fetch_compare_and_swap_write)
602 : # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
603 : AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
604 : # define AO_HAVE_short_fetch_compare_and_swap
605 : #endif
606 : #if !defined(AO_HAVE_short_fetch_compare_and_swap) \
607 : && defined(AO_HAVE_short_fetch_compare_and_swap_read)
608 : # define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
609 : AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
610 : # define AO_HAVE_short_fetch_compare_and_swap
611 : #endif
612 :
613 : #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
614 : && defined(AO_HAVE_nop_full) \
615 : && !defined(AO_HAVE_short_fetch_compare_and_swap_full)
616 : # define AO_short_fetch_compare_and_swap_full(addr, old_val, new_val) \
617 : (AO_nop_full(), \
618 : AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val))
619 : # define AO_HAVE_short_fetch_compare_and_swap_full
620 : #endif
621 :
622 : #if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
623 : && defined(AO_HAVE_short_fetch_compare_and_swap_write)
624 : # define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
625 : AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
626 : # define AO_HAVE_short_fetch_compare_and_swap_release_write
627 : #endif
628 : #if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
629 : && defined(AO_HAVE_short_fetch_compare_and_swap_release)
630 : # define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
631 : AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
632 : # define AO_HAVE_short_fetch_compare_and_swap_release_write
633 : #endif
634 : #if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
635 : && defined(AO_HAVE_short_fetch_compare_and_swap_read)
636 : # define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
637 : AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
638 : # define AO_HAVE_short_fetch_compare_and_swap_acquire_read
639 : #endif
640 : #if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
641 : && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
642 : # define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
643 : AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
644 : # define AO_HAVE_short_fetch_compare_and_swap_acquire_read
645 : #endif
646 :
647 : #ifdef AO_NO_DD_ORDERING
648 : # if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read)
649 : # define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
650 : AO_short_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
651 : # define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
652 : # endif
653 : #else
654 : # if defined(AO_HAVE_short_fetch_compare_and_swap)
655 : # define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
656 : AO_short_fetch_compare_and_swap(addr, old_val, new_val)
657 : # define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
658 : # endif
659 : #endif /* !AO_NO_DD_ORDERING */
660 :
661 : /* short_compare_and_swap */
662 : #if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
663 : && !defined(AO_HAVE_short_compare_and_swap_acquire)
664 : AO_INLINE int
665 : AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old,
666 : unsigned/**/short new_val)
667 : {
668 : int result = AO_short_compare_and_swap(addr, old, new_val);
669 : AO_nop_full();
670 : return result;
671 : }
672 : # define AO_HAVE_short_compare_and_swap_acquire
673 : #endif
674 : #if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
675 : && !defined(AO_HAVE_short_compare_and_swap_release)
676 : # define AO_short_compare_and_swap_release(addr, old, new_val) \
677 : (AO_nop_full(), AO_short_compare_and_swap(addr, old, new_val))
678 : # define AO_HAVE_short_compare_and_swap_release
679 : #endif
680 : #if defined(AO_HAVE_short_compare_and_swap_full)
681 : # if !defined(AO_HAVE_short_compare_and_swap_release)
682 : # define AO_short_compare_and_swap_release(addr, old, new_val) \
683 : AO_short_compare_and_swap_full(addr, old, new_val)
684 : # define AO_HAVE_short_compare_and_swap_release
685 : # endif
686 : # if !defined(AO_HAVE_short_compare_and_swap_acquire)
687 : # define AO_short_compare_and_swap_acquire(addr, old, new_val) \
688 : AO_short_compare_and_swap_full(addr, old, new_val)
689 : # define AO_HAVE_short_compare_and_swap_acquire
690 : # endif
691 : # if !defined(AO_HAVE_short_compare_and_swap_write)
692 : # define AO_short_compare_and_swap_write(addr, old, new_val) \
693 : AO_short_compare_and_swap_full(addr, old, new_val)
694 : # define AO_HAVE_short_compare_and_swap_write
695 : # endif
696 : # if !defined(AO_HAVE_short_compare_and_swap_read)
697 : # define AO_short_compare_and_swap_read(addr, old, new_val) \
698 : AO_short_compare_and_swap_full(addr, old, new_val)
699 : # define AO_HAVE_short_compare_and_swap_read
700 : # endif
701 : #endif /* AO_HAVE_short_compare_and_swap_full */
702 :
703 : #if !defined(AO_HAVE_short_compare_and_swap) \
704 : && defined(AO_HAVE_short_compare_and_swap_release)
705 : # define AO_short_compare_and_swap(addr, old, new_val) \
706 : AO_short_compare_and_swap_release(addr, old, new_val)
707 : # define AO_HAVE_short_compare_and_swap
708 : #endif
709 : #if !defined(AO_HAVE_short_compare_and_swap) \
710 : && defined(AO_HAVE_short_compare_and_swap_acquire)
711 : # define AO_short_compare_and_swap(addr, old, new_val) \
712 : AO_short_compare_and_swap_acquire(addr, old, new_val)
713 : # define AO_HAVE_short_compare_and_swap
714 : #endif
715 : #if !defined(AO_HAVE_short_compare_and_swap) \
716 : && defined(AO_HAVE_short_compare_and_swap_write)
717 : # define AO_short_compare_and_swap(addr, old, new_val) \
718 : AO_short_compare_and_swap_write(addr, old, new_val)
719 : # define AO_HAVE_short_compare_and_swap
720 : #endif
721 : #if !defined(AO_HAVE_short_compare_and_swap) \
722 : && defined(AO_HAVE_short_compare_and_swap_read)
723 : # define AO_short_compare_and_swap(addr, old, new_val) \
724 : AO_short_compare_and_swap_read(addr, old, new_val)
725 : # define AO_HAVE_short_compare_and_swap
726 : #endif
727 :
728 : #if defined(AO_HAVE_short_compare_and_swap_acquire) \
729 : && defined(AO_HAVE_nop_full) \
730 : && !defined(AO_HAVE_short_compare_and_swap_full)
731 : # define AO_short_compare_and_swap_full(addr, old, new_val) \
732 : (AO_nop_full(), \
733 : AO_short_compare_and_swap_acquire(addr, old, new_val))
734 : # define AO_HAVE_short_compare_and_swap_full
735 : #endif
736 :
737 : #if !defined(AO_HAVE_short_compare_and_swap_release_write) \
738 : && defined(AO_HAVE_short_compare_and_swap_write)
739 : # define AO_short_compare_and_swap_release_write(addr, old, new_val) \
740 : AO_short_compare_and_swap_write(addr, old, new_val)
741 : # define AO_HAVE_short_compare_and_swap_release_write
742 : #endif
743 : #if !defined(AO_HAVE_short_compare_and_swap_release_write) \
744 : && defined(AO_HAVE_short_compare_and_swap_release)
745 : # define AO_short_compare_and_swap_release_write(addr, old, new_val) \
746 : AO_short_compare_and_swap_release(addr, old, new_val)
747 : # define AO_HAVE_short_compare_and_swap_release_write
748 : #endif
749 : #if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
750 : && defined(AO_HAVE_short_compare_and_swap_read)
751 : # define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
752 : AO_short_compare_and_swap_read(addr, old, new_val)
753 : # define AO_HAVE_short_compare_and_swap_acquire_read
754 : #endif
755 : #if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
756 : && defined(AO_HAVE_short_compare_and_swap_acquire)
757 : # define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
758 : AO_short_compare_and_swap_acquire(addr, old, new_val)
759 : # define AO_HAVE_short_compare_and_swap_acquire_read
760 : #endif
761 :
762 : #ifdef AO_NO_DD_ORDERING
763 : # if defined(AO_HAVE_short_compare_and_swap_acquire_read)
764 : # define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
765 : AO_short_compare_and_swap_acquire_read(addr, old, new_val)
766 : # define AO_HAVE_short_compare_and_swap_dd_acquire_read
767 : # endif
768 : #else
769 : # if defined(AO_HAVE_short_compare_and_swap)
770 : # define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
771 : AO_short_compare_and_swap(addr, old, new_val)
772 : # define AO_HAVE_short_compare_and_swap_dd_acquire_read
773 : # endif
774 : #endif /* !AO_NO_DD_ORDERING */
775 :
776 : /* short_load */
777 : #if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire)
778 : # define AO_short_load_acquire(addr) AO_short_load_full(addr)
779 : # define AO_HAVE_short_load_acquire
780 : #endif
781 :
782 : #if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load)
783 : # define AO_short_load(addr) AO_short_load_acquire(addr)
784 : # define AO_HAVE_short_load
785 : #endif
786 :
787 : #if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read)
788 : # define AO_short_load_read(addr) AO_short_load_full(addr)
789 : # define AO_HAVE_short_load_read
790 : #endif
791 :
792 : #if !defined(AO_HAVE_short_load_acquire_read) \
793 : && defined(AO_HAVE_short_load_acquire)
794 : # define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr)
795 : # define AO_HAVE_short_load_acquire_read
796 : #endif
797 :
798 : #if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \
799 : && !defined(AO_HAVE_short_load_acquire)
800 : AO_INLINE unsigned/**/short
801 : AO_short_load_acquire(const volatile unsigned/**/short *addr)
802 : {
803 : unsigned/**/short result = AO_short_load(addr);
804 :
805 : /* Acquire barrier would be useless, since the load could be delayed */
806 : /* beyond it. */
807 : AO_nop_full();
808 : return result;
809 : }
810 : # define AO_HAVE_short_load_acquire
811 : #endif
812 :
813 : #if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \
814 : && !defined(AO_HAVE_short_load_read)
815 : AO_INLINE unsigned/**/short
816 : AO_short_load_read(const volatile unsigned/**/short *addr)
817 : {
818 : unsigned/**/short result = AO_short_load(addr);
819 :
820 : AO_nop_read();
821 : return result;
822 : }
823 : # define AO_HAVE_short_load_read
824 : #endif
825 :
826 : #if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \
827 : && !defined(AO_HAVE_short_load_full)
828 : # define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr))
829 : # define AO_HAVE_short_load_full
830 : #endif
831 :
832 : #if defined(AO_HAVE_short_compare_and_swap_read) \
833 : && !defined(AO_HAVE_short_load_read)
834 : # define AO_short_CAS_BASED_LOAD_READ
835 : AO_INLINE unsigned/**/short
836 : AO_short_load_read(const volatile unsigned/**/short *addr)
837 : {
838 : unsigned/**/short result;
839 :
840 : do {
841 : result = *(const unsigned/**/short *)addr;
842 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_read(
843 : (volatile unsigned/**/short *)addr,
844 : result, result)));
845 : return result;
846 : }
847 : # define AO_HAVE_short_load_read
848 : #endif
849 :
850 : #if !defined(AO_HAVE_short_load_acquire_read) \
851 : && defined(AO_HAVE_short_load_read)
852 : # define AO_short_load_acquire_read(addr) AO_short_load_read(addr)
853 : # define AO_HAVE_short_load_acquire_read
854 : #endif
855 :
856 : #if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) \
857 : && (!defined(AO_short_CAS_BASED_LOAD_READ) \
858 : || !defined(AO_HAVE_short_compare_and_swap))
859 : # define AO_short_load(addr) AO_short_load_acquire_read(addr)
860 : # define AO_HAVE_short_load
861 : #endif
862 :
863 : #if defined(AO_HAVE_short_compare_and_swap_full) \
864 : && !defined(AO_HAVE_short_load_full)
865 : AO_INLINE unsigned/**/short
866 : AO_short_load_full(const volatile unsigned/**/short *addr)
867 : {
868 : unsigned/**/short result;
869 :
870 : do {
871 : result = *(const unsigned/**/short *)addr;
872 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(
873 : (volatile unsigned/**/short *)addr,
874 : result, result)));
875 : return result;
876 : }
877 : # define AO_HAVE_short_load_full
878 : #endif
879 :
880 : #if defined(AO_HAVE_short_compare_and_swap_acquire) \
881 : && !defined(AO_HAVE_short_load_acquire)
882 : AO_INLINE unsigned/**/short
883 : AO_short_load_acquire(const volatile unsigned/**/short *addr)
884 : {
885 : unsigned/**/short result;
886 :
887 : do {
888 : result = *(const unsigned/**/short *)addr;
889 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(
890 : (volatile unsigned/**/short *)addr,
891 : result, result)));
892 : return result;
893 : }
894 : # define AO_HAVE_short_load_acquire
895 : #endif
896 :
897 : #if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_load)
898 : AO_INLINE unsigned/**/short
899 : AO_short_load(const volatile unsigned/**/short *addr)
900 : {
901 : unsigned/**/short result;
902 :
903 : do {
904 : result = *(const unsigned/**/short *)addr;
905 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(
906 : (volatile unsigned/**/short *)addr,
907 : result, result)));
908 : return result;
909 : }
910 : # define AO_HAVE_short_load
911 : #endif
912 :
913 : #ifdef AO_NO_DD_ORDERING
914 : # if defined(AO_HAVE_short_load_acquire_read)
915 : # define AO_short_load_dd_acquire_read(addr) \
916 : AO_short_load_acquire_read(addr)
917 : # define AO_HAVE_short_load_dd_acquire_read
918 : # endif
919 : #else
920 : # if defined(AO_HAVE_short_load)
921 : # define AO_short_load_dd_acquire_read(addr) AO_short_load(addr)
922 : # define AO_HAVE_short_load_dd_acquire_read
923 : # endif
924 : #endif /* !AO_NO_DD_ORDERING */
925 :
926 : /* short_store */
927 : #if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release)
928 : # define AO_short_store_release(addr, val) AO_short_store_full(addr, val)
929 : # define AO_HAVE_short_store_release
930 : #endif
931 :
932 : #if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store)
933 : # define AO_short_store(addr, val) AO_short_store_release(addr, val)
934 : # define AO_HAVE_short_store
935 : #endif
936 :
937 : #if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write)
938 : # define AO_short_store_write(addr, val) AO_short_store_full(addr, val)
939 : # define AO_HAVE_short_store_write
940 : #endif
941 :
942 : #if defined(AO_HAVE_short_store_release) \
943 : && !defined(AO_HAVE_short_store_release_write)
944 : # define AO_short_store_release_write(addr, val) \
945 : AO_short_store_release(addr, val)
946 : # define AO_HAVE_short_store_release_write
947 : #endif
948 :
949 : #if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store)
950 : # define AO_short_store(addr, val) AO_short_store_write(addr, val)
951 : # define AO_HAVE_short_store
952 : #endif
953 :
954 : #if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \
955 : && !defined(AO_HAVE_short_store_release)
956 : # define AO_short_store_release(addr, val) \
957 : (AO_nop_full(), AO_short_store(addr, val))
958 : # define AO_HAVE_short_store_release
959 : #endif
960 :
961 : #if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_write) \
962 : && !defined(AO_HAVE_short_store_write)
963 : # define AO_short_store_write(addr, val) \
964 : (AO_nop_write(), AO_short_store(addr, val))
965 : # define AO_HAVE_short_store_write
966 : #endif
967 :
968 : #if defined(AO_HAVE_short_compare_and_swap_write) \
969 : && !defined(AO_HAVE_short_store_write)
970 : AO_INLINE void
971 : AO_short_store_write(volatile unsigned/**/short *addr, unsigned/**/short new_val)
972 : {
973 : unsigned/**/short old_val;
974 :
975 : do {
976 : old_val = *(unsigned/**/short *)addr;
977 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_write(addr, old_val,
978 : new_val)));
979 : }
980 : # define AO_HAVE_short_store_write
981 : #endif
982 :
983 : #if defined(AO_HAVE_short_store_write) \
984 : && !defined(AO_HAVE_short_store_release_write)
985 : # define AO_short_store_release_write(addr, val) \
986 : AO_short_store_write(addr, val)
987 : # define AO_HAVE_short_store_release_write
988 : #endif
989 :
990 : #if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \
991 : && !defined(AO_HAVE_short_store_full)
992 : # define AO_short_store_full(addr, val) \
993 : (AO_short_store_release(addr, val), \
994 : AO_nop_full())
995 : # define AO_HAVE_short_store_full
996 : #endif
997 :
998 : #if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_store)
999 : AO_INLINE void
1000 : AO_short_store(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1001 : {
1002 : unsigned/**/short old_val;
1003 :
1004 : do {
1005 : old_val = *(unsigned/**/short *)addr;
1006 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr,
1007 : old_val, new_val)));
1008 : }
1009 : # define AO_HAVE_short_store
1010 : #endif
1011 :
1012 : #if defined(AO_HAVE_short_compare_and_swap_release) \
1013 : && !defined(AO_HAVE_short_store_release)
1014 : AO_INLINE void
1015 : AO_short_store_release(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1016 : {
1017 : unsigned/**/short old_val;
1018 :
1019 : do {
1020 : old_val = *(unsigned/**/short *)addr;
1021 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old_val,
1022 : new_val)));
1023 : }
1024 : # define AO_HAVE_short_store_release
1025 : #endif
1026 :
1027 : #if defined(AO_HAVE_short_compare_and_swap_full) \
1028 : && !defined(AO_HAVE_short_store_full)
1029 : AO_INLINE void
1030 : AO_short_store_full(volatile unsigned/**/short *addr, unsigned/**/short new_val)
1031 : {
1032 : unsigned/**/short old_val;
1033 :
1034 : do {
1035 : old_val = *(unsigned/**/short *)addr;
1036 : } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old_val,
1037 : new_val)));
1038 : }
1039 : # define AO_HAVE_short_store_full
1040 : #endif
1041 : /*
1042 : * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1043 : *
1044 : * Permission is hereby granted, free of charge, to any person obtaining a copy
1045 : * of this software and associated documentation files (the "Software"), to deal
1046 : * in the Software without restriction, including without limitation the rights
1047 : * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1048 : * copies of the Software, and to permit persons to whom the Software is
1049 : * furnished to do so, subject to the following conditions:
1050 : *
1051 : * The above copyright notice and this permission notice shall be included in
1052 : * all copies or substantial portions of the Software.
1053 : *
1054 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1055 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1056 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1057 : * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1058 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1059 : * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1060 : * SOFTWARE.
1061 : */
1062 :
1063 : /* int_fetch_compare_and_swap */
1064 : #if defined(AO_HAVE_int_fetch_compare_and_swap) \
1065 : && defined(AO_HAVE_nop_full) \
1066 : && !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1067 : AO_INLINE unsigned
1068 : AO_int_fetch_compare_and_swap_acquire(volatile unsigned *addr,
1069 : unsigned old_val, unsigned new_val)
1070 : {
1071 : unsigned result = AO_int_fetch_compare_and_swap(addr, old_val, new_val);
1072 : AO_nop_full();
1073 : return result;
1074 : }
1075 : # define AO_HAVE_int_fetch_compare_and_swap_acquire
1076 : #endif
1077 : #if defined(AO_HAVE_int_fetch_compare_and_swap) \
1078 : && defined(AO_HAVE_nop_full) \
1079 : && !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1080 : # define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1081 : (AO_nop_full(), \
1082 : AO_int_fetch_compare_and_swap(addr, old_val, new_val))
1083 : # define AO_HAVE_int_fetch_compare_and_swap_release
1084 : #endif
1085 : #if defined(AO_HAVE_int_fetch_compare_and_swap_full)
1086 : # if !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1087 : # define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1088 : AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1089 : # define AO_HAVE_int_fetch_compare_and_swap_release
1090 : # endif
1091 : # if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1092 : # define AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1093 : AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1094 : # define AO_HAVE_int_fetch_compare_and_swap_acquire
1095 : # endif
1096 : # if !defined(AO_HAVE_int_fetch_compare_and_swap_write)
1097 : # define AO_int_fetch_compare_and_swap_write(addr, old_val, new_val) \
1098 : AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1099 : # define AO_HAVE_int_fetch_compare_and_swap_write
1100 : # endif
1101 : # if !defined(AO_HAVE_int_fetch_compare_and_swap_read)
1102 : # define AO_int_fetch_compare_and_swap_read(addr, old_val, new_val) \
1103 : AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1104 : # define AO_HAVE_int_fetch_compare_and_swap_read
1105 : # endif
1106 : #endif /* AO_HAVE_int_fetch_compare_and_swap_full */
1107 :
1108 : #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1109 : && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1110 : # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1111 : AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1112 : # define AO_HAVE_int_fetch_compare_and_swap
1113 : #endif
1114 : #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1115 : && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1116 : # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1117 : AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1118 : # define AO_HAVE_int_fetch_compare_and_swap
1119 : #endif
1120 : #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1121 : && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1122 : # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1123 : AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1124 : # define AO_HAVE_int_fetch_compare_and_swap
1125 : #endif
1126 : #if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1127 : && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1128 : # define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1129 : AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1130 : # define AO_HAVE_int_fetch_compare_and_swap
1131 : #endif
1132 :
1133 : #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
1134 : && defined(AO_HAVE_nop_full) \
1135 : && !defined(AO_HAVE_int_fetch_compare_and_swap_full)
1136 : # define AO_int_fetch_compare_and_swap_full(addr, old_val, new_val) \
1137 : (AO_nop_full(), \
1138 : AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1139 : # define AO_HAVE_int_fetch_compare_and_swap_full
1140 : #endif
1141 :
1142 : #if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1143 : && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1144 : # define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1145 : AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1146 : # define AO_HAVE_int_fetch_compare_and_swap_release_write
1147 : #endif
1148 : #if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1149 : && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1150 : # define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1151 : AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1152 : # define AO_HAVE_int_fetch_compare_and_swap_release_write
1153 : #endif
1154 : #if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1155 : && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1156 : # define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1157 : AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1158 : # define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1159 : #endif
1160 : #if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1161 : && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1162 : # define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1163 : AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1164 : # define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1165 : #endif
1166 :
1167 : #ifdef AO_NO_DD_ORDERING
1168 : # if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read)
1169 : # define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1170 : AO_int_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1171 : # define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1172 : # endif
1173 : #else
1174 : # if defined(AO_HAVE_int_fetch_compare_and_swap)
1175 : # define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1176 : AO_int_fetch_compare_and_swap(addr, old_val, new_val)
1177 : # define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1178 : # endif
1179 : #endif /* !AO_NO_DD_ORDERING */
1180 :
1181 : /* int_compare_and_swap */
1182 : #if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1183 : && !defined(AO_HAVE_int_compare_and_swap_acquire)
1184 : AO_INLINE int
1185 : AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old,
1186 : unsigned new_val)
1187 : {
1188 : int result = AO_int_compare_and_swap(addr, old, new_val);
1189 : AO_nop_full();
1190 : return result;
1191 : }
1192 : # define AO_HAVE_int_compare_and_swap_acquire
1193 : #endif
1194 : #if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1195 : && !defined(AO_HAVE_int_compare_and_swap_release)
1196 : # define AO_int_compare_and_swap_release(addr, old, new_val) \
1197 : (AO_nop_full(), AO_int_compare_and_swap(addr, old, new_val))
1198 : # define AO_HAVE_int_compare_and_swap_release
1199 : #endif
1200 : #if defined(AO_HAVE_int_compare_and_swap_full)
1201 : # if !defined(AO_HAVE_int_compare_and_swap_release)
1202 : # define AO_int_compare_and_swap_release(addr, old, new_val) \
1203 : AO_int_compare_and_swap_full(addr, old, new_val)
1204 : # define AO_HAVE_int_compare_and_swap_release
1205 : # endif
1206 : # if !defined(AO_HAVE_int_compare_and_swap_acquire)
1207 : # define AO_int_compare_and_swap_acquire(addr, old, new_val) \
1208 : AO_int_compare_and_swap_full(addr, old, new_val)
1209 : # define AO_HAVE_int_compare_and_swap_acquire
1210 : # endif
1211 : # if !defined(AO_HAVE_int_compare_and_swap_write)
1212 : # define AO_int_compare_and_swap_write(addr, old, new_val) \
1213 : AO_int_compare_and_swap_full(addr, old, new_val)
1214 : # define AO_HAVE_int_compare_and_swap_write
1215 : # endif
1216 : # if !defined(AO_HAVE_int_compare_and_swap_read)
1217 : # define AO_int_compare_and_swap_read(addr, old, new_val) \
1218 : AO_int_compare_and_swap_full(addr, old, new_val)
1219 : # define AO_HAVE_int_compare_and_swap_read
1220 : # endif
1221 : #endif /* AO_HAVE_int_compare_and_swap_full */
1222 :
1223 : #if !defined(AO_HAVE_int_compare_and_swap) \
1224 : && defined(AO_HAVE_int_compare_and_swap_release)
1225 : # define AO_int_compare_and_swap(addr, old, new_val) \
1226 : AO_int_compare_and_swap_release(addr, old, new_val)
1227 : # define AO_HAVE_int_compare_and_swap
1228 : #endif
1229 : #if !defined(AO_HAVE_int_compare_and_swap) \
1230 : && defined(AO_HAVE_int_compare_and_swap_acquire)
1231 : # define AO_int_compare_and_swap(addr, old, new_val) \
1232 : AO_int_compare_and_swap_acquire(addr, old, new_val)
1233 : # define AO_HAVE_int_compare_and_swap
1234 : #endif
1235 : #if !defined(AO_HAVE_int_compare_and_swap) \
1236 : && defined(AO_HAVE_int_compare_and_swap_write)
1237 : # define AO_int_compare_and_swap(addr, old, new_val) \
1238 : AO_int_compare_and_swap_write(addr, old, new_val)
1239 : # define AO_HAVE_int_compare_and_swap
1240 : #endif
1241 : #if !defined(AO_HAVE_int_compare_and_swap) \
1242 : && defined(AO_HAVE_int_compare_and_swap_read)
1243 : # define AO_int_compare_and_swap(addr, old, new_val) \
1244 : AO_int_compare_and_swap_read(addr, old, new_val)
1245 : # define AO_HAVE_int_compare_and_swap
1246 : #endif
1247 :
1248 : #if defined(AO_HAVE_int_compare_and_swap_acquire) \
1249 : && defined(AO_HAVE_nop_full) \
1250 : && !defined(AO_HAVE_int_compare_and_swap_full)
1251 : # define AO_int_compare_and_swap_full(addr, old, new_val) \
1252 : (AO_nop_full(), \
1253 : AO_int_compare_and_swap_acquire(addr, old, new_val))
1254 : # define AO_HAVE_int_compare_and_swap_full
1255 : #endif
1256 :
1257 : #if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1258 : && defined(AO_HAVE_int_compare_and_swap_write)
1259 : # define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1260 : AO_int_compare_and_swap_write(addr, old, new_val)
1261 : # define AO_HAVE_int_compare_and_swap_release_write
1262 : #endif
1263 : #if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1264 : && defined(AO_HAVE_int_compare_and_swap_release)
1265 : # define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1266 : AO_int_compare_and_swap_release(addr, old, new_val)
1267 : # define AO_HAVE_int_compare_and_swap_release_write
1268 : #endif
1269 : #if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1270 : && defined(AO_HAVE_int_compare_and_swap_read)
1271 : # define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1272 : AO_int_compare_and_swap_read(addr, old, new_val)
1273 : # define AO_HAVE_int_compare_and_swap_acquire_read
1274 : #endif
1275 : #if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1276 : && defined(AO_HAVE_int_compare_and_swap_acquire)
1277 : # define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1278 : AO_int_compare_and_swap_acquire(addr, old, new_val)
1279 : # define AO_HAVE_int_compare_and_swap_acquire_read
1280 : #endif
1281 :
1282 : #ifdef AO_NO_DD_ORDERING
1283 : # if defined(AO_HAVE_int_compare_and_swap_acquire_read)
1284 : # define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1285 : AO_int_compare_and_swap_acquire_read(addr, old, new_val)
1286 : # define AO_HAVE_int_compare_and_swap_dd_acquire_read
1287 : # endif
1288 : #else
1289 : # if defined(AO_HAVE_int_compare_and_swap)
1290 : # define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1291 : AO_int_compare_and_swap(addr, old, new_val)
1292 : # define AO_HAVE_int_compare_and_swap_dd_acquire_read
1293 : # endif
1294 : #endif /* !AO_NO_DD_ORDERING */
1295 :
1296 : /* int_load */
1297 : #if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
1298 : # define AO_int_load_acquire(addr) AO_int_load_full(addr)
1299 : # define AO_HAVE_int_load_acquire
1300 : #endif
1301 :
1302 : #if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
1303 : # define AO_int_load(addr) AO_int_load_acquire(addr)
1304 : # define AO_HAVE_int_load
1305 : #endif
1306 :
1307 : #if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
1308 : # define AO_int_load_read(addr) AO_int_load_full(addr)
1309 : # define AO_HAVE_int_load_read
1310 : #endif
1311 :
1312 : #if !defined(AO_HAVE_int_load_acquire_read) \
1313 : && defined(AO_HAVE_int_load_acquire)
1314 : # define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
1315 : # define AO_HAVE_int_load_acquire_read
1316 : #endif
1317 :
1318 : #if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
1319 : && !defined(AO_HAVE_int_load_acquire)
1320 : AO_INLINE unsigned
1321 : AO_int_load_acquire(const volatile unsigned *addr)
1322 : {
1323 : unsigned result = AO_int_load(addr);
1324 :
1325 : /* Acquire barrier would be useless, since the load could be delayed */
1326 : /* beyond it. */
1327 : AO_nop_full();
1328 : return result;
1329 : }
1330 : # define AO_HAVE_int_load_acquire
1331 : #endif
1332 :
1333 : #if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
1334 : && !defined(AO_HAVE_int_load_read)
1335 : AO_INLINE unsigned
1336 : AO_int_load_read(const volatile unsigned *addr)
1337 : {
1338 : unsigned result = AO_int_load(addr);
1339 :
1340 : AO_nop_read();
1341 : return result;
1342 : }
1343 : # define AO_HAVE_int_load_read
1344 : #endif
1345 :
1346 : #if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \
1347 : && !defined(AO_HAVE_int_load_full)
1348 : # define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr))
1349 : # define AO_HAVE_int_load_full
1350 : #endif
1351 :
1352 : #if defined(AO_HAVE_int_compare_and_swap_read) \
1353 : && !defined(AO_HAVE_int_load_read)
1354 : # define AO_int_CAS_BASED_LOAD_READ
1355 : AO_INLINE unsigned
1356 : AO_int_load_read(const volatile unsigned *addr)
1357 : {
1358 : unsigned result;
1359 :
1360 : do {
1361 : result = *(const unsigned *)addr;
1362 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_read(
1363 : (volatile unsigned *)addr,
1364 : result, result)));
1365 : return result;
1366 : }
1367 : # define AO_HAVE_int_load_read
1368 : #endif
1369 :
1370 : #if !defined(AO_HAVE_int_load_acquire_read) \
1371 : && defined(AO_HAVE_int_load_read)
1372 : # define AO_int_load_acquire_read(addr) AO_int_load_read(addr)
1373 : # define AO_HAVE_int_load_acquire_read
1374 : #endif
1375 :
1376 : #if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) \
1377 : && (!defined(AO_int_CAS_BASED_LOAD_READ) \
1378 : || !defined(AO_HAVE_int_compare_and_swap))
1379 : # define AO_int_load(addr) AO_int_load_acquire_read(addr)
1380 : # define AO_HAVE_int_load
1381 : #endif
1382 :
1383 : #if defined(AO_HAVE_int_compare_and_swap_full) \
1384 : && !defined(AO_HAVE_int_load_full)
1385 : AO_INLINE unsigned
1386 : AO_int_load_full(const volatile unsigned *addr)
1387 : {
1388 : unsigned result;
1389 :
1390 : do {
1391 : result = *(const unsigned *)addr;
1392 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(
1393 : (volatile unsigned *)addr,
1394 : result, result)));
1395 : return result;
1396 : }
1397 : # define AO_HAVE_int_load_full
1398 : #endif
1399 :
1400 : #if defined(AO_HAVE_int_compare_and_swap_acquire) \
1401 : && !defined(AO_HAVE_int_load_acquire)
1402 : AO_INLINE unsigned
1403 : AO_int_load_acquire(const volatile unsigned *addr)
1404 : {
1405 : unsigned result;
1406 :
1407 : do {
1408 : result = *(const unsigned *)addr;
1409 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(
1410 : (volatile unsigned *)addr,
1411 : result, result)));
1412 : return result;
1413 : }
1414 : # define AO_HAVE_int_load_acquire
1415 : #endif
1416 :
1417 : #if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_load)
1418 : AO_INLINE unsigned
1419 : AO_int_load(const volatile unsigned *addr)
1420 : {
1421 : unsigned result;
1422 :
1423 : do {
1424 : result = *(const unsigned *)addr;
1425 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(
1426 : (volatile unsigned *)addr,
1427 : result, result)));
1428 : return result;
1429 : }
1430 : # define AO_HAVE_int_load
1431 : #endif
1432 :
1433 : #ifdef AO_NO_DD_ORDERING
1434 : # if defined(AO_HAVE_int_load_acquire_read)
1435 : # define AO_int_load_dd_acquire_read(addr) \
1436 : AO_int_load_acquire_read(addr)
1437 : # define AO_HAVE_int_load_dd_acquire_read
1438 : # endif
1439 : #else
1440 : # if defined(AO_HAVE_int_load)
1441 : # define AO_int_load_dd_acquire_read(addr) AO_int_load(addr)
1442 : # define AO_HAVE_int_load_dd_acquire_read
1443 : # endif
1444 : #endif /* !AO_NO_DD_ORDERING */
1445 :
1446 : /* int_store */
1447 : #if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release)
1448 : # define AO_int_store_release(addr, val) AO_int_store_full(addr, val)
1449 : # define AO_HAVE_int_store_release
1450 : #endif
1451 :
1452 : #if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store)
1453 : # define AO_int_store(addr, val) AO_int_store_release(addr, val)
1454 : # define AO_HAVE_int_store
1455 : #endif
1456 :
1457 : #if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write)
1458 : # define AO_int_store_write(addr, val) AO_int_store_full(addr, val)
1459 : # define AO_HAVE_int_store_write
1460 : #endif
1461 :
1462 : #if defined(AO_HAVE_int_store_release) \
1463 : && !defined(AO_HAVE_int_store_release_write)
1464 : # define AO_int_store_release_write(addr, val) \
1465 : AO_int_store_release(addr, val)
1466 : # define AO_HAVE_int_store_release_write
1467 : #endif
1468 :
1469 : #if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store)
1470 : # define AO_int_store(addr, val) AO_int_store_write(addr, val)
1471 : # define AO_HAVE_int_store
1472 : #endif
1473 :
1474 : #if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \
1475 : && !defined(AO_HAVE_int_store_release)
1476 : # define AO_int_store_release(addr, val) \
1477 : (AO_nop_full(), AO_int_store(addr, val))
1478 : # define AO_HAVE_int_store_release
1479 : #endif
1480 :
1481 : #if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_write) \
1482 : && !defined(AO_HAVE_int_store_write)
1483 : # define AO_int_store_write(addr, val) \
1484 : (AO_nop_write(), AO_int_store(addr, val))
1485 : # define AO_HAVE_int_store_write
1486 : #endif
1487 :
1488 : #if defined(AO_HAVE_int_compare_and_swap_write) \
1489 : && !defined(AO_HAVE_int_store_write)
1490 : AO_INLINE void
1491 : AO_int_store_write(volatile unsigned *addr, unsigned new_val)
1492 : {
1493 : unsigned old_val;
1494 :
1495 : do {
1496 : old_val = *(unsigned *)addr;
1497 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_write(addr, old_val,
1498 : new_val)));
1499 : }
1500 : # define AO_HAVE_int_store_write
1501 : #endif
1502 :
1503 : #if defined(AO_HAVE_int_store_write) \
1504 : && !defined(AO_HAVE_int_store_release_write)
1505 : # define AO_int_store_release_write(addr, val) \
1506 : AO_int_store_write(addr, val)
1507 : # define AO_HAVE_int_store_release_write
1508 : #endif
1509 :
1510 : #if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \
1511 : && !defined(AO_HAVE_int_store_full)
1512 : # define AO_int_store_full(addr, val) \
1513 : (AO_int_store_release(addr, val), \
1514 : AO_nop_full())
1515 : # define AO_HAVE_int_store_full
1516 : #endif
1517 :
1518 : #if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_store)
1519 : AO_INLINE void
1520 : AO_int_store(volatile unsigned *addr, unsigned new_val)
1521 : {
1522 : unsigned old_val;
1523 :
1524 : do {
1525 : old_val = *(unsigned *)addr;
1526 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr,
1527 : old_val, new_val)));
1528 : }
1529 : # define AO_HAVE_int_store
1530 : #endif
1531 :
1532 : #if defined(AO_HAVE_int_compare_and_swap_release) \
1533 : && !defined(AO_HAVE_int_store_release)
1534 : AO_INLINE void
1535 : AO_int_store_release(volatile unsigned *addr, unsigned new_val)
1536 : {
1537 : unsigned old_val;
1538 :
1539 : do {
1540 : old_val = *(unsigned *)addr;
1541 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old_val,
1542 : new_val)));
1543 : }
1544 : # define AO_HAVE_int_store_release
1545 : #endif
1546 :
1547 : #if defined(AO_HAVE_int_compare_and_swap_full) \
1548 : && !defined(AO_HAVE_int_store_full)
1549 : AO_INLINE void
1550 : AO_int_store_full(volatile unsigned *addr, unsigned new_val)
1551 : {
1552 : unsigned old_val;
1553 :
1554 : do {
1555 : old_val = *(unsigned *)addr;
1556 : } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old_val,
1557 : new_val)));
1558 : }
1559 : # define AO_HAVE_int_store_full
1560 : #endif
1561 : /*
1562 : * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1563 : *
1564 : * Permission is hereby granted, free of charge, to any person obtaining a copy
1565 : * of this software and associated documentation files (the "Software"), to deal
1566 : * in the Software without restriction, including without limitation the rights
1567 : * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1568 : * copies of the Software, and to permit persons to whom the Software is
1569 : * furnished to do so, subject to the following conditions:
1570 : *
1571 : * The above copyright notice and this permission notice shall be included in
1572 : * all copies or substantial portions of the Software.
1573 : *
1574 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1575 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1576 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1577 : * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1578 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1579 : * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1580 : * SOFTWARE.
1581 : */
1582 :
1583 : /* fetch_compare_and_swap */
1584 : #if defined(AO_HAVE_fetch_compare_and_swap) \
1585 : && defined(AO_HAVE_nop_full) \
1586 : && !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1587 : AO_INLINE AO_t
1588 : AO_fetch_compare_and_swap_acquire(volatile AO_t *addr,
1589 : AO_t old_val, AO_t new_val)
1590 : {
1591 : AO_t result = AO_fetch_compare_and_swap(addr, old_val, new_val);
1592 : AO_nop_full();
1593 : return result;
1594 : }
1595 : # define AO_HAVE_fetch_compare_and_swap_acquire
1596 : #endif
1597 : #if defined(AO_HAVE_fetch_compare_and_swap) \
1598 : && defined(AO_HAVE_nop_full) \
1599 : && !defined(AO_HAVE_fetch_compare_and_swap_release)
1600 : # define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1601 : (AO_nop_full(), \
1602 : AO_fetch_compare_and_swap(addr, old_val, new_val))
1603 : # define AO_HAVE_fetch_compare_and_swap_release
1604 : #endif
1605 : #if defined(AO_HAVE_fetch_compare_and_swap_full)
1606 : # if !defined(AO_HAVE_fetch_compare_and_swap_release)
1607 : # define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1608 : AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1609 : # define AO_HAVE_fetch_compare_and_swap_release
1610 : # endif
1611 : # if !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1612 : # define AO_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1613 : AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1614 : # define AO_HAVE_fetch_compare_and_swap_acquire
1615 : # endif
1616 : # if !defined(AO_HAVE_fetch_compare_and_swap_write)
1617 : # define AO_fetch_compare_and_swap_write(addr, old_val, new_val) \
1618 : AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1619 : # define AO_HAVE_fetch_compare_and_swap_write
1620 : # endif
1621 : # if !defined(AO_HAVE_fetch_compare_and_swap_read)
1622 : # define AO_fetch_compare_and_swap_read(addr, old_val, new_val) \
1623 : AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1624 : # define AO_HAVE_fetch_compare_and_swap_read
1625 : # endif
1626 : #endif /* AO_HAVE_fetch_compare_and_swap_full */
1627 :
1628 : #if !defined(AO_HAVE_fetch_compare_and_swap) \
1629 : && defined(AO_HAVE_fetch_compare_and_swap_release)
1630 : # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1631 : AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1632 : # define AO_HAVE_fetch_compare_and_swap
1633 : #endif
1634 : #if !defined(AO_HAVE_fetch_compare_and_swap) \
1635 : && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1636 : # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1637 : AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1638 : # define AO_HAVE_fetch_compare_and_swap
1639 : #endif
1640 : #if !defined(AO_HAVE_fetch_compare_and_swap) \
1641 : && defined(AO_HAVE_fetch_compare_and_swap_write)
1642 : # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1643 : AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1644 : # define AO_HAVE_fetch_compare_and_swap
1645 : #endif
1646 : #if !defined(AO_HAVE_fetch_compare_and_swap) \
1647 : && defined(AO_HAVE_fetch_compare_and_swap_read)
1648 : # define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1649 : AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1650 : # define AO_HAVE_fetch_compare_and_swap
1651 : #endif
1652 :
1653 : #if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
1654 : && defined(AO_HAVE_nop_full) \
1655 : && !defined(AO_HAVE_fetch_compare_and_swap_full)
1656 : # define AO_fetch_compare_and_swap_full(addr, old_val, new_val) \
1657 : (AO_nop_full(), \
1658 : AO_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1659 : # define AO_HAVE_fetch_compare_and_swap_full
1660 : #endif
1661 :
1662 : #if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1663 : && defined(AO_HAVE_fetch_compare_and_swap_write)
1664 : # define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1665 : AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1666 : # define AO_HAVE_fetch_compare_and_swap_release_write
1667 : #endif
1668 : #if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1669 : && defined(AO_HAVE_fetch_compare_and_swap_release)
1670 : # define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1671 : AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1672 : # define AO_HAVE_fetch_compare_and_swap_release_write
1673 : #endif
1674 : #if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1675 : && defined(AO_HAVE_fetch_compare_and_swap_read)
1676 : # define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1677 : AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1678 : # define AO_HAVE_fetch_compare_and_swap_acquire_read
1679 : #endif
1680 : #if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1681 : && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1682 : # define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1683 : AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1684 : # define AO_HAVE_fetch_compare_and_swap_acquire_read
1685 : #endif
1686 :
1687 : #ifdef AO_NO_DD_ORDERING
1688 : # if defined(AO_HAVE_fetch_compare_and_swap_acquire_read)
1689 : # define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1690 : AO_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1691 : # define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1692 : # endif
1693 : #else
1694 : # if defined(AO_HAVE_fetch_compare_and_swap)
1695 : # define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1696 : AO_fetch_compare_and_swap(addr, old_val, new_val)
1697 : # define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1698 : # endif
1699 : #endif /* !AO_NO_DD_ORDERING */
1700 :
1701 : /* compare_and_swap */
1702 : #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1703 : && !defined(AO_HAVE_compare_and_swap_acquire)
1704 : AO_INLINE int
1705 : AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old,
1706 : AO_t new_val)
1707 : {
1708 : int result = AO_compare_and_swap(addr, old, new_val);
1709 : AO_nop_full();
1710 : return result;
1711 : }
1712 : # define AO_HAVE_compare_and_swap_acquire
1713 : #endif
1714 : #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1715 : && !defined(AO_HAVE_compare_and_swap_release)
1716 : # define AO_compare_and_swap_release(addr, old, new_val) \
1717 : (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
1718 : # define AO_HAVE_compare_and_swap_release
1719 : #endif
1720 : #if defined(AO_HAVE_compare_and_swap_full)
1721 : # if !defined(AO_HAVE_compare_and_swap_release)
1722 : # define AO_compare_and_swap_release(addr, old, new_val) \
1723 : AO_compare_and_swap_full(addr, old, new_val)
1724 : # define AO_HAVE_compare_and_swap_release
1725 : # endif
1726 : # if !defined(AO_HAVE_compare_and_swap_acquire)
1727 : # define AO_compare_and_swap_acquire(addr, old, new_val) \
1728 : AO_compare_and_swap_full(addr, old, new_val)
1729 : # define AO_HAVE_compare_and_swap_acquire
1730 : # endif
1731 : # if !defined(AO_HAVE_compare_and_swap_write)
1732 : # define AO_compare_and_swap_write(addr, old, new_val) \
1733 : AO_compare_and_swap_full(addr, old, new_val)
1734 : # define AO_HAVE_compare_and_swap_write
1735 : # endif
1736 : # if !defined(AO_HAVE_compare_and_swap_read)
1737 : # define AO_compare_and_swap_read(addr, old, new_val) \
1738 : AO_compare_and_swap_full(addr, old, new_val)
1739 : # define AO_HAVE_compare_and_swap_read
1740 : # endif
1741 : #endif /* AO_HAVE_compare_and_swap_full */
1742 :
1743 : #if !defined(AO_HAVE_compare_and_swap) \
1744 : && defined(AO_HAVE_compare_and_swap_release)
1745 : # define AO_compare_and_swap(addr, old, new_val) \
1746 : AO_compare_and_swap_release(addr, old, new_val)
1747 : # define AO_HAVE_compare_and_swap
1748 : #endif
1749 : #if !defined(AO_HAVE_compare_and_swap) \
1750 : && defined(AO_HAVE_compare_and_swap_acquire)
1751 : # define AO_compare_and_swap(addr, old, new_val) \
1752 : AO_compare_and_swap_acquire(addr, old, new_val)
1753 : # define AO_HAVE_compare_and_swap
1754 : #endif
1755 : #if !defined(AO_HAVE_compare_and_swap) \
1756 : && defined(AO_HAVE_compare_and_swap_write)
1757 : # define AO_compare_and_swap(addr, old, new_val) \
1758 : AO_compare_and_swap_write(addr, old, new_val)
1759 : # define AO_HAVE_compare_and_swap
1760 : #endif
1761 : #if !defined(AO_HAVE_compare_and_swap) \
1762 : && defined(AO_HAVE_compare_and_swap_read)
1763 : # define AO_compare_and_swap(addr, old, new_val) \
1764 : AO_compare_and_swap_read(addr, old, new_val)
1765 : # define AO_HAVE_compare_and_swap
1766 : #endif
1767 :
1768 : #if defined(AO_HAVE_compare_and_swap_acquire) \
1769 : && defined(AO_HAVE_nop_full) \
1770 : && !defined(AO_HAVE_compare_and_swap_full)
1771 : # define AO_compare_and_swap_full(addr, old, new_val) \
1772 : (AO_nop_full(), \
1773 : AO_compare_and_swap_acquire(addr, old, new_val))
1774 : # define AO_HAVE_compare_and_swap_full
1775 : #endif
1776 :
1777 : #if !defined(AO_HAVE_compare_and_swap_release_write) \
1778 : && defined(AO_HAVE_compare_and_swap_write)
1779 : # define AO_compare_and_swap_release_write(addr, old, new_val) \
1780 : AO_compare_and_swap_write(addr, old, new_val)
1781 : # define AO_HAVE_compare_and_swap_release_write
1782 : #endif
1783 : #if !defined(AO_HAVE_compare_and_swap_release_write) \
1784 : && defined(AO_HAVE_compare_and_swap_release)
1785 : # define AO_compare_and_swap_release_write(addr, old, new_val) \
1786 : AO_compare_and_swap_release(addr, old, new_val)
1787 : # define AO_HAVE_compare_and_swap_release_write
1788 : #endif
1789 : #if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1790 : && defined(AO_HAVE_compare_and_swap_read)
1791 : # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1792 : AO_compare_and_swap_read(addr, old, new_val)
1793 : # define AO_HAVE_compare_and_swap_acquire_read
1794 : #endif
1795 : #if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1796 : && defined(AO_HAVE_compare_and_swap_acquire)
1797 : # define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1798 : AO_compare_and_swap_acquire(addr, old, new_val)
1799 : # define AO_HAVE_compare_and_swap_acquire_read
1800 : #endif
1801 :
1802 : #ifdef AO_NO_DD_ORDERING
1803 : # if defined(AO_HAVE_compare_and_swap_acquire_read)
1804 : # define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1805 : AO_compare_and_swap_acquire_read(addr, old, new_val)
1806 : # define AO_HAVE_compare_and_swap_dd_acquire_read
1807 : # endif
1808 : #else
1809 : # if defined(AO_HAVE_compare_and_swap)
1810 : # define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1811 : AO_compare_and_swap(addr, old, new_val)
1812 : # define AO_HAVE_compare_and_swap_dd_acquire_read
1813 : # endif
1814 : #endif /* !AO_NO_DD_ORDERING */
1815 :
1816 : /* load */
1817 : #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
1818 : # define AO_load_acquire(addr) AO_load_full(addr)
1819 : # define AO_HAVE_load_acquire
1820 : #endif
1821 :
1822 : #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
1823 : # define AO_load(addr) AO_load_acquire(addr)
1824 : # define AO_HAVE_load
1825 : #endif
1826 :
1827 : #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
1828 : # define AO_load_read(addr) AO_load_full(addr)
1829 : # define AO_HAVE_load_read
1830 : #endif
1831 :
1832 : #if !defined(AO_HAVE_load_acquire_read) \
1833 : && defined(AO_HAVE_load_acquire)
1834 : # define AO_load_acquire_read(addr) AO_load_acquire(addr)
1835 : # define AO_HAVE_load_acquire_read
1836 : #endif
1837 :
1838 : #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
1839 : && !defined(AO_HAVE_load_acquire)
1840 : AO_INLINE AO_t
1841 : AO_load_acquire(const volatile AO_t *addr)
1842 : {
1843 : AO_t result = AO_load(addr);
1844 :
1845 : /* Acquire barrier would be useless, since the load could be delayed */
1846 : /* beyond it. */
1847 : AO_nop_full();
1848 : return result;
1849 : }
1850 : # define AO_HAVE_load_acquire
1851 : #endif
1852 :
1853 : #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
1854 : && !defined(AO_HAVE_load_read)
1855 : AO_INLINE AO_t
1856 32200 : AO_load_read(const volatile AO_t *addr)
1857 : {
1858 32200 : AO_t result = AO_load(addr);
1859 :
1860 32232 : AO_nop_read();
1861 32184 : return result;
1862 : }
1863 : # define AO_HAVE_load_read
1864 : #endif
1865 :
1866 : #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
1867 : && !defined(AO_HAVE_load_full)
1868 : # define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
1869 : # define AO_HAVE_load_full
1870 : #endif
1871 :
1872 : #if defined(AO_HAVE_compare_and_swap_read) \
1873 : && !defined(AO_HAVE_load_read)
1874 : # define AO_CAS_BASED_LOAD_READ
1875 : AO_INLINE AO_t
1876 : AO_load_read(const volatile AO_t *addr)
1877 : {
1878 : AO_t result;
1879 :
1880 : do {
1881 : result = *(const AO_t *)addr;
1882 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_read(
1883 : (volatile AO_t *)addr,
1884 : result, result)));
1885 : return result;
1886 : }
1887 : # define AO_HAVE_load_read
1888 : #endif
1889 :
1890 : #if !defined(AO_HAVE_load_acquire_read) \
1891 : && defined(AO_HAVE_load_read)
1892 : # define AO_load_acquire_read(addr) AO_load_read(addr)
1893 : # define AO_HAVE_load_acquire_read
1894 : #endif
1895 :
1896 : #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) \
1897 : && (!defined(AO_CAS_BASED_LOAD_READ) \
1898 : || !defined(AO_HAVE_compare_and_swap))
1899 : # define AO_load(addr) AO_load_acquire_read(addr)
1900 : # define AO_HAVE_load
1901 : #endif
1902 :
1903 : #if defined(AO_HAVE_compare_and_swap_full) \
1904 : && !defined(AO_HAVE_load_full)
1905 : AO_INLINE AO_t
1906 : AO_load_full(const volatile AO_t *addr)
1907 : {
1908 : AO_t result;
1909 :
1910 : do {
1911 : result = *(const AO_t *)addr;
1912 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(
1913 : (volatile AO_t *)addr,
1914 : result, result)));
1915 : return result;
1916 : }
1917 : # define AO_HAVE_load_full
1918 : #endif
1919 :
1920 : #if defined(AO_HAVE_compare_and_swap_acquire) \
1921 : && !defined(AO_HAVE_load_acquire)
1922 : AO_INLINE AO_t
1923 : AO_load_acquire(const volatile AO_t *addr)
1924 : {
1925 : AO_t result;
1926 :
1927 : do {
1928 : result = *(const AO_t *)addr;
1929 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(
1930 : (volatile AO_t *)addr,
1931 : result, result)));
1932 : return result;
1933 : }
1934 : # define AO_HAVE_load_acquire
1935 : #endif
1936 :
1937 : #if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_load)
1938 : AO_INLINE AO_t
1939 : AO_load(const volatile AO_t *addr)
1940 : {
1941 : AO_t result;
1942 :
1943 : do {
1944 : result = *(const AO_t *)addr;
1945 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap(
1946 : (volatile AO_t *)addr,
1947 : result, result)));
1948 : return result;
1949 : }
1950 : # define AO_HAVE_load
1951 : #endif
1952 :
1953 : #ifdef AO_NO_DD_ORDERING
1954 : # if defined(AO_HAVE_load_acquire_read)
1955 : # define AO_load_dd_acquire_read(addr) \
1956 : AO_load_acquire_read(addr)
1957 : # define AO_HAVE_load_dd_acquire_read
1958 : # endif
1959 : #else
1960 : # if defined(AO_HAVE_load)
1961 : # define AO_load_dd_acquire_read(addr) AO_load(addr)
1962 : # define AO_HAVE_load_dd_acquire_read
1963 : # endif
1964 : #endif /* !AO_NO_DD_ORDERING */
1965 :
1966 : /* store */
1967 : #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
1968 : # define AO_store_release(addr, val) AO_store_full(addr, val)
1969 : # define AO_HAVE_store_release
1970 : #endif
1971 :
1972 : #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
1973 : # define AO_store(addr, val) AO_store_release(addr, val)
1974 : # define AO_HAVE_store
1975 : #endif
1976 :
1977 : #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
1978 : # define AO_store_write(addr, val) AO_store_full(addr, val)
1979 : # define AO_HAVE_store_write
1980 : #endif
1981 :
1982 : #if defined(AO_HAVE_store_release) \
1983 : && !defined(AO_HAVE_store_release_write)
1984 : # define AO_store_release_write(addr, val) \
1985 : AO_store_release(addr, val)
1986 : # define AO_HAVE_store_release_write
1987 : #endif
1988 :
1989 : #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
1990 : # define AO_store(addr, val) AO_store_write(addr, val)
1991 : # define AO_HAVE_store
1992 : #endif
1993 :
1994 : #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
1995 : && !defined(AO_HAVE_store_release)
1996 : # define AO_store_release(addr, val) \
1997 : (AO_nop_full(), AO_store(addr, val))
1998 : # define AO_HAVE_store_release
1999 : #endif
2000 :
2001 : #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_write) \
2002 : && !defined(AO_HAVE_store_write)
2003 : # define AO_store_write(addr, val) \
2004 : (AO_nop_write(), AO_store(addr, val))
2005 : # define AO_HAVE_store_write
2006 : #endif
2007 :
2008 : #if defined(AO_HAVE_compare_and_swap_write) \
2009 : && !defined(AO_HAVE_store_write)
2010 : AO_INLINE void
2011 : AO_store_write(volatile AO_t *addr, AO_t new_val)
2012 : {
2013 : AO_t old_val;
2014 :
2015 : do {
2016 : old_val = *(AO_t *)addr;
2017 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_write(addr, old_val,
2018 : new_val)));
2019 : }
2020 : # define AO_HAVE_store_write
2021 : #endif
2022 :
2023 : #if defined(AO_HAVE_store_write) \
2024 : && !defined(AO_HAVE_store_release_write)
2025 : # define AO_store_release_write(addr, val) \
2026 : AO_store_write(addr, val)
2027 : # define AO_HAVE_store_release_write
2028 : #endif
2029 :
2030 : #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
2031 : && !defined(AO_HAVE_store_full)
2032 : # define AO_store_full(addr, val) \
2033 : (AO_store_release(addr, val), \
2034 : AO_nop_full())
2035 : # define AO_HAVE_store_full
2036 : #endif
2037 :
2038 : #if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_store)
2039 : AO_INLINE void
2040 : AO_store(volatile AO_t *addr, AO_t new_val)
2041 : {
2042 : AO_t old_val;
2043 :
2044 : do {
2045 : old_val = *(AO_t *)addr;
2046 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr,
2047 : old_val, new_val)));
2048 : }
2049 : # define AO_HAVE_store
2050 : #endif
2051 :
2052 : #if defined(AO_HAVE_compare_and_swap_release) \
2053 : && !defined(AO_HAVE_store_release)
2054 : AO_INLINE void
2055 : AO_store_release(volatile AO_t *addr, AO_t new_val)
2056 : {
2057 : AO_t old_val;
2058 :
2059 : do {
2060 : old_val = *(AO_t *)addr;
2061 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old_val,
2062 : new_val)));
2063 : }
2064 : # define AO_HAVE_store_release
2065 : #endif
2066 :
2067 : #if defined(AO_HAVE_compare_and_swap_full) \
2068 : && !defined(AO_HAVE_store_full)
2069 : AO_INLINE void
2070 : AO_store_full(volatile AO_t *addr, AO_t new_val)
2071 : {
2072 : AO_t old_val;
2073 :
2074 : do {
2075 : old_val = *(AO_t *)addr;
2076 : } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old_val,
2077 : new_val)));
2078 : }
2079 : # define AO_HAVE_store_full
2080 : #endif
2081 : /*
2082 : * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
2083 : *
2084 : * Permission is hereby granted, free of charge, to any person obtaining a copy
2085 : * of this software and associated documentation files (the "Software"), to deal
2086 : * in the Software without restriction, including without limitation the rights
2087 : * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
2088 : * copies of the Software, and to permit persons to whom the Software is
2089 : * furnished to do so, subject to the following conditions:
2090 : *
2091 : * The above copyright notice and this permission notice shall be included in
2092 : * all copies or substantial portions of the Software.
2093 : *
2094 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2095 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2096 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
2097 : * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2098 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2099 : * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2100 : * SOFTWARE.
2101 : */
2102 :
2103 : /* double_fetch_compare_and_swap */
2104 : #if defined(AO_HAVE_double_fetch_compare_and_swap) \
2105 : && defined(AO_HAVE_nop_full) \
2106 : && !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2107 : AO_INLINE AO_double_t
2108 : AO_double_fetch_compare_and_swap_acquire(volatile AO_double_t *addr,
2109 : AO_double_t old_val, AO_double_t new_val)
2110 : {
2111 : AO_double_t result = AO_double_fetch_compare_and_swap(addr, old_val, new_val);
2112 : AO_nop_full();
2113 : return result;
2114 : }
2115 : # define AO_HAVE_double_fetch_compare_and_swap_acquire
2116 : #endif
2117 : #if defined(AO_HAVE_double_fetch_compare_and_swap) \
2118 : && defined(AO_HAVE_nop_full) \
2119 : && !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2120 : # define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2121 : (AO_nop_full(), \
2122 : AO_double_fetch_compare_and_swap(addr, old_val, new_val))
2123 : # define AO_HAVE_double_fetch_compare_and_swap_release
2124 : #endif
2125 : #if defined(AO_HAVE_double_fetch_compare_and_swap_full)
2126 : # if !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2127 : # define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2128 : AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2129 : # define AO_HAVE_double_fetch_compare_and_swap_release
2130 : # endif
2131 : # if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2132 : # define AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
2133 : AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2134 : # define AO_HAVE_double_fetch_compare_and_swap_acquire
2135 : # endif
2136 : # if !defined(AO_HAVE_double_fetch_compare_and_swap_write)
2137 : # define AO_double_fetch_compare_and_swap_write(addr, old_val, new_val) \
2138 : AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2139 : # define AO_HAVE_double_fetch_compare_and_swap_write
2140 : # endif
2141 : # if !defined(AO_HAVE_double_fetch_compare_and_swap_read)
2142 : # define AO_double_fetch_compare_and_swap_read(addr, old_val, new_val) \
2143 : AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2144 : # define AO_HAVE_double_fetch_compare_and_swap_read
2145 : # endif
2146 : #endif /* AO_HAVE_double_fetch_compare_and_swap_full */
2147 :
2148 : #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2149 : && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2150 : # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2151 : AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2152 : # define AO_HAVE_double_fetch_compare_and_swap
2153 : #endif
2154 : #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2155 : && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2156 : # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2157 : AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2158 : # define AO_HAVE_double_fetch_compare_and_swap
2159 : #endif
2160 : #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2161 : && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2162 : # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2163 : AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2164 : # define AO_HAVE_double_fetch_compare_and_swap
2165 : #endif
2166 : #if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2167 : && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2168 : # define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2169 : AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2170 : # define AO_HAVE_double_fetch_compare_and_swap
2171 : #endif
2172 :
2173 : #if defined(AO_HAVE_double_fetch_compare_and_swap_acquire) \
2174 : && defined(AO_HAVE_nop_full) \
2175 : && !defined(AO_HAVE_double_fetch_compare_and_swap_full)
2176 : # define AO_double_fetch_compare_and_swap_full(addr, old_val, new_val) \
2177 : (AO_nop_full(), \
2178 : AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val))
2179 : # define AO_HAVE_double_fetch_compare_and_swap_full
2180 : #endif
2181 :
2182 : #if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2183 : && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2184 : # define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2185 : AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2186 : # define AO_HAVE_double_fetch_compare_and_swap_release_write
2187 : #endif
2188 : #if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2189 : && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2190 : # define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2191 : AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2192 : # define AO_HAVE_double_fetch_compare_and_swap_release_write
2193 : #endif
2194 : #if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2195 : && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2196 : # define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2197 : AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2198 : # define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2199 : #endif
2200 : #if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2201 : && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2202 : # define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2203 : AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2204 : # define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2205 : #endif
2206 :
2207 : #ifdef AO_NO_DD_ORDERING
2208 : # if defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read)
2209 : # define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2210 : AO_double_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
2211 : # define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2212 : # endif
2213 : #else
2214 : # if defined(AO_HAVE_double_fetch_compare_and_swap)
2215 : # define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2216 : AO_double_fetch_compare_and_swap(addr, old_val, new_val)
2217 : # define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2218 : # endif
2219 : #endif /* !AO_NO_DD_ORDERING */
2220 :
2221 : /* double_compare_and_swap */
2222 : #if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2223 : && !defined(AO_HAVE_double_compare_and_swap_acquire)
2224 : AO_INLINE int
2225 : AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, AO_double_t old,
2226 : AO_double_t new_val)
2227 : {
2228 : int result = AO_double_compare_and_swap(addr, old, new_val);
2229 : AO_nop_full();
2230 : return result;
2231 : }
2232 : # define AO_HAVE_double_compare_and_swap_acquire
2233 : #endif
2234 : #if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2235 : && !defined(AO_HAVE_double_compare_and_swap_release)
2236 : # define AO_double_compare_and_swap_release(addr, old, new_val) \
2237 : (AO_nop_full(), AO_double_compare_and_swap(addr, old, new_val))
2238 : # define AO_HAVE_double_compare_and_swap_release
2239 : #endif
2240 : #if defined(AO_HAVE_double_compare_and_swap_full)
2241 : # if !defined(AO_HAVE_double_compare_and_swap_release)
2242 : # define AO_double_compare_and_swap_release(addr, old, new_val) \
2243 : AO_double_compare_and_swap_full(addr, old, new_val)
2244 : # define AO_HAVE_double_compare_and_swap_release
2245 : # endif
2246 : # if !defined(AO_HAVE_double_compare_and_swap_acquire)
2247 : # define AO_double_compare_and_swap_acquire(addr, old, new_val) \
2248 : AO_double_compare_and_swap_full(addr, old, new_val)
2249 : # define AO_HAVE_double_compare_and_swap_acquire
2250 : # endif
2251 : # if !defined(AO_HAVE_double_compare_and_swap_write)
2252 : # define AO_double_compare_and_swap_write(addr, old, new_val) \
2253 : AO_double_compare_and_swap_full(addr, old, new_val)
2254 : # define AO_HAVE_double_compare_and_swap_write
2255 : # endif
2256 : # if !defined(AO_HAVE_double_compare_and_swap_read)
2257 : # define AO_double_compare_and_swap_read(addr, old, new_val) \
2258 : AO_double_compare_and_swap_full(addr, old, new_val)
2259 : # define AO_HAVE_double_compare_and_swap_read
2260 : # endif
2261 : #endif /* AO_HAVE_double_compare_and_swap_full */
2262 :
2263 : #if !defined(AO_HAVE_double_compare_and_swap) \
2264 : && defined(AO_HAVE_double_compare_and_swap_release)
2265 : # define AO_double_compare_and_swap(addr, old, new_val) \
2266 : AO_double_compare_and_swap_release(addr, old, new_val)
2267 : # define AO_HAVE_double_compare_and_swap
2268 : #endif
2269 : #if !defined(AO_HAVE_double_compare_and_swap) \
2270 : && defined(AO_HAVE_double_compare_and_swap_acquire)
2271 : # define AO_double_compare_and_swap(addr, old, new_val) \
2272 : AO_double_compare_and_swap_acquire(addr, old, new_val)
2273 : # define AO_HAVE_double_compare_and_swap
2274 : #endif
2275 : #if !defined(AO_HAVE_double_compare_and_swap) \
2276 : && defined(AO_HAVE_double_compare_and_swap_write)
2277 : # define AO_double_compare_and_swap(addr, old, new_val) \
2278 : AO_double_compare_and_swap_write(addr, old, new_val)
2279 : # define AO_HAVE_double_compare_and_swap
2280 : #endif
2281 : #if !defined(AO_HAVE_double_compare_and_swap) \
2282 : && defined(AO_HAVE_double_compare_and_swap_read)
2283 : # define AO_double_compare_and_swap(addr, old, new_val) \
2284 : AO_double_compare_and_swap_read(addr, old, new_val)
2285 : # define AO_HAVE_double_compare_and_swap
2286 : #endif
2287 :
2288 : #if defined(AO_HAVE_double_compare_and_swap_acquire) \
2289 : && defined(AO_HAVE_nop_full) \
2290 : && !defined(AO_HAVE_double_compare_and_swap_full)
2291 : # define AO_double_compare_and_swap_full(addr, old, new_val) \
2292 : (AO_nop_full(), \
2293 : AO_double_compare_and_swap_acquire(addr, old, new_val))
2294 : # define AO_HAVE_double_compare_and_swap_full
2295 : #endif
2296 :
2297 : #if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2298 : && defined(AO_HAVE_double_compare_and_swap_write)
2299 : # define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2300 : AO_double_compare_and_swap_write(addr, old, new_val)
2301 : # define AO_HAVE_double_compare_and_swap_release_write
2302 : #endif
2303 : #if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2304 : && defined(AO_HAVE_double_compare_and_swap_release)
2305 : # define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2306 : AO_double_compare_and_swap_release(addr, old, new_val)
2307 : # define AO_HAVE_double_compare_and_swap_release_write
2308 : #endif
2309 : #if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2310 : && defined(AO_HAVE_double_compare_and_swap_read)
2311 : # define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2312 : AO_double_compare_and_swap_read(addr, old, new_val)
2313 : # define AO_HAVE_double_compare_and_swap_acquire_read
2314 : #endif
2315 : #if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2316 : && defined(AO_HAVE_double_compare_and_swap_acquire)
2317 : # define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2318 : AO_double_compare_and_swap_acquire(addr, old, new_val)
2319 : # define AO_HAVE_double_compare_and_swap_acquire_read
2320 : #endif
2321 :
2322 : #ifdef AO_NO_DD_ORDERING
2323 : # if defined(AO_HAVE_double_compare_and_swap_acquire_read)
2324 : # define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2325 : AO_double_compare_and_swap_acquire_read(addr, old, new_val)
2326 : # define AO_HAVE_double_compare_and_swap_dd_acquire_read
2327 : # endif
2328 : #else
2329 : # if defined(AO_HAVE_double_compare_and_swap)
2330 : # define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2331 : AO_double_compare_and_swap(addr, old, new_val)
2332 : # define AO_HAVE_double_compare_and_swap_dd_acquire_read
2333 : # endif
2334 : #endif /* !AO_NO_DD_ORDERING */
2335 :
2336 : /* double_load */
2337 : #if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_acquire)
2338 : # define AO_double_load_acquire(addr) AO_double_load_full(addr)
2339 : # define AO_HAVE_double_load_acquire
2340 : #endif
2341 :
2342 : #if defined(AO_HAVE_double_load_acquire) && !defined(AO_HAVE_double_load)
2343 : # define AO_double_load(addr) AO_double_load_acquire(addr)
2344 : # define AO_HAVE_double_load
2345 : #endif
2346 :
2347 : #if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_read)
2348 : # define AO_double_load_read(addr) AO_double_load_full(addr)
2349 : # define AO_HAVE_double_load_read
2350 : #endif
2351 :
2352 : #if !defined(AO_HAVE_double_load_acquire_read) \
2353 : && defined(AO_HAVE_double_load_acquire)
2354 : # define AO_double_load_acquire_read(addr) AO_double_load_acquire(addr)
2355 : # define AO_HAVE_double_load_acquire_read
2356 : #endif
2357 :
2358 : #if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_full) \
2359 : && !defined(AO_HAVE_double_load_acquire)
2360 : AO_INLINE AO_double_t
2361 : AO_double_load_acquire(const volatile AO_double_t *addr)
2362 : {
2363 : AO_double_t result = AO_double_load(addr);
2364 :
2365 : /* Acquire barrier would be useless, since the load could be delayed */
2366 : /* beyond it. */
2367 : AO_nop_full();
2368 : return result;
2369 : }
2370 : # define AO_HAVE_double_load_acquire
2371 : #endif
2372 :
2373 : #if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_read) \
2374 : && !defined(AO_HAVE_double_load_read)
2375 : AO_INLINE AO_double_t
2376 : AO_double_load_read(const volatile AO_double_t *addr)
2377 : {
2378 : AO_double_t result = AO_double_load(addr);
2379 :
2380 : AO_nop_read();
2381 : return result;
2382 : }
2383 : # define AO_HAVE_double_load_read
2384 : #endif
2385 :
2386 : #if defined(AO_HAVE_double_load_acquire) && defined(AO_HAVE_nop_full) \
2387 : && !defined(AO_HAVE_double_load_full)
2388 : # define AO_double_load_full(addr) (AO_nop_full(), AO_double_load_acquire(addr))
2389 : # define AO_HAVE_double_load_full
2390 : #endif
2391 :
2392 : #if defined(AO_HAVE_double_compare_and_swap_read) \
2393 : && !defined(AO_HAVE_double_load_read)
2394 : # define AO_double_CAS_BASED_LOAD_READ
2395 : AO_INLINE AO_double_t
2396 : AO_double_load_read(const volatile AO_double_t *addr)
2397 : {
2398 : AO_double_t result;
2399 :
2400 : do {
2401 : result = *(const AO_double_t *)addr;
2402 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_read(
2403 : (volatile AO_double_t *)addr,
2404 : result, result)));
2405 : return result;
2406 : }
2407 : # define AO_HAVE_double_load_read
2408 : #endif
2409 :
2410 : #if !defined(AO_HAVE_double_load_acquire_read) \
2411 : && defined(AO_HAVE_double_load_read)
2412 : # define AO_double_load_acquire_read(addr) AO_double_load_read(addr)
2413 : # define AO_HAVE_double_load_acquire_read
2414 : #endif
2415 :
2416 : #if defined(AO_HAVE_double_load_acquire_read) && !defined(AO_HAVE_double_load) \
2417 : && (!defined(AO_double_CAS_BASED_LOAD_READ) \
2418 : || !defined(AO_HAVE_double_compare_and_swap))
2419 : # define AO_double_load(addr) AO_double_load_acquire_read(addr)
2420 : # define AO_HAVE_double_load
2421 : #endif
2422 :
2423 : #if defined(AO_HAVE_double_compare_and_swap_full) \
2424 : && !defined(AO_HAVE_double_load_full)
2425 : AO_INLINE AO_double_t
2426 : AO_double_load_full(const volatile AO_double_t *addr)
2427 : {
2428 : AO_double_t result;
2429 :
2430 : do {
2431 : result = *(const AO_double_t *)addr;
2432 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(
2433 : (volatile AO_double_t *)addr,
2434 : result, result)));
2435 : return result;
2436 : }
2437 : # define AO_HAVE_double_load_full
2438 : #endif
2439 :
2440 : #if defined(AO_HAVE_double_compare_and_swap_acquire) \
2441 : && !defined(AO_HAVE_double_load_acquire)
2442 : AO_INLINE AO_double_t
2443 : AO_double_load_acquire(const volatile AO_double_t *addr)
2444 : {
2445 : AO_double_t result;
2446 :
2447 : do {
2448 : result = *(const AO_double_t *)addr;
2449 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_acquire(
2450 : (volatile AO_double_t *)addr,
2451 : result, result)));
2452 : return result;
2453 : }
2454 : # define AO_HAVE_double_load_acquire
2455 : #endif
2456 :
2457 : #if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_load)
2458 : AO_INLINE AO_double_t
2459 : AO_double_load(const volatile AO_double_t *addr)
2460 : {
2461 : AO_double_t result;
2462 :
2463 : do {
2464 : result = *(const AO_double_t *)addr;
2465 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(
2466 : (volatile AO_double_t *)addr,
2467 : result, result)));
2468 : return result;
2469 : }
2470 : # define AO_HAVE_double_load
2471 : #endif
2472 :
2473 : #ifdef AO_NO_DD_ORDERING
2474 : # if defined(AO_HAVE_double_load_acquire_read)
2475 : # define AO_double_load_dd_acquire_read(addr) \
2476 : AO_double_load_acquire_read(addr)
2477 : # define AO_HAVE_double_load_dd_acquire_read
2478 : # endif
2479 : #else
2480 : # if defined(AO_HAVE_double_load)
2481 : # define AO_double_load_dd_acquire_read(addr) AO_double_load(addr)
2482 : # define AO_HAVE_double_load_dd_acquire_read
2483 : # endif
2484 : #endif /* !AO_NO_DD_ORDERING */
2485 :
2486 : /* double_store */
2487 : #if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_release)
2488 : # define AO_double_store_release(addr, val) AO_double_store_full(addr, val)
2489 : # define AO_HAVE_double_store_release
2490 : #endif
2491 :
2492 : #if defined(AO_HAVE_double_store_release) && !defined(AO_HAVE_double_store)
2493 : # define AO_double_store(addr, val) AO_double_store_release(addr, val)
2494 : # define AO_HAVE_double_store
2495 : #endif
2496 :
2497 : #if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_write)
2498 : # define AO_double_store_write(addr, val) AO_double_store_full(addr, val)
2499 : # define AO_HAVE_double_store_write
2500 : #endif
2501 :
2502 : #if defined(AO_HAVE_double_store_release) \
2503 : && !defined(AO_HAVE_double_store_release_write)
2504 : # define AO_double_store_release_write(addr, val) \
2505 : AO_double_store_release(addr, val)
2506 : # define AO_HAVE_double_store_release_write
2507 : #endif
2508 :
2509 : #if defined(AO_HAVE_double_store_write) && !defined(AO_HAVE_double_store)
2510 : # define AO_double_store(addr, val) AO_double_store_write(addr, val)
2511 : # define AO_HAVE_double_store
2512 : #endif
2513 :
2514 : #if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_full) \
2515 : && !defined(AO_HAVE_double_store_release)
2516 : # define AO_double_store_release(addr, val) \
2517 : (AO_nop_full(), AO_double_store(addr, val))
2518 : # define AO_HAVE_double_store_release
2519 : #endif
2520 :
2521 : #if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_write) \
2522 : && !defined(AO_HAVE_double_store_write)
2523 : # define AO_double_store_write(addr, val) \
2524 : (AO_nop_write(), AO_double_store(addr, val))
2525 : # define AO_HAVE_double_store_write
2526 : #endif
2527 :
2528 : #if defined(AO_HAVE_double_compare_and_swap_write) \
2529 : && !defined(AO_HAVE_double_store_write)
2530 : AO_INLINE void
2531 : AO_double_store_write(volatile AO_double_t *addr, AO_double_t new_val)
2532 : {
2533 : AO_double_t old_val;
2534 :
2535 : do {
2536 : old_val = *(AO_double_t *)addr;
2537 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_write(addr, old_val,
2538 : new_val)));
2539 : }
2540 : # define AO_HAVE_double_store_write
2541 : #endif
2542 :
2543 : #if defined(AO_HAVE_double_store_write) \
2544 : && !defined(AO_HAVE_double_store_release_write)
2545 : # define AO_double_store_release_write(addr, val) \
2546 : AO_double_store_write(addr, val)
2547 : # define AO_HAVE_double_store_release_write
2548 : #endif
2549 :
2550 : #if defined(AO_HAVE_double_store_release) && defined(AO_HAVE_nop_full) \
2551 : && !defined(AO_HAVE_double_store_full)
2552 : # define AO_double_store_full(addr, val) \
2553 : (AO_double_store_release(addr, val), \
2554 : AO_nop_full())
2555 : # define AO_HAVE_double_store_full
2556 : #endif
2557 :
2558 : #if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_store)
2559 : AO_INLINE void
2560 : AO_double_store(volatile AO_double_t *addr, AO_double_t new_val)
2561 : {
2562 : AO_double_t old_val;
2563 :
2564 : do {
2565 : old_val = *(AO_double_t *)addr;
2566 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(addr,
2567 : old_val, new_val)));
2568 : }
2569 : # define AO_HAVE_double_store
2570 : #endif
2571 :
2572 : #if defined(AO_HAVE_double_compare_and_swap_release) \
2573 : && !defined(AO_HAVE_double_store_release)
2574 : AO_INLINE void
2575 : AO_double_store_release(volatile AO_double_t *addr, AO_double_t new_val)
2576 : {
2577 : AO_double_t old_val;
2578 :
2579 : do {
2580 : old_val = *(AO_double_t *)addr;
2581 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_release(addr, old_val,
2582 : new_val)));
2583 : }
2584 : # define AO_HAVE_double_store_release
2585 : #endif
2586 :
2587 : #if defined(AO_HAVE_double_compare_and_swap_full) \
2588 : && !defined(AO_HAVE_double_store_full)
2589 : AO_INLINE void
2590 : AO_double_store_full(volatile AO_double_t *addr, AO_double_t new_val)
2591 : {
2592 : AO_double_t old_val;
2593 :
2594 : do {
2595 : old_val = *(AO_double_t *)addr;
2596 : } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(addr, old_val,
2597 : new_val)));
2598 : }
2599 : # define AO_HAVE_double_store_full
2600 : #endif
|
