my_atomic.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. #ifndef MY_ATOMIC_INCLUDED
  2. #define MY_ATOMIC_INCLUDED
  3. /* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  4. This program is free software; you can redistribute it and/or modify
  5. it under the terms of the GNU General Public License as published by
  6. the Free Software Foundation; version 2 of the License.
  7. This program is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU General Public License for more details.
  11. You should have received a copy of the GNU General Public License
  12. along with this program; if not, write to the Free Software
  13. Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
  14. /*
  15. This header defines five atomic operations:
  16. my_atomic_add#(&var, what)
  17. my_atomic_add#_explicit(&var, what, memory_order)
  18. 'Fetch and Add'
  19. add 'what' to *var, and return the old value of *var
  20. All memory orders are valid.
  21. my_atomic_fas#(&var, what)
  22. my_atomic_fas#_explicit(&var, what, memory_order)
  23. 'Fetch And Store'
  24. store 'what' in *var, and return the old value of *var
  25. All memory orders are valid.
  26. my_atomic_cas#(&var, &old, new)
  27. my_atomic_cas#_weak_explicit(&var, &old, new, succ, fail)
  28. my_atomic_cas#_strong_explicit(&var, &old, new, succ, fail)
  29. 'Compare And Swap'
  30. if *var is equal to *old, then store 'new' in *var, and return TRUE
  31. otherwise store *var in *old, and return FALSE
  32. succ - the memory synchronization ordering for the read-modify-write
  33. operation if the comparison succeeds. All memory orders are valid.
  34. fail - the memory synchronization ordering for the load operation if the
  35. comparison fails. Cannot be MY_MEMORY_ORDER_RELEASE or
  36. MY_MEMORY_ORDER_ACQ_REL and cannot specify stronger ordering than succ.
  37. The weak form is allowed to fail spuriously, that is, act as if *var != *old
  38. even if they are equal. When a compare-and-exchange is in a loop, the weak
  39. version will yield better performance on some platforms. When a weak
  40. compare-and-exchange would require a loop and a strong one would not, the
  41. strong one is preferable.
  42. my_atomic_load#(&var)
  43. my_atomic_load#_explicit(&var, memory_order)
  44. return *var
  45. Order must be one of MY_MEMORY_ORDER_RELAXED, MY_MEMORY_ORDER_CONSUME,
  46. MY_MEMORY_ORDER_ACQUIRE, MY_MEMORY_ORDER_SEQ_CST.
  47. my_atomic_store#(&var, what)
  48. my_atomic_store#_explicit(&var, what, memory_order)
  49. store 'what' in *var
  50. Order must be one of MY_MEMORY_ORDER_RELAXED, MY_MEMORY_ORDER_RELEASE,
  51. MY_MEMORY_ORDER_SEQ_CST.
  52. '#' is substituted by a size suffix - 8, 16, 32, 64, or ptr
  53. (e.g. my_atomic_add8, my_atomic_fas32, my_atomic_casptr).
  54. The first version orders memory accesses according to MY_MEMORY_ORDER_SEQ_CST,
  55. the second version (with _explicit suffix) orders memory accesses according to
  56. given memory order.
  57. memory_order specifies how non-atomic memory accesses are to be ordered around
  58. an atomic operation:
  59. MY_MEMORY_ORDER_RELAXED - there are no constraints on reordering of memory
  60. accesses around the atomic variable.
  61. MY_MEMORY_ORDER_CONSUME - no reads in the current thread dependent on the
  62. value currently loaded can be reordered before this
  63. load. This ensures that writes to dependent
  64. variables in other threads that release the same
  65. atomic variable are visible in the current thread.
  66. On most platforms, this affects compiler
  67. optimization only.
  68. MY_MEMORY_ORDER_ACQUIRE - no reads in the current thread can be reordered
  69. before this load. This ensures that all writes in
  70. other threads that release the same atomic variable
  71. are visible in the current thread.
  72. MY_MEMORY_ORDER_RELEASE - no writes in the current thread can be reordered
  73. after this store. This ensures that all writes in
  74. the current thread are visible in other threads that
  75. acquire the same atomic variable.
  76. MY_MEMORY_ORDER_ACQ_REL - no reads in the current thread can be reordered
  77. before this load as well as no writes in the current
  78. thread can be reordered after this store. The
  79. operation is read-modify-write operation. It is
  80. ensured that all writes in another threads that
  81. release the same atomic variable are visible before
  82. the modification and the modification is visible in
  83. other threads that acquire the same atomic variable.
  84. MY_MEMORY_ORDER_SEQ_CST - The operation has the same semantics as
  85. acquire-release operation, and additionally has
  86. sequentially-consistent operation ordering.
  87. 8- and 16-bit atomics aren't implemented for windows (see generic-msvc.h),
  88. but can be added, if necessary.
  89. */
  90. #define intptr void *
  91. /**
  92. Currently we don't support 8-bit and 16-bit operations.
  93. It can be added later if needed.
  94. */
  95. #undef MY_ATOMIC_HAS_8_16
  96. /*
  97. * Attempt to do atomic ops without locks
  98. */
  99. #include "atomic/nolock.h"
  100. #ifndef make_atomic_cas_body
  101. /* nolock.h was not able to generate even a CAS function, fall back */
  102. #error atomic ops for this platform are not implemented
  103. #endif
  104. /* define missing functions by using the already generated ones */
  105. #ifndef make_atomic_add_body
  106. #define make_atomic_add_body(S) \
  107. int ## S tmp=*a; \
  108. while (!my_atomic_cas ## S(a, &tmp, tmp+v)) ; \
  109. v=tmp;
  110. #endif
  111. #ifndef make_atomic_fas_body
  112. #define make_atomic_fas_body(S) \
  113. int ## S tmp=*a; \
  114. while (!my_atomic_cas ## S(a, &tmp, v)) ; \
  115. v=tmp;
  116. #endif
  117. #ifndef make_atomic_load_body
  118. #define make_atomic_load_body(S) \
  119. ret= 0; /* avoid compiler warning */ \
  120. (void)(my_atomic_cas ## S(a, &ret, ret));
  121. #endif
  122. #ifndef make_atomic_store_body
  123. #define make_atomic_store_body(S) \
  124. (void)(my_atomic_fas ## S (a, v));
  125. #endif
  126. /*
  127. transparent_union doesn't work in g++
  128. Bug ?
  129. Darwin's gcc doesn't want to put pointers in a transparent_union
  130. when built with -arch ppc64. Complains:
  131. warning: 'transparent_union' attribute ignored
  132. */
  133. #if defined(__GNUC__) && !defined(__cplusplus) && \
  134. ! (defined(__APPLE__) && (defined(_ARCH_PPC64) ||defined (_ARCH_PPC)))
  135. /*
  136. we want to be able to use my_atomic_xxx functions with
  137. both signed and unsigned integers. But gcc will issue a warning
  138. "passing arg N of `my_atomic_XXX' as [un]signed due to prototype"
  139. if the signedness of the argument doesn't match the prototype, or
  140. "pointer targets in passing argument N of my_atomic_XXX differ in signedness"
  141. if int* is used where uint* is expected (or vice versa).
  142. Let's shut these warnings up
  143. */
  144. #define make_transparent_unions(S) \
  145. typedef union { \
  146. int ## S i; \
  147. uint ## S u; \
  148. } U_ ## S __attribute__ ((transparent_union)); \
  149. typedef union { \
  150. int ## S volatile *i; \
  151. uint ## S volatile *u; \
  152. } Uv_ ## S __attribute__ ((transparent_union));
  153. #define uintptr intptr
  154. make_transparent_unions(8)
  155. make_transparent_unions(16)
  156. make_transparent_unions(32)
  157. make_transparent_unions(64)
  158. make_transparent_unions(ptr)
  159. #undef uintptr
  160. #undef make_transparent_unions
  161. #define a U_a.i
  162. #define cmp U_cmp.i
  163. #define v U_v.i
  164. #define set U_set.i
  165. #else
  166. #define U_8 int8
  167. #define U_16 int16
  168. #define U_32 int32
  169. #define U_64 int64
  170. #define U_ptr intptr
  171. #define Uv_8 int8
  172. #define Uv_16 int16
  173. #define Uv_32 int32
  174. #define Uv_64 int64
  175. #define Uv_ptr intptr
  176. #define U_a volatile *a
  177. #define U_cmp *cmp
  178. #define U_v v
  179. #define U_set set
  180. #endif /* __GCC__ transparent_union magic */
  181. #define make_atomic_cas(S) \
  182. static inline int my_atomic_cas ## S(Uv_ ## S U_a, \
  183. Uv_ ## S U_cmp, U_ ## S U_set) \
  184. { \
  185. int8 ret; \
  186. make_atomic_cas_body(S); \
  187. return ret; \
  188. }
  189. #define make_atomic_add(S) \
  190. static inline int ## S my_atomic_add ## S( \
  191. Uv_ ## S U_a, U_ ## S U_v) \
  192. { \
  193. make_atomic_add_body(S); \
  194. return v; \
  195. }
  196. #define make_atomic_fas(S) \
  197. static inline int ## S my_atomic_fas ## S( \
  198. Uv_ ## S U_a, U_ ## S U_v) \
  199. { \
  200. make_atomic_fas_body(S); \
  201. return v; \
  202. }
  203. #define make_atomic_load(S) \
  204. static inline int ## S my_atomic_load ## S(Uv_ ## S U_a) \
  205. { \
  206. int ## S ret; \
  207. make_atomic_load_body(S); \
  208. return ret; \
  209. }
  210. #define make_atomic_store(S) \
  211. static inline void my_atomic_store ## S( \
  212. Uv_ ## S U_a, U_ ## S U_v) \
  213. { \
  214. make_atomic_store_body(S); \
  215. }
  216. #ifdef MY_ATOMIC_HAS_8_16
  217. make_atomic_cas(8)
  218. make_atomic_cas(16)
  219. #endif
  220. make_atomic_cas(32)
  221. make_atomic_cas(64)
  222. make_atomic_cas(ptr)
  223. #ifdef MY_ATOMIC_HAS_8_16
  224. make_atomic_add(8)
  225. make_atomic_add(16)
  226. #endif
  227. make_atomic_add(32)
  228. make_atomic_add(64)
  229. #ifdef MY_ATOMIC_HAS_8_16
  230. make_atomic_load(8)
  231. make_atomic_load(16)
  232. #endif
  233. make_atomic_load(32)
  234. make_atomic_load(64)
  235. make_atomic_load(ptr)
  236. #ifdef MY_ATOMIC_HAS_8_16
  237. make_atomic_fas(8)
  238. make_atomic_fas(16)
  239. #endif
  240. make_atomic_fas(32)
  241. make_atomic_fas(64)
  242. make_atomic_fas(ptr)
  243. #ifdef MY_ATOMIC_HAS_8_16
  244. make_atomic_store(8)
  245. make_atomic_store(16)
  246. #endif
  247. make_atomic_store(32)
  248. make_atomic_store(64)
  249. make_atomic_store(ptr)
  250. #ifdef _atomic_h_cleanup_
  251. #include _atomic_h_cleanup_
  252. #undef _atomic_h_cleanup_
  253. #endif
  254. #undef U_8
  255. #undef U_16
  256. #undef U_32
  257. #undef U_64
  258. #undef U_ptr
  259. #undef Uv_8
  260. #undef Uv_16
  261. #undef Uv_32
  262. #undef Uv_64
  263. #undef Uv_ptr
  264. #undef a
  265. #undef cmp
  266. #undef v
  267. #undef set
  268. #undef U_a
  269. #undef U_cmp
  270. #undef U_v
  271. #undef U_set
  272. #undef make_atomic_add
  273. #undef make_atomic_cas
  274. #undef make_atomic_load
  275. #undef make_atomic_store
  276. #undef make_atomic_fas
  277. #undef make_atomic_add_body
  278. #undef make_atomic_cas_body
  279. #undef make_atomic_load_body
  280. #undef make_atomic_store_body
  281. #undef make_atomic_fas_body
  282. #undef intptr
  283. /*
  284. the macro below defines (as an expression) the code that
  285. will be run in spin-loops. Intel manuals recummend to have PAUSE there.
  286. It is expected to be defined in include/atomic/ *.h files
  287. */
  288. #ifndef LF_BACKOFF
  289. #define LF_BACKOFF (1)
  290. #endif
  291. #define MY_ATOMIC_OK 0
  292. #define MY_ATOMIC_NOT_1CPU 1
  293. extern int my_atomic_initialize();
  294. #ifdef __ATOMIC_SEQ_CST
  295. #define MY_MEMORY_ORDER_RELAXED __ATOMIC_RELAXED
  296. #define MY_MEMORY_ORDER_CONSUME __ATOMIC_CONSUME
  297. #define MY_MEMORY_ORDER_ACQUIRE __ATOMIC_ACQUIRE
  298. #define MY_MEMORY_ORDER_RELEASE __ATOMIC_RELEASE
  299. #define MY_MEMORY_ORDER_ACQ_REL __ATOMIC_ACQ_REL
  300. #define MY_MEMORY_ORDER_SEQ_CST __ATOMIC_SEQ_CST
  301. #define my_atomic_store32_explicit(P, D, O) __atomic_store_n((P), (D), (O))
  302. #define my_atomic_store64_explicit(P, D, O) __atomic_store_n((P), (D), (O))
  303. #define my_atomic_storeptr_explicit(P, D, O) __atomic_store_n((P), (D), (O))
  304. #define my_atomic_load32_explicit(P, O) __atomic_load_n((P), (O))
  305. #define my_atomic_load64_explicit(P, O) __atomic_load_n((P), (O))
  306. #define my_atomic_loadptr_explicit(P, O) __atomic_load_n((P), (O))
  307. #define my_atomic_fas32_explicit(P, D, O) __atomic_exchange_n((P), (D), (O))
  308. #define my_atomic_fas64_explicit(P, D, O) __atomic_exchange_n((P), (D), (O))
  309. #define my_atomic_fasptr_explicit(P, D, O) __atomic_exchange_n((P), (D), (O))
  310. #define my_atomic_add32_explicit(P, A, O) __atomic_fetch_add((P), (A), (O))
  311. #define my_atomic_add64_explicit(P, A, O) __atomic_fetch_add((P), (A), (O))
  312. #define my_atomic_cas32_weak_explicit(P, E, D, S, F) \
  313. __atomic_compare_exchange_n((P), (E), (D), true, (S), (F))
  314. #define my_atomic_cas64_weak_explicit(P, E, D, S, F) \
  315. __atomic_compare_exchange_n((P), (E), (D), true, (S), (F))
  316. #define my_atomic_casptr_weak_explicit(P, E, D, S, F) \
  317. __atomic_compare_exchange_n((P), (E), (D), true, (S), (F))
  318. #define my_atomic_cas32_strong_explicit(P, E, D, S, F) \
  319. __atomic_compare_exchange_n((P), (E), (D), false, (S), (F))
  320. #define my_atomic_cas64_strong_explicit(P, E, D, S, F) \
  321. __atomic_compare_exchange_n((P), (E), (D), false, (S), (F))
  322. #define my_atomic_casptr_strong_explicit(P, E, D, S, F) \
  323. __atomic_compare_exchange_n((P), (E), (D), false, (S), (F))
  324. #else
  325. #define MY_MEMORY_ORDER_RELAXED
  326. #define MY_MEMORY_ORDER_CONSUME
  327. #define MY_MEMORY_ORDER_ACQUIRE
  328. #define MY_MEMORY_ORDER_RELEASE
  329. #define MY_MEMORY_ORDER_ACQ_REL
  330. #define MY_MEMORY_ORDER_SEQ_CST
  331. #define my_atomic_store32_explicit(P, D, O) my_atomic_store32((P), (D))
  332. #define my_atomic_store64_explicit(P, D, O) my_atomic_store64((P), (D))
  333. #define my_atomic_storeptr_explicit(P, D, O) my_atomic_storeptr((P), (D))
  334. #define my_atomic_load32_explicit(P, O) my_atomic_load32((P))
  335. #define my_atomic_load64_explicit(P, O) my_atomic_load64((P))
  336. #define my_atomic_loadptr_explicit(P, O) my_atomic_loadptr((P))
  337. #define my_atomic_fas32_explicit(P, D, O) my_atomic_fas32((P), (D))
  338. #define my_atomic_fas64_explicit(P, D, O) my_atomic_fas64((P), (D))
  339. #define my_atomic_fasptr_explicit(P, D, O) my_atomic_fasptr((P), (D))
  340. #define my_atomic_add32_explicit(P, A, O) my_atomic_add32((P), (A))
  341. #define my_atomic_add64_explicit(P, A, O) my_atomic_add64((P), (A))
  342. #define my_atomic_addptr_explicit(P, A, O) my_atomic_addptr((P), (A))
  343. #define my_atomic_cas32_weak_explicit(P, E, D, S, F) \
  344. my_atomic_cas32((P), (E), (D))
  345. #define my_atomic_cas64_weak_explicit(P, E, D, S, F) \
  346. my_atomic_cas64((P), (E), (D))
  347. #define my_atomic_casptr_weak_explicit(P, E, D, S, F) \
  348. my_atomic_casptr((P), (E), (D))
  349. #define my_atomic_cas32_strong_explicit(P, E, D, S, F) \
  350. my_atomic_cas32((P), (E), (D))
  351. #define my_atomic_cas64_strong_explicit(P, E, D, S, F) \
  352. my_atomic_cas64((P), (E), (D))
  353. #define my_atomic_casptr_strong_explicit(P, E, D, S, F) \
  354. my_atomic_casptr((P), (E), (D))
  355. #endif
  356. #endif /* MY_ATOMIC_INCLUDED */