thirdparty: update libatomic_ops to commit cc9bc49 from https://github.com/ivmai/libatomic_ops/ (#22235)

thirdparty/libatomic_ops/atomic_ops.c

@@ -45,14 +45,29 @@
 # define AO_USE_NO_SIGNALS
 #endif
 
-#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \
+#if (defined(__CYGWIN__) || defined(__GLIBC__) || defined(__GNU__) \
+     || defined(__linux__)) \
     && !defined(AO_USE_NO_SIGNALS) && !defined(_GNU_SOURCE)
 # define _GNU_SOURCE 1
 #endif
 
+#ifndef AO_BUILD
+# define AO_BUILD
+#endif
+
 #undef AO_REQUIRE_CAS
 #include "atomic_ops.h" /* Without cas emulation! */
 
+#ifdef __cplusplus
+  extern "C" {
+#endif
+
+AO_API void AO_pause(int); /* defined below */
+
+#ifdef __cplusplus
+  } /* extern "C" */
+#endif
+
 #if !defined(_MSC_VER) && !defined(__MINGW32__) && !defined(__BORLANDC__) \
     || defined(AO_USE_NO_SIGNALS)
 
@@ -80,7 +95,29 @@
 # include "atomic_ops/sysdeps/standard_ao_double_t.h"
 #endif
 
+#ifdef __cplusplus
+  extern "C" {
+#endif
+
+AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
+                                                AO_t old_val, AO_t new_val);
+
+AO_API int
+AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2);
+
+AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
+
 /* Lock for pthreads-based implementation.      */
+#ifndef AO_NO_PTHREADS
+  AO_API pthread_mutex_t AO_pt_lock;
+#endif
+
+#ifdef __cplusplus
+  } /* extern "C" */
+#endif
+
 #ifndef AO_NO_PTHREADS
   pthread_mutex_t AO_pt_lock = PTHREAD_MUTEX_INITIALIZER;
 #endif
@@ -99,7 +136,7 @@
  */
 #define AO_HASH_SIZE 16
 
-#define AO_HASH(x) (((unsigned long)(x) >> 12) & (AO_HASH_SIZE-1))
+#define AO_HASH(x) ((unsigned)((AO_uintptr_t)(x) >> 12) & (AO_HASH_SIZE-1))
 
 static AO_TS_t AO_locks[AO_HASH_SIZE] = {
   AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER,
@@ -108,8 +145,6 @@ static AO_TS_t AO_locks[AO_HASH_SIZE] = {
   AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER,
 };
 
-void AO_pause(int); /* defined below */
-
 static void lock_ool(volatile AO_TS_t *l)
 {
   int i = 0;
@@ -155,8 +190,8 @@ AO_INLINE void unlock(volatile AO_TS_t *l)
   }
 #endif /* !AO_USE_NO_SIGNALS */
 
-AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
+AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
-                                         AO_t new_val)
+                                                AO_t old_val, AO_t new_val)
 {
   AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
   AO_t fetched_val;
@@ -176,9 +211,10 @@ AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
   return fetched_val;
 }
 
-int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
+AO_API int
-                                                AO_t old_val1, AO_t old_val2,
+AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
-                                                AO_t new_val1, AO_t new_val2)
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2)
 {
   AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
   int result;
@@ -203,7 +239,7 @@ int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
   return result;
 }
 
-void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
+AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
 {
   AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
   lock(my_lock);
@@ -222,7 +258,7 @@ void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
 
 #endif
 
-static AO_t spin_dummy = 1;
+static volatile AO_t spin_dummy = 0;
 
 /* Spin for 2**n units. */
 static void AO_spin(int n)
@@ -231,35 +267,34 @@ static void AO_spin(int n)
   int i = 2 << n;
 
   while (i-- > 0)
-    j += (j - 1) << 2;
+    j += j << 2;
-  /* Given 'spin_dummy' is initialized to 1, j is 1 after the loop.     */
+  /* Given spin_dummy is initialized to 0, j is 0 after the loop.   */
   AO_store(&spin_dummy, j);
 }
 
-void AO_pause(int n)
+AO_API void AO_pause(int n)
 {
-  if (n < 12)
+  if (n < 12) {
     AO_spin(n);
-  else
+  } else {
-    {
+#   ifdef AO_USE_NANOSLEEP
-#     ifdef AO_USE_NANOSLEEP
+      struct timespec ts;
-        struct timespec ts;
+      ts.tv_sec = 0;
-        ts.tv_sec = 0;
+      ts.tv_nsec = n > 28 ? 100000L * 1000 : 1L << (n - 2);
-        ts.tv_nsec = n > 28 ? 100000L * 1000 : 1L << (n - 2);
+      nanosleep(&ts, 0);
-        nanosleep(&ts, 0);
+#   elif defined(AO_USE_WIN32_PTHREADS)
-#     elif defined(AO_USE_WIN32_PTHREADS)
+      Sleep(n > 28 ? 100 /* millis */
-        Sleep(n > 28 ? 100 /* millis */
+                   : n < 22 ? 1 : (DWORD)1 << (n - 22));
-                     : n < 22 ? 1 : (DWORD)1 << (n - 22));
+#   else
-#     else
+      struct timeval tv;
-        struct timeval tv;
+      /* Short async-signal-safe sleep. */
-        /* Short async-signal-safe sleep. */
+      int usec = n > 28 ? 100000 : 1 << (n - 12);
-        int usec = n > 28 ? 100000 : 1 << (n - 12);
                 /* Use an intermediate variable (of int type) to avoid  */
                 /* "shift followed by widening conversion" warning.     */
 
-        tv.tv_sec = 0;
+      tv.tv_sec = 0;
-        tv.tv_usec = usec;
+      tv.tv_usec = usec;
-        (void)select(0, 0, 0, 0, &tv);
+      (void)select(0, 0, 0, 0, &tv);
-#     endif
+#   endif
-    }
+  }
 }

thirdparty/libatomic_ops/atomic_ops.h

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2008-2021 Ivan Maidanski
+ * Copyright (c) 2008-2022 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -153,8 +153,30 @@
 /* atomic_ops/generalize.h.                                     */
 
 /* Some common defaults.  Overridden for some architectures.    */
+
 #define AO_t size_t
 
+#if defined(__SIZEOF_POINTER__) \
+    && (__SIZEOF_POINTER__ == 2 * __SIZEOF_SIZE_T__)
+  /* Pointers are twice bigger than the machine word.   */
+# define AO_FAT_POINTER
+#endif
+
+#ifndef AO_FAT_POINTER
+# define AO_uintptr_t AO_t
+#elif defined(__e2k__)
+  /* For some reason uintptr_t is 64-bit on E2K in the protected mode.  */
+  typedef unsigned __int128 AO_uintptr_t;
+#else
+# include <inttypes.h>
+# define AO_uintptr_t uintptr_t
+#endif
+
+/* A compile-time assertion for AO_uintptr_t size.      */
+struct AO_uintptr_t_size_static_assert {
+  char dummy[sizeof(AO_uintptr_t) == sizeof(void *) ? 1 : -1];
+};
+
 /* The test_and_set primitive returns an AO_TS_VAL_t value.     */
 /* AO_TS_t is the type of an in-memory test-and-set location.   */
 
@@ -235,12 +257,36 @@
 # define AO_ALIGNOF_SUPPORTED 1
 #endif
 
+#if defined(AO_DLL) && !defined(AO_API)
+# ifdef AO_BUILD
+#   if defined(__CEGCC__) || (defined(__MINGW32__) && !defined(__cplusplus))
+#     define AO_API __declspec(dllexport)
+#   elif defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__) \
+         || defined(__DMC__) || defined(__MINGW32__) || defined(__WATCOMC__)
+#     define AO_API extern __declspec(dllexport)
+#   endif
+# else
+#   if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CEGCC__) \
+       || defined(__CYGWIN__) || defined(__DMC__)
+#     define AO_API __declspec(dllimport)
+#   elif defined(__MINGW32_DELAY_LOAD__)
+#     define AO_API __declspec(dllexport)
+#   elif defined(__MINGW32__) || defined(__WATCOMC__)
+#     define AO_API extern __declspec(dllimport)
+#   endif
+# endif
+#endif /* AO_DLL */
+
+#ifndef AO_API
+# define AO_API extern
+#endif
+
 #ifdef AO_ALIGNOF_SUPPORTED
 # define AO_ASSERT_ADDR_ALIGNED(addr) \
-    assert(((size_t)(addr) & (__alignof__(*(addr)) - 1)) == 0)
+    assert(((AO_uintptr_t)(addr) & (__alignof__(*(addr)) - 1)) == 0)
 #else
 # define AO_ASSERT_ADDR_ALIGNED(addr) \
-    assert(((size_t)(addr) & (sizeof(*(addr)) - 1)) == 0)
+    assert(((AO_uintptr_t)(addr) & (sizeof(*(addr)) - 1)) == 0)
 #endif /* !AO_ALIGNOF_SUPPORTED */
 
 #if defined(__GNUC__) && !defined(__INTEL_COMPILER)
@@ -270,7 +316,7 @@
 #   include <machine/sys/inline.h>
 #   define AO_compiler_barrier() _Asm_sched_fence()
 # else
-    /* FIXME: We do not know how to do this.  This is a guess. */
+    /* FIXME - We do not know how to do this.  This is a guess. */
     /* And probably a bad one.                                  */
     static volatile int AO_barrier_dummy;
 #   define AO_compiler_barrier() (void)(AO_barrier_dummy = AO_barrier_dummy)
@@ -340,6 +386,8 @@
 #   define AO_CAN_EMUL_CAS
 # elif defined(__avr32__)
 #   include "atomic_ops/sysdeps/gcc/avr32.h"
+# elif defined(__e2k__)
+#   include "atomic_ops/sysdeps/gcc/e2k.h"
 # elif defined(__hexagon__)
 #   include "atomic_ops/sysdeps/gcc/hexagon.h"
 # elif defined(__nios2__)
@@ -395,8 +443,10 @@
 
 #if defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \
         || (defined(__WATCOMC__) && defined(__NT__))
-# if defined(_AMD64_) || defined(_M_X64) || defined(_M_ARM64)
+# if defined(_AMD64_) || defined(_M_X64)
 #   include "atomic_ops/sysdeps/msftc/x86_64.h"
+# elif defined(_M_ARM64)
+#   include "atomic_ops/sysdeps/msftc/arm64.h"
 # elif defined(_M_IX86) || defined(x86)
 #   include "atomic_ops/sysdeps/msftc/x86.h"
 # elif defined(_M_ARM) || defined(ARM) || defined(_ARM_)

thirdparty/libatomic_ops/atomic_ops/ao_version.h

@@ -34,5 +34,5 @@
 
 /* The version here should match that in configure.ac and README.       */
 #define AO_VERSION_MAJOR 7
-#define AO_VERSION_MINOR 6
+#define AO_VERSION_MINOR 9
-#define AO_VERSION_MICRO 12 /* 7.6.12 */
+#define AO_VERSION_MICRO 0 /* 7.9.0 */

thirdparty/libatomic_ops/atomic_ops/sysdeps/armcc/arm_v6.h

@@ -13,7 +13,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h

@@ -47,14 +47,15 @@
   extern "C" {
 #endif
 
-AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
+AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
-                                         AO_t new_val);
+                                                AO_t old_val, AO_t new_val);
 
-int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
+AO_API int
-                                                AO_t old_val1, AO_t old_val2,
+AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
-                                                AO_t new_val1, AO_t new_val2);
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2);
 
-void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
+AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
 
 #ifndef AO_HAVE_fetch_compare_and_swap_full
 # define AO_fetch_compare_and_swap_full(addr, old, newval) \

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/aarch64.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/alpha.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/arm.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/e2k.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022 Ivan Maidanski
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* As of clang-9, all __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n are missing.   */
+#define AO_GCC_FORCE_HAVE_CAS
+
+#if (__SIZEOF_SIZE_T__ == 4) \
+    || (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(__clang__)) \
+    || (defined(__clang__) && __iset__ >= 5 /* elbrus-v5 or later */ \
+        && defined(AO_PREFER_BUILTIN_ATOMICS))
+  /* Note for 128-bit operations: clang reports warning                 */
+  /* "large atomic operation may incur significant performance penalty" */
+  /* and requires LDFLAGS="-latomic", thus this is not on by default.   */
+# define AO_GCC_HAVE_double_SYNC_CAS
+# include "../standard_ao_double_t.h"
+#endif
+
+#include "generic.h"
+
+#undef AO_GCC_FORCE_HAVE_CAS
+#undef AO_GCC_HAVE_double_SYNC_CAS

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h

@@ -58,6 +58,10 @@
   }
 # define AO_HAVE_nop_full
 
+#elif defined(AO_THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
+  /* Workaround a compiler warning (reported by gcc-11, at least)   */
+  /* that atomic_thread_fence is unsupported with thread sanitizer. */
+
 #else
   AO_INLINE void
   AO_nop_read(void)
@@ -82,7 +86,7 @@
     __atomic_thread_fence(__ATOMIC_SEQ_CST);
   }
 # define AO_HAVE_nop_full
-#endif /* !AO_UNIPROCESSOR */
+#endif /* !AO_UNIPROCESSOR && !AO_THREAD_SANITIZER */
 
 #include "generic-small.h"
 
@@ -95,28 +99,32 @@
   AO_INLINE AO_TS_VAL_t
   AO_test_and_set(volatile AO_TS_t *addr)
   {
-    return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELAXED);
+    return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELAXED)
+                         ? AO_TS_SET : AO_TS_CLEAR);
   }
 # define AO_HAVE_test_and_set
 
   AO_INLINE AO_TS_VAL_t
   AO_test_and_set_acquire(volatile AO_TS_t *addr)
   {
-    return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_ACQUIRE);
+    return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_ACQUIRE)
+                         ? AO_TS_SET : AO_TS_CLEAR);
   }
 # define AO_HAVE_test_and_set_acquire
 
   AO_INLINE AO_TS_VAL_t
   AO_test_and_set_release(volatile AO_TS_t *addr)
   {
-    return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELEASE);
+    return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELEASE)
+                         ? AO_TS_SET : AO_TS_CLEAR);
   }
 # define AO_HAVE_test_and_set_release
 
   AO_INLINE AO_TS_VAL_t
   AO_test_and_set_full(volatile AO_TS_t *addr)
   {
-    return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_SEQ_CST);
+    return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_SEQ_CST)
+                         ? AO_TS_SET : AO_TS_CLEAR);
   }
 # define AO_HAVE_test_and_set_full
 #endif /* !AO_PREFER_GENERALIZED */

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/hexagon.h

@@ -3,7 +3,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/m68k.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/mips.h

@@ -5,7 +5,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/powerpc.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h

@@ -3,7 +3,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.
@@ -25,8 +25,76 @@
 # endif
 #endif /* !__clang__ */
 
+#if defined(__riscv_zacas) && __riscv_xlen == 64 && !defined(AO_NO_DOUBLE_CAS)
+  /* TODO: Support also rv32, i.e. use amocas.w.        */
+
+# define AO_SKIPATOMIC_double_load
+# define AO_SKIPATOMIC_double_load_acquire
+# define AO_SKIPATOMIC_double_store
+# define AO_SKIPATOMIC_double_store_release
+
+# include "../standard_ao_double_t.h"
+
+  AO_INLINE int
+  AO_double_compare_and_swap(volatile AO_double_t *addr,
+                             AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t expected = old_val;
+
+    __asm__ __volatile__("amocas.q %0, %z2, %1"
+                         : "+rJ" (expected.AO_whole), "+A" (*addr)
+                         : "rJ" (new_val.AO_whole));
+    return expected.AO_whole == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap
+
+  AO_INLINE int
+  AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
+                                     AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t expected = old_val;
+
+    __asm__ __volatile__("amocas.q.aq %0, %z2, %1"
+                         : "+rJ" (expected.AO_whole), "+A" (*addr)
+                         : "rJ" (new_val.AO_whole));
+    return expected.AO_whole == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_acquire
+
+  AO_INLINE int
+  AO_double_compare_and_swap_release(volatile AO_double_t *addr,
+                                     AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t expected = old_val;
+
+    __asm__ __volatile__("amocas.q.rl %0, %z2, %1"
+                         : "+rJ" (expected.AO_whole), "+A" (*addr)
+                         : "rJ" (new_val.AO_whole));
+    return expected.AO_whole == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_release
+
+  AO_INLINE int
+  AO_double_compare_and_swap_full(volatile AO_double_t *addr,
+                                  AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t expected = old_val;
+
+    __asm__ __volatile__("amocas.q.aqrl %0, %z2, %1"
+                         : "+rJ" (expected.AO_whole), "+A" (*addr)
+                         : "rJ" (new_val.AO_whole));
+    return expected.AO_whole == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_full
+
+#endif /* __riscv_zacas */
+
 #include "generic.h"
 
 #undef AO_GCC_FORCE_HAVE_CAS
 #undef AO_NO_char_ARITHM
 #undef AO_NO_short_ARITHM
+#undef AO_SKIPATOMIC_double_load
+#undef AO_SKIPATOMIC_double_load_acquire
+#undef AO_SKIPATOMIC_double_store
+#undef AO_SKIPATOMIC_double_store_release

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/s390.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sh.h

@@ -6,7 +6,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h

@@ -8,15 +8,20 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.
  *
  */
 
-/* TODO: Very incomplete; Add support for sparc64.      */
+#if (AO_GNUC_PREREQ(12, 0) || AO_CLANG_PREREQ(13, 0)) \
-/* Non-ancient SPARCs provide compare-and-swap (casa).  */
+    && !defined(AO_DISABLE_GCC_ATOMICS)
+  /* Probably, it could be enabled for earlier compiler versions as well. */
+
+# include "generic.h"
+
+#else /* AO_DISABLE_GCC_ATOMICS */
 
 #include "../all_atomic_load_store.h"
 
@@ -43,23 +48,17 @@ AO_test_and_set_full(volatile AO_TS_t *addr) {
 /* Returns nonzero if the comparison succeeded. */
 AO_INLINE int
 AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) {
-  AO_t ret;
   __asm__ __volatile__ ("membar #StoreLoad | #LoadLoad\n\t"
 #                       if defined(__arch64__)
-                          "casx [%2],%0,%1\n\t"
+                          "casx [%1],%2,%0\n\t"
 #                       else
-                          "cas [%2],%0,%1\n\t" /* 32-bit version */
+                          "cas [%1],%2,%0\n\t" /* 32-bit version */
 #                       endif
                         "membar #StoreLoad | #StoreStore\n\t"
-                        "cmp %0,%1\n\t"
+                        : "+r" (new_val)
-                        "be,a 0f\n\t"
+                        : "r" (addr), "r" (old)
-                        "mov 1,%0\n\t"/* one insn after branch always executed */
+                        : "memory");
-                        "clr %0\n\t"
+  return new_val == old;
-                        "0:\n\t"
-                        : "=r" (ret), "+r" (new_val)
-                        : "r" (addr), "0" (old)
-                        : "memory", "cc");
-  return (int)ret;
 }
 #   define AO_HAVE_compare_and_swap_full
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
@@ -85,3 +84,5 @@ AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) {
 /* TODO: Extend this for SPARC v8 and v9 (V8 also has swap, V9 has CAS, */
 /* there are barriers like membar #LoadStore, CASA (32-bit) and         */
 /* CASXA (64-bit) instructions added in V9).                            */
+
+#endif /* AO_DISABLE_GCC_ATOMICS */

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h

@@ -9,8 +9,6 @@
  * modified is included with the above copyright notice.
  */
 
-/* Minimal support for tile.    */
-
 #if (AO_GNUC_PREREQ(4, 8) || AO_CLANG_PREREQ(3, 4)) \
     && !defined(AO_DISABLE_GCC_ATOMICS)
 
@@ -18,6 +16,8 @@
 
 #else /* AO_DISABLE_GCC_ATOMICS */
 
+  /* Minimal support for tile.  */
+
 # include "../all_atomic_load_store.h"
 
 # include "../test_and_set_t_is_ao_t.h"

thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.
@@ -68,14 +68,14 @@
 #     endif
 #   endif /* __x86_64__ */
 
-# elif AO_GNUC_PREREQ(7, 0) && !defined(AO_PREFER_BUILTIN_ATOMICS) \
+# elif defined(__x86_64__) && !defined(AO_PREFER_BUILTIN_ATOMICS) \
-       && !defined(AO_THREAD_SANITIZER) && !defined(__MINGW32__)
+       && !defined(AO_THREAD_SANITIZER)
-    /* gcc-7.x/x64 (gcc-7.2, at least) requires -latomic flag in case   */
+    /* gcc/x64 (as of gcc-12.2) requires -latomic flag in case          */
     /* of double-word atomic operations use (but not in case of TSan).  */
-    /* TODO: Revise it for the future gcc-7 releases. */
+    /* TODO: Revise it for the future gcc releases. */
 #   define AO_SKIPATOMIC_double_compare_and_swap_ANY
 #   define AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY
-# endif /* __GNUC__ && !__clang__ */
+# endif /* __x86_64__ && !__clang__ */
 
 # ifdef AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY
 #   define AO_SKIPATOMIC_double_load

thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h

@@ -39,7 +39,7 @@
 
 /* We define only the full barrier variants, and count on the           */
 /* generalization section below to fill in the rest.                    */
-extern pthread_mutex_t AO_pt_lock;
+AO_API pthread_mutex_t AO_pt_lock;
 
 #ifdef __cplusplus
   } /* extern "C" */

thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009-2017 Ivan Maidanski
+ * Copyright (c) 2009-2021 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -21,21 +21,20 @@
  * SOFTWARE.
  */
 
-#ifndef AO_ASSUME_WINDOWS98
-  /* CAS is always available */
-# define AO_ASSUME_WINDOWS98
-#endif
-#include "common32_defs.h"
-
-#include "../test_and_set_t_is_ao_t.h"
-/* AO_test_and_set_full() is emulated using CAS.                        */
-
 /* Some ARM slide set, if it has been read correctly, claims that Loads */
 /* followed by either a Load or a Store are ordered, but nothing else.  */
 /* It is assumed that Windows interrupt handlers clear the LL/SC flag.  */
 /* Unaligned accesses are not guaranteed to be atomic.                  */
 #include "../all_aligned_atomic_load_store.h"
 
+#define AO_T_IS_INT
+
+#ifndef AO_ASSUME_WINDOWS98
+  /* CAS is always available */
+# define AO_ASSUME_WINDOWS98
+#endif
+#include "common32_defs.h"
+
 /* If only a single processor is used, we can define AO_UNIPROCESSOR.   */
 #ifdef AO_UNIPROCESSOR
   AO_INLINE void AO_nop_full(void)
@@ -47,16 +46,67 @@
   /* AO_nop_full() is emulated using AO_test_and_set_full().            */
 #endif
 
-#if _M_ARM >= 6
+#ifndef AO_HAVE_test_and_set_full
-/* ARMv6 is the first architecture providing support for simple LL/SC.  */
+# include "../test_and_set_t_is_ao_t.h"
+  /* AO_test_and_set_full() is emulated. */
+#endif
 
-/* #include "../standard_ao_double_t.h" */
+#if _M_ARM >= 7 && !defined(AO_NO_DOUBLE_CAS)
-/* TODO: implement double-wide operations (similar to x86).     */
 
-#else /* _M_ARM < 6 */
+# include "../standard_ao_double_t.h"
 
-/* TODO: implement AO_test_and_set_full using SWP.      */
+/* These intrinsics are supposed to use LDREXD/STREXD.  */
+# pragma intrinsic (_InterlockedCompareExchange64)
+# pragma intrinsic (_InterlockedCompareExchange64_acq)
+# pragma intrinsic (_InterlockedCompareExchange64_nf)
+# pragma intrinsic (_InterlockedCompareExchange64_rel)
 
-#endif /* _M_ARM < 6 */
+  AO_INLINE int
+  AO_double_compare_and_swap(volatile AO_double_t *addr,
+                             AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    return (double_ptr_storage)_InterlockedCompareExchange64_nf(
+                                        (__int64 volatile *)addr,
+                                        new_val.AO_whole /* exchange */,
+                                        old_val.AO_whole) == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap
 
-#define AO_T_IS_INT
+  AO_INLINE int
+  AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
+                                     AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    return (double_ptr_storage)_InterlockedCompareExchange64_acq(
+                                        (__int64 volatile *)addr,
+                                        new_val.AO_whole /* exchange */,
+                                        old_val.AO_whole) == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_acquire
+
+  AO_INLINE int
+  AO_double_compare_and_swap_release(volatile AO_double_t *addr,
+                                     AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    return (double_ptr_storage)_InterlockedCompareExchange64_rel(
+                                        (__int64 volatile *)addr,
+                                        new_val.AO_whole /* exchange */,
+                                        old_val.AO_whole) == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_release
+
+  AO_INLINE int
+  AO_double_compare_and_swap_full(volatile AO_double_t *addr,
+                                  AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    return (double_ptr_storage)_InterlockedCompareExchange64(
+                                        (__int64 volatile *)addr,
+                                        new_val.AO_whole /* exchange */,
+                                        old_val.AO_whole) == old_val.AO_whole;
+  }
+# define AO_HAVE_double_compare_and_swap_full
+
+#endif /* _M_ARM >= 7 && !AO_NO_DOUBLE_CAS */

thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm64.h

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2021 Ivan Maidanski
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "../all_aligned_atomic_load_store.h"
+
+#ifndef AO_ASSUME_WINDOWS98
+# define AO_ASSUME_WINDOWS98
+#endif
+#ifndef AO_USE_INTERLOCKED_INTRINSICS
+# define AO_USE_INTERLOCKED_INTRINSICS
+#endif
+#include "common32_defs.h"
+
+#ifndef AO_HAVE_test_and_set_full
+# include "../test_and_set_t_is_ao_t.h"
+  /* AO_test_and_set_full() is emulated using word-wide CAS.    */
+#endif
+
+#ifndef AO_NO_DOUBLE_CAS
+
+# include "../standard_ao_double_t.h"
+
+# pragma intrinsic (_InterlockedCompareExchange128)
+# pragma intrinsic (_InterlockedCompareExchange128_acq)
+# pragma intrinsic (_InterlockedCompareExchange128_nf)
+# pragma intrinsic (_InterlockedCompareExchange128_rel)
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double(volatile AO_double_t *addr,
+                                    AO_t old_val1, AO_t old_val2,
+                                    AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_nf((volatile __int64 *)addr,
+                                             new_val2 /* high */,
+                                             new_val1 /* low */,
+                                             comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_acq((volatile __int64 *)addr,
+                                              new_val2 /* high */,
+                                              new_val1 /* low */,
+                                              comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_acquire
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_release(volatile AO_double_t *addr,
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_rel((volatile __int64 *)addr,
+                                              new_val2 /* high */,
+                                              new_val1 /* low */,
+                                              comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_release
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
+                                         AO_t old_val1, AO_t old_val2,
+                                         AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128((volatile __int64 *)addr,
+                                          new_val2 /* high */,
+                                          new_val1 /* low */,
+                                          comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_full
+
+#endif /* !AO_NO_DOUBLE_CAS */

thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009-2021 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,12 +21,13 @@
  * SOFTWARE.
  */
 
-/* If AO_ASSUME_VISTA is defined, we assume Windows Server 2003, Vista  */
-/* or later.                                                            */
-
 #include "../all_aligned_atomic_load_store.h"
 
-#include "../test_and_set_t_is_char.h"
+#if !defined(AO_ASSUME_VISTA) && _MSC_VER >= 1910
+  /* Visual Studio 2017 (15.0) discontinued support of Windows XP.  */
+  /* We assume Windows Server 2003, Vista or later.                 */
+# define AO_ASSUME_VISTA
+#endif
 
 #if !defined(AO_ASSUME_WINDOWS98) \
     && (defined(AO_ASSUME_VISTA) || _MSC_VER >= 1400)
@@ -33,6 +35,13 @@
 # define AO_ASSUME_WINDOWS98
 #endif
 
+#if !defined(AO_USE_PENTIUM4_INSTRS) && _M_IX86_FP >= 2 /* SSE2 */
+  /* "mfence" is a part of SSE2 set (introduced on Intel Pentium 4).    */
+# define AO_USE_PENTIUM4_INSTRS
+#endif
+
+#define AO_T_IS_INT
+
 #ifndef AO_USE_INTERLOCKED_INTRINSICS
   /* _Interlocked primitives (Inc, Dec, Xchg, Add) are always available */
 # define AO_USE_INTERLOCKED_INTRINSICS
@@ -91,9 +100,12 @@ AO_nop_full(void)
 # define AO_HAVE_short_fetch_and_add_full
 #endif /* !AO_NO_ASM_XADD */
 
-AO_INLINE AO_TS_VAL_t
+#ifndef AO_HAVE_test_and_set_full
-AO_test_and_set_full(volatile AO_TS_t *addr)
+# include "../test_and_set_t_is_char.h"
-{
+
+  AO_INLINE AO_TS_VAL_t
+  AO_test_and_set_full(volatile AO_TS_t *addr)
+  {
     __asm
     {
         mov     eax,0xff                ; /* AO_TS_SET */
@@ -101,8 +113,9 @@ AO_test_and_set_full(volatile AO_TS_t *addr)
         xchg    byte ptr [ebx],al       ;
     }
     /* Ignore possible "missing return value" warning here. */
-}
+  }
-#define AO_HAVE_test_and_set_full
+# define AO_HAVE_test_and_set_full
+#endif
 
 #if defined(_WIN64) && !defined(CPPCHECK)
 # error wrong architecture
@@ -134,8 +147,6 @@ AO_test_and_set_full(volatile AO_TS_t *addr)
 # define AO_HAVE_double_compare_and_swap_full
 #endif /* AO_ASSUME_VISTA */
 
-#define AO_T_IS_INT
-
 /* Real X86 implementations, except for some old WinChips, appear       */
 /* to enforce ordering between memory operations, EXCEPT that a later   */
 /* read can pass earlier writes, presumably due to the visible          */

thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009-2021 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -28,185 +29,20 @@
 /* presence of store buffers.                                           */
 /* We ignore the fact that the official specs                           */
 /* seem to be much weaker (and arguably too weak to be usable).         */
-
 #include "../ordered_except_wr.h"
 
-#ifdef AO_ASM_X64_AVAILABLE
+#ifndef AO_ASSUME_WINDOWS98
-# include "../test_and_set_t_is_char.h"
+  /* CAS is always available */
-#else
+# define AO_ASSUME_WINDOWS98
-# include "../test_and_set_t_is_ao_t.h"
 #endif
+#ifndef AO_USE_INTERLOCKED_INTRINSICS
+# define AO_USE_INTERLOCKED_INTRINSICS
+#endif
+#include "common32_defs.h"
 
-/* Assume _MSC_VER >= 1400 */
+#ifdef AO_ASM_X64_AVAILABLE
-#include <intrin.h>
-
-#pragma intrinsic (_InterlockedCompareExchange)
-#pragma intrinsic (_InterlockedCompareExchange64)
-
-#ifndef AO_PREFER_GENERALIZED
-
-# pragma intrinsic (_InterlockedIncrement)
-# pragma intrinsic (_InterlockedIncrement64)
-# pragma intrinsic (_InterlockedDecrement)
-# pragma intrinsic (_InterlockedDecrement64)
-# pragma intrinsic (_InterlockedExchangeAdd)
-# pragma intrinsic (_InterlockedExchangeAdd64)
-
-AO_INLINE AO_t
-AO_fetch_and_add_full (volatile AO_t *p, AO_t incr)
-{
-  return _InterlockedExchangeAdd64((__int64 volatile *)p, incr);
-}
-#define AO_HAVE_fetch_and_add_full
-
-AO_INLINE AO_t
-AO_fetch_and_add1_full (volatile AO_t *p)
-{
-  return _InterlockedIncrement64((__int64 volatile *)p) - 1;
-}
-#define AO_HAVE_fetch_and_add1_full
-
-AO_INLINE AO_t
-AO_fetch_and_sub1_full (volatile AO_t *p)
-{
-  return _InterlockedDecrement64((__int64 volatile *)p) + 1;
-}
-#define AO_HAVE_fetch_and_sub1_full
-#endif /* !AO_PREFER_GENERALIZED */
-
-AO_INLINE AO_t
-AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
-                               AO_t new_val)
-{
-  return (AO_t)_InterlockedCompareExchange64((__int64 volatile *)addr,
-                                             new_val, old_val);
-}
-#define AO_HAVE_fetch_compare_and_swap_full
-
-AO_INLINE unsigned int
-AO_int_fetch_compare_and_swap_full(volatile unsigned int *addr,
-                                   unsigned int old_val, unsigned int new_val)
-{
-  return _InterlockedCompareExchange((long volatile *)addr, new_val, old_val);
-}
-#define AO_HAVE_int_fetch_compare_and_swap_full
-
-#ifndef AO_PREFER_GENERALIZED
-AO_INLINE unsigned int
-AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
-{
-  return _InterlockedExchangeAdd((long volatile *)p, incr);
-}
-#define AO_HAVE_int_fetch_and_add_full
-
-  AO_INLINE unsigned int
-  AO_int_fetch_and_add1_full(volatile unsigned int *p)
-  {
-    return _InterlockedIncrement((long volatile *)p) - 1;
-  }
-# define AO_HAVE_int_fetch_and_add1_full
-
-  AO_INLINE unsigned int
-  AO_int_fetch_and_sub1_full(volatile unsigned int *p)
-  {
-    return _InterlockedDecrement((long volatile *)p) + 1;
-  }
-# define AO_HAVE_int_fetch_and_sub1_full
-#endif /* !AO_PREFER_GENERALIZED */
-
-#if _MSC_VER > 1400
-# pragma intrinsic (_InterlockedAnd8)
-# pragma intrinsic (_InterlockedCompareExchange16)
-# pragma intrinsic (_InterlockedOr8)
-# pragma intrinsic (_InterlockedXor8)
-
-  AO_INLINE void
-  AO_char_and_full(volatile unsigned char *p, unsigned char value)
-  {
-    _InterlockedAnd8((char volatile *)p, value);
-  }
-# define AO_HAVE_char_and_full
-
-  AO_INLINE void
-  AO_char_or_full(volatile unsigned char *p, unsigned char value)
-  {
-    _InterlockedOr8((char volatile *)p, value);
-  }
-# define AO_HAVE_char_or_full
-
-  AO_INLINE void
-  AO_char_xor_full(volatile unsigned char *p, unsigned char value)
-  {
-    _InterlockedXor8((char volatile *)p, value);
-  }
-# define AO_HAVE_char_xor_full
-
-  AO_INLINE unsigned short
-  AO_short_fetch_compare_and_swap_full(volatile unsigned short *addr,
-                                       unsigned short old_val,
-                                       unsigned short new_val)
-  {
-    return _InterlockedCompareExchange16((short volatile *)addr,
-                                         new_val, old_val);
-  }
-# define AO_HAVE_short_fetch_compare_and_swap_full
-
-# ifndef AO_PREFER_GENERALIZED
-#   pragma intrinsic (_InterlockedIncrement16)
-#   pragma intrinsic (_InterlockedDecrement16)
-
-    AO_INLINE unsigned short
-    AO_short_fetch_and_add1_full(volatile unsigned short *p)
-    {
-      return _InterlockedIncrement16((short volatile *)p) - 1;
-    }
-#   define AO_HAVE_short_fetch_and_add1_full
-
-    AO_INLINE unsigned short
-    AO_short_fetch_and_sub1_full(volatile unsigned short *p)
-    {
-      return _InterlockedDecrement16((short volatile *)p) + 1;
-    }
-#   define AO_HAVE_short_fetch_and_sub1_full
-# endif /* !AO_PREFER_GENERALIZED */
-#endif /* _MSC_VER > 1400 */
-
-#if _MSC_VER >= 1800 /* Visual Studio 2013+ */
-
-# pragma intrinsic (_InterlockedCompareExchange8)
-
-  AO_INLINE unsigned char
-  AO_char_fetch_compare_and_swap_full(volatile unsigned char *addr,
-                                      unsigned char old_val,
-                                      unsigned char new_val)
-  {
-    return _InterlockedCompareExchange8((char volatile *)addr,
-                                        new_val, old_val);
-  }
-# define AO_HAVE_char_fetch_compare_and_swap_full
-
-# ifndef AO_PREFER_GENERALIZED
-#   pragma intrinsic (_InterlockedExchangeAdd16)
-#   pragma intrinsic (_InterlockedExchangeAdd8)
-
-    AO_INLINE unsigned char
-    AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr)
-    {
-      return _InterlockedExchangeAdd8((char volatile *)p, incr);
-    }
-#   define AO_HAVE_char_fetch_and_add_full
-
-    AO_INLINE unsigned short
-    AO_short_fetch_and_add_full(volatile unsigned short *p,
-                                unsigned short incr)
-    {
-      return _InterlockedExchangeAdd16((short volatile *)p, incr);
-    }
-#   define AO_HAVE_short_fetch_and_add_full
-# endif /* !AO_PREFER_GENERALIZED */
-
-#elif defined(AO_ASM_X64_AVAILABLE)
 
+#if _MSC_VER < 1800
   AO_INLINE unsigned char
   AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr)
   {
@@ -230,13 +66,10 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
     }
   }
 # define AO_HAVE_short_fetch_and_add_full
+#endif /* _MSC_VER < 1800 */
 
-#endif /* _MSC_VER < 1800 && AO_ASM_X64_AVAILABLE */
+  /* As far as we can tell, the lfence and sfence instructions are not  */
-
+  /* currently needed or useful for cached memory accesses.             */
-#ifdef AO_ASM_X64_AVAILABLE
-
-/* As far as we can tell, the lfence and sfence instructions are not    */
-/* currently needed or useful for cached memory accesses.               */
 
   AO_INLINE void
   AO_nop_full(void)
@@ -246,57 +79,62 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
   }
 # define AO_HAVE_nop_full
 
-  AO_INLINE AO_TS_VAL_t
+# ifndef AO_HAVE_test_and_set_full
-  AO_test_and_set_full(volatile AO_TS_t *addr)
+#   include "../test_and_set_t_is_char.h"
-  {
+
-    __asm
+    AO_INLINE AO_TS_VAL_t
+    AO_test_and_set_full(volatile AO_TS_t *addr)
     {
+      __asm
+      {
         mov     rax,AO_TS_SET           ;
         mov     rbx,addr                ;
         xchg    byte ptr [rbx],al       ;
+      }
     }
-  }
+#   define AO_HAVE_test_and_set_full
-# define AO_HAVE_test_and_set_full
+# endif
 
 #endif /* AO_ASM_X64_AVAILABLE */
 
+#ifndef AO_HAVE_test_and_set_full
+# include "../test_and_set_t_is_ao_t.h"
+  /* AO_test_and_set_full() is emulated using word-wide CAS.    */
+#endif
+
 #ifdef AO_CMPXCHG16B_AVAILABLE
-/* AO_compare_double_and_swap_double_full needs implementation for Win64.
- * Also see ../gcc/x86.h for partial old Opteron workaround.
- */
 
 # if _MSC_VER >= 1500
-
 #   include "../standard_ao_double_t.h"
-
 #   pragma intrinsic (_InterlockedCompareExchange128)
 
-AO_INLINE int
+    AO_INLINE int
-AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
+    AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
-                                       AO_t old_val1, AO_t old_val2,
+                                           AO_t old_val1, AO_t old_val2,
-                                       AO_t new_val1, AO_t new_val2)
+                                           AO_t new_val1, AO_t new_val2)
-{
+    {
-   __int64 comparandResult[2];
+      __int64 comparandResult[2];
 
-   AO_ASSERT_ADDR_ALIGNED(addr);
+      AO_ASSERT_ADDR_ALIGNED(addr);
-   comparandResult[0] = old_val1; /* low */
+      comparandResult[0] = old_val1; /* low */
-   comparandResult[1] = old_val2; /* high */
+      comparandResult[1] = old_val2; /* high */
-   return _InterlockedCompareExchange128((volatile __int64 *)addr,
+      return _InterlockedCompareExchange128((volatile __int64 *)addr,
-                new_val2 /* high */, new_val1 /* low */, comparandResult);
+                                            new_val2 /* high */,
-}
+                                            new_val1 /* low */,
+                                            comparandResult);
+    }
 #   define AO_HAVE_compare_double_and_swap_double_full
 
 # elif defined(AO_ASM_X64_AVAILABLE)
-
 #   include "../standard_ao_double_t.h"
 
     /* If there is no intrinsic _InterlockedCompareExchange128 then we  */
     /* need basically what's given below.                               */
-AO_INLINE int
+    AO_INLINE int
-AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
+    AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
-                                       AO_t old_val1, AO_t old_val2,
+                                           AO_t old_val1, AO_t old_val2,
-                                       AO_t new_val1, AO_t new_val2)
+                                           AO_t new_val1, AO_t new_val2)
-{
+    {
         __asm
         {
                 mov     rdx,QWORD PTR [old_val2]        ;
@@ -306,7 +144,7 @@ AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
                 lock cmpxchg16b [addr]                  ;
                 setz    rax                             ;
         }
-}
+    }
 #   define AO_HAVE_compare_double_and_swap_double_full
 # endif /* AO_ASM_X64_AVAILABLE && (_MSC_VER < 1500) */
 

thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004-2011 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2012-2021 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,13 +23,15 @@
 
 /* For 64-bit systems, we expect the double type to hold two int64's.   */
 
-#if ((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \
+#if (((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \
-     || defined(__aarch64__)) && !defined(__ILP32__)
+      || defined(__aarch64__)) && !defined(__ILP32__)) \
+    || (defined(__e2k__) && __SIZEOF_SIZE_T__ == 8) \
+    || (defined(__riscv) && __riscv_xlen == 64)
   /* x86-64: __m128 is not applicable to atomic intrinsics.     */
 # if AO_GNUC_PREREQ(4, 7) || AO_CLANG_PREREQ(3, 6)
 #   pragma GCC diagnostic push
     /* Suppress warning about __int128 type.      */
-#   if defined(__clang__) || AO_GNUC_PREREQ(6, 4)
+#   if defined(__clang__) || AO_GNUC_PREREQ(6, 0)
 #     pragma GCC diagnostic ignored "-Wpedantic"
 #   else
       /* GCC before ~4.8 does not accept "-Wpedantic" quietly.  */
@@ -39,6 +42,9 @@
 # else /* pragma diagnostic is not supported */
     typedef unsigned __int128 double_ptr_storage;
 # endif
+#elif defined(_M_ARM64) && defined(_MSC_VER)
+  /* __int128 does not seem to be available.    */
+  typedef __declspec(align(16)) unsigned __int64 double_ptr_storage[2];
 #elif ((defined(__x86_64__) && AO_GNUC_PREREQ(4, 0)) || defined(_WIN64)) \
       && !defined(__ILP32__)
   /* x86-64 (except for x32): __m128 serves as a placeholder which also */

thirdparty/libatomic_ops/atomic_ops/sysdeps/sunc/x86.h

@@ -8,7 +8,7 @@
  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  *
  * Permission is hereby granted to use or copy this program
- * for any purpose,  provided the above notices are retained on all copies.
+ * for any purpose, provided the above notices are retained on all copies.
  * Permission to modify the code and to distribute modified code is granted,
  * provided the above notices are retained, and a notice that the code was
  * modified is included with the above copyright notice.

thirdparty/libatomic_ops/atomic_ops_malloc.c

@@ -0,0 +1,428 @@
+/*
+ * Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#if defined(HAVE_CONFIG_H)
+# include "config.h"
+#endif
+
+#ifdef DONT_USE_MMAP /* for testing */
+# undef HAVE_MMAP
+#endif
+
+#ifndef AO_BUILD
+# define AO_BUILD
+#endif
+
+#define AO_REQUIRE_CAS
+#include "atomic_ops_malloc.h"
+
+#include <string.h>     /* for ffs, which is assumed reentrant. */
+#include <stdlib.h>
+#include <assert.h>
+
+#ifdef AO_TRACE_MALLOC
+# include <stdio.h>
+# include <pthread.h>
+#endif
+
+#if defined(AO_ADDRESS_SANITIZER) && !defined(AO_NO_MALLOC_POISON)
+  /* #include "sanitizer/asan_interface.h" */
+  void __asan_poison_memory_region(void *, size_t);
+  void __asan_unpoison_memory_region(void *, size_t);
+# define ASAN_POISON_MEMORY_REGION(addr, size) \
+                __asan_poison_memory_region(addr, size)
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
+                __asan_unpoison_memory_region(addr, size)
+#else
+# define ASAN_POISON_MEMORY_REGION(addr, size) (void)0
+# define ASAN_UNPOISON_MEMORY_REGION(addr, size) (void)0
+#endif /* !AO_ADDRESS_SANITIZER */
+
+#if (defined(_WIN32_WCE) || defined(__MINGW32CE__)) && !defined(AO_HAVE_abort)
+# define abort() _exit(-1) /* there is no abort() in WinCE */
+#endif
+
+/*
+ * We round up each allocation request to the next power of two
+ * minus one word.
+ * We keep one stack of free objects for each size.  Each object has
+ * an initial word (offset from the visible pointer is -sizeof(AO_uintptr_t))
+ * which contains either:
+ * - the binary log of the object size in bytes (for small objects), or
+ * - the object size (a multiple of CHUNK_SIZE) for large objects.
+ * The second case only arises if mmap-based allocation is supported.
+ * We align the user-visible part of each object on an ALIGNMENT
+ * byte boundary.  That means that the actual (hidden) start of
+ * the object starts a word before this boundary.
+ */
+
+#ifndef LOG_MAX_SIZE
+# define LOG_MAX_SIZE 16
+        /* We assume that 2**LOG_MAX_SIZE is a multiple of page size. */
+#endif
+
+#ifndef ALIGNMENT
+# define ALIGNMENT 16
+        /* Assumed to be at least sizeof(AO_uintptr_t).         */
+#endif
+
+#define CHUNK_SIZE (1 << LOG_MAX_SIZE)
+
+#ifndef AO_INITIAL_HEAP_SIZE
+# ifndef AO_INITIAL_HEAP_CHUNKS
+#   define AO_INITIAL_HEAP_CHUNKS 2*(LOG_MAX_SIZE+1)
+# endif
+# define AO_INITIAL_HEAP_SIZE (AO_INITIAL_HEAP_CHUNKS * CHUNK_SIZE)
+#endif /* !AO_INITIAL_HEAP_SIZE */
+
+static char AO_initial_heap[AO_INITIAL_HEAP_SIZE]; /* ~2MB by default */
+
+static AO_internal_ptr_t volatile initial_heap_ptr = 0;
+
+#if defined(HAVE_MMAP)
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+
+#if defined(MAP_ANONYMOUS) || defined(MAP_ANON)
+# define USE_MMAP_ANON
+#endif
+
+#ifdef USE_MMAP_FIXED
+# define GC_MMAP_FLAGS (MAP_FIXED | MAP_PRIVATE)
+        /* Seems to yield better performance on Solaris 2, but can      */
+        /* be unreliable if something is already mapped at the address. */
+#else
+# define GC_MMAP_FLAGS MAP_PRIVATE
+#endif
+
+#ifdef USE_MMAP_ANON
+# if defined(CPPCHECK)
+#   define OPT_MAP_ANON 0x20 /* taken from linux */
+# elif defined(MAP_ANONYMOUS)
+#   define OPT_MAP_ANON MAP_ANONYMOUS
+# else
+#   define OPT_MAP_ANON MAP_ANON
+# endif
+#else
+# include <unistd.h> /* for close() */
+# define OPT_MAP_ANON 0
+#endif
+
+static volatile AO_t mmap_enabled = 0;
+
+AO_API void
+AO_malloc_enable_mmap(void)
+{
+# if defined(__sun)
+    AO_store_release(&mmap_enabled, 1);
+            /* Workaround for Sun CC */
+# else
+    AO_store(&mmap_enabled, 1);
+# endif
+}
+
+static char *get_mmaped(size_t sz)
+{
+  char * result;
+# ifdef USE_MMAP_ANON
+#   define zero_fd -1
+# else
+    int zero_fd;
+# endif
+
+  assert(!(sz & (CHUNK_SIZE - 1)));
+  if (!mmap_enabled)
+    return 0;
+
+# ifndef USE_MMAP_ANON
+    zero_fd = open("/dev/zero", O_RDONLY);
+    if (zero_fd == -1)
+      return 0;
+# endif
+  result = (char *)mmap(0, sz, PROT_READ | PROT_WRITE,
+                        GC_MMAP_FLAGS | OPT_MAP_ANON,
+                        zero_fd, 0 /* offset */);
+# ifndef USE_MMAP_ANON
+    close(zero_fd);
+# endif
+  if (AO_EXPECT_FALSE(result == MAP_FAILED))
+    result = NULL;
+  return result;
+}
+
+#ifndef SIZE_MAX
+# include <limits.h>
+#endif
+#if defined(SIZE_MAX) && !defined(CPPCHECK)
+# define AO_SIZE_MAX ((size_t)SIZE_MAX)
+            /* Extra cast to workaround some buggy SIZE_MAX definitions. */
+#else
+# define AO_SIZE_MAX (~(size_t)0)
+#endif
+
+/* Saturated addition of size_t values.  Used to avoid value wrap       */
+/* around on overflow.  The arguments should have no side effects.      */
+#define SIZET_SAT_ADD(a, b) \
+    (AO_EXPECT_FALSE((a) >= AO_SIZE_MAX - (b)) ? AO_SIZE_MAX : (a) + (b))
+
+/* Allocate an object of size (incl. header) of size > CHUNK_SIZE.      */
+/* sz includes space for a pointer-sized header.                        */
+static char *
+AO_malloc_large(size_t sz)
+{
+  void *result;
+
+  /* The header will force us to waste ALIGNMENT bytes, including the   */
+  /* header.  Round to multiple of CHUNK_SIZE.                          */
+  sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1)
+            & ~(size_t)(CHUNK_SIZE - 1);
+  assert(sz > LOG_MAX_SIZE);
+  result = get_mmaped(sz);
+  if (AO_EXPECT_FALSE(NULL == result))
+    return NULL;
+
+  result = (AO_uintptr_t *)result + ALIGNMENT / sizeof(AO_uintptr_t);
+  ((AO_uintptr_t *)result)[-1] = (AO_uintptr_t)sz;
+  return (char *)result;
+}
+
+static void
+AO_free_large(void *p)
+{
+  size_t sz = (size_t)(((AO_uintptr_t *)p)[-1]);
+
+  if (munmap((AO_uintptr_t *)p - ALIGNMENT / sizeof(AO_uintptr_t), sz) != 0)
+    abort();  /* Programmer error.  Not really async-signal-safe, but ... */
+}
+
+#else /* !HAVE_MMAP */
+
+AO_API void
+AO_malloc_enable_mmap(void)
+{
+}
+
+#define get_mmaped(sz) ((char*)0)
+#define AO_malloc_large(sz) ((char*)0)
+#define AO_free_large(p) abort()
+                /* Programmer error.  Not really async-signal-safe, but ... */
+
+#endif /* !HAVE_MMAP */
+
+/* TODO: Duplicates (partially) the definitions in atomic_ops_stack.c.  */
+#if defined(AO_FAT_POINTER) || defined(AO_STACK_USE_CPTR)
+  AO_INLINE int
+  AO_cptr_compare_and_swap(AO_internal_ptr_t volatile *addr,
+                        AO_internal_ptr_t old_val, AO_internal_ptr_t new_val)
+  {
+    return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+
+  AO_INLINE int
+  AO_cptr_compare_and_swap_acquire(AO_internal_ptr_t volatile *addr,
+                        AO_internal_ptr_t old_val, AO_internal_ptr_t new_val)
+  {
+    return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+  }
+
+# define AO_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
+#else
+# define AO_cptr_compare_and_swap AO_compare_and_swap
+# define AO_cptr_compare_and_swap_acquire AO_compare_and_swap_acquire
+# define AO_cptr_load AO_load
+#endif
+
+static char *
+get_chunk(void)
+{
+  AO_internal_ptr_t my_chunk_ptr;
+
+  for (;;) {
+    AO_internal_ptr_t initial_ptr = AO_cptr_load(&initial_heap_ptr);
+
+    my_chunk_ptr = AO_EXPECT_FALSE(0 == initial_ptr) ?
+                    (AO_internal_ptr_t)AO_initial_heap : initial_ptr;
+    /* Round up the pointer to ALIGNMENT.       */
+#   ifdef AO_STACK_USE_CPTR
+      my_chunk_ptr += ((size_t)ALIGNMENT - (size_t)(AO_uintptr_t)my_chunk_ptr)
+                         & (ALIGNMENT - 1);
+#   else
+      my_chunk_ptr = (AO_internal_ptr_t)(((AO_uintptr_t)my_chunk_ptr
+                                            + ALIGNMENT-1)
+                                         & ~(AO_uintptr_t)(ALIGNMENT-1));
+#   endif
+    if (initial_ptr != my_chunk_ptr) {
+      /* Align correctly.  If this fails, someone else did it for us.   */
+      assert(my_chunk_ptr != 0);
+      (void)AO_cptr_compare_and_swap_acquire(&initial_heap_ptr, initial_ptr,
+                                             my_chunk_ptr);
+    }
+
+    if (AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr
+                            < (AO_uintptr_t)AO_initial_heap)
+        || AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr
+                            > (AO_uintptr_t)(AO_initial_heap
+                                    + AO_INITIAL_HEAP_SIZE - CHUNK_SIZE))) {
+      /* We failed.  The initial heap is used up.       */
+      my_chunk_ptr = (AO_internal_ptr_t)get_mmaped(CHUNK_SIZE);
+#     if !defined(CPPCHECK)
+        assert(((AO_uintptr_t)my_chunk_ptr & (ALIGNMENT - 1)) == 0);
+#     endif
+      break;
+    }
+    if (AO_cptr_compare_and_swap(&initial_heap_ptr, my_chunk_ptr,
+                                 my_chunk_ptr + CHUNK_SIZE)) {
+      break;
+    }
+  }
+  return (char *)my_chunk_ptr;
+}
+
+/* Object free lists.  I-th entry corresponds to objects        */
+/* of total size 2**i bytes.                                    */
+static AO_stack_t AO_free_list[LOG_MAX_SIZE+1];
+
+/* Break up the chunk, and add it to the object free list for   */
+/* the given size.  We have exclusive access to chunk.          */
+static void add_chunk_as(void * chunk, unsigned log_sz)
+{
+  size_t ofs, limit;
+  size_t sz = (size_t)1 << log_sz;
+
+  assert((size_t)CHUNK_SIZE >= sz);
+  assert(sz % sizeof(AO_uintptr_t) == 0);
+  limit = (size_t)CHUNK_SIZE - sz;
+  for (ofs = ALIGNMENT - sizeof(AO_uintptr_t); ofs <= limit; ofs += sz) {
+    ASAN_POISON_MEMORY_REGION((char *)chunk + ofs + sizeof(AO_uintptr_t),
+                              sz - sizeof(AO_uintptr_t));
+    AO_stack_push(&AO_free_list[log_sz],
+                  (AO_uintptr_t *)chunk + ofs / sizeof(AO_uintptr_t));
+  }
+}
+
+static const unsigned char msbs[16] = {
+  0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4
+};
+
+/* Return the position of the most significant set bit in the   */
+/* argument.                                                    */
+/* We follow the conventions of ffs(), i.e. the least           */
+/* significant bit is number one.                               */
+static unsigned msb(size_t s)
+{
+  unsigned result = 0;
+  if ((s & 0xff) != s) {
+#   if (__SIZEOF_SIZE_T__ == 8) && !defined(CPPCHECK)
+      unsigned v = (unsigned)(s >> 32);
+      if (AO_EXPECT_FALSE(v != 0))
+        {
+          s = v;
+          result += 32;
+        }
+#   elif __SIZEOF_SIZE_T__ == 4
+      /* No op. */
+#   else
+      unsigned v;
+      /* The following is a tricky code ought to be equivalent to       */
+      /* "(v = s >> 32) != 0" but suppresses warnings on 32-bit arch's. */
+#     define SIZEOF_SIZE_T_GT_4 (sizeof(size_t) > 4)
+      if (SIZEOF_SIZE_T_GT_4
+          && (v = (unsigned)(s >> (SIZEOF_SIZE_T_GT_4 ? 32 : 0))) != 0)
+        {
+          s = v;
+          result += 32;
+        }
+#   endif /* !defined(__SIZEOF_SIZE_T__) */
+    if (AO_EXPECT_FALSE((s >> 16) != 0))
+      {
+        s >>= 16;
+        result += 16;
+      }
+    if ((s >> 8) != 0)
+      {
+        s >>= 8;
+        result += 8;
+      }
+  }
+  if (s > 15)
+    {
+      s >>= 4;
+      result += 4;
+    }
+  result += msbs[s];
+  return result;
+}
+
+AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1)
+void *
+AO_malloc(size_t sz)
+{
+  AO_uintptr_t *result;
+  unsigned log_sz;
+
+  if (AO_EXPECT_FALSE(sz > CHUNK_SIZE - sizeof(AO_uintptr_t)))
+    return AO_malloc_large(sz);
+  log_sz = msb(sz + sizeof(AO_uintptr_t) - 1);
+  assert(log_sz <= LOG_MAX_SIZE);
+  assert(((size_t)1 << log_sz) >= sz + sizeof(AO_uintptr_t));
+  result = AO_stack_pop(AO_free_list + log_sz);
+  while (AO_EXPECT_FALSE(NULL == result)) {
+    void *chunk = get_chunk();
+
+    if (AO_EXPECT_FALSE(NULL == chunk))
+      return NULL;
+    add_chunk_as(chunk, log_sz);
+    result = AO_stack_pop(AO_free_list + log_sz);
+  }
+  *result = log_sz;
+# ifdef AO_TRACE_MALLOC
+    fprintf(stderr, "%p: AO_malloc(%lu) = %p\n",
+            (void *)pthread_self(), (unsigned long)sz, (void *)(result + 1));
+# endif
+  ASAN_UNPOISON_MEMORY_REGION(result + 1, sz);
+  return result + 1;
+}
+
+AO_API void
+AO_free(void *p)
+{
+  AO_uintptr_t *base;
+  int log_sz;
+
+  if (AO_EXPECT_FALSE(NULL == p))
+    return;
+
+  base = (AO_uintptr_t *)p - 1;
+  log_sz = (int)(*base);
+# ifdef AO_TRACE_MALLOC
+    fprintf(stderr, "%p: AO_free(%p sz:%lu)\n", (void *)pthread_self(), p,
+            log_sz > LOG_MAX_SIZE ? (unsigned)log_sz : 1UL << log_sz);
+# endif
+  if (AO_EXPECT_FALSE(log_sz > LOG_MAX_SIZE)) {
+    AO_free_large(p);
+  } else {
+    ASAN_POISON_MEMORY_REGION(base + 1,
+                              ((size_t)1 << log_sz) - sizeof(AO_uintptr_t));
+    AO_stack_push(AO_free_list + log_sz, base);
+  }
+}

thirdparty/libatomic_ops/atomic_ops_malloc.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* Almost lock-free malloc implementation based on stack implementation. */
+/* See README_malloc.txt file for detailed usage rules.                  */
+
+#ifndef AO_MALLOC_H
+#define AO_MALLOC_H
+
+#include "atomic_ops_stack.h"
+
+#include <stddef.h> /* for size_t */
+
+#ifdef __cplusplus
+  extern "C" {
+#endif
+
+#ifdef AO_STACK_IS_LOCK_FREE
+# define AO_MALLOC_IS_LOCK_FREE
+#endif
+
+#ifndef AO_ATTR_MALLOC
+# if AO_GNUC_PREREQ(3, 1)
+#   define AO_ATTR_MALLOC __attribute__((__malloc__))
+# elif defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(__EDG__)
+#   define AO_ATTR_MALLOC \
+                __declspec(allocator) __declspec(noalias) __declspec(restrict)
+# elif defined(_MSC_VER) && _MSC_VER >= 1400
+#   define AO_ATTR_MALLOC __declspec(noalias) __declspec(restrict)
+# else
+#   define AO_ATTR_MALLOC /* empty */
+# endif
+#endif
+
+#ifndef AO_ATTR_ALLOC_SIZE
+# ifdef __clang__
+#   if __has_attribute(__alloc_size__)
+#     define AO_ATTR_ALLOC_SIZE(argnum) \
+                __attribute__((__alloc_size__(argnum)))
+#   else
+#     define AO_ATTR_ALLOC_SIZE(argnum) /* empty */
+#   endif
+# elif AO_GNUC_PREREQ(4, 3) && !defined(__ICC)
+#   define AO_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum)))
+# else
+#   define AO_ATTR_ALLOC_SIZE(argnum) /* empty */
+# endif
+#endif
+
+AO_API void AO_free(void *);
+
+AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1)
+void * AO_malloc(size_t);
+
+/* Allow use of mmap to grow the heap.  No-op on some platforms.        */
+AO_API void AO_malloc_enable_mmap(void);
+
+#ifdef __cplusplus
+  } /* extern "C" */
+#endif
+
+#endif /* !AO_MALLOC_H */

thirdparty/libatomic_ops/atomic_ops_sysdeps.S

@@ -0,0 +1,9 @@
+/*
+ * Include the appropriate system-dependent assembly file, if any.
+ * This is used only if the platform supports neither inline assembly
+ * code, nor appropriate compiler intrinsics.
+ */
+
+#if !defined(__GNUC__) && (defined(sparc) || defined(__sparc))
+#  include "atomic_ops/sysdeps/sunc/sparc.S"
+#endif