diff --git a/thirdparty/libatomic_ops/atomic_ops.c b/thirdparty/libatomic_ops/atomic_ops.c index 5f33a9056b..8d897c628a 100644 --- a/thirdparty/libatomic_ops/atomic_ops.c +++ b/thirdparty/libatomic_ops/atomic_ops.c @@ -45,14 +45,29 @@ # define AO_USE_NO_SIGNALS #endif -#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \ +#if (defined(__CYGWIN__) || defined(__GLIBC__) || defined(__GNU__) \ + || defined(__linux__)) \ && !defined(AO_USE_NO_SIGNALS) && !defined(_GNU_SOURCE) # define _GNU_SOURCE 1 #endif +#ifndef AO_BUILD +# define AO_BUILD +#endif + #undef AO_REQUIRE_CAS #include "atomic_ops.h" /* Without cas emulation! */ +#ifdef __cplusplus + extern "C" { +#endif + +AO_API void AO_pause(int); /* defined below */ + +#ifdef __cplusplus + } /* extern "C" */ +#endif + #if !defined(_MSC_VER) && !defined(__MINGW32__) && !defined(__BORLANDC__) \ || defined(AO_USE_NO_SIGNALS) @@ -80,7 +95,29 @@ # include "atomic_ops/sysdeps/standard_ao_double_t.h" #endif +#ifdef __cplusplus + extern "C" { +#endif + +AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, + AO_t old_val, AO_t new_val); + +AO_API int +AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2); + +AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val); + /* Lock for pthreads-based implementation. */ +#ifndef AO_NO_PTHREADS + AO_API pthread_mutex_t AO_pt_lock; +#endif + +#ifdef __cplusplus + } /* extern "C" */ +#endif + #ifndef AO_NO_PTHREADS pthread_mutex_t AO_pt_lock = PTHREAD_MUTEX_INITIALIZER; #endif @@ -99,7 +136,7 @@ */ #define AO_HASH_SIZE 16 -#define AO_HASH(x) (((unsigned long)(x) >> 12) & (AO_HASH_SIZE-1)) +#define AO_HASH(x) ((unsigned)((AO_uintptr_t)(x) >> 12) & (AO_HASH_SIZE-1)) static AO_TS_t AO_locks[AO_HASH_SIZE] = { AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, @@ -108,8 +145,6 @@ static AO_TS_t AO_locks[AO_HASH_SIZE] = { AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, }; -void AO_pause(int); /* defined below */ - static void lock_ool(volatile AO_TS_t *l) { int i = 0; @@ -155,8 +190,8 @@ AO_INLINE void unlock(volatile AO_TS_t *l) } #endif /* !AO_USE_NO_SIGNALS */ -AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val, - AO_t new_val) +AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, + AO_t old_val, AO_t new_val) { AO_TS_t *my_lock = AO_locks + AO_HASH(addr); AO_t fetched_val; @@ -176,9 +211,10 @@ AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val, return fetched_val; } -int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, - AO_t old_val1, AO_t old_val2, - AO_t new_val1, AO_t new_val2) +AO_API int +AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) { AO_TS_t *my_lock = AO_locks + AO_HASH(addr); int result; @@ -203,7 +239,7 @@ int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, return result; } -void AO_store_full_emulation(volatile AO_t *addr, AO_t val) +AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val) { AO_TS_t *my_lock = AO_locks + AO_HASH(addr); lock(my_lock); @@ -222,7 +258,7 @@ void AO_store_full_emulation(volatile AO_t *addr, AO_t val) #endif -static AO_t spin_dummy = 1; +static volatile AO_t spin_dummy = 0; /* Spin for 2**n units. */ static void AO_spin(int n) @@ -231,35 +267,34 @@ static void AO_spin(int n) int i = 2 << n; while (i-- > 0) - j += (j - 1) << 2; - /* Given 'spin_dummy' is initialized to 1, j is 1 after the loop. */ + j += j << 2; + /* Given spin_dummy is initialized to 0, j is 0 after the loop. */ AO_store(&spin_dummy, j); } -void AO_pause(int n) +AO_API void AO_pause(int n) { - if (n < 12) + if (n < 12) { AO_spin(n); - else - { -# ifdef AO_USE_NANOSLEEP - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = n > 28 ? 100000L * 1000 : 1L << (n - 2); - nanosleep(&ts, 0); -# elif defined(AO_USE_WIN32_PTHREADS) - Sleep(n > 28 ? 100 /* millis */ - : n < 22 ? 1 : (DWORD)1 << (n - 22)); -# else - struct timeval tv; - /* Short async-signal-safe sleep. */ - int usec = n > 28 ? 100000 : 1 << (n - 12); + } else { +# ifdef AO_USE_NANOSLEEP + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = n > 28 ? 100000L * 1000 : 1L << (n - 2); + nanosleep(&ts, 0); +# elif defined(AO_USE_WIN32_PTHREADS) + Sleep(n > 28 ? 100 /* millis */ + : n < 22 ? 1 : (DWORD)1 << (n - 22)); +# else + struct timeval tv; + /* Short async-signal-safe sleep. */ + int usec = n > 28 ? 100000 : 1 << (n - 12); /* Use an intermediate variable (of int type) to avoid */ /* "shift followed by widening conversion" warning. */ - tv.tv_sec = 0; - tv.tv_usec = usec; - (void)select(0, 0, 0, 0, &tv); -# endif - } + tv.tv_sec = 0; + tv.tv_usec = usec; + (void)select(0, 0, 0, 0, &tv); +# endif + } } diff --git a/thirdparty/libatomic_ops/atomic_ops.h b/thirdparty/libatomic_ops/atomic_ops.h index 106d450592..4a5cfab8bc 100644 --- a/thirdparty/libatomic_ops/atomic_ops.h +++ b/thirdparty/libatomic_ops/atomic_ops.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2008-2021 Ivan Maidanski + * Copyright (c) 2008-2022 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -153,8 +153,30 @@ /* atomic_ops/generalize.h. */ /* Some common defaults. Overridden for some architectures. */ + #define AO_t size_t +#if defined(__SIZEOF_POINTER__) \ + && (__SIZEOF_POINTER__ == 2 * __SIZEOF_SIZE_T__) + /* Pointers are twice bigger than the machine word. */ +# define AO_FAT_POINTER +#endif + +#ifndef AO_FAT_POINTER +# define AO_uintptr_t AO_t +#elif defined(__e2k__) + /* For some reason uintptr_t is 64-bit on E2K in the protected mode. */ + typedef unsigned __int128 AO_uintptr_t; +#else +# include +# define AO_uintptr_t uintptr_t +#endif + +/* A compile-time assertion for AO_uintptr_t size. */ +struct AO_uintptr_t_size_static_assert { + char dummy[sizeof(AO_uintptr_t) == sizeof(void *) ? 1 : -1]; +}; + /* The test_and_set primitive returns an AO_TS_VAL_t value. */ /* AO_TS_t is the type of an in-memory test-and-set location. */ @@ -235,12 +257,36 @@ # define AO_ALIGNOF_SUPPORTED 1 #endif +#if defined(AO_DLL) && !defined(AO_API) +# ifdef AO_BUILD +# if defined(__CEGCC__) || (defined(__MINGW32__) && !defined(__cplusplus)) +# define AO_API __declspec(dllexport) +# elif defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__) \ + || defined(__DMC__) || defined(__MINGW32__) || defined(__WATCOMC__) +# define AO_API extern __declspec(dllexport) +# endif +# else +# if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CEGCC__) \ + || defined(__CYGWIN__) || defined(__DMC__) +# define AO_API __declspec(dllimport) +# elif defined(__MINGW32_DELAY_LOAD__) +# define AO_API __declspec(dllexport) +# elif defined(__MINGW32__) || defined(__WATCOMC__) +# define AO_API extern __declspec(dllimport) +# endif +# endif +#endif /* AO_DLL */ + +#ifndef AO_API +# define AO_API extern +#endif + #ifdef AO_ALIGNOF_SUPPORTED # define AO_ASSERT_ADDR_ALIGNED(addr) \ - assert(((size_t)(addr) & (__alignof__(*(addr)) - 1)) == 0) + assert(((AO_uintptr_t)(addr) & (__alignof__(*(addr)) - 1)) == 0) #else # define AO_ASSERT_ADDR_ALIGNED(addr) \ - assert(((size_t)(addr) & (sizeof(*(addr)) - 1)) == 0) + assert(((AO_uintptr_t)(addr) & (sizeof(*(addr)) - 1)) == 0) #endif /* !AO_ALIGNOF_SUPPORTED */ #if defined(__GNUC__) && !defined(__INTEL_COMPILER) @@ -270,7 +316,7 @@ # include # define AO_compiler_barrier() _Asm_sched_fence() # else - /* FIXME: We do not know how to do this. This is a guess. */ + /* FIXME - We do not know how to do this. This is a guess. */ /* And probably a bad one. */ static volatile int AO_barrier_dummy; # define AO_compiler_barrier() (void)(AO_barrier_dummy = AO_barrier_dummy) @@ -340,6 +386,8 @@ # define AO_CAN_EMUL_CAS # elif defined(__avr32__) # include "atomic_ops/sysdeps/gcc/avr32.h" +# elif defined(__e2k__) +# include "atomic_ops/sysdeps/gcc/e2k.h" # elif defined(__hexagon__) # include "atomic_ops/sysdeps/gcc/hexagon.h" # elif defined(__nios2__) @@ -395,8 +443,10 @@ #if defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \ || (defined(__WATCOMC__) && defined(__NT__)) -# if defined(_AMD64_) || defined(_M_X64) || defined(_M_ARM64) +# if defined(_AMD64_) || defined(_M_X64) # include "atomic_ops/sysdeps/msftc/x86_64.h" +# elif defined(_M_ARM64) +# include "atomic_ops/sysdeps/msftc/arm64.h" # elif defined(_M_IX86) || defined(x86) # include "atomic_ops/sysdeps/msftc/x86.h" # elif defined(_M_ARM) || defined(ARM) || defined(_ARM_) diff --git a/thirdparty/libatomic_ops/atomic_ops/ao_version.h b/thirdparty/libatomic_ops/atomic_ops/ao_version.h index aab6b025f4..ca084752de 100644 --- a/thirdparty/libatomic_ops/atomic_ops/ao_version.h +++ b/thirdparty/libatomic_ops/atomic_ops/ao_version.h @@ -34,5 +34,5 @@ /* The version here should match that in configure.ac and README. */ #define AO_VERSION_MAJOR 7 -#define AO_VERSION_MINOR 6 -#define AO_VERSION_MICRO 12 /* 7.6.12 */ +#define AO_VERSION_MINOR 9 +#define AO_VERSION_MICRO 0 /* 7.9.0 */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/armcc/arm_v6.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/armcc/arm_v6.h index 23ecbc7fef..3c3beb463c 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/armcc/arm_v6.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/armcc/arm_v6.h @@ -13,7 +13,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h index e52f75ad7d..c322a5b2e2 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h @@ -47,14 +47,15 @@ extern "C" { #endif -AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val, - AO_t new_val); +AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, + AO_t old_val, AO_t new_val); -int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, - AO_t old_val1, AO_t old_val2, - AO_t new_val1, AO_t new_val2); +AO_API int +AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2); -void AO_store_full_emulation(volatile AO_t *addr, AO_t val); +AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val); #ifndef AO_HAVE_fetch_compare_and_swap_full # define AO_fetch_compare_and_swap_full(addr, old, newval) \ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/aarch64.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/aarch64.h index bcd2aeb161..7ba6bb7bce 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/aarch64.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/aarch64.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/alpha.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/alpha.h index 3b98becc32..59c669e0bb 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/alpha.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/alpha.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/arm.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/arm.h index 2ae76588f4..e02aad17cb 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/arm.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/arm.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/e2k.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/e2k.h new file mode 100644 index 0000000000..0726204dcb --- /dev/null +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/e2k.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 Ivan Maidanski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* As of clang-9, all __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n are missing. */ +#define AO_GCC_FORCE_HAVE_CAS + +#if (__SIZEOF_SIZE_T__ == 4) \ + || (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(__clang__)) \ + || (defined(__clang__) && __iset__ >= 5 /* elbrus-v5 or later */ \ + && defined(AO_PREFER_BUILTIN_ATOMICS)) + /* Note for 128-bit operations: clang reports warning */ + /* "large atomic operation may incur significant performance penalty" */ + /* and requires LDFLAGS="-latomic", thus this is not on by default. */ +# define AO_GCC_HAVE_double_SYNC_CAS +# include "../standard_ao_double_t.h" +#endif + +#include "generic.h" + +#undef AO_GCC_FORCE_HAVE_CAS +#undef AO_GCC_HAVE_double_SYNC_CAS diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h index 00fd3ace08..032990ca52 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h @@ -58,6 +58,10 @@ } # define AO_HAVE_nop_full +#elif defined(AO_THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE) + /* Workaround a compiler warning (reported by gcc-11, at least) */ + /* that atomic_thread_fence is unsupported with thread sanitizer. */ + #else AO_INLINE void AO_nop_read(void) @@ -82,7 +86,7 @@ __atomic_thread_fence(__ATOMIC_SEQ_CST); } # define AO_HAVE_nop_full -#endif /* !AO_UNIPROCESSOR */ +#endif /* !AO_UNIPROCESSOR && !AO_THREAD_SANITIZER */ #include "generic-small.h" @@ -95,28 +99,32 @@ AO_INLINE AO_TS_VAL_t AO_test_and_set(volatile AO_TS_t *addr) { - return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELAXED); + return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELAXED) + ? AO_TS_SET : AO_TS_CLEAR); } # define AO_HAVE_test_and_set AO_INLINE AO_TS_VAL_t AO_test_and_set_acquire(volatile AO_TS_t *addr) { - return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_ACQUIRE); + return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_ACQUIRE) + ? AO_TS_SET : AO_TS_CLEAR); } # define AO_HAVE_test_and_set_acquire AO_INLINE AO_TS_VAL_t AO_test_and_set_release(volatile AO_TS_t *addr) { - return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELEASE); + return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELEASE) + ? AO_TS_SET : AO_TS_CLEAR); } # define AO_HAVE_test_and_set_release AO_INLINE AO_TS_VAL_t AO_test_and_set_full(volatile AO_TS_t *addr) { - return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_SEQ_CST); + return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_SEQ_CST) + ? AO_TS_SET : AO_TS_CLEAR); } # define AO_HAVE_test_and_set_full #endif /* !AO_PREFER_GENERALIZED */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/hexagon.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/hexagon.h index ca0d8975c2..cffffe81bf 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/hexagon.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/hexagon.h @@ -3,7 +3,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/m68k.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/m68k.h index 1cf4833174..27e62f6db8 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/m68k.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/m68k.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/mips.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/mips.h index 34cc4fbb33..94c4e24d81 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/mips.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/mips.h @@ -5,7 +5,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/powerpc.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/powerpc.h index e70b76d211..509dd800ca 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/powerpc.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/powerpc.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h index 9a0586a27a..261093b728 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h @@ -3,7 +3,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. @@ -25,8 +25,76 @@ # endif #endif /* !__clang__ */ +#if defined(__riscv_zacas) && __riscv_xlen == 64 && !defined(AO_NO_DOUBLE_CAS) + /* TODO: Support also rv32, i.e. use amocas.w. */ + +# define AO_SKIPATOMIC_double_load +# define AO_SKIPATOMIC_double_load_acquire +# define AO_SKIPATOMIC_double_store +# define AO_SKIPATOMIC_double_store_release + +# include "../standard_ao_double_t.h" + + AO_INLINE int + AO_double_compare_and_swap(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_double_t expected = old_val; + + __asm__ __volatile__("amocas.q %0, %z2, %1" + : "+rJ" (expected.AO_whole), "+A" (*addr) + : "rJ" (new_val.AO_whole)); + return expected.AO_whole == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap + + AO_INLINE int + AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_double_t expected = old_val; + + __asm__ __volatile__("amocas.q.aq %0, %z2, %1" + : "+rJ" (expected.AO_whole), "+A" (*addr) + : "rJ" (new_val.AO_whole)); + return expected.AO_whole == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_acquire + + AO_INLINE int + AO_double_compare_and_swap_release(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_double_t expected = old_val; + + __asm__ __volatile__("amocas.q.rl %0, %z2, %1" + : "+rJ" (expected.AO_whole), "+A" (*addr) + : "rJ" (new_val.AO_whole)); + return expected.AO_whole == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_release + + AO_INLINE int + AO_double_compare_and_swap_full(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_double_t expected = old_val; + + __asm__ __volatile__("amocas.q.aqrl %0, %z2, %1" + : "+rJ" (expected.AO_whole), "+A" (*addr) + : "rJ" (new_val.AO_whole)); + return expected.AO_whole == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_full + +#endif /* __riscv_zacas */ + #include "generic.h" #undef AO_GCC_FORCE_HAVE_CAS #undef AO_NO_char_ARITHM #undef AO_NO_short_ARITHM +#undef AO_SKIPATOMIC_double_load +#undef AO_SKIPATOMIC_double_load_acquire +#undef AO_SKIPATOMIC_double_store +#undef AO_SKIPATOMIC_double_store_release diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/s390.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/s390.h index 9b31da881c..a302b7ab8a 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/s390.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/s390.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sh.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sh.h index 098fb8c70d..8534038cf9 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sh.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sh.h @@ -6,7 +6,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h index 2dd28817f1..3f107d8f45 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h @@ -8,15 +8,20 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. * */ -/* TODO: Very incomplete; Add support for sparc64. */ -/* Non-ancient SPARCs provide compare-and-swap (casa). */ +#if (AO_GNUC_PREREQ(12, 0) || AO_CLANG_PREREQ(13, 0)) \ + && !defined(AO_DISABLE_GCC_ATOMICS) + /* Probably, it could be enabled for earlier compiler versions as well. */ + +# include "generic.h" + +#else /* AO_DISABLE_GCC_ATOMICS */ #include "../all_atomic_load_store.h" @@ -43,23 +48,17 @@ AO_test_and_set_full(volatile AO_TS_t *addr) { /* Returns nonzero if the comparison succeeded. */ AO_INLINE int AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) { - AO_t ret; __asm__ __volatile__ ("membar #StoreLoad | #LoadLoad\n\t" # if defined(__arch64__) - "casx [%2],%0,%1\n\t" + "casx [%1],%2,%0\n\t" # else - "cas [%2],%0,%1\n\t" /* 32-bit version */ + "cas [%1],%2,%0\n\t" /* 32-bit version */ # endif "membar #StoreLoad | #StoreStore\n\t" - "cmp %0,%1\n\t" - "be,a 0f\n\t" - "mov 1,%0\n\t"/* one insn after branch always executed */ - "clr %0\n\t" - "0:\n\t" - : "=r" (ret), "+r" (new_val) - : "r" (addr), "0" (old) - : "memory", "cc"); - return (int)ret; + : "+r" (new_val) + : "r" (addr), "r" (old) + : "memory"); + return new_val == old; } # define AO_HAVE_compare_and_swap_full # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ @@ -85,3 +84,5 @@ AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) { /* TODO: Extend this for SPARC v8 and v9 (V8 also has swap, V9 has CAS, */ /* there are barriers like membar #LoadStore, CASA (32-bit) and */ /* CASXA (64-bit) instructions added in V9). */ + +#endif /* AO_DISABLE_GCC_ATOMICS */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h index e42be1e646..6d284b1ae0 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h @@ -9,8 +9,6 @@ * modified is included with the above copyright notice. */ -/* Minimal support for tile. */ - #if (AO_GNUC_PREREQ(4, 8) || AO_CLANG_PREREQ(3, 4)) \ && !defined(AO_DISABLE_GCC_ATOMICS) @@ -18,6 +16,8 @@ #else /* AO_DISABLE_GCC_ATOMICS */ + /* Minimal support for tile. */ + # include "../all_atomic_load_store.h" # include "../test_and_set_t_is_ao_t.h" diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h index e34f6495b4..13f4a7cc5a 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. @@ -68,14 +68,14 @@ # endif # endif /* __x86_64__ */ -# elif AO_GNUC_PREREQ(7, 0) && !defined(AO_PREFER_BUILTIN_ATOMICS) \ - && !defined(AO_THREAD_SANITIZER) && !defined(__MINGW32__) - /* gcc-7.x/x64 (gcc-7.2, at least) requires -latomic flag in case */ +# elif defined(__x86_64__) && !defined(AO_PREFER_BUILTIN_ATOMICS) \ + && !defined(AO_THREAD_SANITIZER) + /* gcc/x64 (as of gcc-12.2) requires -latomic flag in case */ /* of double-word atomic operations use (but not in case of TSan). */ - /* TODO: Revise it for the future gcc-7 releases. */ + /* TODO: Revise it for the future gcc releases. */ # define AO_SKIPATOMIC_double_compare_and_swap_ANY # define AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY -# endif /* __GNUC__ && !__clang__ */ +# endif /* __x86_64__ && !__clang__ */ # ifdef AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY # define AO_SKIPATOMIC_double_load diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h index 854cb7753f..724b148b0c 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h @@ -39,7 +39,7 @@ /* We define only the full barrier variants, and count on the */ /* generalization section below to fill in the rest. */ -extern pthread_mutex_t AO_pt_lock; +AO_API pthread_mutex_t AO_pt_lock; #ifdef __cplusplus } /* extern "C" */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h index 8ea238c044..0458cddcbe 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2003 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2009-2017 Ivan Maidanski + * Copyright (c) 2009-2021 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -21,21 +21,20 @@ * SOFTWARE. */ -#ifndef AO_ASSUME_WINDOWS98 - /* CAS is always available */ -# define AO_ASSUME_WINDOWS98 -#endif -#include "common32_defs.h" - -#include "../test_and_set_t_is_ao_t.h" -/* AO_test_and_set_full() is emulated using CAS. */ - /* Some ARM slide set, if it has been read correctly, claims that Loads */ /* followed by either a Load or a Store are ordered, but nothing else. */ /* It is assumed that Windows interrupt handlers clear the LL/SC flag. */ /* Unaligned accesses are not guaranteed to be atomic. */ #include "../all_aligned_atomic_load_store.h" +#define AO_T_IS_INT + +#ifndef AO_ASSUME_WINDOWS98 + /* CAS is always available */ +# define AO_ASSUME_WINDOWS98 +#endif +#include "common32_defs.h" + /* If only a single processor is used, we can define AO_UNIPROCESSOR. */ #ifdef AO_UNIPROCESSOR AO_INLINE void AO_nop_full(void) @@ -47,16 +46,67 @@ /* AO_nop_full() is emulated using AO_test_and_set_full(). */ #endif -#if _M_ARM >= 6 -/* ARMv6 is the first architecture providing support for simple LL/SC. */ +#ifndef AO_HAVE_test_and_set_full +# include "../test_and_set_t_is_ao_t.h" + /* AO_test_and_set_full() is emulated. */ +#endif -/* #include "../standard_ao_double_t.h" */ -/* TODO: implement double-wide operations (similar to x86). */ +#if _M_ARM >= 7 && !defined(AO_NO_DOUBLE_CAS) -#else /* _M_ARM < 6 */ +# include "../standard_ao_double_t.h" -/* TODO: implement AO_test_and_set_full using SWP. */ +/* These intrinsics are supposed to use LDREXD/STREXD. */ +# pragma intrinsic (_InterlockedCompareExchange64) +# pragma intrinsic (_InterlockedCompareExchange64_acq) +# pragma intrinsic (_InterlockedCompareExchange64_nf) +# pragma intrinsic (_InterlockedCompareExchange64_rel) -#endif /* _M_ARM < 6 */ + AO_INLINE int + AO_double_compare_and_swap(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_ASSERT_ADDR_ALIGNED(addr); + return (double_ptr_storage)_InterlockedCompareExchange64_nf( + (__int64 volatile *)addr, + new_val.AO_whole /* exchange */, + old_val.AO_whole) == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap -#define AO_T_IS_INT + AO_INLINE int + AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_ASSERT_ADDR_ALIGNED(addr); + return (double_ptr_storage)_InterlockedCompareExchange64_acq( + (__int64 volatile *)addr, + new_val.AO_whole /* exchange */, + old_val.AO_whole) == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_acquire + + AO_INLINE int + AO_double_compare_and_swap_release(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_ASSERT_ADDR_ALIGNED(addr); + return (double_ptr_storage)_InterlockedCompareExchange64_rel( + (__int64 volatile *)addr, + new_val.AO_whole /* exchange */, + old_val.AO_whole) == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_release + + AO_INLINE int + AO_double_compare_and_swap_full(volatile AO_double_t *addr, + AO_double_t old_val, AO_double_t new_val) + { + AO_ASSERT_ADDR_ALIGNED(addr); + return (double_ptr_storage)_InterlockedCompareExchange64( + (__int64 volatile *)addr, + new_val.AO_whole /* exchange */, + old_val.AO_whole) == old_val.AO_whole; + } +# define AO_HAVE_double_compare_and_swap_full + +#endif /* _M_ARM >= 7 && !AO_NO_DOUBLE_CAS */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm64.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm64.h new file mode 100644 index 0000000000..367042af6c --- /dev/null +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm64.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2003 Hewlett-Packard Development Company, L.P. + * Copyright (c) 2021 Ivan Maidanski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "../all_aligned_atomic_load_store.h" + +#ifndef AO_ASSUME_WINDOWS98 +# define AO_ASSUME_WINDOWS98 +#endif +#ifndef AO_USE_INTERLOCKED_INTRINSICS +# define AO_USE_INTERLOCKED_INTRINSICS +#endif +#include "common32_defs.h" + +#ifndef AO_HAVE_test_and_set_full +# include "../test_and_set_t_is_ao_t.h" + /* AO_test_and_set_full() is emulated using word-wide CAS. */ +#endif + +#ifndef AO_NO_DOUBLE_CAS + +# include "../standard_ao_double_t.h" + +# pragma intrinsic (_InterlockedCompareExchange128) +# pragma intrinsic (_InterlockedCompareExchange128_acq) +# pragma intrinsic (_InterlockedCompareExchange128_nf) +# pragma intrinsic (_InterlockedCompareExchange128_rel) + + AO_INLINE int + AO_compare_double_and_swap_double(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { + __int64 comparandResult[2]; + + AO_ASSERT_ADDR_ALIGNED(addr); + comparandResult[0] = old_val1; /* low */ + comparandResult[1] = old_val2; /* high */ + return _InterlockedCompareExchange128_nf((volatile __int64 *)addr, + new_val2 /* high */, + new_val1 /* low */, + comparandResult); + } +# define AO_HAVE_compare_double_and_swap_double + + AO_INLINE int + AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { + __int64 comparandResult[2]; + + AO_ASSERT_ADDR_ALIGNED(addr); + comparandResult[0] = old_val1; /* low */ + comparandResult[1] = old_val2; /* high */ + return _InterlockedCompareExchange128_acq((volatile __int64 *)addr, + new_val2 /* high */, + new_val1 /* low */, + comparandResult); + } +# define AO_HAVE_compare_double_and_swap_double_acquire + + AO_INLINE int + AO_compare_double_and_swap_double_release(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { + __int64 comparandResult[2]; + + AO_ASSERT_ADDR_ALIGNED(addr); + comparandResult[0] = old_val1; /* low */ + comparandResult[1] = old_val2; /* high */ + return _InterlockedCompareExchange128_rel((volatile __int64 *)addr, + new_val2 /* high */, + new_val1 /* low */, + comparandResult); + } +# define AO_HAVE_compare_double_and_swap_double_release + + AO_INLINE int + AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { + __int64 comparandResult[2]; + + AO_ASSERT_ADDR_ALIGNED(addr); + comparandResult[0] = old_val1; /* low */ + comparandResult[1] = old_val2; /* high */ + return _InterlockedCompareExchange128((volatile __int64 *)addr, + new_val2 /* high */, + new_val1 /* low */, + comparandResult); + } +# define AO_HAVE_compare_double_and_swap_double_full + +#endif /* !AO_NO_DOUBLE_CAS */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/common32_defs.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/common32_defs.h index fa7ee95308..ece41f743e 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/common32_defs.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/common32_defs.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2009-2018 Ivan Maidanski + * Copyright (c) 2009-2021 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ */ /* This file contains AO primitives based on VC++ built-in intrinsic */ -/* functions commonly available across 32-bit architectures. */ +/* functions commonly available across 32- and 64-bit architectures. */ /* This file should be included from arch-specific header files. */ /* Define AO_USE_INTERLOCKED_INTRINSICS if _Interlocked primitives */ @@ -35,6 +35,9 @@ # include /* Seems like over-kill, but that's what MSDN recommends. */ /* And apparently winbase.h is not always self-contained. */ + /* Optionally, client could define WIN32_LEAN_AND_MEAN before */ + /* include atomic_ops.h to reduce amount of Windows internal */ + /* headers included by windows.h one. */ #endif #if _MSC_VER < 1310 || !defined(AO_USE_INTERLOCKED_INTRINSICS) @@ -71,34 +74,78 @@ # pragma intrinsic (_InterlockedIncrement) # pragma intrinsic (_InterlockedDecrement) # pragma intrinsic (_InterlockedExchangeAdd) +# ifndef AO_T_IS_INT +# pragma intrinsic (_InterlockedIncrement64) +# pragma intrinsic (_InterlockedDecrement64) +# pragma intrinsic (_InterlockedExchangeAdd64) +# endif # endif /* !AO_PREFER_GENERALIZED */ + # pragma intrinsic (_InterlockedCompareExchange) +# ifndef AO_T_IS_INT +# pragma intrinsic (_InterlockedCompareExchange64) +# endif # define AO_INTERLOCKED_VOLATILE volatile #endif /* _MSC_VER >= 1310 */ #if !defined(AO_PREFER_GENERALIZED) || !defined(AO_ASSUME_WINDOWS98) -AO_INLINE AO_t -AO_fetch_and_add_full(volatile AO_t *p, AO_t incr) -{ - return _InterlockedExchangeAdd((long AO_INTERLOCKED_VOLATILE *)p, incr); -} -#define AO_HAVE_fetch_and_add_full + AO_INLINE AO_t + AO_fetch_and_add_full(volatile AO_t *p, AO_t incr) + { +# ifdef AO_T_IS_INT + return _InterlockedExchangeAdd((long AO_INTERLOCKED_VOLATILE *)p, incr); +# else + return _InterlockedExchangeAdd64((__int64 volatile *)p, incr); +# endif + } +# define AO_HAVE_fetch_and_add_full -AO_INLINE AO_t -AO_fetch_and_add1_full(volatile AO_t *p) -{ - return _InterlockedIncrement((long AO_INTERLOCKED_VOLATILE *)p) - 1; -} -#define AO_HAVE_fetch_and_add1_full + AO_INLINE AO_t + AO_fetch_and_add1_full(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedIncrement((long AO_INTERLOCKED_VOLATILE *)p) - 1; +# else + return _InterlockedIncrement64((__int64 volatile *)p) - 1; +# endif + } +# define AO_HAVE_fetch_and_add1_full -AO_INLINE AO_t -AO_fetch_and_sub1_full(volatile AO_t *p) -{ - return _InterlockedDecrement((long AO_INTERLOCKED_VOLATILE *)p) + 1; -} -#define AO_HAVE_fetch_and_sub1_full + AO_INLINE AO_t + AO_fetch_and_sub1_full(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedDecrement((long AO_INTERLOCKED_VOLATILE *)p) + 1; +# else + return _InterlockedDecrement64((__int64 volatile *)p) + 1; +# endif + } +# define AO_HAVE_fetch_and_sub1_full + +# ifndef AO_T_IS_INT + AO_INLINE unsigned int + AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr) + { + return _InterlockedExchangeAdd((long volatile *)p, incr); + } +# define AO_HAVE_int_fetch_and_add_full + + AO_INLINE unsigned int + AO_int_fetch_and_add1_full(volatile unsigned int *p) + { + return _InterlockedIncrement((long volatile *)p) - 1; + } +# define AO_HAVE_int_fetch_and_add1_full + + AO_INLINE unsigned int + AO_int_fetch_and_sub1_full(volatile unsigned int *p) + { + return _InterlockedDecrement((long volatile *)p) + 1; + } +# define AO_HAVE_int_fetch_and_sub1_full +# endif /* !AO_T_IS_INT */ #endif /* !AO_PREFER_GENERALIZED */ #ifdef AO_ASSUME_WINDOWS98 @@ -106,7 +153,10 @@ AO_fetch_and_sub1_full(volatile AO_t *p) AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, AO_t new_val) { -# ifdef AO_OLD_STYLE_INTERLOCKED_COMPARE_EXCHANGE +# ifndef AO_T_IS_INT + return (AO_t)_InterlockedCompareExchange64((__int64 volatile *)addr, + new_val, old_val); +# elif defined(AO_OLD_STYLE_INTERLOCKED_COMPARE_EXCHANGE) return (AO_t)_InterlockedCompareExchange( (void *AO_INTERLOCKED_VOLATILE *)addr, (void *)new_val, (void *)old_val); @@ -116,35 +166,50 @@ AO_fetch_and_sub1_full(volatile AO_t *p) # endif } # define AO_HAVE_fetch_compare_and_swap_full + +# ifndef AO_T_IS_INT + AO_INLINE unsigned int + AO_int_fetch_compare_and_swap_full(volatile unsigned int *addr, + unsigned int old_val, + unsigned int new_val) + { + return _InterlockedCompareExchange((long volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_int_fetch_compare_and_swap_full +# endif /* !AO_T_IS_INT */ #endif /* AO_ASSUME_WINDOWS98 */ #if (_MSC_VER > 1400) && (!defined(_M_ARM) || _MSC_VER >= 1800) -# pragma intrinsic (_InterlockedAnd8) +# if _MSC_VER < 1800 || !defined(AO_PREFER_GENERALIZED) +# pragma intrinsic (_InterlockedAnd8) +# pragma intrinsic (_InterlockedOr8) +# pragma intrinsic (_InterlockedXor8) + + AO_INLINE void + AO_char_and_full(volatile unsigned char *p, unsigned char value) + { + _InterlockedAnd8((char volatile *)p, value); + } +# define AO_HAVE_char_and_full + + AO_INLINE void + AO_char_or_full(volatile unsigned char *p, unsigned char value) + { + _InterlockedOr8((char volatile *)p, value); + } +# define AO_HAVE_char_or_full + + AO_INLINE void + AO_char_xor_full(volatile unsigned char *p, unsigned char value) + { + _InterlockedXor8((char volatile *)p, value); + } +# define AO_HAVE_char_xor_full +# endif /* _MSC_VER < 1800 || !AO_PREFER_GENERALIZED */ + # pragma intrinsic (_InterlockedCompareExchange16) -# pragma intrinsic (_InterlockedOr8) -# pragma intrinsic (_InterlockedXor8) - - AO_INLINE void - AO_char_and_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedAnd8((char volatile *)p, value); - } -# define AO_HAVE_char_and_full - - AO_INLINE void - AO_char_or_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedOr8((char volatile *)p, value); - } -# define AO_HAVE_char_or_full - - AO_INLINE void - AO_char_xor_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedXor8((char volatile *)p, value); - } -# define AO_HAVE_char_xor_full AO_INLINE unsigned short AO_short_fetch_compare_and_swap_full(volatile unsigned short *addr, @@ -174,10 +239,674 @@ AO_fetch_and_sub1_full(volatile AO_t *p) } # define AO_HAVE_short_fetch_and_sub1_full # endif /* !AO_PREFER_GENERALIZED */ + #endif /* _MSC_VER > 1400 */ #if _MSC_VER >= 1800 /* Visual Studio 2013+ */ +# ifndef AO_PREFER_GENERALIZED +# pragma intrinsic (_InterlockedAnd16) +# pragma intrinsic (_InterlockedOr16) +# pragma intrinsic (_InterlockedXor16) + + AO_INLINE void + AO_short_and_full(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedAnd16((short volatile *)p, value); + } +# define AO_HAVE_short_and_full + + AO_INLINE void + AO_short_or_full(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedOr16((short volatile *)p, value); + } +# define AO_HAVE_short_or_full + + AO_INLINE void + AO_short_xor_full(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedXor16((short volatile *)p, value); + } +# define AO_HAVE_short_xor_full + +# pragma intrinsic (_InterlockedAnd) +# pragma intrinsic (_InterlockedOr) +# pragma intrinsic (_InterlockedXor) + +# ifndef AO_T_IS_INT + AO_INLINE void + AO_int_and_full(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedAnd((long volatile *)p, value); + } +# define AO_HAVE_int_and_full + + AO_INLINE void + AO_int_or_full(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedOr((long volatile *)p, value); + } +# define AO_HAVE_int_or_full + + AO_INLINE void + AO_int_xor_full(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedXor((long volatile *)p, value); + } +# define AO_HAVE_int_xor_full + +# pragma intrinsic (_InterlockedAnd64) +# pragma intrinsic (_InterlockedOr64) +# pragma intrinsic (_InterlockedXor64) +# endif /* !AO_T_IS_INT */ + + AO_INLINE void + AO_and_full(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedAnd((long volatile *)p, value); +# else + (void)_InterlockedAnd64((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_and_full + + AO_INLINE void + AO_or_full(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedOr((long volatile *)p, value); +# else + (void)_InterlockedOr64((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_or_full + + AO_INLINE void + AO_xor_full(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedXor((long volatile *)p, value); +# else + (void)_InterlockedXor64((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_xor_full +# endif /* !AO_PREFER_GENERALIZED */ + +# if !defined(AO_PREFER_GENERALIZED) && (defined(_M_ARM) || defined(_M_ARM64)) +# pragma intrinsic (_InterlockedAnd8_acq) +# pragma intrinsic (_InterlockedAnd8_nf) +# pragma intrinsic (_InterlockedAnd8_rel) +# pragma intrinsic (_InterlockedOr8_acq) +# pragma intrinsic (_InterlockedOr8_nf) +# pragma intrinsic (_InterlockedOr8_rel) +# pragma intrinsic (_InterlockedXor8_acq) +# pragma intrinsic (_InterlockedXor8_nf) +# pragma intrinsic (_InterlockedXor8_rel) + + AO_INLINE void + AO_char_and(volatile unsigned char *p, unsigned char value) + { + _InterlockedAnd8_nf((char volatile *)p, value); + } +# define AO_HAVE_char_and + + AO_INLINE void + AO_char_or(volatile unsigned char *p, unsigned char value) + { + _InterlockedOr8_nf((char volatile *)p, value); + } +# define AO_HAVE_char_or + + AO_INLINE void + AO_char_xor(volatile unsigned char *p, unsigned char value) + { + _InterlockedXor8_nf((char volatile *)p, value); + } +# define AO_HAVE_char_xor + + AO_INLINE void + AO_char_and_acquire(volatile unsigned char *p, unsigned char value) + { + _InterlockedAnd8_acq((char volatile *)p, value); + } +# define AO_HAVE_char_and_acquire + + AO_INLINE void + AO_char_or_acquire(volatile unsigned char *p, unsigned char value) + { + _InterlockedOr8_acq((char volatile *)p, value); + } +# define AO_HAVE_char_or_acquire + + AO_INLINE void + AO_char_xor_acquire(volatile unsigned char *p, unsigned char value) + { + _InterlockedXor8_acq((char volatile *)p, value); + } +# define AO_HAVE_char_xor_acquire + + AO_INLINE void + AO_char_and_release(volatile unsigned char *p, unsigned char value) + { + _InterlockedAnd8_rel((char volatile *)p, value); + } +# define AO_HAVE_char_and_release + + AO_INLINE void + AO_char_or_release(volatile unsigned char *p, unsigned char value) + { + _InterlockedOr8_rel((char volatile *)p, value); + } +# define AO_HAVE_char_or_release + + AO_INLINE void + AO_char_xor_release(volatile unsigned char *p, unsigned char value) + { + _InterlockedXor8_rel((char volatile *)p, value); + } +# define AO_HAVE_char_xor_release + +# pragma intrinsic (_InterlockedAnd16_acq) +# pragma intrinsic (_InterlockedAnd16_nf) +# pragma intrinsic (_InterlockedAnd16_rel) +# pragma intrinsic (_InterlockedOr16_acq) +# pragma intrinsic (_InterlockedOr16_nf) +# pragma intrinsic (_InterlockedOr16_rel) +# pragma intrinsic (_InterlockedXor16_acq) +# pragma intrinsic (_InterlockedXor16_nf) +# pragma intrinsic (_InterlockedXor16_rel) + + AO_INLINE void + AO_short_and(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedAnd16_nf((short volatile *)p, value); + } +# define AO_HAVE_short_and + + AO_INLINE void + AO_short_or(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedOr16_nf((short volatile *)p, value); + } +# define AO_HAVE_short_or + + AO_INLINE void + AO_short_xor(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedXor16_nf((short volatile *)p, value); + } +# define AO_HAVE_short_xor + + AO_INLINE void + AO_short_and_acquire(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedAnd16_acq((short volatile *)p, value); + } +# define AO_HAVE_short_and_acquire + + AO_INLINE void + AO_short_or_acquire(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedOr16_acq((short volatile *)p, value); + } +# define AO_HAVE_short_or_acquire + + AO_INLINE void + AO_short_xor_acquire(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedXor16_acq((short volatile *)p, value); + } +# define AO_HAVE_short_xor_acquire + + AO_INLINE void + AO_short_and_release(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedAnd16_rel((short volatile *)p, value); + } +# define AO_HAVE_short_and_release + + AO_INLINE void + AO_short_or_release(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedOr16_rel((short volatile *)p, value); + } +# define AO_HAVE_short_or_release + + AO_INLINE void + AO_short_xor_release(volatile unsigned short *p, unsigned short value) + { + (void)_InterlockedXor16_rel((short volatile *)p, value); + } +# define AO_HAVE_short_xor_release + +# pragma intrinsic (_InterlockedAnd_acq) +# pragma intrinsic (_InterlockedAnd_nf) +# pragma intrinsic (_InterlockedAnd_rel) +# pragma intrinsic (_InterlockedOr_acq) +# pragma intrinsic (_InterlockedOr_nf) +# pragma intrinsic (_InterlockedOr_rel) +# pragma intrinsic (_InterlockedXor_acq) +# pragma intrinsic (_InterlockedXor_nf) +# pragma intrinsic (_InterlockedXor_rel) + +# ifndef AO_T_IS_INT + AO_INLINE void + AO_int_and(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedAnd_nf((long volatile *)p, value); + } +# define AO_HAVE_int_and + + AO_INLINE void + AO_int_or(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedOr_nf((long volatile *)p, value); + } +# define AO_HAVE_int_or + + AO_INLINE void + AO_int_xor(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedXor_nf((long volatile *)p, value); + } +# define AO_HAVE_int_xor + + AO_INLINE void + AO_int_and_acquire(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedAnd_acq((long volatile *)p, value); + } +# define AO_HAVE_int_and_acquire + + AO_INLINE void + AO_int_or_acquire(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedOr_acq((long volatile *)p, value); + } +# define AO_HAVE_int_or_acquire + + AO_INLINE void + AO_int_xor_acquire(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedXor_acq((long volatile *)p, value); + } +# define AO_HAVE_int_xor_acquire + + AO_INLINE void + AO_int_and_release(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedAnd_rel((long volatile *)p, value); + } +# define AO_HAVE_int_and_release + + AO_INLINE void + AO_int_or_release(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedOr_rel((long volatile *)p, value); + } +# define AO_HAVE_int_or_release + + AO_INLINE void + AO_int_xor_release(volatile unsigned int *p, unsigned int value) + { + (void)_InterlockedXor_rel((long volatile *)p, value); + } +# define AO_HAVE_int_xor_release + +# pragma intrinsic (_InterlockedAnd64_acq) +# pragma intrinsic (_InterlockedAnd64_nf) +# pragma intrinsic (_InterlockedAnd64_rel) +# pragma intrinsic (_InterlockedOr64_acq) +# pragma intrinsic (_InterlockedOr64_nf) +# pragma intrinsic (_InterlockedOr64_rel) +# pragma intrinsic (_InterlockedXor64_acq) +# pragma intrinsic (_InterlockedXor64_nf) +# pragma intrinsic (_InterlockedXor64_rel) +# endif /* !AO_T_IS_INT */ + + AO_INLINE void + AO_and(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedAnd_nf((long volatile *)p, value); +# else + (void)_InterlockedAnd64_nf((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_and + + AO_INLINE void + AO_or(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedOr_nf((long volatile *)p, value); +# else + (void)_InterlockedOr64_nf((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_or + + AO_INLINE void + AO_xor(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedXor_nf((long volatile *)p, value); +# else + (void)_InterlockedXor64_nf((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_xor + + AO_INLINE void + AO_and_acquire(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedAnd_acq((long volatile *)p, value); +# else + (void)_InterlockedAnd64_acq((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_and_acquire + + AO_INLINE void + AO_or_acquire(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedOr_acq((long volatile *)p, value); +# else + (void)_InterlockedOr64_acq((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_or_acquire + + AO_INLINE void + AO_xor_acquire(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedXor_acq((long volatile *)p, value); +# else + (void)_InterlockedXor64_acq((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_xor_acquire + + AO_INLINE void + AO_and_release(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedAnd_rel((long volatile *)p, value); +# else + (void)_InterlockedAnd64_rel((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_and_release + + AO_INLINE void + AO_or_release(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedOr_rel((long volatile *)p, value); +# else + (void)_InterlockedOr64_rel((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_or_release + + AO_INLINE void + AO_xor_release(volatile AO_t *p, AO_t value) + { +# ifdef AO_T_IS_INT + (void)_InterlockedXor_rel((long volatile *)p, value); +# else + (void)_InterlockedXor64_rel((__int64 volatile *)p, value); +# endif + } +# define AO_HAVE_xor_release + +# pragma intrinsic (_InterlockedDecrement16_acq) +# pragma intrinsic (_InterlockedDecrement16_nf) +# pragma intrinsic (_InterlockedDecrement16_rel) +# pragma intrinsic (_InterlockedIncrement16_acq) +# pragma intrinsic (_InterlockedIncrement16_nf) +# pragma intrinsic (_InterlockedIncrement16_rel) + + AO_INLINE unsigned short + AO_short_fetch_and_add1(volatile unsigned short *p) + { + return _InterlockedIncrement16_nf((short volatile *)p) - 1; + } +# define AO_HAVE_short_fetch_and_add1 + + AO_INLINE unsigned short + AO_short_fetch_and_sub1(volatile unsigned short *p) + { + return _InterlockedDecrement16_nf((short volatile *)p) + 1; + } +# define AO_HAVE_short_fetch_and_sub1 + + AO_INLINE unsigned short + AO_short_fetch_and_add1_acquire(volatile unsigned short *p) + { + return _InterlockedIncrement16_acq((short volatile *)p) - 1; + } +# define AO_HAVE_short_fetch_and_add1_acquire + + AO_INLINE unsigned short + AO_short_fetch_and_sub1_acquire(volatile unsigned short *p) + { + return _InterlockedDecrement16_acq((short volatile *)p) + 1; + } +# define AO_HAVE_short_fetch_and_sub1_acquire + + AO_INLINE unsigned short + AO_short_fetch_and_add1_release(volatile unsigned short *p) + { + return _InterlockedIncrement16_rel((short volatile *)p) - 1; + } +# define AO_HAVE_short_fetch_and_add1_release + + AO_INLINE unsigned short + AO_short_fetch_and_sub1_release(volatile unsigned short *p) + { + return _InterlockedDecrement16_rel((short volatile *)p) + 1; + } +# define AO_HAVE_short_fetch_and_sub1_release + +# pragma intrinsic (_InterlockedExchangeAdd_acq) +# pragma intrinsic (_InterlockedExchangeAdd_nf) +# pragma intrinsic (_InterlockedExchangeAdd_rel) + +# pragma intrinsic (_InterlockedDecrement_acq) +# pragma intrinsic (_InterlockedDecrement_nf) +# pragma intrinsic (_InterlockedDecrement_rel) +# pragma intrinsic (_InterlockedIncrement_acq) +# pragma intrinsic (_InterlockedIncrement_nf) +# pragma intrinsic (_InterlockedIncrement_rel) + +# ifndef AO_T_IS_INT +# pragma intrinsic (_InterlockedExchangeAdd64_acq) +# pragma intrinsic (_InterlockedExchangeAdd64_nf) +# pragma intrinsic (_InterlockedExchangeAdd64_rel) + +# pragma intrinsic (_InterlockedDecrement64_acq) +# pragma intrinsic (_InterlockedDecrement64_nf) +# pragma intrinsic (_InterlockedDecrement64_rel) +# pragma intrinsic (_InterlockedIncrement64_acq) +# pragma intrinsic (_InterlockedIncrement64_nf) +# pragma intrinsic (_InterlockedIncrement64_rel) +# endif + + AO_INLINE AO_t + AO_fetch_and_add(volatile AO_t *p, AO_t incr) + { +# ifdef AO_T_IS_INT + return _InterlockedExchangeAdd_nf((long volatile *)p, incr); +# else + return _InterlockedExchangeAdd64_nf((__int64 volatile *)p, incr); +# endif + } +# define AO_HAVE_fetch_and_add + + AO_INLINE AO_t + AO_fetch_and_add1(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedIncrement_nf((long volatile *)p) - 1; +# else + return _InterlockedIncrement64_nf((__int64 volatile *)p) - 1; +# endif + } +# define AO_HAVE_fetch_and_add1 + + AO_INLINE AO_t + AO_fetch_and_sub1(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedDecrement_nf((long volatile *)p) + 1; +# else + return _InterlockedDecrement64_nf((__int64 volatile *)p) + 1; +# endif + } +# define AO_HAVE_fetch_and_sub1 + + AO_INLINE AO_t + AO_fetch_and_add_acquire(volatile AO_t *p, AO_t incr) + { +# ifdef AO_T_IS_INT + return _InterlockedExchangeAdd_acq((long volatile *)p, incr); +# else + return _InterlockedExchangeAdd64_acq((__int64 volatile *)p, incr); +# endif + } +# define AO_HAVE_fetch_and_add_acquire + + AO_INLINE AO_t + AO_fetch_and_add1_acquire(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedIncrement_acq((long volatile *)p) - 1; +# else + return _InterlockedIncrement64_acq((__int64 volatile *)p) - 1; +# endif + } +# define AO_HAVE_fetch_and_add1_acquire + + AO_INLINE AO_t + AO_fetch_and_sub1_acquire(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedDecrement_acq((long volatile *)p) + 1; +# else + return _InterlockedDecrement64_acq((__int64 volatile *)p) + 1; +# endif + } +# define AO_HAVE_fetch_and_sub1_acquire + + AO_INLINE AO_t + AO_fetch_and_add_release(volatile AO_t *p, AO_t incr) + { +# ifdef AO_T_IS_INT + return _InterlockedExchangeAdd_rel((long volatile *)p, incr); +# else + return _InterlockedExchangeAdd64_rel((__int64 volatile *)p, incr); +# endif + } +# define AO_HAVE_fetch_and_add_release + + AO_INLINE AO_t + AO_fetch_and_add1_release(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedIncrement_rel((long volatile *)p) - 1; +# else + return _InterlockedIncrement64_rel((__int64 volatile *)p) - 1; +# endif + } +# define AO_HAVE_fetch_and_add1_release + + AO_INLINE AO_t + AO_fetch_and_sub1_release(volatile AO_t *p) + { +# ifdef AO_T_IS_INT + return _InterlockedDecrement_rel((long volatile *)p) + 1; +# else + return _InterlockedDecrement64_rel((__int64 volatile *)p) + 1; +# endif + } +# define AO_HAVE_fetch_and_sub1_release + +# ifndef AO_T_IS_INT + AO_INLINE unsigned int + AO_int_fetch_and_add(volatile unsigned int *p, unsigned int incr) + { + return _InterlockedExchangeAdd_nf((long volatile *)p, incr); + } +# define AO_HAVE_int_fetch_and_add + + AO_INLINE unsigned int + AO_int_fetch_and_add1(volatile unsigned int *p) + { + return _InterlockedIncrement_nf((long volatile *)p) - 1; + } +# define AO_HAVE_int_fetch_and_add1 + + AO_INLINE unsigned int + AO_int_fetch_and_sub1(volatile unsigned int *p) + { + return _InterlockedDecrement_nf((long volatile *)p) + 1; + } +# define AO_HAVE_int_fetch_and_sub1 + + AO_INLINE unsigned int + AO_int_fetch_and_add_acquire(volatile unsigned int *p, + unsigned int incr) + { + return _InterlockedExchangeAdd_acq((long volatile *)p, incr); + } +# define AO_HAVE_int_fetch_and_add_acquire + + AO_INLINE unsigned int + AO_int_fetch_and_add1_acquire(volatile unsigned int *p) + { + return _InterlockedIncrement_acq((long volatile *)p) - 1; + } +# define AO_HAVE_int_fetch_and_add1_acquire + + AO_INLINE unsigned int + AO_int_fetch_and_sub1_acquire(volatile unsigned int *p) + { + return _InterlockedDecrement_acq((long volatile *)p) + 1; + } +# define AO_HAVE_int_fetch_and_sub1_acquire + + AO_INLINE unsigned int + AO_int_fetch_and_add_release(volatile unsigned int *p, + unsigned int incr) + { + return _InterlockedExchangeAdd_rel((long volatile *)p, incr); + } +# define AO_HAVE_int_fetch_and_add_release + + AO_INLINE unsigned int + AO_int_fetch_and_add1_release(volatile unsigned int *p) + { + return _InterlockedIncrement_rel((long volatile *)p) - 1; + } +# define AO_HAVE_int_fetch_and_add1_release + + AO_INLINE unsigned int + AO_int_fetch_and_sub1_release(volatile unsigned int *p) + { + return _InterlockedDecrement_rel((long volatile *)p) + 1; + } +# define AO_HAVE_int_fetch_and_sub1_release +# endif /* !AO_T_IS_INT */ + +# endif /* !AO_PREFER_GENERALIZED && (_M_ARM || _M_ARM64) */ + # pragma intrinsic (_InterlockedCompareExchange8) AO_INLINE unsigned char @@ -190,6 +919,158 @@ AO_fetch_and_sub1_full(volatile AO_t *p) } # define AO_HAVE_char_fetch_compare_and_swap_full +# if defined(_M_ARM) || defined(_M_ARM64) + +# pragma intrinsic (_InterlockedCompareExchange_acq) +# pragma intrinsic (_InterlockedCompareExchange_nf) +# pragma intrinsic (_InterlockedCompareExchange_rel) +# ifndef AO_T_IS_INT +# pragma intrinsic (_InterlockedCompareExchange64_acq) +# pragma intrinsic (_InterlockedCompareExchange64_nf) +# pragma intrinsic (_InterlockedCompareExchange64_rel) +# endif + + AO_INLINE AO_t + AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val) + { +# ifdef AO_T_IS_INT + return _InterlockedCompareExchange_nf((long volatile *)addr, + new_val, old_val); +# else + return (AO_t)_InterlockedCompareExchange64_nf( + (__int64 volatile *)addr, new_val, old_val); +# endif + } +# define AO_HAVE_fetch_compare_and_swap + + AO_INLINE AO_t + AO_fetch_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val, + AO_t new_val) + { +# ifdef AO_T_IS_INT + return _InterlockedCompareExchange_acq((long volatile *)addr, + new_val, old_val); +# else + return (AO_t)_InterlockedCompareExchange64_acq( + (__int64 volatile *)addr, new_val, old_val); +# endif + } +# define AO_HAVE_fetch_compare_and_swap_acquire + + AO_INLINE AO_t + AO_fetch_compare_and_swap_release(volatile AO_t *addr, AO_t old_val, + AO_t new_val) + { +# ifdef AO_T_IS_INT + return _InterlockedCompareExchange_rel((long volatile *)addr, + new_val, old_val); +# else + return (AO_t)_InterlockedCompareExchange64_rel( + (__int64 volatile *)addr, new_val, old_val); +# endif + } +# define AO_HAVE_fetch_compare_and_swap_release + +# ifndef AO_T_IS_INT + AO_INLINE unsigned int + AO_int_fetch_compare_and_swap(volatile unsigned int *addr, + unsigned int old_val, + unsigned int new_val) + { + return _InterlockedCompareExchange_nf((long volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_int_fetch_compare_and_swap + + AO_INLINE unsigned int + AO_int_fetch_compare_and_swap_acquire(volatile unsigned int *addr, + unsigned int old_val, + unsigned int new_val) + { + return _InterlockedCompareExchange_acq((long volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_int_fetch_compare_and_swap_acquire + + AO_INLINE unsigned int + AO_int_fetch_compare_and_swap_release(volatile unsigned int *addr, + unsigned int old_val, + unsigned int new_val) + { + return _InterlockedCompareExchange_rel((long volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_int_fetch_compare_and_swap_release +# endif /* !AO_T_IS_INT */ + +# pragma intrinsic (_InterlockedCompareExchange16_acq) +# pragma intrinsic (_InterlockedCompareExchange16_nf) +# pragma intrinsic (_InterlockedCompareExchange16_rel) +# pragma intrinsic (_InterlockedCompareExchange8_acq) +# pragma intrinsic (_InterlockedCompareExchange8_nf) +# pragma intrinsic (_InterlockedCompareExchange8_rel) + + AO_INLINE unsigned short + AO_short_fetch_compare_and_swap(volatile unsigned short *addr, + unsigned short old_val, + unsigned short new_val) + { + return _InterlockedCompareExchange16_nf((short volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_short_fetch_compare_and_swap + + AO_INLINE unsigned short + AO_short_fetch_compare_and_swap_acquire(volatile unsigned short *addr, + unsigned short old_val, + unsigned short new_val) + { + return _InterlockedCompareExchange16_acq((short volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_short_fetch_compare_and_swap_acquire + + AO_INLINE unsigned short + AO_short_fetch_compare_and_swap_release(volatile unsigned short *addr, + unsigned short old_val, + unsigned short new_val) + { + return _InterlockedCompareExchange16_rel((short volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_short_fetch_compare_and_swap_release + + AO_INLINE unsigned char + AO_char_fetch_compare_and_swap(volatile unsigned char *addr, + unsigned char old_val, + unsigned char new_val) + { + return _InterlockedCompareExchange8_nf((char volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_char_fetch_compare_and_swap + + AO_INLINE unsigned char + AO_char_fetch_compare_and_swap_acquire(volatile unsigned char *addr, + unsigned char old_val, + unsigned char new_val) + { + return _InterlockedCompareExchange8_acq((char volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_char_fetch_compare_and_swap_acquire + + AO_INLINE unsigned char + AO_char_fetch_compare_and_swap_release(volatile unsigned char *addr, + unsigned char old_val, + unsigned char new_val) + { + return _InterlockedCompareExchange8_rel((char volatile *)addr, + new_val, old_val); + } +# define AO_HAVE_char_fetch_compare_and_swap_release +# endif /* _M_ARM || _M_ARM64 */ + # if !defined(AO_PREFER_GENERALIZED) && !defined(_M_ARM) # pragma intrinsic (_InterlockedExchangeAdd16) # pragma intrinsic (_InterlockedExchangeAdd8) @@ -208,5 +1089,109 @@ AO_fetch_and_sub1_full(volatile AO_t *p) return _InterlockedExchangeAdd16((short volatile *)p, incr); } # define AO_HAVE_short_fetch_and_add_full + +# if defined(_M_ARM64) +# pragma intrinsic (_InterlockedExchangeAdd16_acq) +# pragma intrinsic (_InterlockedExchangeAdd16_nf) +# pragma intrinsic (_InterlockedExchangeAdd16_rel) +# pragma intrinsic (_InterlockedExchangeAdd8_acq) +# pragma intrinsic (_InterlockedExchangeAdd8_nf) +# pragma intrinsic (_InterlockedExchangeAdd8_rel) + + AO_INLINE unsigned char + AO_char_fetch_and_add(volatile unsigned char *p, unsigned char incr) + { + return _InterlockedExchangeAdd8_nf((char volatile *)p, incr); + } +# define AO_HAVE_char_fetch_and_add + + AO_INLINE unsigned short + AO_short_fetch_and_add(volatile unsigned short *p, unsigned short incr) + { + return _InterlockedExchangeAdd16_nf((short volatile *)p, incr); + } +# define AO_HAVE_short_fetch_and_add + + AO_INLINE unsigned char + AO_char_fetch_and_add_acquire(volatile unsigned char *p, + unsigned char incr) + { + return _InterlockedExchangeAdd8_acq((char volatile *)p, incr); + } +# define AO_HAVE_char_fetch_and_add_acquire + + AO_INLINE unsigned short + AO_short_fetch_and_add_acquire(volatile unsigned short *p, + unsigned short incr) + { + return _InterlockedExchangeAdd16_acq((short volatile *)p, incr); + } +# define AO_HAVE_short_fetch_and_add_acquire + + AO_INLINE unsigned char + AO_char_fetch_and_add_release(volatile unsigned char *p, + unsigned char incr) + { + return _InterlockedExchangeAdd8_rel((char volatile *)p, incr); + } +# define AO_HAVE_char_fetch_and_add_release + + AO_INLINE unsigned short + AO_short_fetch_and_add_release(volatile unsigned short *p, + unsigned short incr) + { + return _InterlockedExchangeAdd16_rel((short volatile *)p, incr); + } +# define AO_HAVE_short_fetch_and_add_release +# endif /* _M_ARM64 */ + # endif /* !AO_PREFER_GENERALIZED && !_M_ARM */ + +# if !defined(_M_ARM) || _M_ARM >= 6 +# include "../test_and_set_t_is_char.h" + +# pragma intrinsic (_InterlockedExchange8) + + AO_INLINE AO_TS_VAL_t + AO_test_and_set_full(volatile AO_TS_t *addr) + { + return (AO_TS_VAL_t)(_InterlockedExchange8((char volatile *)addr, + (AO_TS_t)AO_TS_SET) & 0xff); + /* Note: bitwise "and 0xff" is applied to the result because cast */ + /* to unsigned char does not work properly (for a reason) if /J */ + /* option is passed to the MS VC compiler. */ + } +# define AO_HAVE_test_and_set_full +# endif /* !_M_ARM || _M_ARM >= 6 */ + +# if _M_ARM >= 6 || defined(_M_ARM64) +# pragma intrinsic (_InterlockedExchange8_acq) +# pragma intrinsic (_InterlockedExchange8_nf) +# pragma intrinsic (_InterlockedExchange8_rel) + + AO_INLINE AO_TS_VAL_t + AO_test_and_set(volatile AO_TS_t *addr) + { + return (AO_TS_VAL_t)(_InterlockedExchange8_nf((char volatile *)addr, + (AO_TS_t)AO_TS_SET) & 0xff); + } +# define AO_HAVE_test_and_set + + AO_INLINE AO_TS_VAL_t + AO_test_and_set_acquire(volatile AO_TS_t *addr) + { + return (AO_TS_VAL_t)(_InterlockedExchange8_acq((char volatile *)addr, + (AO_TS_t)AO_TS_SET) & 0xff); + } +# define AO_HAVE_test_and_set_acquire + + AO_INLINE AO_TS_VAL_t + AO_test_and_set_release(volatile AO_TS_t *addr) + { + return (AO_TS_VAL_t)(_InterlockedExchange8_rel((char volatile *)addr, + (AO_TS_t)AO_TS_SET) & 0xff); + } +# define AO_HAVE_test_and_set_release +# endif /* _M_ARM >= 6 || _M_ARM64 */ + #endif /* _MSC_VER >= 1800 */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h index d290fc7868..ba2400a502 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2003 Hewlett-Packard Development Company, L.P. + * Copyright (c) 2009-2021 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,12 +21,13 @@ * SOFTWARE. */ -/* If AO_ASSUME_VISTA is defined, we assume Windows Server 2003, Vista */ -/* or later. */ - #include "../all_aligned_atomic_load_store.h" -#include "../test_and_set_t_is_char.h" +#if !defined(AO_ASSUME_VISTA) && _MSC_VER >= 1910 + /* Visual Studio 2017 (15.0) discontinued support of Windows XP. */ + /* We assume Windows Server 2003, Vista or later. */ +# define AO_ASSUME_VISTA +#endif #if !defined(AO_ASSUME_WINDOWS98) \ && (defined(AO_ASSUME_VISTA) || _MSC_VER >= 1400) @@ -33,6 +35,13 @@ # define AO_ASSUME_WINDOWS98 #endif +#if !defined(AO_USE_PENTIUM4_INSTRS) && _M_IX86_FP >= 2 /* SSE2 */ + /* "mfence" is a part of SSE2 set (introduced on Intel Pentium 4). */ +# define AO_USE_PENTIUM4_INSTRS +#endif + +#define AO_T_IS_INT + #ifndef AO_USE_INTERLOCKED_INTRINSICS /* _Interlocked primitives (Inc, Dec, Xchg, Add) are always available */ # define AO_USE_INTERLOCKED_INTRINSICS @@ -91,9 +100,12 @@ AO_nop_full(void) # define AO_HAVE_short_fetch_and_add_full #endif /* !AO_NO_ASM_XADD */ -AO_INLINE AO_TS_VAL_t -AO_test_and_set_full(volatile AO_TS_t *addr) -{ +#ifndef AO_HAVE_test_and_set_full +# include "../test_and_set_t_is_char.h" + + AO_INLINE AO_TS_VAL_t + AO_test_and_set_full(volatile AO_TS_t *addr) + { __asm { mov eax,0xff ; /* AO_TS_SET */ @@ -101,8 +113,9 @@ AO_test_and_set_full(volatile AO_TS_t *addr) xchg byte ptr [ebx],al ; } /* Ignore possible "missing return value" warning here. */ -} -#define AO_HAVE_test_and_set_full + } +# define AO_HAVE_test_and_set_full +#endif #if defined(_WIN64) && !defined(CPPCHECK) # error wrong architecture @@ -134,8 +147,6 @@ AO_test_and_set_full(volatile AO_TS_t *addr) # define AO_HAVE_double_compare_and_swap_full #endif /* AO_ASSUME_VISTA */ -#define AO_T_IS_INT - /* Real X86 implementations, except for some old WinChips, appear */ /* to enforce ordering between memory operations, EXCEPT that a later */ /* read can pass earlier writes, presumably due to the visible */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h index 196fc650a6..ad98f0b19f 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. + * Copyright (c) 2009-2021 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,185 +29,20 @@ /* presence of store buffers. */ /* We ignore the fact that the official specs */ /* seem to be much weaker (and arguably too weak to be usable). */ - #include "../ordered_except_wr.h" -#ifdef AO_ASM_X64_AVAILABLE -# include "../test_and_set_t_is_char.h" -#else -# include "../test_and_set_t_is_ao_t.h" +#ifndef AO_ASSUME_WINDOWS98 + /* CAS is always available */ +# define AO_ASSUME_WINDOWS98 #endif +#ifndef AO_USE_INTERLOCKED_INTRINSICS +# define AO_USE_INTERLOCKED_INTRINSICS +#endif +#include "common32_defs.h" -/* Assume _MSC_VER >= 1400 */ -#include - -#pragma intrinsic (_InterlockedCompareExchange) -#pragma intrinsic (_InterlockedCompareExchange64) - -#ifndef AO_PREFER_GENERALIZED - -# pragma intrinsic (_InterlockedIncrement) -# pragma intrinsic (_InterlockedIncrement64) -# pragma intrinsic (_InterlockedDecrement) -# pragma intrinsic (_InterlockedDecrement64) -# pragma intrinsic (_InterlockedExchangeAdd) -# pragma intrinsic (_InterlockedExchangeAdd64) - -AO_INLINE AO_t -AO_fetch_and_add_full (volatile AO_t *p, AO_t incr) -{ - return _InterlockedExchangeAdd64((__int64 volatile *)p, incr); -} -#define AO_HAVE_fetch_and_add_full - -AO_INLINE AO_t -AO_fetch_and_add1_full (volatile AO_t *p) -{ - return _InterlockedIncrement64((__int64 volatile *)p) - 1; -} -#define AO_HAVE_fetch_and_add1_full - -AO_INLINE AO_t -AO_fetch_and_sub1_full (volatile AO_t *p) -{ - return _InterlockedDecrement64((__int64 volatile *)p) + 1; -} -#define AO_HAVE_fetch_and_sub1_full -#endif /* !AO_PREFER_GENERALIZED */ - -AO_INLINE AO_t -AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, - AO_t new_val) -{ - return (AO_t)_InterlockedCompareExchange64((__int64 volatile *)addr, - new_val, old_val); -} -#define AO_HAVE_fetch_compare_and_swap_full - -AO_INLINE unsigned int -AO_int_fetch_compare_and_swap_full(volatile unsigned int *addr, - unsigned int old_val, unsigned int new_val) -{ - return _InterlockedCompareExchange((long volatile *)addr, new_val, old_val); -} -#define AO_HAVE_int_fetch_compare_and_swap_full - -#ifndef AO_PREFER_GENERALIZED -AO_INLINE unsigned int -AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr) -{ - return _InterlockedExchangeAdd((long volatile *)p, incr); -} -#define AO_HAVE_int_fetch_and_add_full - - AO_INLINE unsigned int - AO_int_fetch_and_add1_full(volatile unsigned int *p) - { - return _InterlockedIncrement((long volatile *)p) - 1; - } -# define AO_HAVE_int_fetch_and_add1_full - - AO_INLINE unsigned int - AO_int_fetch_and_sub1_full(volatile unsigned int *p) - { - return _InterlockedDecrement((long volatile *)p) + 1; - } -# define AO_HAVE_int_fetch_and_sub1_full -#endif /* !AO_PREFER_GENERALIZED */ - -#if _MSC_VER > 1400 -# pragma intrinsic (_InterlockedAnd8) -# pragma intrinsic (_InterlockedCompareExchange16) -# pragma intrinsic (_InterlockedOr8) -# pragma intrinsic (_InterlockedXor8) - - AO_INLINE void - AO_char_and_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedAnd8((char volatile *)p, value); - } -# define AO_HAVE_char_and_full - - AO_INLINE void - AO_char_or_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedOr8((char volatile *)p, value); - } -# define AO_HAVE_char_or_full - - AO_INLINE void - AO_char_xor_full(volatile unsigned char *p, unsigned char value) - { - _InterlockedXor8((char volatile *)p, value); - } -# define AO_HAVE_char_xor_full - - AO_INLINE unsigned short - AO_short_fetch_compare_and_swap_full(volatile unsigned short *addr, - unsigned short old_val, - unsigned short new_val) - { - return _InterlockedCompareExchange16((short volatile *)addr, - new_val, old_val); - } -# define AO_HAVE_short_fetch_compare_and_swap_full - -# ifndef AO_PREFER_GENERALIZED -# pragma intrinsic (_InterlockedIncrement16) -# pragma intrinsic (_InterlockedDecrement16) - - AO_INLINE unsigned short - AO_short_fetch_and_add1_full(volatile unsigned short *p) - { - return _InterlockedIncrement16((short volatile *)p) - 1; - } -# define AO_HAVE_short_fetch_and_add1_full - - AO_INLINE unsigned short - AO_short_fetch_and_sub1_full(volatile unsigned short *p) - { - return _InterlockedDecrement16((short volatile *)p) + 1; - } -# define AO_HAVE_short_fetch_and_sub1_full -# endif /* !AO_PREFER_GENERALIZED */ -#endif /* _MSC_VER > 1400 */ - -#if _MSC_VER >= 1800 /* Visual Studio 2013+ */ - -# pragma intrinsic (_InterlockedCompareExchange8) - - AO_INLINE unsigned char - AO_char_fetch_compare_and_swap_full(volatile unsigned char *addr, - unsigned char old_val, - unsigned char new_val) - { - return _InterlockedCompareExchange8((char volatile *)addr, - new_val, old_val); - } -# define AO_HAVE_char_fetch_compare_and_swap_full - -# ifndef AO_PREFER_GENERALIZED -# pragma intrinsic (_InterlockedExchangeAdd16) -# pragma intrinsic (_InterlockedExchangeAdd8) - - AO_INLINE unsigned char - AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr) - { - return _InterlockedExchangeAdd8((char volatile *)p, incr); - } -# define AO_HAVE_char_fetch_and_add_full - - AO_INLINE unsigned short - AO_short_fetch_and_add_full(volatile unsigned short *p, - unsigned short incr) - { - return _InterlockedExchangeAdd16((short volatile *)p, incr); - } -# define AO_HAVE_short_fetch_and_add_full -# endif /* !AO_PREFER_GENERALIZED */ - -#elif defined(AO_ASM_X64_AVAILABLE) +#ifdef AO_ASM_X64_AVAILABLE +#if _MSC_VER < 1800 AO_INLINE unsigned char AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr) { @@ -230,13 +66,10 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr) } } # define AO_HAVE_short_fetch_and_add_full +#endif /* _MSC_VER < 1800 */ -#endif /* _MSC_VER < 1800 && AO_ASM_X64_AVAILABLE */ - -#ifdef AO_ASM_X64_AVAILABLE - -/* As far as we can tell, the lfence and sfence instructions are not */ -/* currently needed or useful for cached memory accesses. */ + /* As far as we can tell, the lfence and sfence instructions are not */ + /* currently needed or useful for cached memory accesses. */ AO_INLINE void AO_nop_full(void) @@ -246,57 +79,62 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr) } # define AO_HAVE_nop_full - AO_INLINE AO_TS_VAL_t - AO_test_and_set_full(volatile AO_TS_t *addr) - { - __asm +# ifndef AO_HAVE_test_and_set_full +# include "../test_and_set_t_is_char.h" + + AO_INLINE AO_TS_VAL_t + AO_test_and_set_full(volatile AO_TS_t *addr) { + __asm + { mov rax,AO_TS_SET ; mov rbx,addr ; xchg byte ptr [rbx],al ; + } } - } -# define AO_HAVE_test_and_set_full +# define AO_HAVE_test_and_set_full +# endif #endif /* AO_ASM_X64_AVAILABLE */ +#ifndef AO_HAVE_test_and_set_full +# include "../test_and_set_t_is_ao_t.h" + /* AO_test_and_set_full() is emulated using word-wide CAS. */ +#endif + #ifdef AO_CMPXCHG16B_AVAILABLE -/* AO_compare_double_and_swap_double_full needs implementation for Win64. - * Also see ../gcc/x86.h for partial old Opteron workaround. - */ # if _MSC_VER >= 1500 - # include "../standard_ao_double_t.h" - # pragma intrinsic (_InterlockedCompareExchange128) -AO_INLINE int -AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, - AO_t old_val1, AO_t old_val2, - AO_t new_val1, AO_t new_val2) -{ - __int64 comparandResult[2]; + AO_INLINE int + AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { + __int64 comparandResult[2]; - AO_ASSERT_ADDR_ALIGNED(addr); - comparandResult[0] = old_val1; /* low */ - comparandResult[1] = old_val2; /* high */ - return _InterlockedCompareExchange128((volatile __int64 *)addr, - new_val2 /* high */, new_val1 /* low */, comparandResult); -} + AO_ASSERT_ADDR_ALIGNED(addr); + comparandResult[0] = old_val1; /* low */ + comparandResult[1] = old_val2; /* high */ + return _InterlockedCompareExchange128((volatile __int64 *)addr, + new_val2 /* high */, + new_val1 /* low */, + comparandResult); + } # define AO_HAVE_compare_double_and_swap_double_full # elif defined(AO_ASM_X64_AVAILABLE) - # include "../standard_ao_double_t.h" /* If there is no intrinsic _InterlockedCompareExchange128 then we */ /* need basically what's given below. */ -AO_INLINE int -AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, - AO_t old_val1, AO_t old_val2, - AO_t new_val1, AO_t new_val2) -{ + AO_INLINE int + AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, + AO_t old_val1, AO_t old_val2, + AO_t new_val1, AO_t new_val2) + { __asm { mov rdx,QWORD PTR [old_val2] ; @@ -306,7 +144,7 @@ AO_compare_double_and_swap_double_full(volatile AO_double_t *addr, lock cmpxchg16b [addr] ; setz rax ; } -} + } # define AO_HAVE_compare_double_and_swap_double_full # endif /* AO_ASM_X64_AVAILABLE && (_MSC_VER < 1500) */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h index 630c3762b3..4adf0b3d10 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2004-2011 Hewlett-Packard Development Company, L.P. + * Copyright (c) 2012-2021 Ivan Maidanski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,13 +23,15 @@ /* For 64-bit systems, we expect the double type to hold two int64's. */ -#if ((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \ - || defined(__aarch64__)) && !defined(__ILP32__) +#if (((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \ + || defined(__aarch64__)) && !defined(__ILP32__)) \ + || (defined(__e2k__) && __SIZEOF_SIZE_T__ == 8) \ + || (defined(__riscv) && __riscv_xlen == 64) /* x86-64: __m128 is not applicable to atomic intrinsics. */ # if AO_GNUC_PREREQ(4, 7) || AO_CLANG_PREREQ(3, 6) # pragma GCC diagnostic push /* Suppress warning about __int128 type. */ -# if defined(__clang__) || AO_GNUC_PREREQ(6, 4) +# if defined(__clang__) || AO_GNUC_PREREQ(6, 0) # pragma GCC diagnostic ignored "-Wpedantic" # else /* GCC before ~4.8 does not accept "-Wpedantic" quietly. */ @@ -39,6 +42,9 @@ # else /* pragma diagnostic is not supported */ typedef unsigned __int128 double_ptr_storage; # endif +#elif defined(_M_ARM64) && defined(_MSC_VER) + /* __int128 does not seem to be available. */ + typedef __declspec(align(16)) unsigned __int64 double_ptr_storage[2]; #elif ((defined(__x86_64__) && AO_GNUC_PREREQ(4, 0)) || defined(_WIN64)) \ && !defined(__ILP32__) /* x86-64 (except for x32): __m128 serves as a placeholder which also */ diff --git a/thirdparty/libatomic_ops/atomic_ops/sysdeps/sunc/x86.h b/thirdparty/libatomic_ops/atomic_ops/sysdeps/sunc/x86.h index dfac1e5637..5efda2503b 100644 --- a/thirdparty/libatomic_ops/atomic_ops/sysdeps/sunc/x86.h +++ b/thirdparty/libatomic_ops/atomic_ops/sysdeps/sunc/x86.h @@ -8,7 +8,7 @@ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. * * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. + * for any purpose, provided the above notices are retained on all copies. * Permission to modify the code and to distribute modified code is granted, * provided the above notices are retained, and a notice that the code was * modified is included with the above copyright notice. diff --git a/thirdparty/libatomic_ops/atomic_ops_malloc.c b/thirdparty/libatomic_ops/atomic_ops_malloc.c new file mode 100644 index 0000000000..5581cf51cd --- /dev/null +++ b/thirdparty/libatomic_ops/atomic_ops_malloc.c @@ -0,0 +1,428 @@ +/* + * Copyright (c) 2005 Hewlett-Packard Development Company, L.P. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#if defined(HAVE_CONFIG_H) +# include "config.h" +#endif + +#ifdef DONT_USE_MMAP /* for testing */ +# undef HAVE_MMAP +#endif + +#ifndef AO_BUILD +# define AO_BUILD +#endif + +#define AO_REQUIRE_CAS +#include "atomic_ops_malloc.h" + +#include /* for ffs, which is assumed reentrant. */ +#include +#include + +#ifdef AO_TRACE_MALLOC +# include +# include +#endif + +#if defined(AO_ADDRESS_SANITIZER) && !defined(AO_NO_MALLOC_POISON) + /* #include "sanitizer/asan_interface.h" */ + void __asan_poison_memory_region(void *, size_t); + void __asan_unpoison_memory_region(void *, size_t); +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + __asan_poison_memory_region(addr, size) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + __asan_unpoison_memory_region(addr, size) +#else +# define ASAN_POISON_MEMORY_REGION(addr, size) (void)0 +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) (void)0 +#endif /* !AO_ADDRESS_SANITIZER */ + +#if (defined(_WIN32_WCE) || defined(__MINGW32CE__)) && !defined(AO_HAVE_abort) +# define abort() _exit(-1) /* there is no abort() in WinCE */ +#endif + +/* + * We round up each allocation request to the next power of two + * minus one word. + * We keep one stack of free objects for each size. Each object has + * an initial word (offset from the visible pointer is -sizeof(AO_uintptr_t)) + * which contains either: + * - the binary log of the object size in bytes (for small objects), or + * - the object size (a multiple of CHUNK_SIZE) for large objects. + * The second case only arises if mmap-based allocation is supported. + * We align the user-visible part of each object on an ALIGNMENT + * byte boundary. That means that the actual (hidden) start of + * the object starts a word before this boundary. + */ + +#ifndef LOG_MAX_SIZE +# define LOG_MAX_SIZE 16 + /* We assume that 2**LOG_MAX_SIZE is a multiple of page size. */ +#endif + +#ifndef ALIGNMENT +# define ALIGNMENT 16 + /* Assumed to be at least sizeof(AO_uintptr_t). */ +#endif + +#define CHUNK_SIZE (1 << LOG_MAX_SIZE) + +#ifndef AO_INITIAL_HEAP_SIZE +# ifndef AO_INITIAL_HEAP_CHUNKS +# define AO_INITIAL_HEAP_CHUNKS 2*(LOG_MAX_SIZE+1) +# endif +# define AO_INITIAL_HEAP_SIZE (AO_INITIAL_HEAP_CHUNKS * CHUNK_SIZE) +#endif /* !AO_INITIAL_HEAP_SIZE */ + +static char AO_initial_heap[AO_INITIAL_HEAP_SIZE]; /* ~2MB by default */ + +static AO_internal_ptr_t volatile initial_heap_ptr = 0; + +#if defined(HAVE_MMAP) + +#include +#include +#include +#include + +#if defined(MAP_ANONYMOUS) || defined(MAP_ANON) +# define USE_MMAP_ANON +#endif + +#ifdef USE_MMAP_FIXED +# define GC_MMAP_FLAGS (MAP_FIXED | MAP_PRIVATE) + /* Seems to yield better performance on Solaris 2, but can */ + /* be unreliable if something is already mapped at the address. */ +#else +# define GC_MMAP_FLAGS MAP_PRIVATE +#endif + +#ifdef USE_MMAP_ANON +# if defined(CPPCHECK) +# define OPT_MAP_ANON 0x20 /* taken from linux */ +# elif defined(MAP_ANONYMOUS) +# define OPT_MAP_ANON MAP_ANONYMOUS +# else +# define OPT_MAP_ANON MAP_ANON +# endif +#else +# include /* for close() */ +# define OPT_MAP_ANON 0 +#endif + +static volatile AO_t mmap_enabled = 0; + +AO_API void +AO_malloc_enable_mmap(void) +{ +# if defined(__sun) + AO_store_release(&mmap_enabled, 1); + /* Workaround for Sun CC */ +# else + AO_store(&mmap_enabled, 1); +# endif +} + +static char *get_mmaped(size_t sz) +{ + char * result; +# ifdef USE_MMAP_ANON +# define zero_fd -1 +# else + int zero_fd; +# endif + + assert(!(sz & (CHUNK_SIZE - 1))); + if (!mmap_enabled) + return 0; + +# ifndef USE_MMAP_ANON + zero_fd = open("/dev/zero", O_RDONLY); + if (zero_fd == -1) + return 0; +# endif + result = (char *)mmap(0, sz, PROT_READ | PROT_WRITE, + GC_MMAP_FLAGS | OPT_MAP_ANON, + zero_fd, 0 /* offset */); +# ifndef USE_MMAP_ANON + close(zero_fd); +# endif + if (AO_EXPECT_FALSE(result == MAP_FAILED)) + result = NULL; + return result; +} + +#ifndef SIZE_MAX +# include +#endif +#if defined(SIZE_MAX) && !defined(CPPCHECK) +# define AO_SIZE_MAX ((size_t)SIZE_MAX) + /* Extra cast to workaround some buggy SIZE_MAX definitions. */ +#else +# define AO_SIZE_MAX (~(size_t)0) +#endif + +/* Saturated addition of size_t values. Used to avoid value wrap */ +/* around on overflow. The arguments should have no side effects. */ +#define SIZET_SAT_ADD(a, b) \ + (AO_EXPECT_FALSE((a) >= AO_SIZE_MAX - (b)) ? AO_SIZE_MAX : (a) + (b)) + +/* Allocate an object of size (incl. header) of size > CHUNK_SIZE. */ +/* sz includes space for a pointer-sized header. */ +static char * +AO_malloc_large(size_t sz) +{ + void *result; + + /* The header will force us to waste ALIGNMENT bytes, including the */ + /* header. Round to multiple of CHUNK_SIZE. */ + sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1) + & ~(size_t)(CHUNK_SIZE - 1); + assert(sz > LOG_MAX_SIZE); + result = get_mmaped(sz); + if (AO_EXPECT_FALSE(NULL == result)) + return NULL; + + result = (AO_uintptr_t *)result + ALIGNMENT / sizeof(AO_uintptr_t); + ((AO_uintptr_t *)result)[-1] = (AO_uintptr_t)sz; + return (char *)result; +} + +static void +AO_free_large(void *p) +{ + size_t sz = (size_t)(((AO_uintptr_t *)p)[-1]); + + if (munmap((AO_uintptr_t *)p - ALIGNMENT / sizeof(AO_uintptr_t), sz) != 0) + abort(); /* Programmer error. Not really async-signal-safe, but ... */ +} + +#else /* !HAVE_MMAP */ + +AO_API void +AO_malloc_enable_mmap(void) +{ +} + +#define get_mmaped(sz) ((char*)0) +#define AO_malloc_large(sz) ((char*)0) +#define AO_free_large(p) abort() + /* Programmer error. Not really async-signal-safe, but ... */ + +#endif /* !HAVE_MMAP */ + +/* TODO: Duplicates (partially) the definitions in atomic_ops_stack.c. */ +#if defined(AO_FAT_POINTER) || defined(AO_STACK_USE_CPTR) + AO_INLINE int + AO_cptr_compare_and_swap(AO_internal_ptr_t volatile *addr, + AO_internal_ptr_t old_val, AO_internal_ptr_t new_val) + { + return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + } + + AO_INLINE int + AO_cptr_compare_and_swap_acquire(AO_internal_ptr_t volatile *addr, + AO_internal_ptr_t old_val, AO_internal_ptr_t new_val) + { + return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE); + } + +# define AO_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED) +#else +# define AO_cptr_compare_and_swap AO_compare_and_swap +# define AO_cptr_compare_and_swap_acquire AO_compare_and_swap_acquire +# define AO_cptr_load AO_load +#endif + +static char * +get_chunk(void) +{ + AO_internal_ptr_t my_chunk_ptr; + + for (;;) { + AO_internal_ptr_t initial_ptr = AO_cptr_load(&initial_heap_ptr); + + my_chunk_ptr = AO_EXPECT_FALSE(0 == initial_ptr) ? + (AO_internal_ptr_t)AO_initial_heap : initial_ptr; + /* Round up the pointer to ALIGNMENT. */ +# ifdef AO_STACK_USE_CPTR + my_chunk_ptr += ((size_t)ALIGNMENT - (size_t)(AO_uintptr_t)my_chunk_ptr) + & (ALIGNMENT - 1); +# else + my_chunk_ptr = (AO_internal_ptr_t)(((AO_uintptr_t)my_chunk_ptr + + ALIGNMENT-1) + & ~(AO_uintptr_t)(ALIGNMENT-1)); +# endif + if (initial_ptr != my_chunk_ptr) { + /* Align correctly. If this fails, someone else did it for us. */ + assert(my_chunk_ptr != 0); + (void)AO_cptr_compare_and_swap_acquire(&initial_heap_ptr, initial_ptr, + my_chunk_ptr); + } + + if (AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr + < (AO_uintptr_t)AO_initial_heap) + || AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr + > (AO_uintptr_t)(AO_initial_heap + + AO_INITIAL_HEAP_SIZE - CHUNK_SIZE))) { + /* We failed. The initial heap is used up. */ + my_chunk_ptr = (AO_internal_ptr_t)get_mmaped(CHUNK_SIZE); +# if !defined(CPPCHECK) + assert(((AO_uintptr_t)my_chunk_ptr & (ALIGNMENT - 1)) == 0); +# endif + break; + } + if (AO_cptr_compare_and_swap(&initial_heap_ptr, my_chunk_ptr, + my_chunk_ptr + CHUNK_SIZE)) { + break; + } + } + return (char *)my_chunk_ptr; +} + +/* Object free lists. I-th entry corresponds to objects */ +/* of total size 2**i bytes. */ +static AO_stack_t AO_free_list[LOG_MAX_SIZE+1]; + +/* Break up the chunk, and add it to the object free list for */ +/* the given size. We have exclusive access to chunk. */ +static void add_chunk_as(void * chunk, unsigned log_sz) +{ + size_t ofs, limit; + size_t sz = (size_t)1 << log_sz; + + assert((size_t)CHUNK_SIZE >= sz); + assert(sz % sizeof(AO_uintptr_t) == 0); + limit = (size_t)CHUNK_SIZE - sz; + for (ofs = ALIGNMENT - sizeof(AO_uintptr_t); ofs <= limit; ofs += sz) { + ASAN_POISON_MEMORY_REGION((char *)chunk + ofs + sizeof(AO_uintptr_t), + sz - sizeof(AO_uintptr_t)); + AO_stack_push(&AO_free_list[log_sz], + (AO_uintptr_t *)chunk + ofs / sizeof(AO_uintptr_t)); + } +} + +static const unsigned char msbs[16] = { + 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4 +}; + +/* Return the position of the most significant set bit in the */ +/* argument. */ +/* We follow the conventions of ffs(), i.e. the least */ +/* significant bit is number one. */ +static unsigned msb(size_t s) +{ + unsigned result = 0; + if ((s & 0xff) != s) { +# if (__SIZEOF_SIZE_T__ == 8) && !defined(CPPCHECK) + unsigned v = (unsigned)(s >> 32); + if (AO_EXPECT_FALSE(v != 0)) + { + s = v; + result += 32; + } +# elif __SIZEOF_SIZE_T__ == 4 + /* No op. */ +# else + unsigned v; + /* The following is a tricky code ought to be equivalent to */ + /* "(v = s >> 32) != 0" but suppresses warnings on 32-bit arch's. */ +# define SIZEOF_SIZE_T_GT_4 (sizeof(size_t) > 4) + if (SIZEOF_SIZE_T_GT_4 + && (v = (unsigned)(s >> (SIZEOF_SIZE_T_GT_4 ? 32 : 0))) != 0) + { + s = v; + result += 32; + } +# endif /* !defined(__SIZEOF_SIZE_T__) */ + if (AO_EXPECT_FALSE((s >> 16) != 0)) + { + s >>= 16; + result += 16; + } + if ((s >> 8) != 0) + { + s >>= 8; + result += 8; + } + } + if (s > 15) + { + s >>= 4; + result += 4; + } + result += msbs[s]; + return result; +} + +AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1) +void * +AO_malloc(size_t sz) +{ + AO_uintptr_t *result; + unsigned log_sz; + + if (AO_EXPECT_FALSE(sz > CHUNK_SIZE - sizeof(AO_uintptr_t))) + return AO_malloc_large(sz); + log_sz = msb(sz + sizeof(AO_uintptr_t) - 1); + assert(log_sz <= LOG_MAX_SIZE); + assert(((size_t)1 << log_sz) >= sz + sizeof(AO_uintptr_t)); + result = AO_stack_pop(AO_free_list + log_sz); + while (AO_EXPECT_FALSE(NULL == result)) { + void *chunk = get_chunk(); + + if (AO_EXPECT_FALSE(NULL == chunk)) + return NULL; + add_chunk_as(chunk, log_sz); + result = AO_stack_pop(AO_free_list + log_sz); + } + *result = log_sz; +# ifdef AO_TRACE_MALLOC + fprintf(stderr, "%p: AO_malloc(%lu) = %p\n", + (void *)pthread_self(), (unsigned long)sz, (void *)(result + 1)); +# endif + ASAN_UNPOISON_MEMORY_REGION(result + 1, sz); + return result + 1; +} + +AO_API void +AO_free(void *p) +{ + AO_uintptr_t *base; + int log_sz; + + if (AO_EXPECT_FALSE(NULL == p)) + return; + + base = (AO_uintptr_t *)p - 1; + log_sz = (int)(*base); +# ifdef AO_TRACE_MALLOC + fprintf(stderr, "%p: AO_free(%p sz:%lu)\n", (void *)pthread_self(), p, + log_sz > LOG_MAX_SIZE ? (unsigned)log_sz : 1UL << log_sz); +# endif + if (AO_EXPECT_FALSE(log_sz > LOG_MAX_SIZE)) { + AO_free_large(p); + } else { + ASAN_POISON_MEMORY_REGION(base + 1, + ((size_t)1 << log_sz) - sizeof(AO_uintptr_t)); + AO_stack_push(AO_free_list + log_sz, base); + } +} diff --git a/thirdparty/libatomic_ops/atomic_ops_malloc.h b/thirdparty/libatomic_ops/atomic_ops_malloc.h new file mode 100644 index 0000000000..be997f5bb4 --- /dev/null +++ b/thirdparty/libatomic_ops/atomic_ops_malloc.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2005 Hewlett-Packard Development Company, L.P. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Almost lock-free malloc implementation based on stack implementation. */ +/* See README_malloc.txt file for detailed usage rules. */ + +#ifndef AO_MALLOC_H +#define AO_MALLOC_H + +#include "atomic_ops_stack.h" + +#include /* for size_t */ + +#ifdef __cplusplus + extern "C" { +#endif + +#ifdef AO_STACK_IS_LOCK_FREE +# define AO_MALLOC_IS_LOCK_FREE +#endif + +#ifndef AO_ATTR_MALLOC +# if AO_GNUC_PREREQ(3, 1) +# define AO_ATTR_MALLOC __attribute__((__malloc__)) +# elif defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(__EDG__) +# define AO_ATTR_MALLOC \ + __declspec(allocator) __declspec(noalias) __declspec(restrict) +# elif defined(_MSC_VER) && _MSC_VER >= 1400 +# define AO_ATTR_MALLOC __declspec(noalias) __declspec(restrict) +# else +# define AO_ATTR_MALLOC /* empty */ +# endif +#endif + +#ifndef AO_ATTR_ALLOC_SIZE +# ifdef __clang__ +# if __has_attribute(__alloc_size__) +# define AO_ATTR_ALLOC_SIZE(argnum) \ + __attribute__((__alloc_size__(argnum))) +# else +# define AO_ATTR_ALLOC_SIZE(argnum) /* empty */ +# endif +# elif AO_GNUC_PREREQ(4, 3) && !defined(__ICC) +# define AO_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum))) +# else +# define AO_ATTR_ALLOC_SIZE(argnum) /* empty */ +# endif +#endif + +AO_API void AO_free(void *); + +AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1) +void * AO_malloc(size_t); + +/* Allow use of mmap to grow the heap. No-op on some platforms. */ +AO_API void AO_malloc_enable_mmap(void); + +#ifdef __cplusplus + } /* extern "C" */ +#endif + +#endif /* !AO_MALLOC_H */ diff --git a/thirdparty/libatomic_ops/atomic_ops_sysdeps.S b/thirdparty/libatomic_ops/atomic_ops_sysdeps.S new file mode 100644 index 0000000000..f586f23bae --- /dev/null +++ b/thirdparty/libatomic_ops/atomic_ops_sysdeps.S @@ -0,0 +1,9 @@ +/* + * Include the appropriate system-dependent assembly file, if any. + * This is used only if the platform supports neither inline assembly + * code, nor appropriate compiler intrinsics. + */ + +#if !defined(__GNUC__) && (defined(sparc) || defined(__sparc)) +# include "atomic_ops/sysdeps/sunc/sparc.S" +#endif