mirrors/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2025-09-13 22:42:26 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

thirdparty: update libatomic_ops to commit cc9bc49 from https://github.com/ivmai/libatomic_ops/ (#22235)

Browse source
This commit is contained in:
Delyan Angelov 2024-09-16 21:20:42 +03:00 • committed by GitHub
parent c3b55a68a5
commit f53b5d737f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 2102 additions and 375 deletions
Download patch file Download diff file Expand all files Collapse all files

65
thirdparty/libatomic_ops/atomic_ops.c vendored
View file

@ -45,14 +45,29 @@
# define AO_USE_NO_SIGNALS
#endif
#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \
#if (defined(__CYGWIN__) || defined(__GLIBC__) || defined(__GNU__) \
|| defined(__linux__)) \
&& !defined(AO_USE_NO_SIGNALS) && !defined(_GNU_SOURCE)
# define _GNU_SOURCE 1
#endif
#ifndef AO_BUILD
# define AO_BUILD
#endif
#undef AO_REQUIRE_CAS
#include "atomic_ops.h" /* Without cas emulation! */
#ifdef __cplusplus
extern "C" {
#endif
AO_API void AO_pause(int); /* defined below */
#ifdef __cplusplus
} /* extern "C" */
#endif
#if !defined(_MSC_VER) && !defined(__MINGW32__) && !defined(__BORLANDC__) \
|| defined(AO_USE_NO_SIGNALS)
@ -80,7 +95,29 @@
# include "atomic_ops/sysdeps/standard_ao_double_t.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
AO_t old_val, AO_t new_val);
AO_API int
AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2);
AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
/* Lock for pthreads-based implementation. */
#ifndef AO_NO_PTHREADS
AO_API pthread_mutex_t AO_pt_lock;
#endif
#ifdef __cplusplus
} /* extern "C" */
#endif
#ifndef AO_NO_PTHREADS
pthread_mutex_t AO_pt_lock = PTHREAD_MUTEX_INITIALIZER;
#endif
@ -99,7 +136,7 @@
*/
#define AO_HASH_SIZE 16
#define AO_HASH(x) (((unsigned long)(x) >> 12) & (AO_HASH_SIZE-1))
#define AO_HASH(x) ((unsigned)((AO_uintptr_t)(x) >> 12) & (AO_HASH_SIZE-1))
static AO_TS_t AO_locks[AO_HASH_SIZE] = {
AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER,
@ -108,8 +145,6 @@ static AO_TS_t AO_locks[AO_HASH_SIZE] = {
AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER, AO_TS_INITIALIZER,
};
void AO_pause(int); /* defined below */
static void lock_ool(volatile AO_TS_t *l)
{
int i = 0;
@ -155,8 +190,8 @@ AO_INLINE void unlock(volatile AO_TS_t *l)
}
#endif /* !AO_USE_NO_SIGNALS */
AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
AO_t new_val)
AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
AO_t old_val, AO_t new_val)
{
AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
AO_t fetched_val;
@ -176,7 +211,8 @@ AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
return fetched_val;
}
int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
AO_API int
AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
@ -203,7 +239,7 @@ int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
return result;
}
void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
{
AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
lock(my_lock);
@ -222,7 +258,7 @@ void AO_store_full_emulation(volatile AO_t *addr, AO_t val)
#endif
static AO_t spin_dummy = 1;
static volatile AO_t spin_dummy = 0;
/* Spin for 2**n units. */
static void AO_spin(int n)
@ -231,17 +267,16 @@ static void AO_spin(int n)
int i = 2 << n;
while (i-- > 0)
j += (j - 1) << 2;
/* Given 'spin_dummy' is initialized to 1, j is 1 after the loop. */
j += j << 2;
/* Given spin_dummy is initialized to 0, j is 0 after the loop. */
AO_store(&spin_dummy, j);
}
void AO_pause(int n)
AO_API void AO_pause(int n)
{
if (n < 12)
if (n < 12) {
AO_spin(n);
else
{
} else {
# ifdef AO_USE_NANOSLEEP
struct timespec ts;
ts.tv_sec = 0;

60
thirdparty/libatomic_ops/atomic_ops.h vendored
View file

@ -1,6 +1,6 @@
/*
* Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2008-2021 Ivan Maidanski
* Copyright (c) 2008-2022 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -153,8 +153,30 @@
/* atomic_ops/generalize.h. */
/* Some common defaults. Overridden for some architectures. */
#define AO_t size_t
#if defined(__SIZEOF_POINTER__) \
&& (__SIZEOF_POINTER__ == 2 * __SIZEOF_SIZE_T__)
/* Pointers are twice bigger than the machine word. */
# define AO_FAT_POINTER
#endif
#ifndef AO_FAT_POINTER
# define AO_uintptr_t AO_t
#elif defined(__e2k__)
/* For some reason uintptr_t is 64-bit on E2K in the protected mode. */
typedef unsigned __int128 AO_uintptr_t;
#else
# include <inttypes.h>
# define AO_uintptr_t uintptr_t
#endif
/* A compile-time assertion for AO_uintptr_t size. */
struct AO_uintptr_t_size_static_assert {
char dummy[sizeof(AO_uintptr_t) == sizeof(void *) ? 1 : -1];
};
/* The test_and_set primitive returns an AO_TS_VAL_t value. */
/* AO_TS_t is the type of an in-memory test-and-set location. */
@ -235,12 +257,36 @@
# define AO_ALIGNOF_SUPPORTED 1
#endif
#if defined(AO_DLL) && !defined(AO_API)
# ifdef AO_BUILD
# if defined(__CEGCC__) || (defined(__MINGW32__) && !defined(__cplusplus))
# define AO_API __declspec(dllexport)
# elif defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__) \
|| defined(__DMC__) || defined(__MINGW32__) || defined(__WATCOMC__)
# define AO_API extern __declspec(dllexport)
# endif
# else
# if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CEGCC__) \
|| defined(__CYGWIN__) || defined(__DMC__)
# define AO_API __declspec(dllimport)
# elif defined(__MINGW32_DELAY_LOAD__)
# define AO_API __declspec(dllexport)
# elif defined(__MINGW32__) || defined(__WATCOMC__)
# define AO_API extern __declspec(dllimport)
# endif
# endif
#endif /* AO_DLL */
#ifndef AO_API
# define AO_API extern
#endif
#ifdef AO_ALIGNOF_SUPPORTED
# define AO_ASSERT_ADDR_ALIGNED(addr) \
assert(((size_t)(addr) & (__alignof__(*(addr)) - 1)) == 0)
assert(((AO_uintptr_t)(addr) & (__alignof__(*(addr)) - 1)) == 0)
#else
# define AO_ASSERT_ADDR_ALIGNED(addr) \
assert(((size_t)(addr) & (sizeof(*(addr)) - 1)) == 0)
assert(((AO_uintptr_t)(addr) & (sizeof(*(addr)) - 1)) == 0)
#endif /* !AO_ALIGNOF_SUPPORTED */
#if defined(__GNUC__) && !defined(__INTEL_COMPILER)
@ -270,7 +316,7 @@
# include <machine/sys/inline.h>
# define AO_compiler_barrier() _Asm_sched_fence()
# else
/* FIXME: We do not know how to do this. This is a guess. */
/* FIXME - We do not know how to do this. This is a guess. */
/* And probably a bad one. */
static volatile int AO_barrier_dummy;
# define AO_compiler_barrier() (void)(AO_barrier_dummy = AO_barrier_dummy)
@ -340,6 +386,8 @@
# define AO_CAN_EMUL_CAS
# elif defined(__avr32__)
# include "atomic_ops/sysdeps/gcc/avr32.h"
# elif defined(__e2k__)
# include "atomic_ops/sysdeps/gcc/e2k.h"
# elif defined(__hexagon__)
# include "atomic_ops/sysdeps/gcc/hexagon.h"
# elif defined(__nios2__)
@ -395,8 +443,10 @@
#if defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \
|| (defined(__WATCOMC__) && defined(__NT__))
# if defined(_AMD64_) || defined(_M_X64) || defined(_M_ARM64)
# if defined(_AMD64_) || defined(_M_X64)
# include "atomic_ops/sysdeps/msftc/x86_64.h"
# elif defined(_M_ARM64)
# include "atomic_ops/sysdeps/msftc/arm64.h"
# elif defined(_M_IX86) || defined(x86)
# include "atomic_ops/sysdeps/msftc/x86.h"
# elif defined(_M_ARM) || defined(ARM) || defined(_ARM_)

4
thirdparty/libatomic_ops/atomic_ops/ao_version.h vendored
View file

@ -34,5 +34,5 @@
/* The version here should match that in configure.ac and README. */
#define AO_VERSION_MAJOR 7
#define AO_VERSION_MINOR 6
#define AO_VERSION_MICRO 12 /* 7.6.12 */
#define AO_VERSION_MINOR 9
#define AO_VERSION_MICRO 0 /* 7.9.0 */

9
thirdparty/libatomic_ops/atomic_ops/sysdeps/emul_cas.h vendored
View file

@ -47,14 +47,15 @@
extern "C" {
#endif
AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
AO_t new_val);
AO_API AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr,
AO_t old_val, AO_t new_val);
int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
AO_API int
AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2);
void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
AO_API void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
#ifndef AO_HAVE_fetch_compare_and_swap_full
# define AO_fetch_compare_and_swap_full(addr, old, newval) \

40
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/e2k.h vendored Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright (c) 2022 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* As of clang-9, all __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n are missing. */
#define AO_GCC_FORCE_HAVE_CAS
#if (__SIZEOF_SIZE_T__ == 4) \
|| (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !defined(__clang__)) \
|| (defined(__clang__) && __iset__ >= 5 /* elbrus-v5 or later */ \
&& defined(AO_PREFER_BUILTIN_ATOMICS))
/* Note for 128-bit operations: clang reports warning */
/* "large atomic operation may incur significant performance penalty" */
/* and requires LDFLAGS="-latomic", thus this is not on by default. */
# define AO_GCC_HAVE_double_SYNC_CAS
# include "../standard_ao_double_t.h"
#endif
#include "generic.h"
#undef AO_GCC_FORCE_HAVE_CAS
#undef AO_GCC_HAVE_double_SYNC_CAS

18
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/generic.h vendored
View file

@ -58,6 +58,10 @@
}
# define AO_HAVE_nop_full
#elif defined(AO_THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
/* Workaround a compiler warning (reported by gcc-11, at least) */
/* that atomic_thread_fence is unsupported with thread sanitizer. */
#else
AO_INLINE void
AO_nop_read(void)
@ -82,7 +86,7 @@
__atomic_thread_fence(__ATOMIC_SEQ_CST);
}
# define AO_HAVE_nop_full
#endif /* !AO_UNIPROCESSOR */
#endif /* !AO_UNIPROCESSOR && !AO_THREAD_SANITIZER */
#include "generic-small.h"
@ -95,28 +99,32 @@
AO_INLINE AO_TS_VAL_t
AO_test_and_set(volatile AO_TS_t *addr)
{
return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELAXED);
return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELAXED)
? AO_TS_SET : AO_TS_CLEAR);
}
# define AO_HAVE_test_and_set
AO_INLINE AO_TS_VAL_t
AO_test_and_set_acquire(volatile AO_TS_t *addr)
{
return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_ACQUIRE);
return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_ACQUIRE)
? AO_TS_SET : AO_TS_CLEAR);
}
# define AO_HAVE_test_and_set_acquire
AO_INLINE AO_TS_VAL_t
AO_test_and_set_release(volatile AO_TS_t *addr)
{
return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELEASE);
return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_RELEASE)
? AO_TS_SET : AO_TS_CLEAR);
}
# define AO_HAVE_test_and_set_release
AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t *addr)
{
return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_SEQ_CST);
return (AO_TS_VAL_t)(__atomic_test_and_set(addr, __ATOMIC_SEQ_CST)
? AO_TS_SET : AO_TS_CLEAR);
}
# define AO_HAVE_test_and_set_full
#endif /* !AO_PREFER_GENERALIZED */

68
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/riscv.h vendored
View file

@ -25,8 +25,76 @@
# endif
#endif /* !__clang__ */
#if defined(__riscv_zacas) && __riscv_xlen == 64 && !defined(AO_NO_DOUBLE_CAS)
/* TODO: Support also rv32, i.e. use amocas.w. */
# define AO_SKIPATOMIC_double_load
# define AO_SKIPATOMIC_double_load_acquire
# define AO_SKIPATOMIC_double_store
# define AO_SKIPATOMIC_double_store_release
# include "../standard_ao_double_t.h"
AO_INLINE int
AO_double_compare_and_swap(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_double_t expected = old_val;
__asm__ __volatile__("amocas.q %0, %z2, %1"
: "+rJ" (expected.AO_whole), "+A" (*addr)
: "rJ" (new_val.AO_whole));
return expected.AO_whole == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap
AO_INLINE int
AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_double_t expected = old_val;
__asm__ __volatile__("amocas.q.aq %0, %z2, %1"
: "+rJ" (expected.AO_whole), "+A" (*addr)
: "rJ" (new_val.AO_whole));
return expected.AO_whole == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_acquire
AO_INLINE int
AO_double_compare_and_swap_release(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_double_t expected = old_val;
__asm__ __volatile__("amocas.q.rl %0, %z2, %1"
: "+rJ" (expected.AO_whole), "+A" (*addr)
: "rJ" (new_val.AO_whole));
return expected.AO_whole == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_release
AO_INLINE int
AO_double_compare_and_swap_full(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_double_t expected = old_val;
__asm__ __volatile__("amocas.q.aqrl %0, %z2, %1"
: "+rJ" (expected.AO_whole), "+A" (*addr)
: "rJ" (new_val.AO_whole));
return expected.AO_whole == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_full
#endif /* __riscv_zacas */
#include "generic.h"
#undef AO_GCC_FORCE_HAVE_CAS
#undef AO_NO_char_ARITHM
#undef AO_NO_short_ARITHM
#undef AO_SKIPATOMIC_double_load
#undef AO_SKIPATOMIC_double_load_acquire
#undef AO_SKIPATOMIC_double_store
#undef AO_SKIPATOMIC_double_store_release

29
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/sparc.h vendored
View file

@ -15,8 +15,13 @@
*
*/
/* TODO: Very incomplete; Add support for sparc64. */
/* Non-ancient SPARCs provide compare-and-swap (casa). */
#if (AO_GNUC_PREREQ(12, 0) || AO_CLANG_PREREQ(13, 0)) \
&& !defined(AO_DISABLE_GCC_ATOMICS)
/* Probably, it could be enabled for earlier compiler versions as well. */
# include "generic.h"
#else /* AO_DISABLE_GCC_ATOMICS */
#include "../all_atomic_load_store.h"
@ -43,23 +48,17 @@ AO_test_and_set_full(volatile AO_TS_t *addr) {
/* Returns nonzero if the comparison succeeded. */
AO_INLINE int
AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) {
AO_t ret;
__asm__ __volatile__ ("membar #StoreLoad | #LoadLoad\n\t"
# if defined(__arch64__)
"casx [%2],%0,%1\n\t"
"casx [%1],%2,%0\n\t"
# else
"cas [%2],%0,%1\n\t" /* 32-bit version */
"cas [%1],%2,%0\n\t" /* 32-bit version */
# endif
"membar #StoreLoad | #StoreStore\n\t"
"cmp %0,%1\n\t"
"be,a 0f\n\t"
"mov 1,%0\n\t"/* one insn after branch always executed */
"clr %0\n\t"
"0:\n\t"
: "=r" (ret), "+r" (new_val)
: "r" (addr), "0" (old)
: "memory", "cc");
return (int)ret;
: "+r" (new_val)
: "r" (addr), "r" (old)
: "memory");
return new_val == old;
}
# define AO_HAVE_compare_and_swap_full
# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
@ -85,3 +84,5 @@ AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val) {
/* TODO: Extend this for SPARC v8 and v9 (V8 also has swap, V9 has CAS, */
/* there are barriers like membar #LoadStore, CASA (32-bit) and */
/* CASXA (64-bit) instructions added in V9). */
#endif /* AO_DISABLE_GCC_ATOMICS */

4
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/tile.h vendored
View file

@ -9,8 +9,6 @@
* modified is included with the above copyright notice.
*/
/* Minimal support for tile. */
#if (AO_GNUC_PREREQ(4, 8) || AO_CLANG_PREREQ(3, 4)) \
&& !defined(AO_DISABLE_GCC_ATOMICS)
@ -18,6 +16,8 @@
#else /* AO_DISABLE_GCC_ATOMICS */
/* Minimal support for tile. */
# include "../all_atomic_load_store.h"
# include "../test_and_set_t_is_ao_t.h"

10
thirdparty/libatomic_ops/atomic_ops/sysdeps/gcc/x86.h vendored
View file

@ -68,14 +68,14 @@
# endif
# endif /* __x86_64__ */
# elif AO_GNUC_PREREQ(7, 0) && !defined(AO_PREFER_BUILTIN_ATOMICS) \
&& !defined(AO_THREAD_SANITIZER) && !defined(__MINGW32__)
/* gcc-7.x/x64 (gcc-7.2, at least) requires -latomic flag in case */
# elif defined(__x86_64__) && !defined(AO_PREFER_BUILTIN_ATOMICS) \
&& !defined(AO_THREAD_SANITIZER)
/* gcc/x64 (as of gcc-12.2) requires -latomic flag in case */
/* of double-word atomic operations use (but not in case of TSan). */
/* TODO: Revise it for the future gcc-7 releases. */
/* TODO: Revise it for the future gcc releases. */
# define AO_SKIPATOMIC_double_compare_and_swap_ANY
# define AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY
# endif /* __GNUC__ && !__clang__ */
# endif /* __x86_64__ && !__clang__ */
# ifdef AO_SKIPATOMIC_DOUBLE_LOAD_STORE_ANY
# define AO_SKIPATOMIC_double_load

2
thirdparty/libatomic_ops/atomic_ops/sysdeps/generic_pthread.h vendored
View file

@ -39,7 +39,7 @@
/* We define only the full barrier variants, and count on the */
/* generalization section below to fill in the rest. */
extern pthread_mutex_t AO_pt_lock;
AO_API pthread_mutex_t AO_pt_lock;
#ifdef __cplusplus
} /* extern "C" */

86
thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm.h vendored
View file

@ -1,6 +1,6 @@
/*
* Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2009-2017 Ivan Maidanski
* Copyright (c) 2009-2021 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -21,21 +21,20 @@
* SOFTWARE.
*/
#ifndef AO_ASSUME_WINDOWS98
/* CAS is always available */
# define AO_ASSUME_WINDOWS98
#endif
#include "common32_defs.h"
#include "../test_and_set_t_is_ao_t.h"
/* AO_test_and_set_full() is emulated using CAS. */
/* Some ARM slide set, if it has been read correctly, claims that Loads */
/* followed by either a Load or a Store are ordered, but nothing else. */
/* It is assumed that Windows interrupt handlers clear the LL/SC flag. */
/* Unaligned accesses are not guaranteed to be atomic. */
#include "../all_aligned_atomic_load_store.h"
#define AO_T_IS_INT
#ifndef AO_ASSUME_WINDOWS98
/* CAS is always available */
# define AO_ASSUME_WINDOWS98
#endif
#include "common32_defs.h"
/* If only a single processor is used, we can define AO_UNIPROCESSOR. */
#ifdef AO_UNIPROCESSOR
AO_INLINE void AO_nop_full(void)
@ -47,16 +46,67 @@
/* AO_nop_full() is emulated using AO_test_and_set_full(). */
#endif
#if _M_ARM >= 6
/* ARMv6 is the first architecture providing support for simple LL/SC. */
#ifndef AO_HAVE_test_and_set_full
# include "../test_and_set_t_is_ao_t.h"
/* AO_test_and_set_full() is emulated. */
#endif
/* #include "../standard_ao_double_t.h" */
/* TODO: implement double-wide operations (similar to x86). */
#if _M_ARM >= 7 && !defined(AO_NO_DOUBLE_CAS)
#else /* _M_ARM < 6 */
# include "../standard_ao_double_t.h"
/* TODO: implement AO_test_and_set_full using SWP. */
/* These intrinsics are supposed to use LDREXD/STREXD. */
# pragma intrinsic (_InterlockedCompareExchange64)
# pragma intrinsic (_InterlockedCompareExchange64_acq)
# pragma intrinsic (_InterlockedCompareExchange64_nf)
# pragma intrinsic (_InterlockedCompareExchange64_rel)
#endif /* _M_ARM < 6 */
AO_INLINE int
AO_double_compare_and_swap(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_ASSERT_ADDR_ALIGNED(addr);
return (double_ptr_storage)_InterlockedCompareExchange64_nf(
(__int64 volatile *)addr,
new_val.AO_whole /* exchange */,
old_val.AO_whole) == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap
#define AO_T_IS_INT
AO_INLINE int
AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_ASSERT_ADDR_ALIGNED(addr);
return (double_ptr_storage)_InterlockedCompareExchange64_acq(
(__int64 volatile *)addr,
new_val.AO_whole /* exchange */,
old_val.AO_whole) == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_acquire
AO_INLINE int
AO_double_compare_and_swap_release(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_ASSERT_ADDR_ALIGNED(addr);
return (double_ptr_storage)_InterlockedCompareExchange64_rel(
(__int64 volatile *)addr,
new_val.AO_whole /* exchange */,
old_val.AO_whole) == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_release
AO_INLINE int
AO_double_compare_and_swap_full(volatile AO_double_t *addr,
AO_double_t old_val, AO_double_t new_val)
{
AO_ASSERT_ADDR_ALIGNED(addr);
return (double_ptr_storage)_InterlockedCompareExchange64(
(__int64 volatile *)addr,
new_val.AO_whole /* exchange */,
old_val.AO_whole) == old_val.AO_whole;
}
# define AO_HAVE_double_compare_and_swap_full
#endif /* _M_ARM >= 7 && !AO_NO_DOUBLE_CAS */

116
thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/arm64.h vendored Normal file
View file

@ -0,0 +1,116 @@
/*
* Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2021 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "../all_aligned_atomic_load_store.h"
#ifndef AO_ASSUME_WINDOWS98
# define AO_ASSUME_WINDOWS98
#endif
#ifndef AO_USE_INTERLOCKED_INTRINSICS
# define AO_USE_INTERLOCKED_INTRINSICS
#endif
#include "common32_defs.h"
#ifndef AO_HAVE_test_and_set_full
# include "../test_and_set_t_is_ao_t.h"
/* AO_test_and_set_full() is emulated using word-wide CAS. */
#endif
#ifndef AO_NO_DOUBLE_CAS
# include "../standard_ao_double_t.h"
# pragma intrinsic (_InterlockedCompareExchange128)
# pragma intrinsic (_InterlockedCompareExchange128_acq)
# pragma intrinsic (_InterlockedCompareExchange128_nf)
# pragma intrinsic (_InterlockedCompareExchange128_rel)
AO_INLINE int
AO_compare_double_and_swap_double(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
__int64 comparandResult[2];
AO_ASSERT_ADDR_ALIGNED(addr);
comparandResult[0] = old_val1; /* low */
comparandResult[1] = old_val2; /* high */
return _InterlockedCompareExchange128_nf((volatile __int64 *)addr,
new_val2 /* high */,
new_val1 /* low */,
comparandResult);
}
# define AO_HAVE_compare_double_and_swap_double
AO_INLINE int
AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
__int64 comparandResult[2];
AO_ASSERT_ADDR_ALIGNED(addr);
comparandResult[0] = old_val1; /* low */
comparandResult[1] = old_val2; /* high */
return _InterlockedCompareExchange128_acq((volatile __int64 *)addr,
new_val2 /* high */,
new_val1 /* low */,
comparandResult);
}
# define AO_HAVE_compare_double_and_swap_double_acquire
AO_INLINE int
AO_compare_double_and_swap_double_release(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
__int64 comparandResult[2];
AO_ASSERT_ADDR_ALIGNED(addr);
comparandResult[0] = old_val1; /* low */
comparandResult[1] = old_val2; /* high */
return _InterlockedCompareExchange128_rel((volatile __int64 *)addr,
new_val2 /* high */,
new_val1 /* low */,
comparandResult);
}
# define AO_HAVE_compare_double_and_swap_double_release
AO_INLINE int
AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
AO_t old_val1, AO_t old_val2,
AO_t new_val1, AO_t new_val2)
{
__int64 comparandResult[2];
AO_ASSERT_ADDR_ALIGNED(addr);
comparandResult[0] = old_val1; /* low */
comparandResult[1] = old_val2; /* high */
return _InterlockedCompareExchange128((volatile __int64 *)addr,
new_val2 /* high */,
new_val1 /* low */,
comparandResult);
}
# define AO_HAVE_compare_double_and_swap_double_full
#endif /* !AO_NO_DOUBLE_CAS */

993
thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/common32_defs.h vendored
View file

File diff suppressed because it is too large Load diff

23
thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86.h vendored
View file

@ -1,5 +1,6 @@
/*
* Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2009-2021 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -20,12 +21,13 @@
* SOFTWARE.
*/
/* If AO_ASSUME_VISTA is defined, we assume Windows Server 2003, Vista */
/* or later. */
#include "../all_aligned_atomic_load_store.h"
#include "../test_and_set_t_is_char.h"
#if !defined(AO_ASSUME_VISTA) && _MSC_VER >= 1910
/* Visual Studio 2017 (15.0) discontinued support of Windows XP. */
/* We assume Windows Server 2003, Vista or later. */
# define AO_ASSUME_VISTA
#endif
#if !defined(AO_ASSUME_WINDOWS98) \
&& (defined(AO_ASSUME_VISTA) || _MSC_VER >= 1400)
@ -33,6 +35,13 @@
# define AO_ASSUME_WINDOWS98
#endif
#if !defined(AO_USE_PENTIUM4_INSTRS) && _M_IX86_FP >= 2 /* SSE2 */
/* "mfence" is a part of SSE2 set (introduced on Intel Pentium 4). */
# define AO_USE_PENTIUM4_INSTRS
#endif
#define AO_T_IS_INT
#ifndef AO_USE_INTERLOCKED_INTRINSICS
/* _Interlocked primitives (Inc, Dec, Xchg, Add) are always available */
# define AO_USE_INTERLOCKED_INTRINSICS
@ -91,6 +100,9 @@ AO_nop_full(void)
# define AO_HAVE_short_fetch_and_add_full
#endif /* !AO_NO_ASM_XADD */
#ifndef AO_HAVE_test_and_set_full
# include "../test_and_set_t_is_char.h"
AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t *addr)
{
@ -103,6 +115,7 @@ AO_test_and_set_full(volatile AO_TS_t *addr)
/* Ignore possible "missing return value" warning here. */
}
# define AO_HAVE_test_and_set_full
#endif
#if defined(_WIN64) && !defined(CPPCHECK)
# error wrong architecture
@ -134,8 +147,6 @@ AO_test_and_set_full(volatile AO_TS_t *addr)
# define AO_HAVE_double_compare_and_swap_full
#endif /* AO_ASSUME_VISTA */
#define AO_T_IS_INT
/* Real X86 implementations, except for some old WinChips, appear */
/* to enforce ordering between memory operations, EXCEPT that a later */
/* read can pass earlier writes, presumably due to the visible */

208
thirdparty/libatomic_ops/atomic_ops/sysdeps/msftc/x86_64.h vendored
View file

@ -1,5 +1,6 @@
/*
* Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2009-2021 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -28,185 +29,20 @@
/* presence of store buffers. */
/* We ignore the fact that the official specs */
/* seem to be much weaker (and arguably too weak to be usable). */
#include "../ordered_except_wr.h"
#ifdef AO_ASM_X64_AVAILABLE
# include "../test_and_set_t_is_char.h"
#else
# include "../test_and_set_t_is_ao_t.h"
#ifndef AO_ASSUME_WINDOWS98
/* CAS is always available */
# define AO_ASSUME_WINDOWS98
#endif
#ifndef AO_USE_INTERLOCKED_INTRINSICS
# define AO_USE_INTERLOCKED_INTRINSICS
#endif
#include "common32_defs.h"
/* Assume _MSC_VER >= 1400 */
#include <intrin.h>
#pragma intrinsic (_InterlockedCompareExchange)
#pragma intrinsic (_InterlockedCompareExchange64)
#ifndef AO_PREFER_GENERALIZED
# pragma intrinsic (_InterlockedIncrement)
# pragma intrinsic (_InterlockedIncrement64)
# pragma intrinsic (_InterlockedDecrement)
# pragma intrinsic (_InterlockedDecrement64)
# pragma intrinsic (_InterlockedExchangeAdd)
# pragma intrinsic (_InterlockedExchangeAdd64)
AO_INLINE AO_t
AO_fetch_and_add_full (volatile AO_t *p, AO_t incr)
{
return _InterlockedExchangeAdd64((__int64 volatile *)p, incr);
}
#define AO_HAVE_fetch_and_add_full
AO_INLINE AO_t
AO_fetch_and_add1_full (volatile AO_t *p)
{
return _InterlockedIncrement64((__int64 volatile *)p) - 1;
}
#define AO_HAVE_fetch_and_add1_full
AO_INLINE AO_t
AO_fetch_and_sub1_full (volatile AO_t *p)
{
return _InterlockedDecrement64((__int64 volatile *)p) + 1;
}
#define AO_HAVE_fetch_and_sub1_full
#endif /* !AO_PREFER_GENERALIZED */
AO_INLINE AO_t
AO_fetch_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
AO_t new_val)
{
return (AO_t)_InterlockedCompareExchange64((__int64 volatile *)addr,
new_val, old_val);
}
#define AO_HAVE_fetch_compare_and_swap_full
AO_INLINE unsigned int
AO_int_fetch_compare_and_swap_full(volatile unsigned int *addr,
unsigned int old_val, unsigned int new_val)
{
return _InterlockedCompareExchange((long volatile *)addr, new_val, old_val);
}
#define AO_HAVE_int_fetch_compare_and_swap_full
#ifndef AO_PREFER_GENERALIZED
AO_INLINE unsigned int
AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
{
return _InterlockedExchangeAdd((long volatile *)p, incr);
}
#define AO_HAVE_int_fetch_and_add_full
AO_INLINE unsigned int
AO_int_fetch_and_add1_full(volatile unsigned int *p)
{
return _InterlockedIncrement((long volatile *)p) - 1;
}
# define AO_HAVE_int_fetch_and_add1_full
AO_INLINE unsigned int
AO_int_fetch_and_sub1_full(volatile unsigned int *p)
{
return _InterlockedDecrement((long volatile *)p) + 1;
}
# define AO_HAVE_int_fetch_and_sub1_full
#endif /* !AO_PREFER_GENERALIZED */
#if _MSC_VER > 1400
# pragma intrinsic (_InterlockedAnd8)
# pragma intrinsic (_InterlockedCompareExchange16)
# pragma intrinsic (_InterlockedOr8)
# pragma intrinsic (_InterlockedXor8)
AO_INLINE void
AO_char_and_full(volatile unsigned char *p, unsigned char value)
{
_InterlockedAnd8((char volatile *)p, value);
}
# define AO_HAVE_char_and_full
AO_INLINE void
AO_char_or_full(volatile unsigned char *p, unsigned char value)
{
_InterlockedOr8((char volatile *)p, value);
}
# define AO_HAVE_char_or_full
AO_INLINE void
AO_char_xor_full(volatile unsigned char *p, unsigned char value)
{
_InterlockedXor8((char volatile *)p, value);
}
# define AO_HAVE_char_xor_full
AO_INLINE unsigned short
AO_short_fetch_compare_and_swap_full(volatile unsigned short *addr,
unsigned short old_val,
unsigned short new_val)
{
return _InterlockedCompareExchange16((short volatile *)addr,
new_val, old_val);
}
# define AO_HAVE_short_fetch_compare_and_swap_full
# ifndef AO_PREFER_GENERALIZED
# pragma intrinsic (_InterlockedIncrement16)
# pragma intrinsic (_InterlockedDecrement16)
AO_INLINE unsigned short
AO_short_fetch_and_add1_full(volatile unsigned short *p)
{
return _InterlockedIncrement16((short volatile *)p) - 1;
}
# define AO_HAVE_short_fetch_and_add1_full
AO_INLINE unsigned short
AO_short_fetch_and_sub1_full(volatile unsigned short *p)
{
return _InterlockedDecrement16((short volatile *)p) + 1;
}
# define AO_HAVE_short_fetch_and_sub1_full
# endif /* !AO_PREFER_GENERALIZED */
#endif /* _MSC_VER > 1400 */
#if _MSC_VER >= 1800 /* Visual Studio 2013+ */
# pragma intrinsic (_InterlockedCompareExchange8)
AO_INLINE unsigned char
AO_char_fetch_compare_and_swap_full(volatile unsigned char *addr,
unsigned char old_val,
unsigned char new_val)
{
return _InterlockedCompareExchange8((char volatile *)addr,
new_val, old_val);
}
# define AO_HAVE_char_fetch_compare_and_swap_full
# ifndef AO_PREFER_GENERALIZED
# pragma intrinsic (_InterlockedExchangeAdd16)
# pragma intrinsic (_InterlockedExchangeAdd8)
AO_INLINE unsigned char
AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr)
{
return _InterlockedExchangeAdd8((char volatile *)p, incr);
}
# define AO_HAVE_char_fetch_and_add_full
AO_INLINE unsigned short
AO_short_fetch_and_add_full(volatile unsigned short *p,
unsigned short incr)
{
return _InterlockedExchangeAdd16((short volatile *)p, incr);
}
# define AO_HAVE_short_fetch_and_add_full
# endif /* !AO_PREFER_GENERALIZED */
#elif defined(AO_ASM_X64_AVAILABLE)
#ifdef AO_ASM_X64_AVAILABLE
#if _MSC_VER < 1800
AO_INLINE unsigned char
AO_char_fetch_and_add_full(volatile unsigned char *p, unsigned char incr)
{
@ -230,10 +66,7 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
}
}
# define AO_HAVE_short_fetch_and_add_full
#endif /* _MSC_VER < 1800 && AO_ASM_X64_AVAILABLE */
#ifdef AO_ASM_X64_AVAILABLE
#endif /* _MSC_VER < 1800 */
/* As far as we can tell, the lfence and sfence instructions are not */
/* currently needed or useful for cached memory accesses. */
@ -246,6 +79,9 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
}
# define AO_HAVE_nop_full
# ifndef AO_HAVE_test_and_set_full
# include "../test_and_set_t_is_char.h"
AO_INLINE AO_TS_VAL_t
AO_test_and_set_full(volatile AO_TS_t *addr)
{
@ -257,18 +93,19 @@ AO_int_fetch_and_add_full(volatile unsigned int *p, unsigned int incr)
}
}
# define AO_HAVE_test_and_set_full
# endif
#endif /* AO_ASM_X64_AVAILABLE */
#ifndef AO_HAVE_test_and_set_full
# include "../test_and_set_t_is_ao_t.h"
/* AO_test_and_set_full() is emulated using word-wide CAS. */
#endif
#ifdef AO_CMPXCHG16B_AVAILABLE
/* AO_compare_double_and_swap_double_full needs implementation for Win64.
* Also see ../gcc/x86.h for partial old Opteron workaround.
*/
# if _MSC_VER >= 1500
# include "../standard_ao_double_t.h"
# pragma intrinsic (_InterlockedCompareExchange128)
AO_INLINE int
@ -282,12 +119,13 @@ AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
comparandResult[0] = old_val1; /* low */
comparandResult[1] = old_val2; /* high */
return _InterlockedCompareExchange128((volatile __int64 *)addr,
new_val2 /* high */, new_val1 /* low */, comparandResult);
new_val2 /* high */,
new_val1 /* low */,
comparandResult);
}
# define AO_HAVE_compare_double_and_swap_double_full
# elif defined(AO_ASM_X64_AVAILABLE)
# include "../standard_ao_double_t.h"
/* If there is no intrinsic _InterlockedCompareExchange128 then we */

12
thirdparty/libatomic_ops/atomic_ops/sysdeps/standard_ao_double_t.h vendored
View file

@ -1,5 +1,6 @@
/*
* Copyright (c) 2004-2011 Hewlett-Packard Development Company, L.P.
* Copyright (c) 2012-2021 Ivan Maidanski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -22,13 +23,15 @@
/* For 64-bit systems, we expect the double type to hold two int64's. */
#if ((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \
|| defined(__aarch64__)) && !defined(__ILP32__)
#if (((defined(__x86_64__) && defined(AO_GCC_ATOMIC_TEST_AND_SET)) \
|| defined(__aarch64__)) && !defined(__ILP32__)) \
|| (defined(__e2k__) && __SIZEOF_SIZE_T__ == 8) \
|| (defined(__riscv) && __riscv_xlen == 64)
/* x86-64: __m128 is not applicable to atomic intrinsics. */
# if AO_GNUC_PREREQ(4, 7) || AO_CLANG_PREREQ(3, 6)
# pragma GCC diagnostic push
/* Suppress warning about __int128 type. */
# if defined(__clang__) || AO_GNUC_PREREQ(6, 4)
# if defined(__clang__) || AO_GNUC_PREREQ(6, 0)
# pragma GCC diagnostic ignored "-Wpedantic"
# else
/* GCC before ~4.8 does not accept "-Wpedantic" quietly. */
@ -39,6 +42,9 @@
# else /* pragma diagnostic is not supported */
typedef unsigned __int128 double_ptr_storage;
# endif
#elif defined(_M_ARM64) && defined(_MSC_VER)
/* __int128 does not seem to be available. */
typedef __declspec(align(16)) unsigned __int64 double_ptr_storage[2];
#elif ((defined(__x86_64__) && AO_GNUC_PREREQ(4, 0)) || defined(_WIN64)) \
&& !defined(__ILP32__)
/* x86-64 (except for x32): __m128 serves as a placeholder which also */

428
thirdparty/libatomic_ops/atomic_ops_malloc.c vendored Normal file
View file

@ -0,0 +1,428 @@
/*
* Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#if defined(HAVE_CONFIG_H)
# include "config.h"
#endif
#ifdef DONT_USE_MMAP /* for testing */
# undef HAVE_MMAP
#endif
#ifndef AO_BUILD
# define AO_BUILD
#endif
#define AO_REQUIRE_CAS
#include "atomic_ops_malloc.h"
#include <string.h> /* for ffs, which is assumed reentrant. */
#include <stdlib.h>
#include <assert.h>
#ifdef AO_TRACE_MALLOC
# include <stdio.h>
# include <pthread.h>
#endif
#if defined(AO_ADDRESS_SANITIZER) && !defined(AO_NO_MALLOC_POISON)
/* #include "sanitizer/asan_interface.h" */
void __asan_poison_memory_region(void *, size_t);
void __asan_unpoison_memory_region(void *, size_t);
# define ASAN_POISON_MEMORY_REGION(addr, size) \
__asan_poison_memory_region(addr, size)
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
__asan_unpoison_memory_region(addr, size)
#else
# define ASAN_POISON_MEMORY_REGION(addr, size) (void)0
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) (void)0
#endif /* !AO_ADDRESS_SANITIZER */
#if (defined(_WIN32_WCE) || defined(__MINGW32CE__)) && !defined(AO_HAVE_abort)
# define abort() _exit(-1) /* there is no abort() in WinCE */
#endif
/*
* We round up each allocation request to the next power of two
* minus one word.
* We keep one stack of free objects for each size. Each object has
* an initial word (offset from the visible pointer is -sizeof(AO_uintptr_t))
* which contains either:
* - the binary log of the object size in bytes (for small objects), or
* - the object size (a multiple of CHUNK_SIZE) for large objects.
* The second case only arises if mmap-based allocation is supported.
* We align the user-visible part of each object on an ALIGNMENT
* byte boundary. That means that the actual (hidden) start of
* the object starts a word before this boundary.
*/
#ifndef LOG_MAX_SIZE
# define LOG_MAX_SIZE 16
/* We assume that 2**LOG_MAX_SIZE is a multiple of page size. */
#endif
#ifndef ALIGNMENT
# define ALIGNMENT 16
/* Assumed to be at least sizeof(AO_uintptr_t). */
#endif
#define CHUNK_SIZE (1 << LOG_MAX_SIZE)
#ifndef AO_INITIAL_HEAP_SIZE
# ifndef AO_INITIAL_HEAP_CHUNKS
# define AO_INITIAL_HEAP_CHUNKS 2*(LOG_MAX_SIZE+1)
# endif
# define AO_INITIAL_HEAP_SIZE (AO_INITIAL_HEAP_CHUNKS * CHUNK_SIZE)
#endif /* !AO_INITIAL_HEAP_SIZE */
static char AO_initial_heap[AO_INITIAL_HEAP_SIZE]; /* ~2MB by default */
static AO_internal_ptr_t volatile initial_heap_ptr = 0;
#if defined(HAVE_MMAP)
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#if defined(MAP_ANONYMOUS) || defined(MAP_ANON)
# define USE_MMAP_ANON
#endif
#ifdef USE_MMAP_FIXED
# define GC_MMAP_FLAGS (MAP_FIXED | MAP_PRIVATE)
/* Seems to yield better performance on Solaris 2, but can */
/* be unreliable if something is already mapped at the address. */
#else
# define GC_MMAP_FLAGS MAP_PRIVATE
#endif
#ifdef USE_MMAP_ANON
# if defined(CPPCHECK)
# define OPT_MAP_ANON 0x20 /* taken from linux */
# elif defined(MAP_ANONYMOUS)
# define OPT_MAP_ANON MAP_ANONYMOUS
# else
# define OPT_MAP_ANON MAP_ANON
# endif
#else
# include <unistd.h> /* for close() */
# define OPT_MAP_ANON 0
#endif
static volatile AO_t mmap_enabled = 0;
AO_API void
AO_malloc_enable_mmap(void)
{
# if defined(__sun)
AO_store_release(&mmap_enabled, 1);
/* Workaround for Sun CC */
# else
AO_store(&mmap_enabled, 1);
# endif
}
static char *get_mmaped(size_t sz)
{
char * result;
# ifdef USE_MMAP_ANON
# define zero_fd -1
# else
int zero_fd;
# endif
assert(!(sz & (CHUNK_SIZE - 1)));
if (!mmap_enabled)
return 0;
# ifndef USE_MMAP_ANON
zero_fd = open("/dev/zero", O_RDONLY);
if (zero_fd == -1)
return 0;
# endif
result = (char *)mmap(0, sz, PROT_READ | PROT_WRITE,
GC_MMAP_FLAGS | OPT_MAP_ANON,
zero_fd, 0 /* offset */);
# ifndef USE_MMAP_ANON
close(zero_fd);
# endif
if (AO_EXPECT_FALSE(result == MAP_FAILED))
result = NULL;
return result;
}
#ifndef SIZE_MAX
# include <limits.h>
#endif
#if defined(SIZE_MAX) && !defined(CPPCHECK)
# define AO_SIZE_MAX ((size_t)SIZE_MAX)
/* Extra cast to workaround some buggy SIZE_MAX definitions. */
#else
# define AO_SIZE_MAX (~(size_t)0)
#endif
/* Saturated addition of size_t values. Used to avoid value wrap */
/* around on overflow. The arguments should have no side effects. */
#define SIZET_SAT_ADD(a, b) \
(AO_EXPECT_FALSE((a) >= AO_SIZE_MAX - (b)) ? AO_SIZE_MAX : (a) + (b))
/* Allocate an object of size (incl. header) of size > CHUNK_SIZE. */
/* sz includes space for a pointer-sized header. */
static char *
AO_malloc_large(size_t sz)
{
void *result;
/* The header will force us to waste ALIGNMENT bytes, including the */
/* header. Round to multiple of CHUNK_SIZE. */
sz = SIZET_SAT_ADD(sz, ALIGNMENT + CHUNK_SIZE - 1)
& ~(size_t)(CHUNK_SIZE - 1);
assert(sz > LOG_MAX_SIZE);
result = get_mmaped(sz);
if (AO_EXPECT_FALSE(NULL == result))
return NULL;
result = (AO_uintptr_t *)result + ALIGNMENT / sizeof(AO_uintptr_t);
((AO_uintptr_t *)result)[-1] = (AO_uintptr_t)sz;
return (char *)result;
}
static void
AO_free_large(void *p)
{
size_t sz = (size_t)(((AO_uintptr_t *)p)[-1]);
if (munmap((AO_uintptr_t *)p - ALIGNMENT / sizeof(AO_uintptr_t), sz) != 0)
abort(); /* Programmer error. Not really async-signal-safe, but ... */
}
#else /* !HAVE_MMAP */
AO_API void
AO_malloc_enable_mmap(void)
{
}
#define get_mmaped(sz) ((char*)0)
#define AO_malloc_large(sz) ((char*)0)
#define AO_free_large(p) abort()
/* Programmer error. Not really async-signal-safe, but ... */
#endif /* !HAVE_MMAP */
/* TODO: Duplicates (partially) the definitions in atomic_ops_stack.c. */
#if defined(AO_FAT_POINTER) || defined(AO_STACK_USE_CPTR)
AO_INLINE int
AO_cptr_compare_and_swap(AO_internal_ptr_t volatile *addr,
AO_internal_ptr_t old_val, AO_internal_ptr_t new_val)
{
return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
__ATOMIC_RELAXED, __ATOMIC_RELAXED);
}
AO_INLINE int
AO_cptr_compare_and_swap_acquire(AO_internal_ptr_t volatile *addr,
AO_internal_ptr_t old_val, AO_internal_ptr_t new_val)
{
return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
__ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
}
# define AO_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
#else
# define AO_cptr_compare_and_swap AO_compare_and_swap
# define AO_cptr_compare_and_swap_acquire AO_compare_and_swap_acquire
# define AO_cptr_load AO_load
#endif
static char *
get_chunk(void)
{
AO_internal_ptr_t my_chunk_ptr;
for (;;) {
AO_internal_ptr_t initial_ptr = AO_cptr_load(&initial_heap_ptr);
my_chunk_ptr = AO_EXPECT_FALSE(0 == initial_ptr) ?
(AO_internal_ptr_t)AO_initial_heap : initial_ptr;
/* Round up the pointer to ALIGNMENT. */
# ifdef AO_STACK_USE_CPTR
my_chunk_ptr += ((size_t)ALIGNMENT - (size_t)(AO_uintptr_t)my_chunk_ptr)
& (ALIGNMENT - 1);
# else
my_chunk_ptr = (AO_internal_ptr_t)(((AO_uintptr_t)my_chunk_ptr
+ ALIGNMENT-1)
& ~(AO_uintptr_t)(ALIGNMENT-1));
# endif
if (initial_ptr != my_chunk_ptr) {
/* Align correctly. If this fails, someone else did it for us. */
assert(my_chunk_ptr != 0);
(void)AO_cptr_compare_and_swap_acquire(&initial_heap_ptr, initial_ptr,
my_chunk_ptr);
}
if (AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr
< (AO_uintptr_t)AO_initial_heap)
|| AO_EXPECT_FALSE((AO_uintptr_t)my_chunk_ptr
> (AO_uintptr_t)(AO_initial_heap
+ AO_INITIAL_HEAP_SIZE - CHUNK_SIZE))) {
/* We failed. The initial heap is used up. */
my_chunk_ptr = (AO_internal_ptr_t)get_mmaped(CHUNK_SIZE);
# if !defined(CPPCHECK)
assert(((AO_uintptr_t)my_chunk_ptr & (ALIGNMENT - 1)) == 0);
# endif
break;
}
if (AO_cptr_compare_and_swap(&initial_heap_ptr, my_chunk_ptr,
my_chunk_ptr + CHUNK_SIZE)) {
break;
}
}
return (char *)my_chunk_ptr;
}
/* Object free lists. I-th entry corresponds to objects */
/* of total size 2**i bytes. */
static AO_stack_t AO_free_list[LOG_MAX_SIZE+1];
/* Break up the chunk, and add it to the object free list for */
/* the given size. We have exclusive access to chunk. */
static void add_chunk_as(void * chunk, unsigned log_sz)
{
size_t ofs, limit;
size_t sz = (size_t)1 << log_sz;
assert((size_t)CHUNK_SIZE >= sz);
assert(sz % sizeof(AO_uintptr_t) == 0);
limit = (size_t)CHUNK_SIZE - sz;
for (ofs = ALIGNMENT - sizeof(AO_uintptr_t); ofs <= limit; ofs += sz) {
ASAN_POISON_MEMORY_REGION((char *)chunk + ofs + sizeof(AO_uintptr_t),
sz - sizeof(AO_uintptr_t));
AO_stack_push(&AO_free_list[log_sz],
(AO_uintptr_t *)chunk + ofs / sizeof(AO_uintptr_t));
}
}
static const unsigned char msbs[16] = {
0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4
};
/* Return the position of the most significant set bit in the */
/* argument. */
/* We follow the conventions of ffs(), i.e. the least */
/* significant bit is number one. */
static unsigned msb(size_t s)
{
unsigned result = 0;
if ((s & 0xff) != s) {
# if (__SIZEOF_SIZE_T__ == 8) && !defined(CPPCHECK)
unsigned v = (unsigned)(s >> 32);
if (AO_EXPECT_FALSE(v != 0))
{
s = v;
result += 32;
}
# elif __SIZEOF_SIZE_T__ == 4
/* No op. */
# else
unsigned v;
/* The following is a tricky code ought to be equivalent to */
/* "(v = s >> 32) != 0" but suppresses warnings on 32-bit arch's. */
# define SIZEOF_SIZE_T_GT_4 (sizeof(size_t) > 4)
if (SIZEOF_SIZE_T_GT_4
&& (v = (unsigned)(s >> (SIZEOF_SIZE_T_GT_4 ? 32 : 0))) != 0)
{
s = v;
result += 32;
}
# endif /* !defined(__SIZEOF_SIZE_T__) */
if (AO_EXPECT_FALSE((s >> 16) != 0))
{
s >>= 16;
result += 16;
}
if ((s >> 8) != 0)
{
s >>= 8;
result += 8;
}
}
if (s > 15)
{
s >>= 4;
result += 4;
}
result += msbs[s];
return result;
}
AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1)
void *
AO_malloc(size_t sz)
{
AO_uintptr_t *result;
unsigned log_sz;
if (AO_EXPECT_FALSE(sz > CHUNK_SIZE - sizeof(AO_uintptr_t)))
return AO_malloc_large(sz);
log_sz = msb(sz + sizeof(AO_uintptr_t) - 1);
assert(log_sz <= LOG_MAX_SIZE);
assert(((size_t)1 << log_sz) >= sz + sizeof(AO_uintptr_t));
result = AO_stack_pop(AO_free_list + log_sz);
while (AO_EXPECT_FALSE(NULL == result)) {
void *chunk = get_chunk();
if (AO_EXPECT_FALSE(NULL == chunk))
return NULL;
add_chunk_as(chunk, log_sz);
result = AO_stack_pop(AO_free_list + log_sz);
}
*result = log_sz;
# ifdef AO_TRACE_MALLOC
fprintf(stderr, "%p: AO_malloc(%lu) = %p\n",
(void *)pthread_self(), (unsigned long)sz, (void *)(result + 1));
# endif
ASAN_UNPOISON_MEMORY_REGION(result + 1, sz);
return result + 1;
}
AO_API void
AO_free(void *p)
{
AO_uintptr_t *base;
int log_sz;
if (AO_EXPECT_FALSE(NULL == p))
return;
base = (AO_uintptr_t *)p - 1;
log_sz = (int)(*base);
# ifdef AO_TRACE_MALLOC
fprintf(stderr, "%p: AO_free(%p sz:%lu)\n", (void *)pthread_self(), p,
log_sz > LOG_MAX_SIZE ? (unsigned)log_sz : 1UL << log_sz);
# endif
if (AO_EXPECT_FALSE(log_sz > LOG_MAX_SIZE)) {
AO_free_large(p);
} else {
ASAN_POISON_MEMORY_REGION(base + 1,
((size_t)1 << log_sz) - sizeof(AO_uintptr_t));
AO_stack_push(AO_free_list + log_sz, base);
}
}

81
thirdparty/libatomic_ops/atomic_ops_malloc.h vendored Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright (c) 2005 Hewlett-Packard Development Company, L.P.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* Almost lock-free malloc implementation based on stack implementation. */
/* See README_malloc.txt file for detailed usage rules. */
#ifndef AO_MALLOC_H
#define AO_MALLOC_H
#include "atomic_ops_stack.h"
#include <stddef.h> /* for size_t */
#ifdef __cplusplus
extern "C" {
#endif
#ifdef AO_STACK_IS_LOCK_FREE
# define AO_MALLOC_IS_LOCK_FREE
#endif
#ifndef AO_ATTR_MALLOC
# if AO_GNUC_PREREQ(3, 1)
# define AO_ATTR_MALLOC __attribute__((__malloc__))
# elif defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(__EDG__)
# define AO_ATTR_MALLOC \
__declspec(allocator) __declspec(noalias) __declspec(restrict)
# elif defined(_MSC_VER) && _MSC_VER >= 1400
# define AO_ATTR_MALLOC __declspec(noalias) __declspec(restrict)
# else
# define AO_ATTR_MALLOC /* empty */
# endif
#endif
#ifndef AO_ATTR_ALLOC_SIZE
# ifdef __clang__
# if __has_attribute(__alloc_size__)
# define AO_ATTR_ALLOC_SIZE(argnum) \
__attribute__((__alloc_size__(argnum)))
# else
# define AO_ATTR_ALLOC_SIZE(argnum) /* empty */
# endif
# elif AO_GNUC_PREREQ(4, 3) && !defined(__ICC)
# define AO_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum)))
# else
# define AO_ATTR_ALLOC_SIZE(argnum) /* empty */
# endif
#endif
AO_API void AO_free(void *);
AO_API AO_ATTR_MALLOC AO_ATTR_ALLOC_SIZE(1)
void * AO_malloc(size_t);
/* Allow use of mmap to grow the heap. No-op on some platforms. */
AO_API void AO_malloc_enable_mmap(void);
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* !AO_MALLOC_H */

9
thirdparty/libatomic_ops/atomic_ops_sysdeps.S vendored Normal file
View file

@ -0,0 +1,9 @@
/*
* Include the appropriate system-dependent assembly file, if any.
* This is used only if the platform supports neither inline assembly
* code, nor appropriate compiler intrinsics.
*/
#if !defined(__GNUC__) && (defined(sparc) || defined(__sparc))
# include "atomic_ops/sysdeps/sunc/sparc.S"
#endif